zerdovzad commited on
Commit
c8ba9c6
·
verified ·
1 Parent(s): c877dbf

Upload 139 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +25 -0
  2. DATA_NOTICE.md +14 -0
  3. PACKAGE_MANIFEST.json +42 -0
  4. README.md +280 -1
  5. README_UK.md +170 -0
  6. SOURCE_PACKAGE_NOTES.md +29 -0
  7. VALIDATION.json +19 -0
  8. assets/warpstate-architecture.svg +48 -0
  9. assets/warpstate-banner.svg +29 -0
  10. chat.py +341 -0
  11. dpo.py +647 -0
  12. ops/B200_HANDOFF.md +214 -0
  13. ops/B200_OPTIMIZATION_PLAN.md +120 -0
  14. ops/B200_PERFORMANCE.md +231 -0
  15. ops/B200_SCHEDULING.md +83 -0
  16. ops/B200_SETUP.md +176 -0
  17. ops/BACKWARD_LAYOUT_EXPERIMENT.md +204 -0
  18. ops/BACKWARD_LAYOUT_INTEGRATION.md +307 -0
  19. ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md +147 -0
  20. ops/CHAT.md +134 -0
  21. ops/COMMUNICATION_EXPLORATION_20260912.md +254 -0
  22. ops/DATA_PREPARATION.md +78 -0
  23. ops/DPO_CHAT.md +57 -0
  24. ops/DPO_HANDOFF.md +144 -0
  25. ops/LOCAL_RESUME.md +207 -0
  26. ops/LOSS_FORECAST_20260913.md +208 -0
  27. ops/MATMUL_PROFILE_20260912.md +316 -0
  28. ops/MATMUL_TUNING.md +342 -0
  29. ops/OPTIMIZATION_20260912.md +349 -0
  30. ops/OPTIMIZED_TRAINING.md +228 -0
  31. ops/PERFORMANCE.md +236 -0
  32. ops/POINTWISE_HEADROOM_20260913.md +195 -0
  33. ops/POSTTRAIN.md +165 -0
  34. ops/RESTART_HANDOFF.md +97 -0
  35. ops/RTX_ASYNC_TRAINING.md +114 -0
  36. ops/RTX_PROFILE_20260910.md +453 -0
  37. ops/TRAINING.md +148 -0
  38. ops/activation_execution.py +253 -0
  39. ops/analyze_communication_profile.py +227 -0
  40. ops/archive_dpo.py +302 -0
  41. ops/b200-requirements.txt +2 -0
  42. ops/b200-supervisord.conf +35 -0
  43. ops/b200ctl.sh +5 -0
  44. ops/backward_layout.py +283 -0
  45. ops/backward_layout_baseline.py +201 -0
  46. ops/backward_layout_benchmark.py +164 -0
  47. ops/backward_layout_execution.py +355 -0
  48. ops/benchmark-supervisord.conf +16 -0
  49. ops/benchmark.sh +13 -0
  50. ops/benchmark_gemm.py +267 -0
.gitignore ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ .mypy_cache/
6
+ .ruff_cache/
7
+ .venv/
8
+ .server-venv/
9
+ venv/
10
+
11
+ # Model / training outputs
12
+ work/
13
+ checkpoints/
14
+ *.pt
15
+ *.pth
16
+ *.ckpt
17
+ *.safetensors
18
+ *.bin
19
+ *.log
20
+ *.pid
21
+
22
+ # Secrets and local environment
23
+ .env
24
+ .env.*
25
+ !.env.example
DATA_NOTICE.md ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Dataset attribution
2
+
3
+ Default upstream dataset: **FineWeb-Edu**, Hugging Face / HuggingFaceFW.
4
+ Repository: https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu
5
+ Configuration: sample-100BT. Language: English. Dataset card lists ODC-BY.
6
+ Read the upstream dataset card and license for applicable attribution and
7
+ underlying web-content terms. This ZIP distributes no dataset texts.
8
+ The immutable source revision is recorded in work/data/manifest.json at preparation.
9
+
10
+ The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale:
11
+ https://arxiv.org/abs/2406.17557
12
+
13
+ The WarpState code and tokenizer are copied from the user's existing project.
14
+ No new ownership or license over those user-provided assets is asserted here.
PACKAGE_MANIFEST.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_40b.py": {
3
+ "sha256": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
4
+ "bytes": 50988
5
+ },
6
+ "README_UK.md": {
7
+ "sha256": "b401dc313e25aea48ab1a0bd24945a9923cd98adcb254c77cb1e130dd16c1335",
8
+ "bytes": 13562
9
+ },
10
+ "DATA_NOTICE.md": {
11
+ "sha256": "6b11284648bcb6fe9d1c74204d54e64d228b05d78749330c5caab119806b2a79",
12
+ "bytes": 735
13
+ },
14
+ "test_server.py": {
15
+ "sha256": "66ebfd733f8d71a462108c5d99dc469a99d1023bb7a0cd611b501a58b9269bff",
16
+ "bytes": 6351
17
+ },
18
+ "tokenizer.model": {
19
+ "sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
20
+ "bytes": 545080
21
+ },
22
+ "configs/warpstate_2b.json": {
23
+ "sha256": "884ba9e523af2a065ff0e11a4fd86eddfdc5ef656196dc318f94ec455be63009",
24
+ "bytes": 269
25
+ },
26
+ "warpstate/__init__.py": {
27
+ "sha256": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
28
+ "bytes": 113
29
+ },
30
+ "warpstate/config.py": {
31
+ "sha256": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
32
+ "bytes": 1187
33
+ },
34
+ "warpstate/model.py": {
35
+ "sha256": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161",
36
+ "bytes": 15500
37
+ },
38
+ "VALIDATION.json": {
39
+ "sha256": "168bb8233b04db144021f185b6a3890c06c137c3c5c29c1f18464ee8474ce1b8",
40
+ "bytes": 614
41
+ }
42
+ }
README.md CHANGED
@@ -1,3 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
- license: mit
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <div align="center">
2
+ <img src="assets/warpstate-banner.svg" alt="WarpState — memory-native language modeling" width="100%" />
3
+
4
+ <h1>⚡ WarpState</h1>
5
+ <p><strong>A memory-native language model built around local attention, fast/slow temporal states, and depth-wise computation.</strong></p>
6
+ <p><em>Experimental architecture · Original 2.005B-parameter research run · Pretraining → SFT → DPO</em></p>
7
+
8
+ <p>
9
+ <img alt="Research" src="https://img.shields.io/badge/status-research%20prototype-8B5CF6?style=for-the-badge" />
10
+ <img alt="Documented model size" src="https://img.shields.io/badge/model-2.005B%20parameters-22D3EE?style=for-the-badge" />
11
+ <img alt="Tokens trained" src="https://img.shields.io/badge/pretraining-10B%20tokens-0EA5E9?style=for-the-badge" />
12
+ <img alt="PyTorch" src="https://img.shields.io/badge/framework-PyTorch-EE4C2C?style=for-the-badge&logo=pytorch&logoColor=white" />
13
+ </p>
14
+ <p><strong>Created by <a href="https://github.com/gtausa197-svg">Volodymyr Makarenko</a> · Co-authored &amp; sponsored by <a href="https://github.com/ykim078">@ykim078</a></strong></p>
15
+ <p>
16
+ <a href="#-what-is-warpstate">Overview</a> ·
17
+ <a href="#-architecture">Architecture</a> ·
18
+ <a href="#-training--evaluation">Results</a> ·
19
+ <a href="#-getting-started">Quick start</a> ·
20
+ <a href="#-project-leadership--sponsorship">Team &amp; sponsor</a>
21
+ </p>
22
+ </div>
23
+
24
  ---
25
+
26
+ ## 🧠 What is WarpState?
27
+
28
+ **WarpState** is an experimental autoregressive language model exploring a different way to balance *local precision* and *long-range continuity*. Instead of building a full sequence-wide attention matrix at every layer, it combines:
29
+
30
+ - **Causal local attention** within fixed **128-token chunks**.
31
+ - **Two decaying temporal memory states** (fast and slow) that carry compressed information from completed chunks.
32
+ - **Learned gating** that blends locally attended features with historical memory.
33
+ - **Depth-wise computation** through reusable model cores with depth-specific modulation.
34
+ - A full language-model training stack, including **BF16 computation / FP32 optimizer state**, checkpoint-resumable pretraining, supervised fine-tuning (SFT), and direct preference optimization (DPO).
35
+
36
+ The architecture is implemented in [`warpstate/model.py`](warpstate/model.py) and [`warpstate/config.py`](warpstate/config.py). **WarpState is not Project Nord's SNN architecture**: the supplied Project Nord README was used as a *presentation reference*, not as a source of WarpState features or benchmark claims.
37
+
38
+ > [!IMPORTANT]
39
+ > **Release contents:** The supplied source archives include Python code, the SentencePiece tokenizer, tests, and operational documentation, **but not pretrained weights, training data, or the `configs/` directory** referenced by several launchers. This README documents the historical research run described in the included records; the uploaded source package alone is **not** a one-command pretrained-model download. A complete runnable release must include the correct configuration files and access to trusted checkpoints.
40
+
41
+ ## ✨ At a glance
42
+
43
+ | Research dimension | Documented state |
44
+ | :--- | :--- |
45
+ | **Model** | 2,005,111,424 parameters in the recorded full-size training run |
46
+ | **Tokenizer** | SentencePiece, 32,768-token vocabulary |
47
+ | **Training sequence** | 2,048 tokens; not equivalent to an exact 2,048-token persistent memory |
48
+ | **Local attention** | Causal SDPA, 128 tokens per chunk |
49
+ | **Persistent state** | Fast + slow fixed-size matrix state per attention head and core execution |
50
+ | **Training corpus prepared** | 40,000,000,000 train tokens + 1,048,576 validation tokens |
51
+ | **Pretraining completed** | 10,000,007,168 tokens, at step 76,294, per the September 2026 handoff |
52
+ | **Post-training** | 10,312 SFT updates; 2,334 DPO updates, reported complete |
53
+ | **Compute format** | BF16 forward/backward; FP32 parameters, gradients, and AdamW states |
54
+ | **Project stage** | Research implementation with historical training logs and tests; no independently verified general-purpose LLM benchmark in this package |
55
+
56
+ <sub>Numbers are sourced from the included source files and operational handoff documents, primarily `ops/RESTART_HANDOFF.md`, `ops/TRAINING.md`, `ops/DPO_HANDOFF.md`, and `ops/DPO_CHAT.md`. Training outcomes are historical records, not fresh measurements performed for this README.</sub>
57
+
58
  ---
59
+
60
+ ## 🧬 Architecture
61
+
62
+ <div align="center">
63
+ <img src="assets/warpstate-architecture.svg" alt="WarpState: token input, local causal attention, fast and slow memory, gated fusion, repeated cores and language modeling head" width="100%" />
64
+ </div>
65
+
66
+ ### How one WarpState core works
67
+
68
+ 1. **Normalize and project.** RMSNorm feeds one fused projection into queries, keys, values, a gate, and memory-write features.
69
+ 2. **Attend locally.** Rotary position embeddings and causal scaled dot-product attention operate within each 128-token tile.
70
+ 3. **Read compressed history.** Queries read from two memory matrices containing information written by *previously completed* tiles.
71
+ 4. **Fuse the paths.** A learned sigmoid gate mixes local attention and fast/slow historical memory.
72
+ 5. **Update memory.** Bounded, outer-product-like writes update the fast/slow states with separate learned decay coefficients.
73
+ 6. **Refine the representation.** Residual projection, RMSNorm, and SwiGLU complete the block; the language model reuses cores across logical depth passes.
74
+
75
+ ### Memory, not magic
76
+
77
+ The streaming path retains compact fast/slow state instead of an ever-growing full-attention key/value history. That does **not** imply infinite context, lossless recall, perfect retrieval, or constant runtime in every implementation mode. The optional parallel-memory path materializes tile-level interactions during training, and end-to-end cost depends on the kernel and execution configuration.
78
+
79
+ ### Implementation highlights
80
+
81
+ | Module | What it does | Source |
82
+ | :--- | :--- | :--- |
83
+ | **WarpStateCore** | Local attention + two-timescale associative memory | [`warpstate/model.py`](warpstate/model.py) |
84
+ | **WarpStateLM** | Token embeddings, depth modulation, shared cores, tied output head | [`warpstate/model.py`](warpstate/model.py) |
85
+ | **Training runner** | Streaming data preparation, AdamW, validation, checkpoint/resume | [`run_40b.py`](run_40b.py) |
86
+ | **Distributed and kernel paths** | DDP, execution optimizations, matmul and backward experiments | [`ops/`](ops/) |
87
+ | **SFT** | Assistant-token-masked supervised fine-tuning | [`posttrain.py`](posttrain.py) |
88
+ | **DPO** | Preference optimization with a fixed reference model | [`dpo.py`](dpo.py) |
89
+ | **Inference probes** | CPU base-model chat and archived DPO inference tooling | [`chat.py`](chat.py), [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) |
90
+
91
+ ---
92
+
93
+ ## 📈 Training & evaluation
94
+
95
+ ### Documented experimental milestones
96
+
97
+ | Phase | Recorded output | Interpretation |
98
+ | :--- | :--- | :--- |
99
+ | **Raw pretraining** | **10,000,007,168 tokens** · **76,294 updates** · validation loss **2.3497** | Pretraining token-level cross-entropy; **not** a claim that all 40B prepared tokens were trained |
100
+ | **SFT** | **10,312 updates** · **329,973 examples** · validation loss **1.1157** | Assistant-token-masked fine-tuning; its loss uses a **different objective** |
101
+ | **DPO** | **2,334 updates** · **74,661 preference pairs** · held-out DPO loss **0.5790** | Preference objective; **not** model perplexity or accuracy |
102
+
103
+ The included DPO record also reports positive relative reward margins for **245 / 339 held-out pairs (72.27%)**. This is a *margin statistic*, **not** an end-user generation win rate. Source: [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md).
104
+
105
+ ### Selected historical throughput observations
106
+
107
+ | Setup | Recorded observation | Scope / caveat |
108
+ | :--- | :--- | :--- |
109
+ | 2 × RTX PRO 6000 Blackwell Max-Q | **31,199 tokens/s** | September 13 analysis of a repeated 20-update, non-profiled configuration; excludes validation/checkpoint pauses |
110
+ | 1 × NVIDIA B200 | **20,752 sustained tokens/s**; **72,499 median tokens/s** | Specific 20-update autotuned benchmark with provider scheduling waits; excludes initialization, validation and saves |
111
+
112
+ These measurements describe **training throughput on the named server GPUs**, not inference speed, consumer GPU performance, or a reproducible speed promise. See [`ops/POINTWISE_HEADROOM_20260913.md`](ops/POINTWISE_HEADROOM_20260913.md) and [`ops/B200_PERFORMANCE.md`](ops/B200_PERFORMANCE.md) for test conditions and the discrepancy between median and sustained rates.
113
+
114
+ **What remains to be demonstrated publicly:** standardized language tasks, instruction following, multilingual quality, reasoning, independent reproducibility, and memory-retention tests. A low training loss alone cannot establish any of these capabilities.
115
+
116
+ ---
117
+
118
+ ## 🚀 Getting started
119
+
120
+ ### 1. Assemble the source tree
121
+
122
+ The distributed archives contain complementary portions of the same project. Combine their contents under **one project root**, preserving `warpstate/`, `ops/`, root Python files, and `tokenizer.model`:
123
+
124
+ ```text
125
+ WarpState/
126
+ ├── README.md # This document
127
+ ├── assets/ # README visuals
128
+ ├── warpstate/
129
+ │ ├── __init__.py
130
+ │ ├── config.py
131
+ │ └── model.py
132
+ ├── ops/ # Training, optimization, SFT/DPO tooling
133
+ ├── configs/ # REQUIRED for full training; not in supplied ZIPs
134
+ ├── run_40b.py # Pretraining entry point
135
+ ├── posttrain.py # SFT runner
136
+ ├── dpo.py # DPO runner
137
+ ├── chat.py # CPU base-model probe
138
+ ├── tokenizer.model
139
+ ├── README_UK.md # Existing Ukrainian server guide
140
+ └── test_*.py # Test suite
141
+ ```
142
+
143
+ > [!WARNING]
144
+ > The 2.005B training configuration and the SFT/DPO plans under `configs/` were **not present** in the supplied archives. `WarpStateConfig()` has smaller development defaults and does **not** instantiate the documented 2.005B run. Restore the original configuration/plan files from your trusted project copy before attempting to reproduce the historical results.
145
+
146
+ ### 2. Set up a Python environment
147
+
148
+ Use **Linux, Python 3.12, a supported NVIDIA GPU and a CUDA-compatible PyTorch build** for the documented full training workflow. The original server setup used high-memory RTX PRO 6000 or B200-class hardware; the 2B full-optimizer recipe is **not** an 8 GB VRAM training preset.
149
+
150
+ ```bash
151
+ cd WarpState
152
+ python3.12 -m venv .venv
153
+ source .venv/bin/activate
154
+
155
+ # Select a PyTorch/CUDA build compatible with YOUR GPU and driver:
156
+ # https://pytorch.org/get-started/locally/
157
+ python -m pip install torch sentencepiece numpy
158
+ ```
159
+
160
+ The historical automatic bootstrap is implemented in `run_40b.py`. Detailed, pinned deployment requirements are in `ops/training-requirements-resolved.txt`, `ops/B200_SETUP.md`, and `README_UK.md`; package resolution depends on your actual host.
161
+
162
+ ### 3. Inspect the architecture without a checkpoint
163
+
164
+ ```python
165
+ from warpstate import WarpStateConfig, WarpStateLM
166
+
167
+ config = WarpStateConfig() # Development-sized defaults; NOT the historical 2.005B config
168
+ model = WarpStateLM(config)
169
+ print(f"Parameters: {model.num_parameters():,}")
170
+ ```
171
+
172
+ This inspects randomly initialized code only. It does **not** provide a trained model.
173
+
174
+ ### 4. Run a trusted base-model checkpoint
175
+
176
+ With the original compatible tokenizer and full `warpstate-server-v1` checkpoint available:
177
+
178
+ ```bash
179
+ python3.12 chat.py \
180
+ --checkpoint /path/to/trusted/latest.pt \
181
+ --tokenizer ./tokenizer.model \
182
+ --prompt "Explain how memory works." \
183
+ --temperature 0
184
+ ```
185
+
186
+ `chat.py` is a **CPU FP32 base-pretraining probe**. The DPO-tuned assistant uses a **different interface and archived snapshot format**; follow [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) and [`ops/RESTART_HANDOFF.md`](ops/RESTART_HANDOFF.md) on a host containing the original archives. Some operational scripts embed the historical `/workspace/WarpState/rtx6000_40b` layout and require path adaptation elsewhere.
187
+
188
+ ### 5. Start or resume training (advanced)
189
+
190
+ Only after restoring the **matching** 2.005B config, tokenizer, trusted weights or explicit from-scratch initialization, GPU environment, disk space and dataset permissions:
191
+
192
+ ```bash
193
+ # Example for a NEW run initialized from an existing trusted WarpState checkpoint
194
+ python3.12 run_40b.py \
195
+ --weights /path/to/trusted/latest.pt \
196
+ --config ./configs/warpstate_2b.json \
197
+ --work-dir /path/to/warpstate-work
198
+ ```
199
+
200
+ Alternatively, `--from-scratch` requests random initialization; it is **not** a way to load the trained 2B model. By default, the runner prepares **40B train tokens (~80 GB of `uint16` token data)** before full training. Even `--max-steps 10` does **not** skip full corpus preparation. Preserve checkpoint, tokenizer and data provenance when resuming. Never load untrusted `.pt`/pickle files.
201
+
202
+ Read [`README_UK.md`](README_UK.md), [`ops/TRAINING.md`](ops/TRAINING.md), and [`ops/LOCAL_RESUME.md`](ops/LOCAL_RESUME.md) before using production launchers.
203
+
204
+ ---
205
+
206
+ ## 🔍 Reproducibility & engineering
207
+
208
+ - **Deterministic corpus recipe:** FineWeb-Edu streaming, SentencePiece token counting, train/holdout split, provenance and SHA-256 manifests (see [`README_UK.md`](README_UK.md)).
209
+ - **Recoverable checkpoints:** complete FP32 model and AdamW state, RNG state, training cursor, run signature and guarded resume.
210
+ - **Precision discipline:** BF16 compute with FP32 master weights and optimizer; kernel changes require numerical comparisons.
211
+ - **Multiple execution modes:** compiled cores, gradient/communication overlap experiments, distributed training, asynchronous checkpoint writing and optional kernel tuning.
212
+ - **Testing:** unit and integration tests for training, checkpointing, transfer, post-training and distributed execution are included. Historical validation reports are not a substitute for rerunning them on a new host.
213
+
214
+ The repository also contains dated operational documents. **They describe specific machines and historical run states**, not universally safe commands to execute automatically on a new machine.
215
+
216
+ ### Data and model safety
217
+
218
+ The original base data used [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu). Respect upstream dataset licenses and documentation, keep private checkpoints/logs out of Git, and treat PyTorch pickle checkpoints as trusted-code inputs. No datasets or trained weights are embedded in these README assets.
219
+
220
+ ---
221
+
222
+ ## 🗺️ Roadmap
223
+
224
+ - [x] Implement chunked causal attention with fast/slow temporal memory.
225
+ - [x] Implement depth-modulated reusable cores and tied language-model head.
226
+ - [x] Build resumable pretraining and GPU execution/optimization tooling.
227
+ - [x] Record a 10B-token pretraining milestone for the 2.005B experiment.
228
+ - [x] Record completed SFT and DPO stages in historical run documentation.
229
+ - [ ] Publish complete, reproducible configs and a verified release manifest.
230
+ - [ ] Publish licensed, integrity-checked downloadable checkpoints (if distribution is authorized).
231
+ - [ ] Add standardized benchmarks, memory-retention evaluations and representative generations.
232
+ - [ ] Consolidate deployment scripts and add CI for supported hardware paths.
233
+ - [ ] Document scaling, memory behavior and real inference performance across devices.
234
+
235
+ ---
236
+
237
+ ## 🤝 Project leadership & sponsorship
238
+
239
+ <table>
240
+ <tr>
241
+ <td align="center" width="50%">
242
+ <strong>🧠 Project creator &amp; lead developer</strong><br/>
243
+ <a href="https://github.com/gtausa197-svg"><strong>Volodymyr Makarenko</strong></a><br/>
244
+ <sub>Core research direction, architecture and development</sub>
245
+ </td>
246
+ <td align="center" width="50%">
247
+ <strong>💜 Project sponsor &amp; co-author</strong><br/>
248
+ <a href="https://github.com/ykim078"><strong>@ykim078</strong></a><br/>
249
+ <sub>Recognized as both sponsor and co-author of WarpState</sub>
250
+ </td>
251
+ </tr>
252
+ </table>
253
+
254
+ **Special thanks to [@ykim078](https://github.com/ykim078)** for supporting the project and participating as a **co-author**. Sponsorship and co-authorship are distinct roles; both are intentionally credited here at the project owner's request. Specific financial terms and individual code contributions are not claimed.
255
+
256
+ Contributions are welcome through issues and pull requests once the repository is published with a contribution policy and clear license.
257
+
258
+ ## 📚 Documentation
259
+
260
+ | Guide | Purpose |
261
+ | :--- | :--- |
262
+ | [`README_UK.md`](README_UK.md) | Ukrainian source-package and GPU training guide |
263
+ | [`ops/RESTART_HANDOFF.md`](ops/RESTART_HANDOFF.md) | Recorded completed raw / SFT / DPO run and archive state |
264
+ | [`ops/B200_PERFORMANCE.md`](ops/B200_PERFORMANCE.md) | B200 benchmark conditions and results |
265
+ | [`ops/OPTIMIZED_TRAINING.md`](ops/OPTIMIZED_TRAINING.md) | Dual RTX PRO 6000 training measurements |
266
+ | [`ops/POSTTRAIN.md`](ops/POSTTRAIN.md) | SFT execution and data provenance |
267
+ | [`ops/DPO_HANDOFF.md`](ops/DPO_HANDOFF.md) | Preference optimization design |
268
+ | [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) | Post-DPO inference instructions and caveats |
269
+
270
+ ## 📄 License & citation
271
+
272
+ **License:** No root `LICENSE` file was included in the supplied WarpState archives. **Do not assume an open-source license or reuse permission** until the repository owner publishes an explicit license. The Project Nord reference's Apache-2.0 label **does not automatically apply** to WarpState.
273
+
274
+ If citing the project, link to the **actual WarpState repository URL** once published; do not cite the unrelated Nord repository as its code source. Credit the creator and co-author as shown above.
275
+
276
+ ---
277
+
278
+ <div align="center">
279
+ <strong>⚡ WarpState</strong><br/>
280
+ <em>Local attention for immediate detail. Temporal memory for continuity.</em><br/><br/>
281
+ <sub>Built as an experimental research system. Documented results, explicit limitations, reproducible engineering.</sub>
282
+ </div>
README_UK.md ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WarpState 2B: один запуск на RTX PRO 6000 Blackwell 96 ГБ
2
+
3
+ Архів містить код вашої WarpState, точний наявний токенізатор і єдиний
4
+ керівний скрипт `run_40b.py`. Ваг моделі та 80 ГБ даних в архіві немає.
5
+ Ваги беруться з вашого сервера, дані готуються на сервері автоматично.
6
+
7
+ ## Запуск
8
+
9
+ Потрібні Linux x86_64, **Python 3.12–3.14 з модулем venv**, робочий NVIDIA
10
+ драйвер, сумісний із CUDA 13.0, одна RTX PRO 6000 Blackwell 96 ГБ,
11
+ інтернет і рекомендовано **250 ГБ вільного SSD/NVMe**. Бажано 128 ГБ RAM;
12
+ фактичні потреби залежать від завантаження checkpoint і підготовки даних.
13
+ Скрипт не встановлює системний драйвер і не змінює інші Python-середовища.
14
+
15
+ Розпакуйте ZIP, перейдіть у його папку та підставте справжній шлях до ваг:
16
+
17
+ ```bash
18
+ cd rtx6000_40b
19
+ python3.12 run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b
20
+ ```
21
+
22
+ Ця **одна команда**:
23
+
24
+ 1. Створює `.server-venv` та встановлює CUDA PyTorch і залежності.
25
+ 2. Перевіряє GPU, формат/архітектуру ваг, розмір словника й доступне місце.
26
+ 3. Потоково читає FineWeb-Edu та готує **рівно 40 000 000 000 train-токенів**
27
+ вашим токенізатором плюс 1 048 576 validation-токенів.
28
+ 4. Запускає повне навчання всіх параметрів із GPU AdamW та BF16-обчисленнями.
29
+ 5. Пише метрики, validation і checkpoint з можливістю відновлення.
30
+
31
+ Для запуску з від’єднанням від SSH використовуйте вже встановлений tmux
32
+ або стандартний nohup. Наприклад:
33
+
34
+ ```bash
35
+ nohup python3.12 -u run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b > server.log 2>&1 &
36
+ echo $! > server.pid
37
+ tail -f server.log
38
+ ```
39
+
40
+ Завершення через Ctrl+C у передньому плані або `kill -TERM <PID>` для процесу
41
+ скрипта зберігає прогрес. Дочекайтеся `Saved ...latest.pt` та виходу процесу.
42
+ У підготовці корпусу зберігається позиція читання й записані токени.
43
+ Не використовуйте `kill -9`, коли потрібні останні незбережені оновлення.
44
+ Повторна **та сама команда** відновлює запуск. Одночасний запуск у тому
45
+ самому `--work-dir` захищено блокуванням.
46
+
47
+ ## Які ваги підходять
48
+
49
+ Це код **вашої WarpState 2B**, а не універсальний тренер для Llama/Qwen.
50
+ Очікується ваш PyTorch `latest.pt` з полями `model` і `config`.
51
+ Для старого CPU-offload checkpoint використовуються точні FP32 master-ваги,
52
+ якщо вони є. Токенізатор в архіві — той самий `tokenizer.model`, що у проєкті.
53
+ Не замінюйте його на інший зі схожим розміром словника: перевірка розміру
54
+ не доводить однакової відповідності ID словам.
55
+
56
+ Перше завантаження через `--weights` — **початок нового 40B-запуску з ваших
57
+ ваг**: старий 300M-розклад і старі моменти AdamW не переносяться.
58
+ Записується походження ваг і попередній лічильник токенів.
59
+ У наступних перезапусках `work/run/latest.pt` відновлюються ваги,
60
+ AdamW, RNG, позиція корпусу, лічильник та розклад нового запуску.
61
+ Оригінальний файл `--weights` не перезаписується.
62
+
63
+ ## Дані та очікування якості
64
+
65
+ Джерело: [HuggingFaceFW/fineweb-edu, sample-100BT](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu).
66
+ Це відфільтрований **англомовний** корпус для попереднього навчання.
67
+ Число токенів у його назві пораховане іншим токенізатором; скрипт сам
68
+ відміряє 40 млрд саме вашим SentencePiece. Ці 40 млрд — додаткові токени
69
+ нового запуску, незалежно від кількості токенів у початкових вагах.
70
+
71
+ У кож��н документ додається EOS. Незмінний SHA256 тексту визначає train
72
+ або holdout; усі holdout-документи виключаються з train навіть після
73
+ заповнення validation. Повтор того самого тексту потрапляє в той самий
74
+ розділ; це не додаткова глобальна дедуплікація майже однакових документів.
75
+ Ревізія джерела фіксується як commit SHA. Підготовка відновлюється через
76
+ стан HF IterableDataset; незавершений хвіст вихідного файла відкочується
77
+ до останньої узгодженої точки. Якщо джерело закінчиться раніше, скрипт
78
+ зупиниться, а не доповнюватиме корпус нескінченними повтореннями.
79
+
80
+ Два uint16-файли займають приблизно **80,0 ГБ / 74,5 GiB**. Обсяг мережевого
81
+ трафіку більший; вихідні тексти не завантажуються до ZIP. Підготовка може
82
+ зайняти години або дні. Це не вже готовий 40B-корпус у маленькому архіві.
83
+ Наприкінці обчислюються SHA256 файлів. Повна повторна перевірка даних:
84
+
85
+ ```bash
86
+ python3.12 run_40b.py --stage train --work-dir /srv/warpstate40b --weights /srv/models/latest.pt --verify-data
87
+ ```
88
+
89
+ 40 млрд токенів — суттєвий бюджет для експерименту з 2B, але кількість токенів
90
+ не гарантує якість. Тут експериментальна архітектура; оцінюйте validation,
91
+ мовні/фактологічні тести й реальні відповіді. Англомовний pretraining сам
92
+ по собі не робить модель якісним українським чат-асистентом. Для цього
93
+ потрібні відповідні українські дані, instruction tuning та окрема оцінка;
94
+ вони не підмінені випадковою сумішшю в цьому пакеті.
95
+
96
+ ## Параметри й продуктивність
97
+
98
+ За замовчуванням: sequence **2048**, micro-batch **8**, накопичення **8**,
99
+ **131 072 токени/оновлення**, повний FP32 GPU AdamW, BF16 autocast,
100
+ activation checkpointing, TF32 для відповідних FP32 матричних операцій,
101
+ loss блоками 1024, clipping 1,0, betas 0,9/0,95, weight decay 0,1.
102
+ LR 0,00015 → 0,000015, warmup 1000 кроків, cosine decay. Приблизно
103
+ 305 176 оновлень; останнє маскується, щоб не перевищити рівно 40 млрд токенів.
104
+ Циклічне читання проходить корпус послідовно, а не вибирає випадкові
105
+ послідовності з повторенням. На останній межі останній токен передбачає перший.
106
+
107
+ Локальний блок архітектури залишається 128 токенів; 2048 — довжина
108
+ послідовності навчання, а не обіцянка точного зберігання всього контексту.
109
+ Тривала пам’ять лишається стисненими fast/slow станами вашої архітектури.
110
+
111
+ Перед довгим навчанням можна обмежити перший запуск десятьма оновленнями:
112
+
113
+ ```bash
114
+ python3.12 run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b --max-steps 10
115
+ ```
116
+
117
+ Підготовка все одно створить повний 40B-корпус. Потім приберіть `--max-steps`
118
+ і повторіть команду: продовжиться той самий 40B-розклад.
119
+ Опційний `--compile` вмикає Linux Inductor; потрібні сумісні компілятори,
120
+ перший запуск довший. Вибирайте його **до першого навчального запуску**.
121
+ За замовчуванням він вимкнений, оскільки повний compile цього сценарію на
122
+ RTX PRO 6000 тут не перевірявся. Так само не виміряно оптимальний micro-batch.
123
+ За OOM зменшіть micro-batch і відповідно збільште накопичення **до початку
124
+ нового запуску**; відновлення навмисно не дозволяє мовчки змінювати налаштування.
125
+
126
+ Фактична швидкість виводиться після кожного кроку й включає forward,
127
+ backward, clipping, AdamW ��а отримання пакета. Вона не включає підготовку
128
+ корпусу, validation, компіляцію й запис checkpoint. **10 000 токенів/с не
129
+ гарантуються**: повна модель на цій серверній карті тут не запускалась.
130
+ Для масштабу, суто арифметично: 40B при 10 000 токенів/с — 46,3 доби
131
+ чистого навчання, при 20 000 — 23,1 доби, без додаткових витрат.
132
+
133
+ Validation виконується кожні 500 кроків на незмінних позиціях; checkpoint —
134
+ кожні 1000 та при штатній зупинці. Одне збереження моделі з GPU AdamW
135
+ займає приблизно 24 ГБ; атомарна заміна потребує місця для двох файлів.
136
+ Довге навчання означає великий сумарний обсяг записів на диск.
137
+
138
+ ## Файли й перевірки
139
+
140
+ - `work/data/train.bin`, `val.bin`, `manifest.json`, `prepare-state.pkl` — дані та їхній прогрес.
141
+ - `work/run/latest.pt` — відновлюваний серверний checkpoint.
142
+ - `work/run/metrics.jsonl`, `validation.jsonl`, `settings.json` — журнал і параметри.
143
+ - `.server-venv/requirements-resolved.txt` — точний список встановлених залежностей.
144
+ - `PACKAGE_MANIFEST.json` — контрольні суми файлів архіву.
145
+
146
+ Код підготовки й навчання локально перевірений на маленькій моделі:
147
+ маска останнього пакета, відновлення підготовки з відкиданням незбереженого
148
+ хвоста, точний збіг ваг і станів AdamW між безперервним і відновленим
149
+ CUDA-навчанням, відхилення зміненого LR. Це не серверний бенчмарк 2B/96 ГБ.
150
+ Також перевірено справжній HF streaming на 10 000 train + 64 validation
151
+ токенах: відновлення дає побайтово ті самі файли, SHA256 збігаються.
152
+ Локальна перевірка виконувалася на Windows/RTX 5070; Linux bootstrap та
153
+ повне навчання 2B на 96 ГБ потрібно перевірити на вашому сервері.
154
+
155
+ Для відтворення тестів у встановленому середовищі:
156
+
157
+ ```bash
158
+ .server-venv/bin/python -m pip install pytest==9.1.1
159
+ .server-venv/bin/python -m pytest test_server.py -q
160
+ ```
161
+
162
+ Лише trusted локальні checkpoint/state-файли: PyTorch та pickle можуть
163
+ виконувати код під час читання. Не підставляйте невідомі файли з інтернету.
164
+
165
+ ## Джерела
166
+
167
+ - [FineWeb-Edu: опис, англійська мова, sample-100BT, ODC-BY](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu).
168
+ - [Hugging Face: streaming та відновлення IterableDataset](https://huggingface.co/docs/datasets/stream).
169
+ - [NVIDIA: RTX PRO 6000 Blackwell 96 ГБ](https://www.nvidia.com/en-sg/data-center/rtx-pro-6000-blackwell-server-edition/).
170
+ - [CUDA-версія PyTorch](https://download.pytorch.org/whl/cu130/torch/).
SOURCE_PACKAGE_NOTES.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WarpState — assembled source bundle
2
+
3
+ This bundle combines the **three source archives you provided** with the newly designed README and SVG assets. Existing source files were preserved byte-for-byte. No Python implementation was rewritten. Python bytecode caches (`__pycache__`, `.pyc`) were omitted.
4
+
5
+ ## Included
6
+
7
+ - `warpstate/model.py`, `warpstate/config.py`, `warpstate/__init__.py` — model implementation
8
+ - `run_40b.py` — pretraining
9
+ - `posttrain.py` — supervised fine-tuning
10
+ - `dpo.py` — preference optimization
11
+ - `chat.py` — chat inference/probe
12
+ - `ops/*.py` and `ops/*.sh` — execution and training operations
13
+ - `test_*.py` — tests
14
+ - `tokenizer.model` — supplied tokenizer
15
+ - `README.md` and `assets/` — updated GitHub presentation
16
+
17
+ ## Not supplied in the uploaded archives
18
+
19
+ - Trained model checkpoints / weights
20
+ - Training datasets
21
+ - `configs/` directory, including `configs/warpstate_2b.json` referenced by supplied `PACKAGE_MANIFEST.json` and training documents
22
+ - Explicit license file
23
+
24
+ **Note:** This is a *complete assembly of the available source*, not a turnkey pretrained model. Before publishing, review all source and documentation, choose a license, restore valid configs, and independently validate environment-specific commands. Scripts with old server-specific paths may require adaptation. Do not upload private secrets or checkpoints.
25
+
26
+ Original archives:
27
+ - `drive-download-20261026T200043Z-1-001.zip`
28
+ - `warpstate-20261026T200055Z-1-001.zip`
29
+ - `ops-20261026T200120Z-1-001.zip`
VALIDATION.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "unit_tests": 3,
3
+ "unit_test_status": "passed",
4
+ "tests": [
5
+ "partial final token batch",
6
+ "preparation interruption and rollback",
7
+ "small GPU full-optimizer resume equals uninterrupted run; changed LR rejected"
8
+ ],
9
+ "live_fineweb_test": {
10
+ "train_tokens": 10000,
11
+ "val_tokens": 64,
12
+ "resume_byte_identical": true,
13
+ "sha256_verified": true
14
+ },
15
+ "test_platform": "Windows / RTX 5070 Laptop / torch 2.14.0+cu130",
16
+ "target_platform": "Linux / one RTX PRO 6000 Blackwell 96 GB",
17
+ "full_target_hardware_tested": false,
18
+ "full_40b_corpus_prepared_locally": false
19
+ }
assets/warpstate-architecture.svg ADDED
assets/warpstate-banner.svg ADDED
chat.py ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """CPU-only chat/completion probe of an independent WarpState weight snapshot.
3
+
4
+ Only load trusted local torch checkpoints. This does not stop training, touch
5
+ CUDA, load Adam into a model, or write to the training checkpoint directory.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import gc
11
+ import hashlib
12
+ import json
13
+ import math
14
+ import os
15
+ from pathlib import Path
16
+ import stat
17
+ import tempfile
18
+ import time
19
+
20
+ import sentencepiece as spm
21
+ import torch
22
+
23
+ from warpstate import WarpStateConfig, WarpStateLM
24
+
25
+ ROOT = Path(__file__).resolve().parent
26
+ FORMAT = 'warpstate-chat-v1'
27
+
28
+
29
+ def sha256(path):
30
+ digest = hashlib.sha256()
31
+ with Path(path).open('rb') as stream:
32
+ for block in iter(lambda: stream.read(8 * 1024**2), b''):
33
+ digest.update(block)
34
+ return digest.hexdigest()
35
+
36
+
37
+ def capture_latest(source, tokenizer, snapshot_root):
38
+ """Pin one atomically published inode, export only its original FP32 model.
39
+
40
+ latest.pt is published by rename, never updated in place. An open descriptor
41
+ remains bound to that complete version even if the trainer publishes again.
42
+ No hardlink to the production checkpoint is created.
43
+ """
44
+ source, tokenizer, snapshot_root = map(Path, (source, tokenizer, snapshot_root))
45
+ if source.suffix != '.pt' or source.is_symlink():
46
+ raise ValueError('Select a completed regular .pt checkpoint, not a temporary file')
47
+ tokenizer_sha = sha256(tokenizer)
48
+ fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW)
49
+ try:
50
+ before = os.fstat(fd)
51
+ if not stat.S_ISREG(before.st_mode):
52
+ raise ValueError('Checkpoint must be a regular file')
53
+ checkpoint = torch.load(f'/proc/self/fd/{fd}', map_location='cpu',
54
+ weights_only=False, mmap=True)
55
+ if checkpoint.get('format') == FORMAT:
56
+ raise ValueError(
57
+ f'Chat snapshot requires its original snapshot.json beside {source}. '
58
+ 'Restore that metadata or export a fresh snapshot from a training checkpoint.')
59
+ if checkpoint.get('format') != 'warpstate-server-v1':
60
+ raise ValueError('Expected a full warpstate-server-v1 training checkpoint')
61
+ recorded_tokenizer = checkpoint['signature']['data']['signature']['tokenizer_sha256']
62
+ if recorded_tokenizer != tokenizer_sha:
63
+ raise ValueError('Tokenizer SHA256 differs from training checkpoint')
64
+ snapshot_root.mkdir(parents=True, exist_ok=True)
65
+ folder = Path(tempfile.mkdtemp(prefix=f'step-{checkpoint["step"]}-', dir=snapshot_root))
66
+ destination = folder / 'model.pt'
67
+ temporary = folder / 'model.pt.tmp'
68
+ metadata = dict(format=FORMAT, inference_only=True, config=checkpoint['config'],
69
+ step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'],
70
+ val_loss=checkpoint.get('val_loss'), tokenizer_sha256=tokenizer_sha,
71
+ captured_at_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
72
+ source=dict(path=str(source.resolve()), device=before.st_dev,
73
+ inode=before.st_ino, size=before.st_size,
74
+ mtime_ns=before.st_mtime_ns))
75
+ print(f'Capturing completed step {metadata["step"]:,} / '
76
+ f'{metadata["tokens_seen"]:,} tokens; model weights only...', flush=True)
77
+ with temporary.open('xb') as stream:
78
+ torch.save(dict(metadata, model=checkpoint['model']), stream)
79
+ stream.flush()
80
+ os.fsync(stream.fileno())
81
+ after = os.fstat(fd)
82
+ # Atomic unlink/rename may change ctime/nlink without changing contents.
83
+ if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
84
+ after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns):
85
+ raise RuntimeError('Pinned checkpoint contents changed; snapshot not published')
86
+ os.chmod(temporary, 0o444)
87
+ os.replace(temporary, destination)
88
+ metadata['snapshot_sha256'] = sha256(destination)
89
+ metadata['snapshot_bytes'] = destination.stat().st_size
90
+ with (folder / 'snapshot.json').open('x') as stream:
91
+ json.dump(metadata, stream, indent=2)
92
+ stream.write('\n')
93
+ stream.flush()
94
+ os.fsync(stream.fileno())
95
+ directory = os.open(folder, os.O_RDONLY | os.O_DIRECTORY)
96
+ try:
97
+ os.fsync(directory)
98
+ finally:
99
+ os.close(directory)
100
+ return destination
101
+ finally:
102
+ os.close(fd)
103
+
104
+
105
+ def resolve_snapshot(source, tokenizer, snapshot_root):
106
+ """Reuse a sealed export, or safely capture a full training checkpoint.
107
+
108
+ Missing metadata is not permission to load unverified inference weights:
109
+ capture_latest accepts only the embedded training-checkpoint format and
110
+ pins its inode across atomic replacements by the trainer.
111
+ """
112
+ source = Path(source)
113
+ if not source.is_file():
114
+ raise ValueError(f'Checkpoint file does not exist or is not a regular file: {source}')
115
+ if source.with_name('snapshot.json').exists():
116
+ return source
117
+ return capture_latest(source, tokenizer, snapshot_root)
118
+
119
+
120
+ def load_model(snapshot, tokenizer):
121
+ snapshot, tokenizer = Path(snapshot), Path(tokenizer)
122
+ proof_path = snapshot.with_name('snapshot.json')
123
+ try:
124
+ proof = json.loads(proof_path.read_text())
125
+ except FileNotFoundError as exc:
126
+ raise ValueError(
127
+ f'Missing snapshot metadata: {proof_path}. '
128
+ 'Use --checkpoint to export training weights, or restore the original metadata.') from exc
129
+ if not isinstance(proof, dict) or not isinstance(proof.get('snapshot_sha256'), str):
130
+ raise ValueError(f'Invalid snapshot metadata: {proof_path}')
131
+ if proof['snapshot_sha256'] != sha256(snapshot):
132
+ raise ValueError('Inference snapshot SHA256 mismatch')
133
+ checkpoint = torch.load(snapshot, map_location='cpu', weights_only=False, mmap=True)
134
+ if checkpoint.get('format') != FORMAT or not checkpoint.get('inference_only'):
135
+ raise ValueError('Use the independent inference snapshot, not live latest.pt')
136
+ if checkpoint['tokenizer_sha256'] != sha256(tokenizer):
137
+ raise ValueError('Tokenizer SHA256 mismatch')
138
+ config = WarpStateConfig(**checkpoint['config'])
139
+ tokenizer_model = spm.SentencePieceProcessor(model_file=str(tokenizer))
140
+ if tokenizer_model.vocab_size() != config.vocab_size:
141
+ raise ValueError('Tokenizer vocabulary differs from model')
142
+ with torch.device('meta'):
143
+ model = WarpStateLM(config)
144
+ model.load_state_dict(checkpoint['model'], strict=True, assign=True)
145
+ model.eval().requires_grad_(False)
146
+ if any(p.device.type != 'cpu' or p.dtype != torch.float32 for p in model.parameters()):
147
+ raise ValueError('This CPU probe requires the original FP32 model parameters')
148
+ metadata = {key: value for key, value in checkpoint.items() if key != 'model'}
149
+ del checkpoint
150
+ gc.collect()
151
+ return model, tokenizer_model, metadata
152
+
153
+
154
+ def banned_ngram_tokens(tokens, size):
155
+ """Block completions of n-grams already present in this reply, not its prompt."""
156
+ if size == 0 or len(tokens) + 1 < size:
157
+ return set()
158
+ prefix = tuple(tokens[-(size - 1):]) if size > 1 else ()
159
+ return {tokens[start + size - 1] for start in range(len(tokens) - size + 1)
160
+ if tuple(tokens[start:start + size - 1]) == prefix}
161
+
162
+
163
+ def sample_token(logits, temperature, top_k, top_p, generator, *,
164
+ previous_tokens=(), repetition_penalty=1.0, banned_tokens=()):
165
+ logits = logits.float().clone()
166
+ if not torch.isfinite(logits).all():
167
+ raise ValueError('Non-finite generation logits')
168
+ if not math.isfinite(temperature) or temperature < 0 or not 0 < top_p <= 1 or top_k < 0:
169
+ raise ValueError('Invalid sampling settings')
170
+ if not math.isfinite(repetition_penalty) or repetition_penalty < 1:
171
+ raise ValueError('Require a finite repetition-penalty >= 1')
172
+ if previous_tokens and repetition_penalty != 1:
173
+ seen = list(set(previous_tokens))
174
+ scores = logits[seen]
175
+ # Multiplying negative scores and dividing positive ones lowers both.
176
+ logits[seen] = torch.where(scores < 0, scores * repetition_penalty,
177
+ scores / repetition_penalty)
178
+ if banned_tokens:
179
+ logits[list(banned_tokens)] = -torch.inf
180
+ if not torch.isfinite(logits).any():
181
+ return None # No legal continuation; never sample a masked token.
182
+ if temperature == 0:
183
+ return int(logits.argmax())
184
+ values = logits / temperature
185
+ if top_k:
186
+ threshold = values.topk(min(top_k, values.numel())).values[-1]
187
+ values = values.masked_fill(values < threshold, -torch.inf)
188
+ ordered, indices = values.sort(descending=True)
189
+ probability = ordered.softmax(dim=-1)
190
+ # Keep the token crossing the cumulative threshold, including at least one.
191
+ ordered = ordered.masked_fill(probability.cumsum(-1) - probability > top_p, -torch.inf)
192
+ picked = torch.multinomial(ordered.softmax(dim=-1), 1, generator=generator)
193
+ return int(indices[picked].item())
194
+
195
+
196
+ @torch.inference_mode()
197
+ def generate(model, sp, prompt, *, max_new_tokens=256, temperature=0.8,
198
+ top_k=40, top_p=0.9, seed=1337, max_context=2048,
199
+ repetition_penalty=1.2, no_repeat_ngram_size=4):
200
+ if max_new_tokens < 1 or max_context <= max_new_tokens:
201
+ raise ValueError('Require 0 < max-new-tokens < max-context')
202
+ if not math.isfinite(repetition_penalty) or repetition_penalty < 1:
203
+ raise ValueError('Require a finite repetition-penalty >= 1')
204
+ if not isinstance(no_repeat_ngram_size, int) or no_repeat_ngram_size < 0:
205
+ raise ValueError('Require an integer no-repeat-ngram-size >= 0')
206
+ ids = sp.encode(prompt, out_type=int)
207
+ if not ids:
208
+ raise ValueError('Prompt contains no tokens')
209
+ ids = ids[-(max_context - max_new_tokens):]
210
+ caches = model.init_generation_cache(1, torch.device('cpu'), torch.float32)
211
+ token = torch.empty((1, 1), dtype=torch.long)
212
+ for value in ids:
213
+ token.fill_(value)
214
+ logits = model.step(token, caches)[0, -1]
215
+ generator = torch.Generator(device='cpu').manual_seed(seed)
216
+ generated = []
217
+ for index in range(max_new_tokens):
218
+ value = sample_token(logits, temperature, top_k, top_p, generator,
219
+ previous_tokens=generated, repetition_penalty=repetition_penalty,
220
+ banned_tokens=banned_ngram_tokens(generated, no_repeat_ngram_size))
221
+ if value is None or value == sp.eos_id():
222
+ break
223
+ generated.append(value)
224
+ yield value
225
+ if index + 1 < max_new_tokens:
226
+ token.fill_(value)
227
+ logits = model.step(token, caches)[0, -1]
228
+
229
+
230
+ def reply(model, sp, prompt, args):
231
+ ids, emitted = [], ''
232
+ begin, first = time.perf_counter(), None
233
+ for value in generate(model, sp, prompt, max_new_tokens=args.max_new_tokens,
234
+ temperature=args.temperature, top_k=args.top_k, top_p=args.top_p,
235
+ seed=args.seed, max_context=args.max_context,
236
+ repetition_penalty=args.repetition_penalty,
237
+ no_repeat_ngram_size=args.no_repeat_ngram_size):
238
+ if first is None:
239
+ first = time.perf_counter()
240
+ ids.append(value)
241
+ text = sp.decode(ids)
242
+ # Delay incomplete UTF-8 byte-fallback characters until they are complete.
243
+ safe = text.split('\ufffd', 1)[0]
244
+ if safe.startswith(emitted):
245
+ print(safe[len(emitted):], end='', flush=True)
246
+ emitted = safe
247
+ text = sp.decode(ids)
248
+ if text.startswith(emitted):
249
+ print(text[len(emitted):], end='', flush=True)
250
+ elapsed = time.perf_counter() - begin
251
+ ttft = '-' if first is None else f'{first - begin:.2f}s'
252
+ print(f'\n[{len(ids)} tokens; {elapsed:.1f}s total; TTFT {ttft}; CPU FP32]', flush=True)
253
+ return text
254
+
255
+
256
+ def main():
257
+ parser = argparse.ArgumentParser(description=__doc__)
258
+ parser.add_argument('--checkpoint', type=Path, default=ROOT / 'work/run/latest.pt')
259
+ parser.add_argument('--snapshot', type=Path,
260
+ help='Reuse a sealed model-only snapshot, or export a training checkpoint')
261
+ parser.add_argument('--snapshot-root', type=Path, default=ROOT / 'work/chat')
262
+ parser.add_argument('--tokenizer', type=Path, default=ROOT / 'tokenizer.model')
263
+ parser.add_argument('--capture-only', action='store_true')
264
+ parser.add_argument('--prompt', help='One-shot probe instead of interactive input')
265
+ parser.add_argument('--raw', action='store_true', help='Plain text completion instead of Q/A labels')
266
+ parser.add_argument('--threads', type=int, default=16)
267
+ parser.add_argument('--max-new-tokens', type=int, default=256)
268
+ parser.add_argument('--max-context', type=int, default=2048)
269
+ parser.add_argument('--temperature', type=float, default=0.8, help='0 = greedy')
270
+ parser.add_argument('--top-k', type=int, default=40)
271
+ parser.add_argument('--top-p', type=float, default=0.9)
272
+ parser.add_argument('--repetition-penalty', type=float, default=1.2,
273
+ help='Lower scores of tokens already generated in this reply; 1 disables')
274
+ parser.add_argument('--no-repeat-ngram-size', type=int, default=4,
275
+ help='Block repeated token sequences within a reply; 0 disables')
276
+ parser.add_argument('--seed', type=int, default=1337)
277
+ args = parser.parse_args()
278
+ if args.threads < 1 or not 0 < args.max_new_tokens < args.max_context:
279
+ parser.error('Require threads >= 1 and 0 < max-new-tokens < max-context')
280
+ if not math.isfinite(args.temperature) or args.temperature < 0 or args.top_k < 0 or not 0 < args.top_p <= 1:
281
+ parser.error('Invalid temperature / top-k / top-p')
282
+ if not math.isfinite(args.repetition_penalty) or args.repetition_penalty < 1 or args.no_repeat_ngram_size < 0:
283
+ parser.error('Require finite repetition-penalty >= 1 and no-repeat-ngram-size >= 0')
284
+ if args.prompt is not None and not args.prompt.strip():
285
+ parser.error('A one-shot prompt must not be empty')
286
+ torch.set_num_threads(args.threads)
287
+ torch.set_num_interop_threads(1)
288
+ try:
289
+ if args.snapshot is not None:
290
+ snapshot = resolve_snapshot(args.snapshot, args.tokenizer, args.snapshot_root)
291
+ else:
292
+ snapshot = capture_latest(args.checkpoint, args.tokenizer, args.snapshot_root)
293
+ print(f'Snapshot: {snapshot}', flush=True)
294
+ if args.capture_only:
295
+ return
296
+ model, sp, metadata = load_model(snapshot, args.tokenizer)
297
+ except (OSError, ValueError) as exc:
298
+ parser.error(str(exc))
299
+ print(f'WarpState {model.num_parameters() / 1e9:.3f}B | CPU FP32 | '
300
+ f'step {metadata["step"]:,} | {metadata["tokens_seen"]:,} training tokens', flush=True)
301
+ print('Base pretrained model, not instruction-tuned. Q/A labels are only a probe.\n'
302
+ 'Commands: /new, /raw, /chat, /info, /help, /exit. Ctrl-C cancels a reply.\n'
303
+ 'Each turn replays up to the latest context tokens; long prompts take time on CPU.', flush=True)
304
+ print(f'Repetition protection: penalty={args.repetition_penalty:g}, '
305
+ f'no-repeat-ngram-size={args.no_repeat_ngram_size} (per reply).', flush=True)
306
+ history = ''
307
+ while True:
308
+ try:
309
+ user = args.prompt if args.prompt is not None else input('\nYou: ').strip()
310
+ if user is None or (args.prompt is None and user in ('/exit', '/quit')):
311
+ break
312
+ if not user:
313
+ continue
314
+ if args.prompt is None and user in ('/new', '/raw', '/chat'):
315
+ history = ''
316
+ if user != '/new':
317
+ args.raw = user == '/raw'
318
+ print('Context cleared. Mode: ' + ('raw completion' if args.raw else 'Q/A probe'))
319
+ continue
320
+ if args.prompt is None and user == '/info':
321
+ print(json.dumps(metadata, indent=2))
322
+ continue
323
+ if args.prompt is None and user == '/help':
324
+ print('/new clears context; /raw plain completion; /chat Q/A; /info snapshot; /exit quit')
325
+ continue
326
+ prompt = history + (user if args.raw else f'Question: {user}\nAnswer:')
327
+ print('Model: ', end='', flush=True)
328
+ answer = reply(model, sp, prompt, args)
329
+ history = prompt + ('' if args.raw else ' ') + answer + '\n\n'
330
+ if args.prompt is not None:
331
+ break
332
+ except EOFError:
333
+ break
334
+ except KeyboardInterrupt:
335
+ print('\nReply cancelled; training was not affected.', flush=True)
336
+ if args.prompt is not None:
337
+ break
338
+
339
+
340
+ if __name__ == '__main__':
341
+ main()
dpo.py ADDED
@@ -0,0 +1,647 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """BF16 DPO with an immutable SFT reference, exact resume and response-only loss.
3
+
4
+ The reference is scored once, before allocating the policy optimizer. Its cache
5
+ is bound to weights, data, masking, execution and source hashes. Policy training
6
+ uses fresh FP32 Adam state and saves only into its own output directory.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import contextlib
13
+ import gc
14
+ import hashlib
15
+ import json
16
+ import math
17
+ import os
18
+ from pathlib import Path
19
+ import random
20
+ import signal
21
+ import time
22
+ import types
23
+
24
+ import numpy as np
25
+ import torch
26
+ import torch.distributed as dist
27
+ import torch.nn.functional as F
28
+ from torch.utils.checkpoint import checkpoint as activation_checkpoint
29
+
30
+ from posttrain import (WORKSPACE, atomic_json, base_pipeline_lock, digest,
31
+ restore_rng, rng_state, update_indices, validate_paths)
32
+ from ops.dpo_data import MASK_FORMAT, collate_pairs, load_data
33
+ from ops.posttrain_data import RENDER_FORMAT, renderer_sha256
34
+
35
+ FORMAT = 'warpstate-dpo-v1'
36
+ CACHE_FORMAT = 'warpstate-dpo-reference-v1'
37
+ STOP = False
38
+ DEFAULTS = dict(sequence_length=2048, micro_batch=2, grad_accum=8, lr=5e-7,
39
+ min_lr=0., warmup_ratio=.1, epochs=1, max_steps=None,
40
+ weight_decay=0., seed=42, eval_every=100, save_every=100,
41
+ eval_batches=32, compile_core=True, bf16_residuals=True,
42
+ gradient_checkpointing=False, workers=4, deterministic=True,
43
+ clip_grad_norm=1., beta=5., objective='length-normalized-dpo',
44
+ head_chunk_size=256)
45
+
46
+
47
+ def load_plan(path):
48
+ plan = json.loads(Path(path).read_text())
49
+ if plan.get('format') != 'warpstate-dpo-plan-v1' or plan.get('stage') != 'dpo':
50
+ raise ValueError('Expected warpstate-dpo-plan-v1 stage dpo')
51
+ supplied = plan.get('training', {})
52
+ unknown = set(supplied) - set(DEFAULTS)
53
+ if unknown:
54
+ raise ValueError(f'Unknown DPO training settings: {sorted(unknown)}')
55
+ settings = DEFAULTS | supplied
56
+ for key in ('sequence_length', 'micro_batch', 'grad_accum', 'epochs', 'eval_every',
57
+ 'save_every', 'eval_batches', 'workers', 'head_chunk_size'):
58
+ if type(settings[key]) is not int or settings[key] < 1:
59
+ raise ValueError(f'{key} must be a positive integer')
60
+ if settings['max_steps'] is not None and (type(settings['max_steps']) is not int or settings['max_steps'] < 1):
61
+ raise ValueError('max_steps must be null or a positive integer')
62
+ for key in ('lr', 'min_lr', 'warmup_ratio', 'weight_decay', 'clip_grad_norm', 'beta'):
63
+ value = settings[key]
64
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
65
+ raise ValueError(f'{key} must be finite and nonnegative')
66
+ if not 0 <= settings['warmup_ratio'] < 1 or not 0 <= settings['min_lr'] <= settings['lr']:
67
+ raise ValueError('Invalid LR schedule')
68
+ if not settings['lr'] or not settings['clip_grad_norm'] or not settings['beta']:
69
+ raise ValueError('lr, clip_grad_norm and beta must be positive')
70
+ for key in ('compile_core', 'bf16_residuals', 'gradient_checkpointing', 'deterministic'):
71
+ if type(settings[key]) is not bool:
72
+ raise ValueError(f'{key} must be a boolean')
73
+ if type(settings['seed']) is not int or settings['seed'] < 0:
74
+ raise ValueError('seed must be a nonnegative integer')
75
+ if settings['objective'] not in ('length-normalized-dpo', 'summed-dpo'):
76
+ raise ValueError('Unsupported DPO objective')
77
+ return plan, settings
78
+
79
+
80
+ def source_identity():
81
+ from posttrain import source_identity as shared_sources
82
+ root = Path(__file__).parent
83
+ return shared_sources() | {str(path.relative_to(root)): digest(path)
84
+ for path in (Path(__file__), root / 'ops/dpo_data.py')}
85
+
86
+
87
+ def identity_digest(identity):
88
+ return hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(',', ':'),
89
+ allow_nan=False).encode()).hexdigest()
90
+
91
+
92
+ def learning_rate(step, total_steps, settings):
93
+ """Fixed linear warmup/decay schedule, independent of a pause boundary."""
94
+ warmup = min(math.ceil(settings['warmup_ratio'] * total_steps), max(0, total_steps - 1))
95
+ if step < warmup:
96
+ return settings['lr'] * (step + 1) / warmup
97
+ progress = min(1., (step - warmup) / max(1, total_steps - warmup - 1))
98
+ return settings['min_lr'] + (settings['lr'] - settings['min_lr']) * (1. - progress)
99
+
100
+
101
+ def reference_identity(weights_hash, manifest_hash, tokenizer_hash, settings, device, world):
102
+ return dict(format=CACHE_FORMAT, weights_sha256=weights_hash, data_sha256=manifest_hash,
103
+ tokenizer_sha256=tokenizer_hash, renderer_sha256=renderer_sha256(),
104
+ mask_format=MASK_FORMAT, sources=source_identity(), settings={key: settings[key]
105
+ for key in ('sequence_length', 'micro_batch', 'head_chunk_size', 'compile_core',
106
+ 'bf16_residuals', 'deterministic', 'workers')},
107
+ dropout=0., logps='response-token-sum-float32', dtype='bfloat16',
108
+ device_type=device.type, world_size=world, torch_version=torch.__version__)
109
+
110
+
111
+ def validate_cache(directory, identity, sizes):
112
+ directory = Path(directory)
113
+ manifest = json.loads((directory / 'manifest.json').read_text())
114
+ if (manifest.get('format') != CACHE_FORMAT or manifest.get('complete') is not True or
115
+ manifest.get('identity') != identity or manifest.get('identity_sha256') != identity_digest(identity)):
116
+ raise ValueError('Reference cache identity/completion differs')
117
+ result = []
118
+ for name, size in zip(('train.npy', 'val.npy'), sizes):
119
+ path = directory / name
120
+ info = manifest.get('files', {}).get(name, {})
121
+ if not path.is_file() or path.is_symlink() or digest(path) != info.get('sha256') or info.get('rows') != size:
122
+ raise ValueError(f'Reference cache file changed: {name}')
123
+ values = np.load(path, allow_pickle=False, mmap_mode='r')
124
+ if values.shape != (size, 2) or values.dtype != np.float64 or not np.isfinite(values).all():
125
+ raise ValueError(f'Invalid reference log probabilities: {name}')
126
+ result.append(values)
127
+ return *result, digest(directory / 'manifest.json')
128
+
129
+
130
+ def apply_sequence_logps(model, chunk_size):
131
+ """Return per-sequence log-probability sums without retaining vocabulary logits.
132
+
133
+ Vocabulary matmuls use BF16 autocast; cross-entropy reductions use FP32. Each
134
+ head chunk is recomputed in backward, so DPO keeps a vector of branch scores.
135
+ The backbone modules and canonical parameter/state names are unchanged.
136
+ """
137
+ def forward(self, input_ids, targets=None, *, return_logits=False):
138
+ if targets is None or return_logits:
139
+ raise ValueError('DPO forward requires response targets and no returned logits')
140
+ hidden = self.embed(input_ids)
141
+ for depth in range(self.cfg.logical_depth):
142
+ hidden = hidden * (1. + self.depth_scale[depth]) + self.depth_bias[depth]
143
+ core = self.cores[depth % self.cfg.n_cores]
144
+ if self.training and self.cfg.gradient_checkpointing:
145
+ hidden = activation_checkpoint(core, hidden, use_reentrant=False)
146
+ else:
147
+ hidden = core(hidden)
148
+ hidden = self.final_norm(hidden)
149
+
150
+ def score(h, weight, labels):
151
+ logits = F.linear(h, weight).float()
152
+ losses = F.cross_entropy(logits.reshape(-1, logits.size(-1)), labels.reshape(-1),
153
+ reduction='none', ignore_index=-100)
154
+ return -losses.reshape(labels.shape).sum(dim=1)
155
+
156
+ logps = hidden.new_zeros((hidden.shape[0],), dtype=torch.float32)
157
+ for start in range(0, hidden.shape[1], chunk_size):
158
+ args = (hidden[:, start:start + chunk_size], self.embed.weight,
159
+ targets[:, start:start + chunk_size])
160
+ value = activation_checkpoint(score, *args, use_reentrant=False) if torch.is_grad_enabled() else score(*args)
161
+ logps = logps + value
162
+ return None, logps
163
+ model.forward = types.MethodType(forward, model)
164
+
165
+
166
+ def preference_metrics(policy, reference, counts, beta, objective):
167
+ """Pair vectors, retaining gradients only through policy log probabilities."""
168
+ reference = reference.detach().to(policy.dtype)
169
+ if objective == 'length-normalized-dpo':
170
+ policy = policy / counts.clamp_min(1)
171
+ reference = reference / counts.clamp_min(1)
172
+ elif objective != 'summed-dpo':
173
+ raise ValueError('Unsupported DPO objective')
174
+ rewards = beta * (policy - reference)
175
+ margin = rewards[:, 0] - rewards[:, 1]
176
+ return -F.logsigmoid(margin), margin, (margin > 0).to(policy.dtype)
177
+
178
+
179
+ def stop_requested(args, device, world):
180
+ value = STOP or bool(args.stop_file and Path(args.stop_file).exists())
181
+ if world > 1:
182
+ flag = torch.tensor(int(value), device=device)
183
+ dist.all_reduce(flag, op=dist.ReduceOp.MAX)
184
+ value = bool(flag.item())
185
+ return value
186
+
187
+
188
+ def load_model(checkpoint, settings, device, tokenizer):
189
+ from warpstate import WarpStateConfig, WarpStateLM
190
+ from run_40b import compile_training_cores
191
+ from ops.residual_experiment import apply_bf16_residuals
192
+ cfg = WarpStateConfig(**checkpoint['config'])
193
+ cfg.gradient_checkpointing = settings['gradient_checkpointing']
194
+ # DPO compares a deterministic policy/reference scoring rule, with dropout disabled.
195
+ cfg.dropout = 0.
196
+ cfg.validate()
197
+ if cfg.vocab_size != tokenizer.vocab_size() or settings['sequence_length'] % cfg.chunk_size:
198
+ raise ValueError('Model/tokenizer vocabulary or sequence/chunk mismatch')
199
+ with torch.device('meta'):
200
+ raw = WarpStateLM(cfg)
201
+ raw.load_state_dict(checkpoint['model'], strict=True, assign=True)
202
+ # Clone on CPU as well: mmap initialization is immutable and must not alias training.
203
+ raw.to(device=device, dtype=torch.float32)
204
+ if device.type == 'cpu':
205
+ for parameter in raw.parameters():
206
+ parameter.data = parameter.data.clone()
207
+ if settings['compile_core']:
208
+ compile_training_cores(raw)
209
+ apply_sequence_logps(raw, settings['head_chunk_size'])
210
+ if settings['bf16_residuals']:
211
+ apply_bf16_residuals(raw)
212
+ return raw, cfg
213
+
214
+
215
+ def atomic_array(path, array):
216
+ temporary = Path(str(path) + '.tmp')
217
+ with temporary.open('wb') as stream:
218
+ np.save(stream, array, allow_pickle=False)
219
+ stream.flush()
220
+ os.fsync(stream.fileno())
221
+ os.replace(temporary, path)
222
+
223
+
224
+ def prepare_reference(args, identity, records, settings, device, rank, world, tokenizer):
225
+ """Create/reuse distributed resumable score shards, then seal a checked cache."""
226
+ directory = Path(args.reference_cache_dir)
227
+ if (directory / 'manifest.json').exists():
228
+ return validate_cache(directory, identity, [len(rows) for rows in records])
229
+ directory.mkdir(parents=True, exist_ok=True)
230
+ marker = directory / 'identity.json'
231
+ if marker.exists() and json.loads(marker.read_text()) != identity:
232
+ raise ValueError('Partial reference cache identity differs')
233
+ if rank == 0:
234
+ atomic_json(marker, identity)
235
+ if world > 1:
236
+ dist.barrier()
237
+ checkpoint = torch.load(args.weights, map_location='cpu', weights_only=False, mmap=True)
238
+ raw, _ = load_model(checkpoint, settings, device, tokenizer)
239
+ raw.requires_grad_(False).eval()
240
+ del checkpoint
241
+ pad_id = tokenizer.pad_id() if tokenizer.pad_id() >= 0 else tokenizer.eos_id()
242
+ started = time.monotonic()
243
+ try:
244
+ for split, rows in zip(('train', 'val'), records):
245
+ indices = list(range(rank * settings['micro_batch'], len(rows), world * settings['micro_batch']))
246
+ shard = directory / f'{split}-rank-{rank}.npz'
247
+ done, scored_indices, scores = 0, [], []
248
+ if shard.exists():
249
+ with np.load(shard, allow_pickle=False) as old:
250
+ if str(old['identity']) != identity_digest(identity):
251
+ raise ValueError('Partial reference shard identity differs')
252
+ done = int(old['batches'])
253
+ scored_indices = old['indices'].tolist()
254
+ scores = old['scores'].tolist()
255
+ expected_indices = [i for start in indices[:done]
256
+ for i in range(start, min(start + settings['micro_batch'], len(rows)))]
257
+ if scored_indices != expected_indices or len(scores) != len(scored_indices):
258
+ raise ValueError('Partial reference shard cursor differs')
259
+
260
+ def save_shard():
261
+ temporary = Path(str(shard) + '.tmp')
262
+ with temporary.open('wb') as stream:
263
+ np.savez(stream, identity=identity_digest(identity), batches=done,
264
+ indices=np.asarray(scored_indices, dtype=np.int64),
265
+ scores=np.asarray(scores, dtype=np.float64).reshape(-1, 2))
266
+ stream.flush()
267
+ os.fsync(stream.fileno())
268
+ os.replace(temporary, shard)
269
+
270
+ # Every rank executes equal stop/barrier counts, even on uneven final shards.
271
+ max_batches = math.ceil(len(rows) / (world * settings['micro_batch']))
272
+ for batch in range(max_batches):
273
+ if stop_requested(args, device, world):
274
+ save_shard()
275
+ return None
276
+ if batch < done or batch >= len(indices):
277
+ continue
278
+ start = indices[batch]
279
+ selected = list(range(start, min(start + settings['micro_batch'], len(rows))))
280
+ padded = [rows[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected))
281
+ x, y, _ = collate_pairs(padded, settings['sequence_length'], pad_id)
282
+ with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False):
283
+ _, logps = raw(x.to(device), y.to(device), return_logits=False)
284
+ values = logps.reshape(-1, 2).double().cpu().numpy()[:len(selected)]
285
+ if not np.isfinite(values).all():
286
+ raise RuntimeError('Non-finite frozen reference scores')
287
+ scored_indices.extend(selected)
288
+ scores.extend(values.tolist())
289
+ done = batch + 1
290
+ if done % 128 == 0 or done == len(indices):
291
+ save_shard()
292
+ print(json.dumps(dict(event='reference_progress', rank=rank, split=split,
293
+ batches=done, total_batches=len(indices), scored_pairs=len(scores),
294
+ elapsed_seconds=time.monotonic() - started)), flush=True)
295
+ save_shard()
296
+ if world > 1:
297
+ dist.barrier()
298
+ if rank == 0:
299
+ files = {}
300
+ for split, rows in zip(('train', 'val'), records):
301
+ combined = np.full((len(rows), 2), np.nan, dtype=np.float64)
302
+ seen = np.zeros(len(rows), dtype=np.bool_)
303
+ for owner in range(world):
304
+ with np.load(directory / f'{split}-rank-{owner}.npz', allow_pickle=False) as shard:
305
+ if str(shard['identity']) != identity_digest(identity):
306
+ raise ValueError('Reference shard identity differs')
307
+ index = shard['indices']
308
+ if seen[index].any():
309
+ raise ValueError('Duplicate reference indices')
310
+ combined[index] = shard['scores']
311
+ seen[index] = True
312
+ if not seen.all() or not np.isfinite(combined).all():
313
+ raise ValueError('Reference cache is incomplete')
314
+ path = directory / f'{split}.npy'
315
+ atomic_array(path, combined)
316
+ files[path.name] = dict(sha256=digest(path), rows=len(rows))
317
+ atomic_json(directory / 'manifest.json', dict(format=CACHE_FORMAT, complete=True,
318
+ identity=identity, identity_sha256=identity_digest(identity), files=files))
319
+ if world > 1:
320
+ dist.barrier()
321
+ return validate_cache(directory, identity, [len(rows) for rows in records])
322
+ finally:
323
+ del raw
324
+ # Patched bound forwards and functional BF16 wrappers form cycles;
325
+ # collect them before allocating the policy and FP32 Adam state.
326
+ gc.collect()
327
+ if device.type == 'cuda':
328
+ torch.cuda.empty_cache()
329
+
330
+
331
+ def training_update(model, optimizer, microbatches, global_pairs, world, device, settings):
332
+ optimizer.zero_grad(set_to_none=True)
333
+ totals = torch.zeros(5, device=device, dtype=torch.float64)
334
+ for index, (x, y, active, reference) in enumerate(microbatches):
335
+ sync = model.no_sync() if world > 1 and index < len(microbatches) - 1 else contextlib.nullcontext()
336
+ x, y, active, reference = (value.to(device) for value in (x, y, active, reference))
337
+ counts = y.ne(-100).sum(dim=1).reshape(-1, 2)
338
+ with sync:
339
+ with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False):
340
+ _, logps = model(x, y, return_logits=False)
341
+ logps = logps.reshape(-1, 2)
342
+ losses, margins, accuracy = preference_metrics(logps, reference, counts,
343
+ settings['beta'], settings['objective'])
344
+ if not torch.isfinite(losses).all():
345
+ raise RuntimeError('Non-finite DPO training loss')
346
+ # DDP averages rank gradients; this is the true global pair mean,
347
+ # including a partial final update and entirely empty rank slots.
348
+ (losses * active).sum().mul(world / global_pairs).backward()
349
+ totals += torch.stack(((losses.detach() * active).double().sum(),
350
+ (margins.detach() * active).double().sum(),
351
+ (accuracy.detach() * active).double().sum(),
352
+ ((-logps[:, 0].detach() / counts[:, 0].clamp_min(1)) * active).double().sum(),
353
+ active.double().sum()))
354
+ norm = torch.nn.utils.clip_grad_norm_(model.parameters(), settings['clip_grad_norm'], error_if_nonfinite=True)
355
+ optimizer.step()
356
+ if world > 1:
357
+ dist.all_reduce(totals)
358
+ if int(totals[-1]) != global_pairs:
359
+ raise RuntimeError('Distributed DPO pair count differs')
360
+ values = (totals[:4] / totals[-1]).tolist()
361
+ return dict(zip(('loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)), float(norm)
362
+
363
+
364
+ def evaluate(model, records, reference, settings, device, pad_id, *, full=False):
365
+ state = rng_state(device)
366
+ was_training = model.training
367
+ model.eval()
368
+ totals = torch.zeros(4, dtype=torch.float64, device=device)
369
+ count = 0
370
+ limit = len(records) if full else min(len(records), settings['eval_batches'] * settings['micro_batch'])
371
+ try:
372
+ with torch.no_grad():
373
+ for start in range(0, limit, settings['micro_batch']):
374
+ selected = records[start:min(start + settings['micro_batch'], limit)]
375
+ n = len(selected)
376
+ selected += [None] * (settings['micro_batch'] - n)
377
+ x, y, active = collate_pairs(selected, settings['sequence_length'], pad_id)
378
+ x, y, active = x.to(device), y.to(device), active.to(device)
379
+ ref = torch.zeros((settings['micro_batch'], 2), device=device)
380
+ ref[:n] = torch.tensor(np.array(reference[start:start + n]), device=device)
381
+ with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False):
382
+ _, logps = model(x, y, return_logits=False)
383
+ logps = logps.reshape(-1, 2)
384
+ counts = y.ne(-100).sum(dim=1).reshape(-1, 2)
385
+ losses, margins, accuracy = preference_metrics(logps, ref, counts,
386
+ settings['beta'], settings['objective'])
387
+ totals += torch.stack(((losses * active).double().sum(), (margins * active).double().sum(),
388
+ (accuracy * active).double().sum(),
389
+ ((-logps[:, 0] / counts[:, 0].clamp_min(1)) * active).double().sum()))
390
+ count += n
391
+ finally:
392
+ model.train(was_training)
393
+ restore_rng(state, device)
394
+ values = (totals / count).tolist()
395
+ if not all(math.isfinite(value) for value in values):
396
+ raise RuntimeError('Non-finite DPO validation')
397
+ return dict(zip(('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)) | dict(evaluated_samples=count)
398
+
399
+
400
+ def validate_resume(checkpoint, signature, world):
401
+ if checkpoint.get('format') != FORMAT or checkpoint.get('signature') != signature:
402
+ raise ValueError('DPO resume signature differs; use a new output directory')
403
+ if len(checkpoint.get('rng_by_rank', [])) != world:
404
+ raise ValueError('DPO checkpoint lacks exact per-rank RNG state')
405
+
406
+
407
+ def validate_reference_path(args):
408
+ cache = Path(args.reference_cache_dir).resolve()
409
+ paths = [Path(path).resolve() for path in (args.work_dir, args.data_dir, args.weights.parent)]
410
+ if not cache.is_relative_to(WORKSPACE) or cache == WORKSPACE:
411
+ raise ValueError('Reference cache must stay inside the workspace')
412
+ if any(cache == path or cache.is_relative_to(path) or path.is_relative_to(cache) for path in paths):
413
+ raise ValueError('Reference cache must be separate from training outputs and inputs')
414
+ return cache
415
+
416
+
417
+ def train(args):
418
+ import sentencepiece as spm
419
+ from ops.checkpoint_writer import DistributedCheckpointWriter
420
+ work, data_dir, weights, tokenizer_path, config = validate_paths(
421
+ args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config)
422
+ cache = validate_reference_path(args)
423
+ plan, settings = load_plan(config)
424
+ rank = int(os.environ.get('RANK', '0'))
425
+ world = int(os.environ.get('WORLD_SIZE', '1'))
426
+ local_rank = int(os.environ.get('LOCAL_RANK', '0'))
427
+ device = torch.device('cuda', local_rank) if args.device == 'cuda' else torch.device('cpu')
428
+ if device.type == 'cuda':
429
+ torch.cuda.set_device(device)
430
+ if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported():
431
+ raise RuntimeError('BF16-capable CUDA GPU required')
432
+ torch.set_num_threads(settings['workers'])
433
+ torch.use_deterministic_algorithms(settings['deterministic'])
434
+ random.seed(settings['seed'] + rank)
435
+ np.random.seed(settings['seed'] + rank)
436
+ torch.manual_seed(settings['seed'] + rank)
437
+ tokenizer_hash, base_hash = digest(tokenizer_path), digest(weights)
438
+ sp = spm.SentencePieceProcessor(model_file=str(tokenizer_path))
439
+ train_data, val_data, manifest, manifest_hash = load_data(data_dir, sp, tokenizer_hash, settings['sequence_length'])
440
+ source = torch.load(weights, map_location='cpu', weights_only=False, mmap=True)
441
+ if (source.get('format') != 'warpstate-chat-v1' or source.get('inference_only') is not True or
442
+ source.get('stage') != 'sft' or source.get('tokenizer_sha256') != tokenizer_hash):
443
+ raise ValueError('Initialize DPO from an independent, tokenizer-matched SFT model export')
444
+ expected = plan.get('sft', plan.get('base', {}))
445
+ for key in ('step', 'tokens_seen', 'tokenizer_sha256'):
446
+ if key in expected and expected[key] != source.get(key):
447
+ raise ValueError(f'SFT source {key} differs from plan')
448
+ ancestor = source.get('origin', {})
449
+ for key in ('step', 'tokens_seen'):
450
+ expected_key = 'base_' + key
451
+ value = ancestor.get(key, ancestor.get(expected_key))
452
+ if expected_key in expected and expected[expected_key] != value:
453
+ raise ValueError(f'SFT source {expected_key} differs from plan')
454
+ origin = {key: value for key, value in source.items() if key != 'model'}
455
+ origin.update(weights_sha256=base_hash, weights_path=str(weights))
456
+ del source
457
+ identity = reference_identity(base_hash, manifest_hash, tokenizer_hash, settings, device, world)
458
+ if world > 1:
459
+ dist.init_process_group(backend='nccl' if device.type == 'cuda' else 'gloo')
460
+ try:
461
+ reference = prepare_reference(args, identity, (train_data, val_data), settings, device, rank, world, sp)
462
+ if reference is None:
463
+ print(json.dumps(dict(event='reference_paused', rank=rank)), flush=True)
464
+ return
465
+ reference_train, reference_val, reference_hash = reference
466
+ if getattr(args, 'reference_only', False):
467
+ print(json.dumps(dict(event='reference_complete', reference_sha256=reference_hash)), flush=True)
468
+ return
469
+ signature = dict(plan=plan, settings=settings, data_sha256=manifest_hash,
470
+ tokenizer_sha256=tokenizer_hash, renderer=RENDER_FORMAT, mask=MASK_FORMAT,
471
+ base_sha256=base_hash, reference_sha256=reference_hash,
472
+ world_size=world, device_type=device.type, torch_version=torch.__version__, sources=source_identity())
473
+ latest = work / 'latest.pt'
474
+ resuming = latest.exists()
475
+ saved = torch.load(latest if resuming else weights, map_location='cpu', weights_only=False, mmap=True)
476
+ if resuming:
477
+ validate_resume(saved, signature, world)
478
+ origin = saved['origin']
479
+ raw, cfg = load_model(saved, settings, device, sp)
480
+ raw.requires_grad_(True)
481
+ optimizer = torch.optim.AdamW(raw.parameters(), lr=settings['lr'], betas=(.9, .95),
482
+ eps=1e-8, weight_decay=settings['weight_decay'], fused=device.type == 'cuda')
483
+ step = cursor = tokens_seen = response_tokens_seen = 0
484
+ if resuming:
485
+ optimizer.load_state_dict(saved['optimizer'])
486
+ step, cursor = saved['step'], saved['sample_cursor']
487
+ tokens_seen, response_tokens_seen = saved['tokens_seen'], saved['response_tokens_seen']
488
+ resume_rng = saved['rng_by_rank'][rank] if resuming else None
489
+ del saved
490
+ if world > 1:
491
+ model = torch.nn.parallel.DistributedDataParallel(raw,
492
+ device_ids=[local_rank] if device.type == 'cuda' else None,
493
+ broadcast_buffers=False, gradient_as_bucket_view=True, bucket_cap_mb=128)
494
+ else:
495
+ model = raw
496
+ if resume_rng is not None:
497
+ restore_rng(resume_rng, device)
498
+ global_batch = settings['micro_batch'] * settings['grad_accum'] * world
499
+ total_steps = math.ceil(len(train_data) / global_batch) * settings['epochs']
500
+ if settings['max_steps'] is not None:
501
+ total_steps = min(total_steps, settings['max_steps'])
502
+ end_step = min(total_steps, step + args.max_steps) if args.max_steps is not None else total_steps
503
+ work.mkdir(parents=True, exist_ok=True)
504
+ if rank == 0:
505
+ atomic_json(work / 'settings.json', dict(signature=signature, origin=origin,
506
+ total_steps=total_steps, train_rows=len(train_data), val_rows=len(val_data)))
507
+ if step < total_steps:
508
+ (work / 'completion.json').unlink(missing_ok=True)
509
+ pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id()
510
+ last_validation = None
511
+ last_validation_step = None
512
+
513
+ def validation(full=False):
514
+ values = [evaluate(raw, val_data, reference_val, settings, device, pad_id, full=full) if rank == 0 else None]
515
+ if world > 1:
516
+ dist.broadcast_object_list(values, src=0, device=device)
517
+ return values[0]
518
+
519
+ def metric(values):
520
+ if rank == 0:
521
+ values.update(timestamp=time.time(), stage='dpo', step=step, tokens_seen=tokens_seen,
522
+ response_tokens_seen=response_tokens_seen, sample_cursor=cursor,
523
+ objective=settings['objective'])
524
+ with (work / 'metrics.jsonl').open('a') as stream:
525
+ stream.write(json.dumps(values, allow_nan=False) + '\n')
526
+ if values.get('event') == 'validation':
527
+ atomic_json(work / 'evaluation.json', dict(values, data_sha256=manifest_hash,
528
+ base_sha256=base_hash, reference_sha256=reference_hash,
529
+ tokenizer_sha256=tokenizer_hash, mask_format=MASK_FORMAT,
530
+ available_validation_samples=len(val_data)))
531
+ print(json.dumps(values, allow_nan=False), flush=True)
532
+
533
+ def save(writer):
534
+ state = rng_state(device)
535
+ states = [None] * world
536
+ if world > 1:
537
+ dist.all_gather_object(states, state)
538
+ else:
539
+ states = [state]
540
+ writer.submit(state_factory=lambda: dict(format=FORMAT, signature=signature, origin=origin,
541
+ config=vars(cfg), model=raw.state_dict(), optimizer=optimizer.state_dict(), step=step,
542
+ sample_cursor=cursor, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen,
543
+ val_loss=last_validation['val_loss'], validation=last_validation,
544
+ val_step=last_validation_step, rng_by_rank=states, completed=step >= total_steps,
545
+ total_steps=total_steps))
546
+
547
+ with DistributedCheckpointWriter(latest, rank=rank, device=device, distributed=world > 1) as writer:
548
+ last_validation = validation(full=True)
549
+ last_validation_step = step
550
+ metric(dict(event='validation', full_holdout=True, **last_validation))
551
+ last_saved_step = step if resuming else -1
552
+ model.train()
553
+ while step < end_step and not stop_requested(args, device, world):
554
+ writer.check()
555
+ indices, next_cursor = update_indices(len(train_data), settings['seed'], cursor,
556
+ settings['epochs'], global_batch)
557
+ if not indices:
558
+ break
559
+ microbatches = []
560
+ for micro in range(settings['grad_accum']):
561
+ offset = (micro * world + rank) * settings['micro_batch']
562
+ selected = indices[offset:offset + settings['micro_batch']]
563
+ padded = [train_data[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected))
564
+ x, y, active = collate_pairs(padded, settings['sequence_length'], pad_id)
565
+ ref = torch.zeros((settings['micro_batch'], 2), dtype=torch.float32)
566
+ if selected:
567
+ ref[:len(selected)] = torch.tensor(np.array(reference_train[selected]), dtype=torch.float32)
568
+ microbatches.append((x, y, active, ref))
569
+ lr = learning_rate(step, total_steps, settings)
570
+ for group in optimizer.param_groups:
571
+ group['lr'] = lr
572
+ started = time.perf_counter()
573
+ values, norm = training_update(model, optimizer, microbatches, len(indices), world, device, settings)
574
+ if device.type == 'cuda':
575
+ torch.cuda.synchronize(device)
576
+ seconds = time.perf_counter() - started
577
+ step += 1
578
+ cursor = next_cursor
579
+ actual_tokens = sum(len(train_data[i][branch]['input_ids']) - 1
580
+ for i in indices for branch in ('chosen', 'rejected'))
581
+ tokens_seen += actual_tokens
582
+ response_tokens_seen += sum(sum(value != -100 for value in train_data[i][branch]['labels'][1:])
583
+ for i in indices for branch in ('chosen', 'rejected'))
584
+ metric(dict(event='train', **values, lr=lr, grad_norm=norm, update_seconds=seconds,
585
+ tokens_per_second=actual_tokens / seconds, update_samples=len(indices)))
586
+ if step % settings['eval_every'] == 0 or step == end_step:
587
+ full = step >= total_steps
588
+ last_validation = validation(full=full)
589
+ last_validation_step = step
590
+ metric(dict(event='validation', full_holdout=full, **last_validation))
591
+ if step % settings['save_every'] == 0 or step == end_step:
592
+ save(writer)
593
+ last_saved_step = step
594
+ if last_saved_step != step:
595
+ save(writer)
596
+ writer.flush()
597
+ if rank == 0 and step >= total_steps:
598
+ atomic_json(work / 'completion.json', dict(format='warpstate-dpo-completion-v1',
599
+ stage='dpo', completed=True, paused_for_evaluation=True, step=step,
600
+ total_steps=total_steps, sample_cursor=cursor, tokens_seen=tokens_seen,
601
+ response_tokens_seen=response_tokens_seen, val_loss=last_validation['val_loss'],
602
+ evaluation_file=str(work / 'evaluation.json'), checkpoint=str(latest),
603
+ reference_sha256=reference_hash, timestamp=time.time()))
604
+ metric(dict(event='complete' if step >= total_steps else 'paused', total_steps=total_steps,
605
+ checkpoint=str(latest)))
606
+ validate_cache(cache, identity, (len(train_data), len(val_data)))
607
+ if digest(weights) != base_hash:
608
+ raise RuntimeError('Frozen SFT initialization changed during DPO')
609
+ finally:
610
+ if world > 1 and dist.is_initialized():
611
+ dist.destroy_process_group()
612
+
613
+
614
+ def main():
615
+ parser = argparse.ArgumentParser(description=__doc__)
616
+ for name in ('config', 'work-dir', 'data-dir', 'weights', 'tokenizer', 'reference-cache-dir'):
617
+ parser.add_argument('--' + name, required=True, type=Path)
618
+ parser.add_argument('--max-steps', type=int)
619
+ parser.add_argument('--stop-file', type=Path)
620
+ parser.add_argument('--device', choices=('cuda', 'cpu'), default='cuda')
621
+ parser.add_argument('--reference-only', action='store_true')
622
+ args = parser.parse_args()
623
+ if args.max_steps is not None and args.max_steps < 1:
624
+ parser.error('--max-steps must be positive')
625
+ if args.stop_file is None:
626
+ args.stop_file = args.work_dir / 'control/training-stop-request'
627
+ if not args.stop_file.resolve().is_relative_to(WORKSPACE):
628
+ parser.error('--stop-file must stay inside the workspace')
629
+ def stop(signum, frame):
630
+ global STOP
631
+ STOP = True
632
+ for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1):
633
+ signal.signal(signum, stop)
634
+ from run_40b import process_lock
635
+ validate_paths(args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config)
636
+ validate_reference_path(args)
637
+ args.work_dir.mkdir(parents=True, exist_ok=True)
638
+ args.reference_cache_dir.mkdir(parents=True, exist_ok=True)
639
+ plan, _ = load_plan(args.config)
640
+ rank = int(os.environ.get('RANK', '0'))
641
+ with process_lock(args.work_dir / f'rank-{rank}.lock'), \
642
+ process_lock(args.reference_cache_dir / f'rank-{rank}.lock'), base_pipeline_lock(plan):
643
+ train(args)
644
+
645
+
646
+ if __name__ == '__main__':
647
+ main()
ops/B200_HANDOFF.md ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Continue WarpState on the B200
2
+
3
+ This is the starting point for sessions running directly in
4
+ `/workspace/WarpState` on the **single visible B200** instance. Use the current
5
+ session state below; the original transfer details remain for provenance.
6
+
7
+ **Latest session state, September 10, 2026:** training is **STOPPED** at the
8
+ user's request. The graceful stop finished step **680 / 89,128,960 tokens**,
9
+ evaluated validation loss **4.39469**, and durably published the full checkpoint
10
+ at `work/run/latest.pt` before exit. This is the checkpoint for the next resume.
11
+
12
+ Checkpoint SHA256:
13
+ `bf689a10a030842699bfa3a25ff62e488cfa2d5b022b2b6e3bfc860359eb8b4f`.
14
+ Inspection verified all 2,005,111,424 finite FP32 parameters, 256 full AdamW
15
+ entries at step 680, FP32 moments, rank-zero RNG structure, unchanged data and
16
+ resume signature, and scheduled optimizer LR **0.000102**. The full inspection
17
+ is `work/validation/b200/paused-checkpoint-step-680.json`. The final CPU staging
18
+ took 4.93 seconds and background publication took 17.74 seconds; shutdown
19
+ waited for publication. No training GPU process remains.
20
+
21
+ Earlier snapshots `work/run/paused-step-488.pt` and
22
+ `work/run/validated-step-500.pt` remain preserved. The step-500 verification is
23
+ historical; use the newer step-680 report for the current resume state.
24
+ Resume only after the user requests it, using the controls below.
25
+
26
+ The B200 launcher now uses microbatch 16 / accumulation 4, compiled full loss,
27
+ BF16 kernel autotuning, and asynchronous full checkpoints. See
28
+ [B200_PERFORMANCE.md](B200_PERFORMANCE.md) for all measurements and validation,
29
+ and [B200_SCHEDULING.md](B200_SCHEDULING.md) for evidence linking recurring waits
30
+ to provider GPU scheduling. Check Supervisor status before starting another
31
+ job. The deployment record and handover tables below describe the original
32
+ step-448 transfer, not the current training cursor.
33
+
34
+ ## First commands
35
+
36
+ ```bash
37
+ cd /workspace/WarpState
38
+ source rtx6000_40b/ops/training-env.sh
39
+ source rtx6000_40b/ops/workspace-tools.sh
40
+ git status --short --branch
41
+ git log -3 --oneline
42
+ rtx6000_40b/ops/b200ctl.sh status
43
+ nvidia-smi
44
+ df -h /workspace/WarpState
45
+ rtx6000_40b/.server-venv/bin/python -m json.tool \
46
+ rtx6000_40b/work/validation/b200/deployment.json
47
+ ```
48
+
49
+ The generated `work/validation/b200/deployment.json` is the final deployment
50
+ record: use it to check artifact hashes, environment validation, and the
51
+ handover state. If it is missing or reports pending work, finish those checks
52
+ before training. Reading this document alone does not establish that a large
53
+ file transfer completed. Published `train.bin` and `latest.pt` must have
54
+ verified full SHA256 hashes; `.uploading` files are incomplete transfers.
55
+
56
+ The shell setup keeps caches and temporary files inside the workspace and
57
+ adds the project-local Git installation to `PATH`. Source both scripts in
58
+ each new shell. All source and Git history have been transferred; imported
59
+ history includes `a7b25a9`, followed by the handoff documentation commit. Use
60
+ `git log` for the actual current revision. Keep the repository on `main`,
61
+ follow the root [AGENTS.md](../../AGENTS.md), and commit completed source
62
+ changes using `WarpState Automation <warpstate@localhost>`. Generated data,
63
+ environments, checkpoints, and validation reports remain ignored.
64
+ This is an independent Git copy with no push remote configured; commits on
65
+ the B200 are local to that instance until explicitly synchronized elsewhere.
66
+
67
+ ## Environment and preserved training state
68
+
69
+ | Item | Handover state |
70
+ | --- | --- |
71
+ | Hardware | One NVIDIA B200; Ubuntu 24.04; Python 3.12 |
72
+ | Python environment | `/workspace/WarpState/rtx6000_40b/.server-venv` |
73
+ | Dependencies | PyTorch `2.14.0+cu130`; pinned [training requirements](training-requirements-resolved.txt) and [B200 requirements](b200-requirements.txt) |
74
+ | Model | 2,005,111,424 parameters; original tracked 32,768-token tokenizer |
75
+ | Training corpus | `work/data/train.bin`: 40,000,000,000 `uint16` token IDs; 80,000,000,000 bytes |
76
+ | Validation corpus | `work/data/val.bin`: 1,048,576 `uint16` token IDs; 2,097,152 bytes |
77
+ | Checkpoint | `work/run/latest.pt`: full checkpoint at step **448**, with **58,720,256** tokens consumed |
78
+ | Restored state | Model, full FP32 AdamW optimizer, token cursor, learning-rate schedule, and rank-zero random state |
79
+ | Precision | **BF16 computation only**, as explicitly required by the user; parameters, gradients, and AdamW state stay FP32 |
80
+
81
+ Paths beginning with `work/` above are relative to `rtx6000_40b/`. Preserve
82
+ the copied data manifest and preparation state. The original request was
83
+ revision `main`; the data recipe pins the immutable upstream revision
84
+ separately. Changing the requested revision for this existing work directory
85
+ breaks its resume signature. The existing data requires no new download or
86
+ tokenization.
87
+
88
+ The host's GCC/G++ 13.2 and Python development headers passed actual Inductor
89
+ compilation. No additional compiler setup is needed. The launcher also
90
+ supports an optional `tmp/remote-setup/toolchain-env.sh` if the toolchain is
91
+ changed later. Provisioning details are in [B200_SETUP.md](B200_SETUP.md).
92
+
93
+ ## Resume and pause controls
94
+
95
+ Use [b200ctl.sh](b200ctl.sh) with [b200-supervisord.conf](b200-supervisord.conf).
96
+ Both `autostart` and `autorestart` are disabled. Starting the Supervisor alone
97
+ leaves training stopped.
98
+
99
+ If `b200ctl.sh status` reports that the control socket is absent, first check
100
+ whether the daemon is already starting. Initialize it only when absent:
101
+
102
+ ```bash
103
+ mkdir -p rtx6000_40b/work/control rtx6000_40b/work/logs
104
+ rtx6000_40b/.server-venv/bin/supervisord \
105
+ -c rtx6000_40b/ops/b200-supervisord.conf
106
+ rtx6000_40b/ops/b200ctl.sh status
107
+ ```
108
+
109
+ After deployment verification and a user request to resume training:
110
+
111
+ ```bash
112
+ rtx6000_40b/ops/b200ctl.sh start warpstate-train
113
+ rtx6000_40b/ops/b200ctl.sh status
114
+ tail -n 30 rtx6000_40b/work/logs/b200-train.log
115
+ tail -n 5 rtx6000_40b/work/run/metrics.jsonl
116
+ ```
117
+
118
+ Initial core compilation can take several minutes. Confirm the log restores
119
+ step 448, or a later checkpoint after subsequent training, and then reports
120
+ finite loss and completed optimizer updates. Measure throughput after warmup.
121
+ To follow the live log, use `tail -f` on
122
+ `rtx6000_40b/work/logs/b200-train.log`.
123
+
124
+ For interactive monitoring, the separate `warpstate-b200` tmux session has a
125
+ `training` window with the live training log and a second pane showing GPU
126
+ usage. Attach from the repository root:
127
+
128
+ ```bash
129
+ rtx6000_40b/ops/tmux-workspace.sh attach -t warpstate-b200
130
+ ```
131
+
132
+ Detach with `Ctrl-b`, then `d`. Training remains managed by Supervisor, so
133
+ detaching or closing a monitoring pane does not stop training. Continue to
134
+ use `b200ctl.sh` for start/stop controls. The wrapper keeps the tmux socket
135
+ under `work/control`; workspace-local installation details are recorded in
136
+ [tmux-workspace-install.md](tmux-workspace-install.md).
137
+
138
+ To pause and preserve the full continuation state:
139
+
140
+ ```bash
141
+ rtx6000_40b/ops/b200ctl.sh stop warpstate-train
142
+ rtx6000_40b/ops/b200ctl.sh status
143
+ tail -n 30 rtx6000_40b/work/logs/b200-train.log
144
+ ```
145
+
146
+ Stopping requests a graceful save after the current optimizer update and
147
+ allows up to 15 minutes. Verify the saved-checkpoint message and `STOPPED`
148
+ status. The next start resumes `latest.pt`. Preserve this checkpoint;
149
+ **never replace it with a fresh run or use `--from-scratch` to resume**.
150
+
151
+ The original `train-fast.sh` hardcodes the workstation's two GPU UUIDs.
152
+ The original `supervisord.conf`/`datactl.sh` belongs to that workstation,
153
+ specifies user `root`, and can automatically start preparation. Use the B200
154
+ controls on this instance. Run only one training launcher against `work`.
155
+
156
+ ## Current single-GPU configuration
157
+
158
+ [train-b200.sh](train-b200.sh) selects one GPU and uses `--reconfigure` to
159
+ continue the existing full checkpoint with its data, objective, and global batch
160
+ unchanged:
161
+
162
+ | Setting | Value |
163
+ | --- | --- |
164
+ | Sequence length | 2,048 |
165
+ | Microbatch / accumulation | 16 / 4 |
166
+ | Tokens per optimizer update | **131,072** |
167
+ | Core compilation | Enabled; `max-autotune-no-cudagraphs` |
168
+ | Core activation checkpointing | Disabled |
169
+ | Loss | Compiled full vocabulary loss |
170
+ | BF16 autocast cache | Enabled |
171
+ | Allocator cap | 150 GiB |
172
+ | Additional required free VRAM at startup | 3 GiB beyond the cap |
173
+ | Required free SSD space at startup | At least 53 GiB beyond existing files |
174
+ | Save / evaluation interval | Every 50 / 100 updates |
175
+ | Checkpoint writing | Immutable CPU staging, then asynchronous serialization and atomic publication |
176
+
177
+ The allocator cap excludes CUDA context and library memory. The selected
178
+ preset measured about 72.5K median tokens/s and 139.3 GiB peak reserved memory.
179
+ Sustained throughput is much lower and varies with provider scheduling waits;
180
+ see [B200_PERFORMANCE.md](B200_PERFORMANCE.md) for complete measurements and
181
+ limitations. The dataset, 40B token budget, global batch and LR schedule remain
182
+ unchanged. Autotuning can extend initial compilation; eight compiler workers
183
+ are used by default.
184
+
185
+ Asynchronous saves need approximately 24 GB of additional host memory. CPU
186
+ staging finishes before the next update; only serialization and disk I/O overlap
187
+ training. One outstanding save bounds memory, failures stop the job, and graceful
188
+ exit waits for publication. Wait for the **Saved** message, not just **Staged**.
189
+ `--no-async-checkpoint` restores synchronous saves. CUDA graphs are validated
190
+ as an optional `--compile-mode reduce-overhead` override with persistent FP32
191
+ gradient buffers; they were not selected as the default. cuBLASLt preference
192
+ remains an optional experiment rather than a default override.
193
+
194
+ ## Original deployment validation
195
+
196
+ The remote CPU suite passed **92 tests**, with **5 GPU cases deselected**.
197
+ This provider hung during collection with `CUDA_VISIBLE_DEVICES=''`; keep
198
+ normal GPU visibility and use the explicit test selection documented in
199
+ [B200_SETUP.md](B200_SETUP.md). Do not copy the workstation's empty-mask CPU
200
+ test command into this environment.
201
+
202
+ Both single-GPU full-state resume cases also passed (**2 passed, 3
203
+ deselected**, 80.37 seconds). Actual BF16 Inductor compilation passed with
204
+ one graph and 38 generated kernels; two fused AdamW updates retained finite
205
+ FP32 parameters, gradients and moments. These checks used small fixtures;
206
+ the production checkpoint was preserved. The large transfers' final hashes,
207
+ checkpoint inspection and final readiness state are recorded in
208
+ `work/validation/b200/deployment.json`; consult that record for completion.
209
+ Multi-GPU tests require more than this instance's one GPU.
210
+
211
+ The deployment checks above describe the original transfer. Current source
212
+ checks, full-state continuation tests, and production checkpoint validation are
213
+ recorded in [B200_PERFORMANCE.md](B200_PERFORMANCE.md). Read the current
214
+ checkpoint inspection and confirm job status before any later resume.
ops/B200_OPTIMIZATION_PLAN.md ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # B200 optimization research, September 10, 2026
2
+
3
+ This records the initial research before implementation. The subsequent
4
+ measurements, selected launcher settings, CUDA graph validation, backend
5
+ comparison and asynchronous checkpoint implementation are in
6
+ [B200_PERFORMANCE.md](B200_PERFORMANCE.md). See
7
+ [B200_SCHEDULING.md](B200_SCHEDULING.md) for the completed stall investigation.
8
+
9
+ At the time of this initial research, training was paused at step **488**, with
10
+ **63,963,136** tokens consumed and validation loss **4.7938699126**.
11
+ `work/run/latest.pt` and `work/run/paused-step-488.pt` then referenced
12
+ the same verified 24,061,671,905 bytes. Subsequent atomic replacements of
13
+ `latest.pt` preserved the step-488 hardlink. Consult [B200_HANDOFF.md](B200_HANDOFF.md)
14
+ for the current paused checkpoint.
15
+
16
+ Checkpoint SHA256:
17
+ `69d37e902c41f541d55a07b3d10dcb7643721240cc6aae032455cd8904c64608`.
18
+ Inspection verified 2,005,111,424 FP32 parameters, all 256 AdamW state entries
19
+ and their step-488 counters, FP32 moments, rank-zero RNG, token cursor, and
20
+ learning rate 0.0000732. Details are in the ignored generated report
21
+ `work/validation/b200/paused-checkpoint-step-488.json`.
22
+
23
+ The dataset and production execution settings are preserved. The constraints
24
+ for all candidates remain BF16 computation, FP32 master parameters and AdamW,
25
+ sequence length 2,048, and **131,072 global tokens per optimizer update**.
26
+ The original learning-rate schedule, objective, tokenizer, and data order
27
+ remain unchanged. No optimization benchmark or production restart was run
28
+ for this research request.
29
+
30
+ ## Measured baseline and first priority
31
+
32
+ Production updates 450–488 processed 5,111,808 tokens in 351.161 seconds
33
+ inside the update timer: **14,557 sustained tokens/s**. The median individual
34
+ update ran at **61,888 tokens/s**. Nine updates took approximately 30–32
35
+ seconds; ordinary updates took approximately 2.12 seconds. The timer excludes
36
+ checkpoint writes and evaluation. Source: `work/run/metrics.jsonl`.
37
+
38
+ Peak allocation was **59.40 GiB**, with **60.48 GiB reserved**. The B200 reports
39
+ 182,631 MiB total memory; the current allocator cap is 150 GiB. Raising that
40
+ cap alone does not increase useful work.
41
+
42
+ During sampled stalls, GPU utilization fell to zero and the trainer's main
43
+ thread waited on a futex without accumulating CPU execution time. CPU quota
44
+ throttling did not increase; cgroup CPU/I/O pressure averages were zero.
45
+ There is no matching periodic branch in the training loop. This does not
46
+ identify the cause or eliminate data/pinning, compiler, autograd, driver,
47
+ or provider waits. Removing the waits is the largest observed opportunity;
48
+ the median rate is evidence of active compute speed, not a promised sustained
49
+ rate after a fix.
50
+
51
+ Use the benchmark's optional `--diagnose-stalls` to attribute host time to
52
+ batch transfer, forward, backward, clipping, AdamW, and final synchronization.
53
+ It adds no CUDA synchronization and emits Python stacks every ten seconds
54
+ during measured updates. Its timings include implicit waits and are not GPU
55
+ kernel timings. Then correlate the waiting call with CUDA and OS-runtime
56
+ traces across several updates. The existing `--profile` captures only one
57
+ measured update and can miss these stalls. Installed Nsight Systems is
58
+ 2024.3.2.0, so verify B200/tool compatibility before using options documented
59
+ for newer releases. Sources: [NVIDIA Nsight Systems](https://docs.nvidia.com/nsight-systems/UserGuide/index.html#os-runtime-libraries-trace),
60
+ [PyTorch profiler](https://docs.pytorch.org/docs/2.14/profiler.html).
61
+
62
+ ## Candidates, in order
63
+
64
+ | Candidate | Concrete experiment | Why it may help and what remains unverified |
65
+ | --- | --- | --- |
66
+ | Larger microbatch | Compare microbatch/accumulation **4/16**, **8/8**, **16/4** at the same sequence length | Larger GEMMs and fewer microbatch dispatches can improve compute efficiency. Prior workstation memory was about 86 GiB at microbatch 8; roughly 137 GiB at 16 is an extrapolation, not a B200 measurement. Measure startup and steady peaks below the existing cap. [NVIDIA GEMM guidance](https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html) |
67
+ | Keep loss activations instead of recomputing | Existing `--loss-mode compiled-full` at the best fitting microbatch | Uses more memory to avoid repeating the vocabulary projection during backward. At microbatch 16, BF16 logits alone are 2 GiB, before FP32 loss intermediates. Earlier workstation gains were about 3%; B200 gains are unmeasured. [PyTorch recomputation tradeoff](https://pytorch.org/blog/activation-checkpointing-techniques/), [local prior results](PERFORMANCE.md) |
68
+ | Tune matrix kernels for the chosen shape | Existing `--compile-mode max-autotune-no-cudagraphs` | Profiles candidate implementations while preserving BF16. Compile time increases; exclude it from timing and compare numerical results. [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html) |
69
+ | Compare cuBLASLt and workspace choices | Compare `TORCH_BLAS_PREFER_CUBLASLT=1` against the current backend, recording effective workspace | NVIDIA recommends 32 MiB workspace for Blackwell sm10x. Current `CUBLAS_WORKSPACE_CONFIG=:4096:8` already provides 32 MiB. PyTorch's unified workspace can cap cuBLASLt at the cuBLAS allocation; changing only `CUBLASLT_WORKSPACE_SIZE` may have no effect. Library preference may fall back. [NVIDIA cuBLAS](https://docs.nvidia.com/cuda/archive/13.0.0/cublas/index.html#cublassetworkspace), [PyTorch BLAS controls](https://docs.pytorch.org/docs/2.14/backends.html#torch.backends.cuda.preferred_blas_library) |
70
+ | CUDA graphs when traces show dispatch overhead | Existing `--compile-mode reduce-overhead` | Can trade additional persistent GPU memory for lower host launch overhead. Static training shapes help, but accumulation, evaluation, and partial final updates require validation. Graphs are not an established remedy for the observed waits. [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html), [graph constraints](https://docs.pytorch.org/docs/2.14/notes/cuda.html#constraints) |
71
+
72
+ The model already uses compiled cores, BF16 autocast caching, fused AdamW,
73
+ and disabled core activation checkpointing. Recommending those as new changes
74
+ would not explain further gains. The prior workstation profile attributes
75
+ most matrix time to FFN and input projections; attention and the output head
76
+ are smaller fractions. Those proportions need confirmation on B200 before
77
+ investing in custom attention or loss kernels.
78
+
79
+ As a later wall-clock improvement, investigate checkpoint staging and
80
+ asynchronous serialization. PyTorch documents copying a stable model/optimizer
81
+ snapshot into CPU buffers before background writing, with one outstanding
82
+ save at a time. This needs approximately another checkpoint's worth of host
83
+ memory and careful completion/error handling. It requires integration work
84
+ to retain this runner's full-state format, atomic publication and graceful
85
+ stop guarantee. It would address save pauses, not the recurring waits inside
86
+ the measured updates. Keep the existing save cadence during initial compute
87
+ comparisons. [PyTorch asynchronous checkpoint recipe](https://docs.pytorch.org/tutorials/recipes/distributed_async_checkpoint_recipe.html).
88
+
89
+ ## Comparison and promotion requirements
90
+
91
+ Use the preserved step-488 checkpoint for every disposable benchmark. The
92
+ existing benchmark copies model and optimizer tensors to separate CUDA
93
+ storage, reads prepared data in place, and writes no training checkpoints.
94
+ Its learning rate stays fixed at the loaded value and RNG is reseeded; its
95
+ updates are timing experiments rather than production continuation.
96
+
97
+ Start with two warmup updates and at least twenty measured updates, extending
98
+ the window if needed to capture several stalls. Report total tokens divided
99
+ by total update time, median update speed, slow-update count, VRAM peaks,
100
+ loss and gradient norm. Keep profiling runs separate from throughput runs.
101
+ Use fresh `work/benchmark-*` output directories and identical checkpoint/data
102
+ inputs for each candidate. Benchmark the baseline again beside the winner
103
+ to distinguish configuration effects from changing host conditions.
104
+
105
+ Before promoting a candidate, compare loss and gradients with the baseline
106
+ on identical tokens and validate full-state continuation. The existing CPU
107
+ loss tests preserve objective and parameter identity, but their compiled
108
+ case uses an eager backend. The compiled-full resume test currently requires
109
+ two GPUs. Add and run a single-GPU CUDA regression for compiled-full before
110
+ promoting it: continuous versus interrupted/resumed updates, canonical model
111
+ keys, full optimizer and RNG state, token cursor, and the final partial batch.
112
+ Microbatch-only changes still require the existing resume/signature checks
113
+ and a short production continuation with finite loss and a verified saved
114
+ checkpoint. Preserve schedule and data invariants through `--reconfigure`.
115
+
116
+ The currently committed diagnostics have only CPU validation: syntax and CLI
117
+ parsing, accumulated stage timing including exception cleanup, and watchdog
118
+ cancellation. GPU profiling and the candidate grid remain pending. Resume
119
+ optimization work when the user asks to continue; keep the B200 job stopped
120
+ until then.
ops/B200_PERFORMANCE.md ADDED
@@ -0,0 +1,231 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # B200 BF16 execution and checkpoint validation
2
+
3
+ Measurements from September 10, 2026 use disposable copies of the full
4
+ step-488 checkpoint (63,963,136 tokens), including FP32 AdamW moments.
5
+ The prepared dataset, tokenizer, model architecture, sequence length 2,048,
6
+ and global batch of 131,072 tokens remain unchanged. No FP8 or FP16 training
7
+ is used. Production resumed from step 488 with validation loss 4.79387 after
8
+ the user requested continuation; see the production validation section below.
9
+ The preserved input is `work/run/paused-step-488.pt`, SHA256
10
+ `69d37e902c41f541d55a07b3d10dcb7643721240cc6aae032455cd8904c64608`.
11
+
12
+ ## Latest pause
13
+
14
+ Training stopped gracefully at the user's request after step **680 /
15
+ 89,128,960 tokens**, with training loss **4.60232** and validation loss
16
+ **4.39469**. The full `work/run/latest.pt` checkpoint was published before the
17
+ process exited. Final staging took **4.93 seconds**, followed by **17.74
18
+ seconds** of background writing. All 256 optimizer entries are at step 680;
19
+ model/AdamW values are finite FP32, the schedule and data signature are
20
+ unchanged, and the saved optimizer LR is **0.000102**. SHA256 and the complete
21
+ inspection are recorded in `work/validation/b200/paused-checkpoint-step-680.json`
22
+ and the current [handoff](B200_HANDOFF.md).
23
+
24
+ The final 100 completed updates (581-680) averaged **16,501 wall-clock
25
+ tokens/s**, including between-update overhead, with a median **72,418 tokens/s**
26
+ and 18 slow updates. This window ends before the final pause evaluation/save.
27
+ Peak reserved memory stayed **139.55 GiB**. The earlier measurements below
28
+ remain historical comparisons. Production is now stopped; source settings,
29
+ prepared data and earlier preserved checkpoint snapshots remain unchanged.
30
+
31
+ ## Selected launcher and measured tradeoffs
32
+
33
+ [train-b200.sh](train-b200.sh) now selects microbatch 16 / accumulation 4,
34
+ compiled full loss, `max-autotune-no-cudagraphs`, and asynchronous full
35
+ checkpoints. BF16 autocast, FP32 master weights/gradients/AdamW, the 40B token
36
+ budget, original learning-rate schedule, seed, and save/evaluation cadence
37
+ remain unchanged. Core activation checkpointing stays disabled and the
38
+ allocator cap remains 150 GiB. Eight compiler workers are the default;
39
+ initial autotuning can take several minutes.
40
+
41
+ Every benchmark below uses two warmup and twenty measured updates. Sustained
42
+ throughput divides measured tokens by measured update seconds, including
43
+ provider scheduling waits. Median speed describes a typical update; it is
44
+ not continuous throughput or a guarantee of exclusive GPU capacity. Loading,
45
+ compilation, validation and checkpoint writes are outside these measurements.
46
+
47
+ | Microbatch / accumulation | Loss | Core mode / BLAS preference | Sustained tokens/s | Median tokens/s | Peak reserved GiB |
48
+ | --- | --- | --- | ---: | ---: | ---: |
49
+ | 4 / 16 | Checkpointed | Default; stall diagnostics | 13,808 | 61,853 | 60.44 |
50
+ | 4 / 16 | Checkpointed | Final baseline control | 15,344 | 61,839 | 60.44 |
51
+ | 8 / 8 | Checkpointed | Default | 16,362 | 65,636 | 86.09 |
52
+ | 16 / 4 | Checkpointed | Default | 16,467 | 67,421 | 137.72 |
53
+ | 16 / 4 | Compiled full | Default | 16,698 | 71,370 | 141.25 |
54
+ | 16 / 4 | Compiled full | Autotune, no graphs | 20,752 | 72,499 | 139.31 |
55
+ | 16 / 4 | Compiled full | Default / cuBLASLt preference | 18,718 | 71,309 | 141.25 |
56
+ | 16 / 4 | Compiled full | CUDA graphs, corrected timer | 17,432 | 70,660 | 141.31 |
57
+
58
+ Autotuning gave the fastest typical update, about 17.2% above the initial
59
+ baseline. Peak allocated memory was 138.18 GiB. Its first warmup took 206.16
60
+ seconds including compilation/tuning and scheduling waits, excluded above.
61
+ The baseline diagnostic run encountered five slow measured updates, versus
62
+ three in the autotune run. Scheduling phase contributes substantially to
63
+ aggregate differences, so sustained gains cannot be attributed solely to the
64
+ chosen execution settings. The final baseline control repeated a median of
65
+ 61,839 tokens/s, within 0.03% of the original diagnostic baseline despite
66
+ different scheduling waits; the typical-speed gain is repeatable.
67
+
68
+ cuBLASLt preference produced exactly the same losses and gradient norms as
69
+ the default compiled-full case and no meaningful median speed improvement.
70
+ It is available for comparison but is not forced in the B200 launcher. The
71
+ recorded preferences were `_BlasBackend.Cublas` and `_BlasBackend.Cublaslt`;
72
+ a preference does not guarantee every operation uses that backend. Current `CUBLAS_WORKSPACE_CONFIG=:4096:8`
73
+ provides 32 MiB workspace. See [PyTorch backend controls](https://docs.pytorch.org/docs/2.14/backends.html#torch.backends.cuda.preferred_blas_library)
74
+ and [NVIDIA workspace guidance](https://docs.nvidia.com/cuda/archive/13.0.0/cublas/index.html#cublassetworkspace).
75
+
76
+ ## CUDA graph implementation and evidence
77
+
78
+ `--compile-mode reduce-overhead` now preallocates FP32 gradient buffers outside
79
+ capture and zeros them in place between updates. Without this, the installed
80
+ PyTorch rejected accumulation into graph-owned gradient outputs. Other modes
81
+ retain `zero_grad(set_to_none=True)`. No per-core step markers were inserted;
82
+ completed forward/backward microbatches use PyTorch's iteration tracking.
83
+
84
+ The full-model graph run recorded **56 captures**, **zero compiler skips**,
85
+ and **4,480 successful nonempty replays during measured updates**. Losses and
86
+ gradient norms match the default compiled-full trajectory exactly. The
87
+ optional [diagnostic helper](cudagraph_diagnostics.py) counts successful tree
88
+ replay calls using private PyTorch 2.14 APIs, restores its wrapper on exit,
89
+ and adds no CUDA operations or synchronization. Counts are not GPU timings.
90
+
91
+ The first exploratory graph benchmark excluded gradient zeroing from the
92
+ timer and logged capture details with `TORCH_LOGS=cudagraphs`. It is retained
93
+ as evidence but not used in the table. The corrected run includes gradient
94
+ zeroing, matching the production update timer. Future benchmark configurations
95
+ all use this timing. First-update peak allocation was 138.15 GiB; steady
96
+ allocator peaks of 34.69 GiB during replay do not represent the complete graph
97
+ memory footprint. Reserved memory, including graph pools, remained 141.31 GiB.
98
+
99
+ Graph mode is validated and available as an override, but was not selected as
100
+ the default because it did not improve typical update speed. It captures cores;
101
+ embedding, depth modulation, output projection, optimizer and the separately
102
+ compiled full-loss reduction are outside those core graphs. See
103
+ [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html)
104
+ and [CUDA graph constraints](https://docs.pytorch.org/docs/2.14/notes/cuda.html#constraints).
105
+
106
+ ## Asynchronous full checkpoints
107
+
108
+ `--async-checkpoint` is optional in the general runner and enabled in the B200
109
+ launcher. It retains the existing `warpstate-server-v1` single-file checkpoint,
110
+ including model, complete AdamW state, RNG, token cursor, schedule/signature,
111
+ configuration and provenance. `--no-async-checkpoint` restores synchronous
112
+ saving. The asynchronous option currently requires a single training rank.
113
+
114
+ The main thread first creates a fully independent CPU snapshot. GPU-to-CPU
115
+ staging completes synchronously before the next optimizer update can mutate
116
+ live state. One background writer then serializes, flushes and fsyncs the file,
117
+ atomically replaces `latest.pt`, and fsyncs the parent directory. Only
118
+ serialization and disk I/O overlap training. Staging needs roughly 24 GB of
119
+ additional host memory; it does not allocate a full extra GPU checkpoint.
120
+
121
+ A new save waits for the previous writer before allocating another snapshot,
122
+ so only one snapshot/write can be outstanding. Background errors surface
123
+ between updates and again on flush/close. Normal exit, graceful stop and
124
+ training exceptions join any already-staged valid save before the pipeline
125
+ lock is released. Exceptions do not snapshot partially updated live weights.
126
+ Logs distinguish staging from completed publication and include snapshot
127
+ step/tokens. A **Staged** message alone is not a durable checkpoint.
128
+ This follows the stable-snapshot and bounded-outstanding-save principles in
129
+ [PyTorch's asynchronous checkpoint recipe](https://docs.pytorch.org/tutorials/recipes/distributed_async_checkpoint_recipe.html)
130
+ while preserving this repository's existing checkpoint format.
131
+
132
+ ## Numerical and continuation checks
133
+
134
+ All 22 updates in each completed benchmark had finite loss and gradient norm.
135
+ Against the microbatch-4 trajectory, maximum absolute loss differences were
136
+ 0.00006819 for microbatch 8, 0.00006199 for microbatch 16, 0.00031376 for compiled
137
+ full loss at microbatch 16, and 0.00006914 for its autotuned version. Maximum
138
+ relative gradient-norm differences were 0.2680%, 0.1688%, 1.1087%, and 0.1760%.
139
+ Executions preserve the objective but need not be bitwise equal across modes.
140
+ Checkpoint and dataset identities remained unchanged throughout the benchmarks.
141
+
142
+ The final CPU runner/loss/checkpoint/launcher selection passed **91 tests**
143
+ with nine GPU cases deselected. Default and autotuned compiled-full CUDA
144
+ continuation cases passed earlier; the graph and asynchronous cases then
145
+ passed **2/2** in 84.41 seconds. These regressions check exact model, full
146
+ optimizer and RNG state, canonical keys, FP32 state, learning-rate schedule,
147
+ finite validation, and the final partial update at token counts 32, 64 and 73.
148
+ The asynchronous case compares asynchronous pause/resume against synchronous
149
+ continuous training and verifies that the asynchronous path actually ran.
150
+ CPU writer tests cover immutable snapshots during AdamW mutation, atomic
151
+ publication and failure cleanup, bounded writes, persistent errors and both
152
+ normal/exceptional exit flushing. Only one GPU workload ran at a time.
153
+
154
+ ## Production continuation and first asynchronous save
155
+
156
+ After the user requested continuation, production resumed under B200
157
+ Supervisor in tmux session `warpstate-b200`, window `training`, new pane `0.1`.
158
+ Initial validation loss matched the pause snapshot at **4.79387**. The first
159
+ update, step 489, took 8.11 seconds including cached compilation startup.
160
+ Steps **490-500** processed 1,441,792 tokens at **19,223 sustained tokens/s**,
161
+ with a median **72,509 tokens/s**, including provider waits in update timing.
162
+ Peak allocated/reserved memory was **138.18 / 139.55 GiB**. This window excludes
163
+ validation, CPU checkpoint staging and background disk-write overhead.
164
+
165
+ At step **500 / 65,536,000 tokens**, training loss was **5.0160** and validation
166
+ loss was **4.74471**. The different training batches explain why individual
167
+ training losses can be above five while the fixed validation loss is lower.
168
+ The complete snapshot took **35.15 seconds** to stage synchronously, followed
169
+ by **44.44 seconds** of background serialization/fsync/publication. Steps
170
+ **501-507** completed while that writer was active. These staging and write
171
+ times are observed wall times on this host; the staging wait was not separately
172
+ profiled and should not be attributed entirely to transfer bandwidth.
173
+
174
+ The published checkpoint retains all **2,005,111,424 FP32 parameters**, **256
175
+ full AdamW entries at step 500**, finite FP32 moments, one rank's RNG state,
176
+ the same data/origin, **131,072 tokens per update**, and scheduled optimizer
177
+ LR **0.000075**. The full SHA256 and inspection are recorded in
178
+ `work/validation/b200/optimized-checkpoint-step-500.json`. The checkpoint is
179
+ preserved as `work/run/validated-step-500.pt`; later saves advanced `latest.pt`.
180
+ The original `paused-step-488.pt` remained unchanged. Training continued after
181
+ this first-save validation, until the later step-680 pause recorded above.
182
+ Preparation, tokenizer, dataset, schedule and precision were preserved.
183
+
184
+ ## Remaining bottleneck
185
+
186
+ The recurring extra 30-second waits are strongly associated with the provider's
187
+ HAIShare GPU scheduler, which waits for ownership beneath CUDA launch calls.
188
+ Data transfer took about 5-6 ms/update; AdamW and CPU quota throttling did not
189
+ account for sampled delays. See [B200_SCHEDULING.md](B200_SCHEDULING.md) for
190
+ evidence and uncertainty. No provider scheduling or interposition settings
191
+ were changed. Confirm exclusive compute with the provider; full visible VRAM
192
+ alone does not establish exclusive allocation.
193
+
194
+ ## Reproduction
195
+
196
+ Use [b200ctl.sh](b200ctl.sh) to stop production gracefully and confirm the GPU
197
+ is free before benchmarking. Preserve the pause snapshot. Benchmarks copy
198
+ model and optimizer state into disposable CUDA storage and never save a
199
+ training checkpoint. Their learning rate stays fixed at the loaded value and
200
+ RNG is reseeded, so benchmark updates are timing experiments.
201
+
202
+ From `/workspace/WarpState`, choose a new output directory for each run:
203
+
204
+ ```bash
205
+ source rtx6000_40b/ops/training-env.sh
206
+ source rtx6000_40b/ops/workspace-tools.sh
207
+ export TORCHINDUCTOR_COMPILE_THREADS=8
208
+ rtx6000_40b/ops/benchmark.sh \
209
+ --checkpoint /workspace/WarpState/rtx6000_40b/work/run/paused-step-488.pt \
210
+ --micro-batch 16 --seq-len 2048 --global-tokens-per-update 131072 \
211
+ --compile-cores --compile-mode max-autotune-no-cudagraphs \
212
+ --no-activation-checkpointing --autocast-cache --loss-mode compiled-full \
213
+ --loss-chunk-size 1024 --vram-limit-gib 150 \
214
+ --warmup-updates 2 --measured-updates 20 \
215
+ --output-dir /workspace/WarpState/rtx6000_40b/work/benchmark-b200-reproduction
216
+ ```
217
+
218
+ For backend comparisons use `--compile-mode default` and select
219
+ `--blas-backend default`, `--blas-backend cublas`, or `--blas-backend cublaslt`
220
+ in otherwise identical cases. The completed cuBLASLt case used the equivalent
221
+ `TORCH_BLAS_PREFER_CUBLASLT=1` environment preference, recorded in its metadata.
222
+ For graph comparison use `--compile-mode reduce-overhead --cudagraph-diagnostics`.
223
+ The same compile-mode override is accepted by `train-b200.sh` on a later
224
+ production launch; the general runner's async flag is independent.
225
+
226
+ Raw `summary.json` and rank records are under `work/benchmark-b200-*`;
227
+ `work/validation/b200/optimization-grid.json` aggregates comparisons.
228
+ Metadata records source hashes, input identities, versions, backend options,
229
+ precision and each update's measurement. Shared logs are shown in tmux session
230
+ `warpstate-b200`, window `optimization`; attach with
231
+ `rtx6000_40b/ops/tmux-workspace.sh attach -t warpstate-b200`.
ops/B200_SCHEDULING.md ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # B200 scheduling investigation
2
+
3
+ On September 10, 2026, a BF16 benchmark reproduced the recurring training
4
+ stalls while production remained paused at step 488 / 63,963,136 tokens.
5
+ The evidence strongly implicates the provider's HAIShare GPU scheduler.
6
+ The current compute entitlement and pool configuration have not been verified.
7
+
8
+ ## Measured behavior
9
+
10
+ The diagnostic benchmark restored the full paused checkpoint, including AdamW
11
+ state, and used microbatch 4, accumulation 16, sequence length 2,048, compiled
12
+ cores, and checkpointed 1,024-token loss chunks. It performed two warmup and
13
+ 20 measured updates of 131,072 tokens each, without checkpoint writes.
14
+ Generated evidence remains in
15
+ `work/benchmark-b200-baseline-diagnostic/rank-0.jsonl` and
16
+ `work/logs/b200-optimization.log`, relative to `rtx6000_40b/`.
17
+
18
+ Sustained measured throughput was **13,807.94 tokens/s**, versus a median
19
+ per-update rate of **61,853.22 tokens/s**. Most updates took about 2.12 seconds.
20
+ Five slow updates account for the large gap:
21
+
22
+ | Benchmark update, including warmup | Total seconds | Forward wall seconds |
23
+ | --- | ---: | ---: |
24
+ | 3 | 29.1877 | 27.8748 |
25
+ | 8 | 32.1939 | 30.7726 |
26
+ | 12 | 32.2163 | 30.8157 |
27
+ | 17 | 32.1830 | 30.8250 |
28
+ | 22 | 32.1977 | 30.8020 |
29
+
30
+ These are host wall times, including implicit synchronization waits, not GPU
31
+ kernel durations. Batch preparation and transfer took only 4.6-6.5 milliseconds
32
+ on those slow updates. During observed stalls, the main process slept in
33
+ `futex_wait_queue` without accumulating CPU work and GPU utilization dropped
34
+ to zero. Diagnostic Python stacks stopped inside the existing Triton launcher:
35
+ compiled model forward -> Inductor generated call -> Triton runtime `run`
36
+ -> generated `launcher`, line 5. This was the steady kernel launch path;
37
+ the captured stack did not show active compilation.
38
+ The final slow update also produced a stack in ordinary eager depth-modulation
39
+ tensor arithmetic, without Triton or Inductor frames. The wait therefore
40
+ occurs across CUDA entrypoints rather than only one compiled kernel path.
41
+
42
+ ## Runtime evidence and interpretation
43
+
44
+ Read-only inspection found `/usr/lib/haishare/libhaishare.so` in
45
+ `/etc/ld.so.preload` and in both benchmark process maps. `nvidia-smi -q` also
46
+ identified a HAIShare wrapper before returning its report. The library exports
47
+ `cuLaunchKernel`, `continue_with_lock`, `own_lock`, `own_lock_cv`, and scheduler
48
+ state symbols. Disassembly of `continue_with_lock` showed a GPU ownership
49
+ check, a lock request, and `pthread_cond_timedwait` on `own_lock_cv`, retried
50
+ at five-second intervals until ownership is granted.
51
+
52
+ Together, CUDA launch interception and the scheduler's condition-variable
53
+ wait provide a concrete mechanism for the observed futex sleep beneath the
54
+ Python launcher. A native stack tracing every intermediate frame was not
55
+ captured, so the association remains an inference rather than a complete
56
+ dynamic call trace. Static symbols alone do not establish the active quota.
57
+
58
+ The GPU report showed zero ECC errors, no active thermal or power slowdown,
59
+ healthy fabric, and no recovery action. MIG and driver virtualization were
60
+ disabled; those settings do not exclude user-space GPU scheduling.
61
+
62
+ The provider's [GPUaaS FAQ](https://docs.hosted.ai/user-panel/faqs/gpuaas-faqs)
63
+ documents time sharing with a default **30-second time quantum**. Guaranteed
64
+ compute depends on the pool's sharing ratio, while idle tenants can allow
65
+ others to use more of the GPU. This closely matches the recurring extra
66
+ 30-second waits. The provider's
67
+ [pool configuration documentation](https://docs.hosted.ai/admin-panel/gpuaas/creating-a-gpuaas-pool)
68
+ describes sharing ratio, scheduling mode, and time-quantum controls.
69
+
70
+ ## Follow-up
71
+
72
+ Ask the provider to confirm this instance's guaranteed compute percentage,
73
+ pool sharing ratio, time quantum, scheduling mode, and exclusive-allocation
74
+ status. Correlating scheduler grants and releases with the benchmark timestamps
75
+ would confirm the cause and distinguish expected allocation from a scheduler
76
+ fault. A full-memory B200 view alone does not establish exclusive compute.
77
+
78
+ No provider configuration or interposition settings were changed. Local BF16
79
+ optimization continues within the assigned allocation; benchmark comparisons
80
+ must report sustained throughput including waits as well as typical update
81
+ speed. See [B200_OPTIMIZATION_PLAN.md](B200_OPTIMIZATION_PLAN.md) for the
82
+ performance experiments and [B200_HANDOFF.md](B200_HANDOFF.md) for checkpoint
83
+ and production controls.
ops/B200_SETUP.md ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Single-B200 training environment
2
+
3
+ A new deployment starts **paused**. See [B200_HANDOFF.md](B200_HANDOFF.md)
4
+ for the latest checkpoint and current job status. Starting its Supervisor
5
+ does not start training. Use this configuration on the B200 host after
6
+ verifying the transferred dataset and checkpoint.
7
+
8
+ The repository and all environments, caches, data, logs, and temporary files
9
+ must remain under `/workspace/WarpState`. The Python 3.12 environment at
10
+ `rtx6000_40b/.server-venv` needs
11
+ [`training-requirements-resolved.txt`](training-requirements-resolved.txt)
12
+ and `supervisor==4.3.0`, combined in
13
+ [`b200-requirements.txt`](b200-requirements.txt). Source
14
+ [`training-env.sh`](training-env.sh) before provisioning. If present, the launcher also sources the generated
15
+ `/workspace/WarpState/tmp/remote-setup/toolchain-env.sh` for the host's
16
+ workspace-local compiler toolchain.
17
+
18
+ For a fresh environment on Ubuntu 24.04 x86-64 with Python 3.12 and a
19
+ CUDA-13-compatible driver:
20
+
21
+ ```bash
22
+ cd /workspace/WarpState
23
+ source rtx6000_40b/ops/training-env.sh
24
+ python3 -m venv rtx6000_40b/.server-venv
25
+ rtx6000_40b/.server-venv/bin/python -m pip install \
26
+ --extra-index-url https://download.pytorch.org/whl/cu130 \
27
+ -r rtx6000_40b/ops/b200-requirements.txt
28
+ rtx6000_40b/.server-venv/bin/python -m pip check
29
+ ```
30
+
31
+ A copy of the existing environment is also usable when the OS architecture,
32
+ Python version and absolute environment path match. Verify its dependency
33
+ versions and CUDA execution on the destination; omit compiler caches from
34
+ the transfer. The system needs GCC/G++, libc and Python development headers
35
+ for compilation. Do not install an additional CUDA driver inside the project.
36
+
37
+ If the host has no Git, download and extract its distribution package under
38
+ the workspace instead of modifying system directories:
39
+
40
+ ```bash
41
+ mkdir -p /workspace/WarpState/tmp/remote-setup /workspace/WarpState/.tools
42
+ cd /workspace/WarpState/tmp/remote-setup
43
+ apt-get download git
44
+ dpkg-deb -x git_*.deb /workspace/WarpState/.tools
45
+ cd /workspace/WarpState
46
+ source rtx6000_40b/ops/workspace-tools.sh
47
+ git --version
48
+ ```
49
+
50
+ This uses the host's existing shared libraries. Source
51
+ [`workspace-tools.sh`](workspace-tools.sh) in subsequent shells to use this
52
+ Git installation. Transfer Git history with a bundle of all refs; exclude
53
+ private keys, local credentials and generated artifacts from that bundle.
54
+
55
+ [`train-b200.sh`](train-b200.sh) uses one visible GPU, defaulting to index 0;
56
+ an existing `CUDA_VISIBLE_DEVICES` value takes precedence. It selects
57
+ sequence length 2,048, microbatch 16, and accumulation 4, preserving
58
+ **131,072 tokens per optimizer update**. Matrix computation uses BF16
59
+ autocast; parameters, gradients, and full AdamW state remain FP32. Core
60
+ compilation uses `max-autotune-no-cudagraphs`, core activation checkpointing
61
+ is disabled, and full vocabulary loss is compiled. Full checkpoints use
62
+ immutable CPU staging followed by asynchronous writing. The 150 GiB allocator
63
+ cap requires another 3 GiB free at startup and excludes CUDA context and
64
+ library memory. Current measurements and validation are recorded in
65
+ [B200_PERFORMANCE.md](B200_PERFORMANCE.md).
66
+
67
+ Copy the complete prepared `work/data/` directory, including its original
68
+ manifest, and `work/run/latest.pt`. The original transferred checkpoint was
69
+ step 448 with 58,720,256 tokens consumed; consult the handoff for the latest
70
+ saved step. Keep the manifest's requested revision `main`; its immutable source revision is recorded separately. The runner
71
+ restores model, full optimizer, token cursor, learning-rate schedule, and
72
+ rank-zero random state. `--reconfigure` permits the transition from two
73
+ GPUs to one while checking the unchanged data, objective, and global batch.
74
+ Different grouping can produce floating-point rounding differences.
75
+
76
+ ## Environment checks
77
+
78
+ On this host, setting `CUDA_VISIBLE_DEVICES=''` caused pytest collection
79
+ to hang. Keep normal device visibility and explicitly deselect the CUDA
80
+ training cases when running the CPU suite:
81
+
82
+ ```bash
83
+ cd /workspace/WarpState
84
+ source rtx6000_40b/ops/training-env.sh
85
+ rtx6000_40b/.server-venv/bin/python -m pytest \
86
+ rtx6000_40b/test_server.py rtx6000_40b/test_training_execution.py \
87
+ rtx6000_40b/test_training_launcher.py rtx6000_40b/test_loss_execution.py \
88
+ rtx6000_40b/test_checkpoint_writer.py \
89
+ rtx6000_40b/test_residual_execution.py rtx6000_40b/test_artifact_transfer.py \
90
+ -k 'not gpu_full_state_resume_and_exact_partial_update and not single_gpu_compiled_full_state_resume and not two_gpu_full_state_resume_and_empty_final_rank and not one_to_two_gpu_reconfiguration_preserves_training_state' \
91
+ --basetemp=/workspace/WarpState/tmp/b200-cpu-tests -q
92
+ ```
93
+
94
+ This runs the CPU tests and performs device-availability queries during
95
+ collection. Small CUDA resume tests can be run separately with the B200
96
+ available, using their own temporary checkpoints:
97
+
98
+ ```bash
99
+ rtx6000_40b/.server-venv/bin/python -m pytest rtx6000_40b/test_server.py \
100
+ -k gpu_full_state_resume_and_exact_partial_update \
101
+ --basetemp=/workspace/WarpState/tmp/b200-cuda-tests -q
102
+ ```
103
+
104
+ The prepared corpus occupies 80,002,097,152 bytes, and the full checkpoint
105
+ about 24.06 GB. Keep at least 53 GiB additionally free for checkpoint writing
106
+ and atomic replacement. Generated files remain ignored by Git.
107
+
108
+ ## Large artifact transfers
109
+
110
+ [`transfer_artifacts.py`](transfer_artifacts.py) sends large files over
111
+ independent SSH connections when one connection limits throughput. Run it
112
+ on the source host with the destination's already verified host-key file:
113
+
114
+ ```bash
115
+ cd /workspace/WarpState
116
+ source rtx6000_40b/ops/training-env.sh
117
+ python3 rtx6000_40b/ops/transfer_artifacts.py \
118
+ /workspace/WarpState/rtx6000_40b/work/data/train.bin \
119
+ /workspace/WarpState/rtx6000_40b/work/data/train.bin \
120
+ --host ubuntu@HOST --port PORT \
121
+ --key /workspace/WarpState/id_ed25519 \
122
+ --known-hosts /workspace/WarpState/tmp/remote-setup/known_hosts \
123
+ --connections 12 --chunk-mib 512 --compress \
124
+ --expected-sha256 237a9ea6a90831769ec22fb8089c2a26c02fb991492629bb28f2b5b10d6b9114
125
+ ```
126
+
127
+ Use the same command with the checkpoint's source/destination paths and
128
+ its actual SHA256. Copy the small data manifest, preparation state,
129
+ validation tokens and run metadata separately. The uploader retains
130
+ `.uploading`, `.upload-state.json`, `.upload-chunks/` and `.upload.lock`
131
+ sidecars next to each destination. These are inside ignored `work/`.
132
+ Interrupted uploads resume by rerunning the same command; completed chunks
133
+ are checked again before reuse. Full source and destination hashes must
134
+ match before the final filename appears. An existing different destination
135
+ is preserved. Keep training stopped until all artifacts have been verified.
136
+
137
+ ## Controls on the remote host
138
+
139
+ Initialize the dedicated Supervisor once after provisioning:
140
+
141
+ ```bash
142
+ cd /workspace/WarpState
143
+ source rtx6000_40b/ops/training-env.sh
144
+ mkdir -p rtx6000_40b/work/control rtx6000_40b/work/logs
145
+ rtx6000_40b/.server-venv/bin/supervisord \
146
+ -c rtx6000_40b/ops/b200-supervisord.conf
147
+ rtx6000_40b/ops/b200ctl.sh status
148
+ ```
149
+
150
+ It runs as the invoking user and uses its own `b200-*` control files and
151
+ logs. It does not manage preparation or benchmarks. Training remains
152
+ stopped until explicitly started and never restarts automatically after
153
+ failure or a daemon restart.
154
+
155
+ When ready to resume on the B200:
156
+
157
+ ```bash
158
+ cd /workspace/WarpState
159
+ nvidia-smi
160
+ rtx6000_40b/ops/b200ctl.sh start warpstate-train
161
+ rtx6000_40b/ops/b200ctl.sh status
162
+ tail -n 10 rtx6000_40b/work/logs/b200-train.log
163
+ tail -n 3 rtx6000_40b/work/run/metrics.jsonl
164
+ ```
165
+
166
+ To pause and save a full checkpoint:
167
+
168
+ ```bash
169
+ rtx6000_40b/ops/b200ctl.sh stop warpstate-train
170
+ ```
171
+
172
+ The stop request waits for the current optimizer update and checkpoint,
173
+ with a 15-minute Supervisor timeout. The next start resumes `latest.pt`.
174
+ Initial compilation can take several minutes. Use `b200ctl.sh` for this
175
+ daemon; the original `datactl.sh` addresses the separate workstation
176
+ Supervisor. Do not run two launchers against the same `work` directory.
ops/BACKWARD_LAYOUT_EXPERIMENT.md ADDED
@@ -0,0 +1,204 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Attention backward layout experiment — September 12, 2026
2
+
3
+ The isolated paired-lane implementation reduces this region from **0.519935 ms
4
+ to 0.357944 ms** on one RTX PRO 6000 Blackwell Max-Q GPU: **31.16% less time**.
5
+ It matches the original profiled kernels byte for byte on the tested production
6
+ shape. This isolated experiment did not include a training adapter or establish
7
+ a full-model speedup. The subsequent opt-in graph integration and its training
8
+ measurements are documented in
9
+ [BACKWARD_LAYOUT_INTEGRATION.md](BACKWARD_LAYOUT_INTEGRATION.md).
10
+
11
+ ## Boundary and provenance
12
+
13
+ The existing compiled-autograd overlap profile contains two adjacent kernels:
14
+
15
+ | Original kernel | Rank-0 time per update | Calls per update |
16
+ | --- | ---: | ---: |
17
+ | `triton_poi_fused_17`: rotary backward and Q/K scratch | 31.907 ms | 224 |
18
+ | `triton_poi_fused_18`: packed Q/K/V/gate/U gradient | 77.576 ms | 224 |
19
+
20
+ The evidence is
21
+ `work/benchmark-opt-20260912/profile-ca-overlap/profile-rank-0-kernels.tsv`.
22
+ Their combined **109.483 ms** is an existing kernel-time budget, not an amount
23
+ already removed from training. The isolated timing uses a different execution
24
+ context; do not scale it into a claimed training throughput improvement.
25
+
26
+ `ops/backward_layout_baseline.py` retains the exact two profiled arithmetic and
27
+ indexing bodies as source, with their original launch dimensions. It records
28
+ the original generated source identifiers
29
+ `cq665jkyfy6uac7wrzvwa5aigwids6rzt4n7vzip6xvufai4ajjw` and
30
+ `cbr6gfx3vjhnzjbykfch3pg5peunfivhgdrfvl5hnzolhvfjfgdy` for provenance.
31
+ Running the experiment recompiles these tracked bodies; it does not load
32
+ ignored compiler-cache files or depend on preserving them.
33
+
34
+ The shape is `[B,N,H,C,D] = [4,16,32,128,64]`: microbatch four, 2,048-token
35
+ sequences, 32 heads, and a 64-element head dimension. Saved activations and
36
+ gradients are BF16. The original saved rotary sine/cosine tables are FP32.
37
+ K-write gradients have `[B,N,H,D,C]` storage; the output gradient is token-major.
38
+ The final packed tensor has shape `[4,2048,10240]`, ordered Q/K/V/gate/U.
39
+
40
+ ## Arithmetic and implementation
41
+
42
+ `ops/backward_layout.py` provides an independent tensor reference, deterministic
43
+ synthetic saved inputs, the validated `launch_layout` interface, and
44
+ `prepare_layout`, which returns an allocation-free callable for timing.
45
+
46
+ The new rotary kernel processes even/odd pairs together. It reads the separate
47
+ local and memory Q gradients and adds them in FP32, applies the rotary backward
48
+ transform, and rounds the final Q result to BF16. It preserves the original
49
+ interleave's inactive-lane `+0.0`, including signed-zero behavior.
50
+
51
+ K first combines the transposed write gradient through the tanh derivative with
52
+ the local gradient in FP32. Its sine/rotary contribution is **rounded to BF16
53
+ before** adding the cosine contribution, as in the original two-kernel path.
54
+ That rounded intermediate stays in registers. K then rounds to BF16 on output.
55
+
56
+ A second kernel copies V, computes the sigmoid gate derivative in the original
57
+ operation order, and applies the U tanh derivative. Both kernels write directly
58
+ to their disjoint slices of the packed output. The `single` variant launches
59
+ both sets of programs in one grid using a uniform per-program branch. Keeping
60
+ two launches was slightly faster in the final comparison.
61
+
62
+ There are no reductions, GEMMs, parameter-gradient updates, collectives, or
63
+ optimizer operations in this boundary. Model parameters, FP32 gradient
64
+ accumulation, FP32 communication, and FP32 optimizer state are untouched.
65
+ Inputs, output, and profile scratch have explicit dtype/shape/device/stride
66
+ checks; output and scratch storage may not overwrite saved inputs or each other.
67
+
68
+ ## Measurement and correctness
69
+
70
+ The initial screen covered two launch algorithms, blocks 256/512/1024, and
71
+ 4/8 warps. All twelve configurations matched the original outputs numerically.
72
+ The largest single-grid configuration was slightly slower than the baseline,
73
+ so fewer launches alone did not establish a win. The initial report's equality
74
+ field compared values; a later stricter byte comparison detected a few signed
75
+ zeros, prompting the explicit inactive-lane zero addition in the final source.
76
+
77
+ The final confirmation used a different seed, block 256, eight warps, nine
78
+ trials, and 100 calls per timing interval. Each candidate interval was bracketed
79
+ by original-kernel intervals. Inputs, output, and original scratch were
80
+ preallocated; both paths were warmed before measurements. CUDA events measure
81
+ the queued execution, including possible launch scheduling gaps.
82
+
83
+ | Final algorithm | Median original | Median candidate | Time reduction |
84
+ | --- | ---: | ---: | ---: |
85
+ | Paired, two launches | 0.519935 ms | 0.357944 ms | 31.16% |
86
+ | Single grid | 0.520078 ms | 0.358521 ms | 31.06% |
87
+
88
+ Both final candidates had **zero byte differences** across every Q/K/V/gate/U
89
+ projection: 83,886,080 BF16 output elements per comparison. Production-shape
90
+ tests use another seed and strict integer-view byte equality. Ragged-shape
91
+ tests compare each projection independently against tensor expressions, use
92
+ unequal even/odd rotary coefficients, and verify saved inputs remain unchanged.
93
+ CPU checks include the model's actual rotary VJP, gate autograd, packing order,
94
+ observable intermediate BF16 rounding, and malformed input rejection.
95
+
96
+ The independent eager FP32 tensor expression and original compiled kernels can
97
+ have different FMA contraction. In the final run their maximum relative RMS
98
+ error was `8.72e-6`; V/gate/U were exactly equal. Therefore the independent
99
+ reference establishes mathematical indexing/arithmetic, while the original
100
+ compiled bodies supply the exact GPU-rounding oracle. These finite synthetic
101
+ fixtures do not prove equivalence for every possible activation distribution.
102
+
103
+ The final GPU suite passed **18 tests**. The focused suite also supports CPU-only
104
+ execution. Reports remain ignored:
105
+
106
+ - `work/benchmark-backward-layout-20260912/results.json`: initial twelve-way screen.
107
+ - `work/benchmark-backward-layout-confirm-20260912/results.json`: signed-zero diagnostic.
108
+ - `work/benchmark-backward-layout-final-20260912/results.json`: final source, strict byte comparison, and confirmation timings.
109
+ - `work/validation/rtx-opt-20260912/backward-layout-tests.log`: final GPU tests.
110
+
111
+ The final timing report records source hashes, arguments, Torch `2.14.0+cu130`,
112
+ CUDA 13.0, device identity, and per-projection errors. Later test/documentation
113
+ additions do not change kernel arithmetic. After measurement, one extra EOF
114
+ blank line was removed from `backward_layout_baseline.py` for the Git whitespace
115
+ check. Its parsed AST is identical; the original and final source hashes are
116
+ recorded in `work/validation/communication-exploration/backward-source-normalization.json`.
117
+
118
+ ## Traffic and integration limits
119
+
120
+ The rewrite avoids the logical stores and subsequent reads of two
121
+ 16,777,216-element BF16 scratch tensors: **134,217,728 bytes (128 MiB) per
122
+ region invocation**. This is a source-level traffic calculation, not a measured
123
+ DRAM counter. Cache behavior and other reads also affect the measured timing.
124
+ The production compiler reuses a dead buffer for one scratch tensor, so this
125
+ does **not** establish two fewer allocations or a corresponding peak-VRAM gain.
126
+
127
+ The prototype accepts separate local and memory gradients at the same backward
128
+ boundary as the profiled graph. A naive custom forward that returns one shared
129
+ BF16 Q tensor can cause autograd to round the local-plus-memory gradient sum
130
+ before rotary backward; the current graph adds those contributions in FP32.
131
+ Integrating the kernel requires preserving that join and the K rounding point,
132
+ then validating real model gradients, optimizer updates, checkpoint resumption,
133
+ and full-model timing. No such integration is selected by this experiment.
134
+
135
+ ## Recommended next integration experiment
136
+
137
+ The narrowest route is a **post-gradient graph replacement**, leaving the
138
+ forward and autograd ownership intact. The installed PyTorch exposes
139
+ `torch._inductor.config.post_grad_custom_post_pass`; its
140
+ `fx_passes/post_grad.py` applies this hook to normalized, functionalized graphs
141
+ before final topological sorting and fake-tensor metadata updates. Implement a
142
+ `CustomGraphPass` that replaces only the existing packed-gradient region with a
143
+ pure opaque `torch.library.custom_op` and a matching fake implementation.
144
+ `ops/matmul_execution.py` already uses that opaque backward-kernel boundary to
145
+ avoid this PyTorch version's Compiled Autograd Triton higher-order tracing issue.
146
+ These are version-dependent compiler interfaces; their availability is a
147
+ proposed route, not evidence that this replacement has been integrated.
148
+
149
+ The pass should match the actual backward dataflow and arithmetic, not generated
150
+ kernel names or an inference flag. Require the exact supported shapes, dtypes,
151
+ strides, projection ordering, separate Q gradient branches, K rounding boundary,
152
+ and gate scale/expression. Verify that removed intermediates have no consumers
153
+ outside the replacement. On any mismatch, leave the original graph intact and
154
+ report the missed replacement; an opt-in validation run must fail if expected
155
+ dispatches are absent. Initially support only the measured parallel-memory,
156
+ dropout-zero production shape. Pass existing native-layout tensors into the
157
+ opaque operator and reshape views inside it, avoiding new contiguous copies
158
+ merely to satisfy a custom-operator input layout.
159
+
160
+ Use `CustomGraphPass.uuid()` with hashes of the matcher, kernel, and wrapper
161
+ sources plus all specialization settings, so a changed replacement cannot reuse
162
+ an incompatible compiled artifact. The installed `custom_graph_pass.py`
163
+ documents `get_hash_for_files` for this purpose. Record the same execution
164
+ identity in benchmark and checkpoint metadata, with explicit reconfiguration
165
+ for resumption changes. A new forward custom-autograd splitter is less suitable:
166
+ separate Q outputs address its gradient join, but the complete gate/packing
167
+ fusion also needs the later blend's raw output gradient and saved local/memory
168
+ values. Capturing that region through a forward boundary would require broader
169
+ ownership of the attention/memory computation or extra materialized gradients.
170
+
171
+ Begin with a match-only graph inspection, then replace one supported core.
172
+ Require generated-kernel/profile evidence that the expected regions actually
173
+ dispatch the paired implementation, with no added layout copies. Compare
174
+ forward outputs, all parameter gradients, accumulated FP32 gradients and Adam
175
+ updates against the same compiled baseline; cover first and later accumulation
176
+ microbatches, the final DDP synchronization, and checkpoint continuation. Then
177
+ measure repeated full-model runs from the same checkpoint/token cursor,
178
+ including peak VRAM and communication overlap. Select the replacement only
179
+ after those correctness and full-model performance checks pass.
180
+
181
+ ## Reproduction
182
+
183
+ Reserve one idle GPU and keep training paused before running CUDA commands.
184
+ Use a new output directory each time. All source, caches, and artifacts stay
185
+ inside `/workspace/WarpState`.
186
+
187
+ ```bash
188
+ cd /workspace/WarpState/rtx6000_40b
189
+ source ops/training-env.sh
190
+ CUDA_VISIBLE_DEVICES='' .server-venv/bin/python -B -m pytest -q test_backward_layout.py
191
+ CUDA_VISIBLE_DEVICES=1 WARPSTATE_TEST_CUDA=1 \
192
+ .server-venv/bin/python -B -m pytest -q test_backward_layout.py
193
+ CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B \
194
+ ops/backward_layout_benchmark.py \
195
+ --output-dir work/benchmark-backward-layout-screen-new
196
+ CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B \
197
+ ops/backward_layout_benchmark.py --algorithms paired single \
198
+ --blocks 256 --warps 8 --seed 9929 --trials 9 --repeats 100 \
199
+ --output-dir work/benchmark-backward-layout-confirm-new
200
+ ```
201
+
202
+ The benchmark needs no dataset or model checkpoint. It reports candidates
203
+ even when their bytes differ, so inspect correctness fields before interpreting
204
+ timings. It does not generate a training recipe or modify the production run.
ops/BACKWARD_LAYOUT_INTEGRATION.md ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BF16 backward-layout integration experiment
2
+
3
+ The paired attention-backward layout prototype is integrated into the compiled
4
+ training graph. Repeated complete-model measurements improve update throughput
5
+ by 0.637%, so `train-local-resume.sh` selects `--backward-layout paired` for the
6
+ next resumption. The CLI default remains `baseline`. The conservative 128 MiB
7
+ communication preset remains selected; the separate producer-scheduling trial
8
+ is available but disabled. Production training remains paused at step 40,081.
9
+
10
+ ## Execution contract
11
+
12
+ Both `run_40b.py` and `ops/benchmark_training.py` accept
13
+ `--backward-layout baseline|inspect|paired`. `baseline` preserves existing
14
+ execution. `inspect` checks the graph without replacing it. `paired` replaces
15
+ only the recognized backward region and requires actual kernel dispatches.
16
+
17
+ The initial supported envelope is microbatch four, sequence length 2,048,
18
+ 32 heads of width 64, chunk length 128, parallel memory, and zero dropout.
19
+ Use BF16 residuals, compiled cores in default mode, baseline activation and
20
+ normalization implementations, no activation checkpointing, and
21
+ `compiled-autograd-overlap`. Separate matmul recipes are excluded from this
22
+ initial experiment. Master parameters, gradient accumulation, collectives, and
23
+ AdamW state retain their existing FP32 representation; model compute uses BF16.
24
+
25
+ The graph pass matches normalized ATen operations, constants, tensor metadata,
26
+ branch relationships, and external consumers. Generated kernel names are not
27
+ match keys. Separate local and memory Q gradients and the K intermediate
28
+ rounding boundary are preserved. Unsupported regions retain the original graph;
29
+ an opted-in training or benchmark update fails if expected dispatches are absent.
30
+
31
+ Checkpoint execution signatures and benchmark metadata contain the implementation
32
+ identity and source hashes. A changed identity requires `--reconfigure` on
33
+ resumption. Compiler cache identity also includes the pass implementation.
34
+ Runtime dispatch counters establish execution even when a compiler cache hit
35
+ bypasses the graph matcher. Benchmarks require exactly one replacement per
36
+ logical layer per accumulation microbatch in each measured update.
37
+
38
+ The opaque operator follows PyTorch's [functional custom-operator contract](https://docs.pytorch.org/tutorials/advanced/python_custom_ops_functional.html):
39
+ it mutates no input, returns a fresh output, and registers fake output metadata.
40
+ The hook and arithmetic specialization are tied to the installed PyTorch
41
+ version. This is a backward graph transformation; it does not replace the
42
+ model's forward autograd ownership.
43
+
44
+ ## Validation and timing protocol
45
+
46
+ The isolated prototype's 31% reduction applies only to its two-kernel region.
47
+ Roughly 34 ms saved per training update, or 0.8% overall, was the pre-integration
48
+ estimate. The measured complete-model result appears below.
49
+
50
+ Validation compares the compiled baseline and candidate at production attention
51
+ geometry using a smaller number of layers. It covers forward outputs, every
52
+ parameter gradient after initial and accumulated backwards, optimizer updates,
53
+ and two-rank continuation through a final partly masked update. Continuation
54
+ within an execution mode is a separate check from numerical agreement between
55
+ different execution modes.
56
+
57
+ Complete-model timings use the retained checkpoint and data cursor, microbatch
58
+ four per rank, eight accumulation microbatches, and 131,072 global tokens per
59
+ update. Both GPUs retain their 300 W power limits. Full resumed AdamW state and
60
+ an 88 GiB allocator cap remain enabled. No production checkpoint is written.
61
+
62
+ Repeated runs use fresh processes and interleave baseline and candidate order.
63
+ Warmup and measured counts are matched. GPU telemetry is sampled outside the
64
+ timed Python region. Compare total tokens divided by total update time, timing
65
+ drift, matched losses/gradient norms, and allocated/reserved VRAM. Profiles are
66
+ separate runs; their instrumented throughput is excluded from timing results.
67
+
68
+ ## Reproduction
69
+
70
+ Keep production training paused and reserve both GPUs before running:
71
+
72
+ ```bash
73
+ cd /workspace/WarpState/rtx6000_40b
74
+ source ops/training-env.sh
75
+ CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO \
76
+ NCCL_DEBUG_SUBSYS=INIT,GRAPH,TUNING,ENV \
77
+ .server-venv/bin/torchrun --standalone --nproc-per-node=2 \
78
+ ops/benchmark_training.py \
79
+ --checkpoint work/run/latest.pt --data work/data/train.bin \
80
+ --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
81
+ --no-activation-checkpointing --autocast-cache \
82
+ --compile-cores --compile-mode default --loss-mode compiled-full \
83
+ --bf16-residuals --gradient-mode compiled-autograd-overlap \
84
+ --ddp-bucket-cap-mb 128 --optimizer-state resume --vram-limit-gib 88 \
85
+ --warmup-updates 8 --measured-updates 40 --backward-layout paired \
86
+ --output-dir work/benchmark-backward-layout-reproduction/paired-1
87
+ ```
88
+
89
+ Use `--backward-layout baseline` for the control and a fresh output directory for
90
+ every process. For dispatch and overlap profiles, use `--profile` with one
91
+ measured update after the same warmup. Existing comparison and communication
92
+ profile tools consume the saved JSON reports without opening model weights.
93
+
94
+ The underlying kernel arithmetic and original isolated measurements are in
95
+ [BACKWARD_LAYOUT_EXPERIMENT.md](BACKWARD_LAYOUT_EXPERIMENT.md).
96
+
97
+ ## Training validation results
98
+
99
+ One production-shaped core matched the compiled baseline exactly for forward
100
+ output, input gradients, and all nine core parameter gradients. The opaque
101
+ operator's schema-purity and fake-metadata checks passed on CUDA.
102
+
103
+ The two-rank training fixture uses production attention/FFN dimensions, one
104
+ logical layer, three accumulation microbatches, two full updates, and a final
105
+ nine-token update with an entirely masked rank. It compares every parameter
106
+ gradient after the first, middle, and final synchronization backward, then
107
+ checks each parameter and Adam-state tensor after every optimizer update.
108
+
109
+ An initial 512-vocabulary fixture with repeated input tokens passed numerical
110
+ comparison but failed byte-exact continuation: 33 embedding-weight elements
111
+ differed by at most `3.7253e-9`. Differences in optimizer state were also confined
112
+ to the embedding. Other model weights, RNG states, token counters, and execution
113
+ identity matched. Its report is retained as a diagnostic of the existing
114
+ embedding accumulation's nondeterminism, not counted as an exact-resume pass.
115
+
116
+ The follow-up `--unique-inputs` fixture uses vocabulary 8,192 and a deterministic
117
+ permutation so each embedding row appears once per backward. It retains the
118
+ same attention geometry and precision without relaxing comparison thresholds.
119
+ Both ranks passed exact continuation for model, full optimizer, RNG, and cursor
120
+ state. All Adam parameters and moments also matched the baseline byte for byte
121
+ at every update. The largest gradient relative L2 difference was `1.22e-7`,
122
+ with maximum absolute difference `2.73e-12`, confined to the small
123
+ `memory_mix_logit` reduction. Both ranks agreed exactly on final model and Adam
124
+ state within each run. Every expected layout dispatch occurred.
125
+
126
+ Reproduce the strict continuation fixture with a new output directory:
127
+
128
+ ```bash
129
+ CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/python -B \
130
+ ops/validate_backward_layout_training.py --world-size 2 --unique-inputs \
131
+ --output-dir work/validation/backward-layout-training-reproduction
132
+ ```
133
+
134
+ The fixture initializes deterministic synthetic weights, tokens, and nonzero
135
+ FP32 Adam moments. This checks integration and continuation at production
136
+ attention geometry; it is not a byte-exact continuation claim for the complete
137
+ 28-layer model on repeated-token corpus data. The reports are
138
+ `work/validation/backward-layout-training-20260912/` and
139
+ `work/validation/backward-layout-training-unique-20260912/`.
140
+
141
+ A two-core follow-up covers the compiler's sharing of an identical zero constant
142
+ across layers. The matcher retains that strictly matched constant while another
143
+ region still uses it, then removes it after its final replacement. Shared
144
+ computed arithmetic still causes rejection. The installed compiler's actual
145
+ common-subexpression pass is exercised by a focused structural test.
146
+
147
+ The final two-core, two-GPU fixture also passed exact model/optimizer/RNG/cursor
148
+ continuation. It required two dispatches per backward and observed every one.
149
+ All updated parameters and Adam states matched baseline byte for byte; maximum
150
+ gradient errors were `3.64e-12` absolute and `1.27e-7` relative L2. Add `--cores 2`
151
+ to the command above to reproduce it. Results are in
152
+ `work/validation/backward-layout-training-two-core-20260912/results.json`.
153
+ See [BACKWARD_LAYOUT_TRAINING_VALIDATION.md](BACKWARD_LAYOUT_TRAINING_VALIDATION.md)
154
+ for the full validation scope and source identities.
155
+
156
+ The separate producer-scheduling fixture also passed the existing two-GPU
157
+ asynchronous-save/stop/resume test, including full model/optimizer/RNG equality
158
+ and a final masked rank. Its log is
159
+ `work/validation/backward-layout-training-producer-resume-20260912.log`.
160
+
161
+ ## Complete-model timing results
162
+
163
+ Four fresh-process runs used the order control, paired, paired, control, each
164
+ with eight warmup and 40 measured updates. All started from step 40,081 and
165
+ token cursor 5,253,496,832 with the same retained checkpoint, data, optimizer,
166
+ and source identities. Profiling and evaluation/checkpoint pauses are
167
+ excluded; these numbers describe training-update throughput.
168
+
169
+ | Run | Layout | Tokens/second |
170
+ | --- | --- | ---: |
171
+ | `control-1` | Baseline | 31,010.48 |
172
+ | `paired-2` | Paired | 31,197.36 |
173
+ | `paired-3` | Paired | 31,200.70 |
174
+ | `control-2` | Baseline | 30,992.67 |
175
+
176
+ Aggregating total tokens over total measured time gives **31,001.57 to
177
+ 31,199.03 tokens/second**, a **0.637%** gain. Mean update time falls from
178
+ 4.22791 to 4.20116 seconds, saving **26.76 ms per update**. The two controls
179
+ differ by 0.0575%; the two candidates differ by 0.011%. Both candidates beat
180
+ both controls. This is a small, repeatable improvement on this hardware and
181
+ preset; the isolated region's 31% gain does not apply to whole-model training.
182
+
183
+ Peak live allocator usage is unchanged at **65.7119 GiB per GPU**. Peak reserved
184
+ allocator memory falls from **66.9785 to 66.4785 GiB per GPU**. The 0.5 GiB
185
+ reduction is reservation, not a reduction in peak live tensors. Power limits
186
+ remain 300 W and measured-window temperatures and clock rates are comparable.
187
+
188
+ Every measured candidate update dispatches exactly 224 paired operations per
189
+ rank: 28 logical layers times eight accumulation microbatches. The second
190
+ candidate run loads compiler caches and still records every runtime dispatch,
191
+ despite the graph matcher itself not being invoked on those cache hits.
192
+
193
+ The first update's loss and gradient norm are exactly equal across all four
194
+ runs. Later trajectories are not byte-identical, including the repeated control.
195
+ Against the first control, the second control's maximum loss difference is
196
+ `1.28e-4` and relative gradient-norm difference is 0.219%. Candidate maxima are
197
+ `1.48e-4` / 0.179% and `1.35e-4` / 0.307%. These small scalar differences are
198
+ consistent with nondeterministic training reductions; they do not establish
199
+ byte equality of the complete model's gradients. The fixture checks above
200
+ provide the separate tensor-level evidence.
201
+
202
+ Run artifacts are under `work/benchmark-backward-layout-integration-20260912/`.
203
+ The comparison and telemetry summary is
204
+ `work/validation/backward-layout-integration/abba-summary.json`.
205
+ The manually stopped `paired-1` compilation probe has no completed update and
206
+ is excluded from all timing results.
207
+
208
+ ## Profile verification
209
+
210
+ Separate baseline and paired runs each use eight warmup updates and one profiled
211
+ update. Each candidate rank records 224 custom-operator calls and 448 actual
212
+ `_paired_kernel` launches: 224 rotary and 224 value/gate/U kernels. The original
213
+ rotary and projection-packing launch signatures are absent. Their combined
214
+ region time falls from **114.95 to 78.96 ms** on rank zero, and from **114.65 to
215
+ 78.91 ms** on rank one, about 31% less CUDA time for this region. This saves
216
+ roughly 36 ms of kernel time; the repeated unprofiled whole-update improvement
217
+ is the smaller 26.76 ms reported above.
218
+
219
+ The total CUDA kernel count stays at 17,477 per rank, and every copy/fill count
220
+ and byte total is unchanged. There are no added layout-materialization kernels.
221
+ The cached synchronized backward graph retains 45 identical generated Triton
222
+ bodies and the same 225 MM, 224 BMM, and one ADDMM calls; only the two recognized
223
+ layout bodies are removed. The replacement consumes native buffers directly.
224
+ These checks are recorded in
225
+ `work/validation/backward-layout-integration/profile-kernel-copy-comparison.json`
226
+ and `generated-sync-graph-comparison.json` in the same directory.
227
+
228
+ The complete captured update retains all 59 collectives and
229
+ 8,020,445,708 logical input bytes per rank, with identical collective signatures.
230
+ The following values are local-device timeline unions, not sums across GPUs:
231
+
232
+ | Profile metric (ms/update) | Baseline rank 0 | Paired rank 0 | Baseline rank 1 | Paired rank 1 |
233
+ | --- | ---: | ---: | ---: | ---: |
234
+ | Noncommunication CUDA activity | 3,957.68 | 3,935.16 | 3,834.21 | 3,796.91 |
235
+ | Compute/communication overlap | 192.63 | 192.56 | 191.92 | 191.57 |
236
+ | Communication with no overlapping compute | 149.87 | 154.23 | 281.09 | 290.27 |
237
+
238
+ Overlap is preserved. The exposed communication duration varies slightly upward
239
+ in this single profile pair; the measured layout gain is not a communication
240
+ speedup. Removing computation can also reduce work available to hide communication.
241
+ Instrumented profile durations are excluded from the repeated timing result.
242
+ The timeline accounting is in
243
+ `work/validation/backward-layout-integration/profile-communication.json`.
244
+
245
+ ## Separate producer scheduling experiment
246
+
247
+ `--overlap-producer-schedule` is an opt-in communication experiment. Its
248
+ two separate runs use `--backward-layout baseline` and 128 MiB buckets, with
249
+ the same checkpoint, cursor, geometry, warmup, timing, and telemetry protocol
250
+ above. `producer-1` reached **31,133.27 tokens/second** (+0.425% against the
251
+ aggregate control); `producer-2` reached **31,090.54** (+0.287%). The repeat is
252
+ 0.137% slower than the first screen, and its gain falls just below the approximate
253
+ 0.3% threshold used to justify testing a combined preset. Peak allocated and
254
+ reserved VRAM match the control's 65.7119 and 66.9785 GiB per GPU.
255
+
256
+ Both runs show a small positive average difference, but the evidence is weaker
257
+ than the layout's repeated gain. The producer option remains available for
258
+ experimentation and is **not selected** in the production launcher. No combined
259
+ layout/producer run was started; additive gains have not been established.
260
+ This is a conservative selection decision, not proof that the scheduler cannot
261
+ help. The run summaries remain under the same benchmark directory as the ABBA
262
+ measurements.
263
+
264
+ The installed scheduler already moves ready collective launches earlier through
265
+ `simple_overlap`. This experiment additionally lets
266
+ `reorder_communication_preserving_peak_memory` move a collective with its
267
+ packing/casting producers, followed by the existing `sink_waits_iterative` pass.
268
+ Without runtime estimates, producer grouping stops at GEMM-producing nodes.
269
+ It therefore targets earlier bucket readiness rather than promising to reorder
270
+ all independent backward GEMMs.
271
+
272
+ The installed `comms.py` docstring names a `sink_comms_and_waits` prerequisite
273
+ that has no callable implementation in this version. The installed `config.py`
274
+ instead recommends producer reordering and iterative wait sinking; ordinary
275
+ peak-memory scheduling runs before these passes. The initial experiment uses one
276
+ producer pass followed by one wait sink, retaining the existing final wait
277
+ scheduling behavior.
278
+
279
+ Both passes explicitly prohibit changing collective launch order and allow
280
+ zero increase in their estimated peak memory. Producer groups cannot include
281
+ another collective. These settings override the producer pass's installed
282
+ unsafe-order and 20% memory defaults. The flag requires
283
+ `compiled-autograd-overlap` and initially rejects wait swapping, runtime
284
+ estimation, and a nonzero memory allowance. Its immutable execution setting is
285
+ recorded in benchmark metadata and checkpoint signatures; changing it on resume
286
+ requires `--reconfigure`. The timing and continuation checks above measure this
287
+ option separately from the paired layout.
288
+
289
+ ## Final checks and resume state
290
+
291
+ The final CPU suite passed **356 tests**, with 24 CUDA tests skipped in that
292
+ CPU-only invocation and three existing Dynamo cache-wrapper warnings. Separate
293
+ CUDA operator checks, production-shaped one-/two-core gradient/optimizer and
294
+ exact-continuation fixtures, and the producer scheduling async-resume test are
295
+ documented above. The final CPU log is
296
+ `work/validation/backward-layout-integration/pytest-final-cpu.log`.
297
+
298
+ All ten package-manifest entries validate. The launcher passes `bash -n`.
299
+ The retained full checkpoint at `work/run/latest.pt` passes the before/after
300
+ SHA256 check, retaining
301
+ `9745d93345dff7c389ac6fa65948dc78ec021c90f56b40e9caed8ddcd84e8c54`.
302
+ The verification is saved in `checkpoint-after.log` beside the CPU log.
303
+ It remains at step **40,081**, **5,253,496,832 tokens**, with recorded validation
304
+ loss **2.4545078724622726**. Production and preparation jobs remain stopped,
305
+ both GPUs are idle, and their power limits remain 300 W. Archived weights are
306
+ untouched. The selected launcher will use the paired layout when training is
307
+ next resumed; this experiment does not resume production.
ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Backward-layout training validation — September 12, 2026
2
+
3
+ The paired graph replacement was validated with two-GPU training and checkpoint
4
+ continuation fixtures with production attention and FFN dimensions. The fixtures
5
+ use one or two physical cores/logical layers, not the complete 28-layer model.
6
+ They verify complete parameter-gradient and optimizer behavior for those fixtures;
7
+ they are not throughput measurements or proof about every trained activation.
8
+
9
+ `ops/validate_backward_layout_training.py` retains microbatch four, sequence
10
+ length 2,048, 32 heads, head dimension 64, chunk size 128, and FFN width 7,168.
11
+ It uses BF16 autocast/residuals, compiled cores and full loss, conservative
12
+ compiled-autograd communication overlap, 128 MiB DDP buckets, and FP32 master
13
+ parameters, gradients, all-reduces, and fused AdamW state. Weights and tokens are
14
+ deterministic synthetic fixtures. Adam starts with deterministic nonzero FP32
15
+ moments, rather than fresh first-update moments or imported production state.
16
+
17
+ Three accumulation microbatches exercise separate backward variants: first
18
+ `no_sync()` with absent gradients, later `no_sync()` accumulation, and final
19
+ accumulation with DDP synchronization. There are two full updates and a final
20
+ nine-token update, during which rank one has entirely masked targets. Each
21
+ candidate backward must execute the paired operation; an earlier dispatch cannot
22
+ hide fallback in a later graph. Expected totals per rank are seven dispatches
23
+ for continuous execution, three for the partial run, and four after resumption.
24
+
25
+ The harness runs baseline, continuous candidate, partial candidate, and resumed
26
+ candidate in separate processes. It compares initial forward logits, every
27
+ parameter gradient after every backward, and every parameter/Adam state after
28
+ every update. It also requires byte-identical final model, optimizer, RNG, token
29
+ cursor, configuration, and execution/source identity across candidate stop/resume.
30
+ Both ranks' final model and optimizer states must agree exactly in each run.
31
+ Source hashes are checked between phases, and outputs require a fresh directory
32
+ under `work/validation`. No production checkpoint or dataset is opened or changed.
33
+
34
+ The later `--cores` option selects a positive number of distinct physical cores
35
+ and the same number of logical layers. It scales both the per-backward dispatch
36
+ requirement and exact final dispatch count with logical depth. This covers
37
+ cross-core compilation that a single-core fixture cannot test.
38
+
39
+ The numerical limits were declared before CUDA measurements. Gradient and Adam
40
+ moment errors must satisfy both L2 error at most 0.5% of reference L2 plus
41
+ `1e-8`, and maximum error at most 1% of reference maximum plus `1e-8`. Updated
42
+ parameters have a `2e-5` maximum absolute limit; losses have a `2e-4` absolute
43
+ limit. Adam step counters, initial forward logits, and complete candidate resume
44
+ state require byte equality. Every comparison separately records exact bytes,
45
+ finite values, and numerical error; a tolerance pass is not labeled byte equality.
46
+
47
+ The initial fixture used a 512-token vocabulary with repeated input indices.
48
+ Its candidate gradient/optimizer comparisons passed, but strict continuation
49
+ failed. Differences were confined to the tied embedding: 33 parameter elements
50
+ with maximum difference `3.725290298461914e-9`, 50 first-moment elements with
51
+ maximum `9.516952559351921e-8`, and 28 second-moment elements with maximum
52
+ `4.3655745685100555e-11`. All other parameter/state tensors, RNG states, and
53
+ execution/cursor metadata matched exactly. Every run's two ranks agreed. This
54
+ is consistent with the existing relaxed embedding atomic accumulation for
55
+ repeated tokens. The original failure and numerical diagnostics remain recorded;
56
+ no comparison limit was relaxed to turn it into a pass.
57
+
58
+ The successful `--unique-inputs` fixture uses an 8,192-token vocabulary and a
59
+ deterministically shuffled permutation of all input indices in every microbatch.
60
+ Each embedding row is therefore visited once per backward. Targets can still
61
+ repeat. This removes that repeated-index accumulation from the exact-continuation
62
+ fixture without changing the model's training implementation. The guarded pass
63
+ also gained a rejection for unsupported `disable_ftz=True` before this fresh run;
64
+ the measured default setting and kernel arithmetic were unchanged.
65
+
66
+ The single-core run passed on both RTX PRO 6000 Blackwell Max-Q GPUs:
67
+
68
+ | Check | Result |
69
+ | --- | --- |
70
+ | Initial forward logits | Byte-identical |
71
+ | Every required backward dispatch | Present; one per microbatch |
72
+ | Gradient comparisons across candidate/partial/resumed ranks | 28 complete parameter sets |
73
+ | Maximum gradient absolute / relative L2 error | `2.7284841053187847e-12` / `1.2185588970205807e-7` |
74
+ | Remaining gradient differences | Only the 32-element memory-mix parameter |
75
+ | Parameters and Adam moments after all 12 compared rank-updates | Byte-identical to baseline |
76
+ | Full candidate checkpoint continuation on both ranks | Byte-identical for every saved field |
77
+ | Model/optimizer agreement between ranks in every run | Byte-identical |
78
+ | Maximum allocated / reserved CUDA memory | 3.064 / 3.445 GiB |
79
+
80
+ For that single-core run, the harness source hash was
81
+ `a2d24d27b81db2f24daac76fb23096a67116e86feed2b43621a866ec5b7d1656`;
82
+ and the guarded graph-pass hash was
83
+ `4029da453f54b470f8c091f3faf825913583d357d0285450fccf82082c4557c3`.
84
+ The JSON reports contain all other source hashes, per-tensor results, and runtime
85
+ dispatch/match counts. That run found four matching compiled backward
86
+ variants per rank with no missed candidates. The initial and final vocabulary
87
+ fixtures are different correctness experiments and must not be compared for speed.
88
+
89
+ While the full-model probe was compiling, a CPU reproduction exposed a limitation
90
+ that a single-core fixture cannot detect: installed AOT common-subexpression
91
+ elimination shares the same zero constant between cores. The original matcher's
92
+ external-user guard rejected both regions in that reproduction. The disposable
93
+ full-model probe was stopped before its first update completed; it supplies no
94
+ timing result. The matcher was corrected to retain only that strictly
95
+ matched, immutable zero constant while another region still uses it; external
96
+ users of all other matched arithmetic remain rejected. The arithmetic kernels
97
+ were unchanged. A separate CPU test applies the installed CSE pass to two regions
98
+ and checks that both replacements succeed.
99
+
100
+ The fresh two-core run used `--cores 2 --unique-inputs` and passed all the same
101
+ checks with unchanged numerical limits. It found eight matched regions per rank
102
+ across four backward variants, with no misses. Each backward dispatched twice;
103
+ the continuous/partial/resumed totals were exactly 14/6/8 per rank. Across 28
104
+ complete gradient comparisons, the largest difference was
105
+ `3.637978807091713e-12` absolute and `1.2654258902165433e-7` relative L2. Every
106
+ parameter and Adam tensor after all 12 compared rank-updates remained
107
+ byte-identical to baseline. Complete candidate stop/resume state and model/Adam
108
+ agreement between ranks were byte-identical for every saved field. Maximum
109
+ allocated/reserved CUDA memory was 5.366/5.736 GiB.
110
+
111
+ For the two-core run, the final harness source hash is
112
+ `5f2d161aa624b670b07205345c63177b3b0f46baa67eaef7e9d4d15482f42f6d`
113
+ and the graph-pass hash is
114
+ `174e97d06602471c258c70b7c0cef9efba7d14e7b78d71251ea012136cbc5ff9`.
115
+ These supersede the single-core hashes above; all older reports are preserved.
116
+ The CUDA kernel source stayed unchanged across both successful fixtures.
117
+
118
+ The expanded CPU oracle suite passed 18 tests. Independently, the producer-scheduling
119
+ option passed the existing two-GPU asynchronous checkpoint/resumption test:
120
+ one passed, five deselected, in 71.06 seconds. That smaller fixture validates the
121
+ producer scheduler's continuation behavior; it does not exercise the paired
122
+ kernel or establish a combined full-model performance gain.
123
+
124
+ Ignored evidence is retained at:
125
+
126
+ - `work/validation/backward-layout-training-20260912/results.json`: repeated-index fixture, including strict-resume failure.
127
+ - `work/validation/backward-layout-training-20260912/resume-differences.json`: embedding-only continuation differences.
128
+ - `work/validation/backward-layout-training-unique-20260912/results.json`: successful single-core fixture and all source hashes.
129
+ - `work/validation/backward-layout-training-two-core-20260912/results.json`: successful two-core fixture after the shared-constant fix.
130
+ - `work/validation/backward-layout-training-producer-resume-20260912.log`: producer-scheduling continuation test.
131
+
132
+ To reproduce, reserve both GPUs, keep production training paused, and use a new
133
+ output directory. Use `--unique-inputs` for the strict continuation fixture;
134
+ omitting it retains the repeated-index diagnostic, which can fail exact resume
135
+ because of the existing embedding accumulation nondeterminism.
136
+
137
+ ```bash
138
+ cd /workspace/WarpState/rtx6000_40b
139
+ source ops/training-env.sh
140
+ CUDA_VISIBLE_DEVICES='' .server-venv/bin/python -B -m pytest -q test_backward_layout_training.py
141
+ CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/python -B \
142
+ ops/validate_backward_layout_training.py --world-size 2 --unique-inputs --cores 2 \
143
+ --output-dir work/validation/backward-layout-training-two-core-new
144
+ CUDA_VISIBLE_DEVICES=0,1 WARPSTATE_TEST_CUDA=1 \
145
+ .server-venv/bin/python -B -m pytest -q test_training_optimization.py \
146
+ -k producer-scheduled-overlap
147
+ ```
ops/CHAT.md ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CPU chat with the latest completed checkpoint
2
+
3
+ This is a text-generation probe of the pretrained **2.005B WarpState** model,
4
+ not a chat-finetuned assistant. The training corpus is English text plus EOS,
5
+ without role tokens. The default `Question: ... / Answer:` wrapper is only a
6
+ prompting convention. Use `/raw` to test ordinary text completion instead.
7
+ Fluency or one good answer does not by itself establish model quality.
8
+
9
+ Run from `/workspace/WarpState`:
10
+
11
+ ```bash
12
+ bash rtx6000_40b/ops/chat-cpu.sh
13
+ ```
14
+
15
+ The launcher uses CPU FP32 inference and 16 intra-op threads; it never
16
+ initializes CUDA or signals the training job. This allows both GPUs to keep
17
+ training. CPU responses are slower, and CPU/disk contention is still possible.
18
+ No training/model implementation, checkpoint writer, optimizer or data setting
19
+ is modified. No quantization, compilation or external service is introduced.
20
+
21
+ ## Snapshot safety
22
+
23
+ At startup the tool opens the completed `work/run/latest.pt` once and pins that
24
+ inode using a read-only descriptor. The existing training writer publishes by
25
+ atomic rename, so a later save does not change the opened version. The tool
26
+ memory-maps that descriptor on CPU and exports only original model weights,
27
+ configuration, step/token boundary and provenance into a fresh directory under
28
+ `work/chat/`. Adam/RNG/dataset state are not exported: this is **inference-only**,
29
+ not a training-resume checkpoint. The training checkpoint is not overwritten,
30
+ hardlinked, chmodded, or otherwise mutated.
31
+
32
+ An export uses about 7.47 GiB. Its new weight file is read-only, and
33
+ `snapshot.json` records its SHA256, tokenizer hash and source inode/boundary.
34
+ The hash is verified before strict model loading. Only load trusted local torch
35
+ checkpoints. A snapshot is frozen at its printed step, not refreshed silently
36
+ while chatting. To test a later checkpoint, quit and launch again without
37
+ `--snapshot`; this creates another independent export. Reuse an existing export
38
+ without consuming another export's storage:
39
+
40
+ ```bash
41
+ bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/work/chat/step-N-suffix/model.pt
42
+ ```
43
+
44
+ `--snapshot` also accepts a full training checkpoint such as `work/run/latest.pt`.
45
+ When no `snapshot.json` is present, the tool validates the embedded training
46
+ format and creates an independent model-only export under `work/chat/` before
47
+ loading it. This creates a fresh export on each launch and prints its path:
48
+
49
+ ```bash
50
+ bash rtx6000_40b/ops/chat-cpu.sh --snapshot /workspace/WarpState/rtx6000_40b/work/run/latest.pt \
51
+ --max-new-tokens 128 --repetition-penalty 1.2 --no-repeat-ngram-size 4
52
+ ```
53
+
54
+ `--checkpoint /path/to/latest.pt` remains the explicit way to capture training
55
+ weights. A chat export whose original `snapshot.json` is missing or invalid is
56
+ rejected; its integrity check is never bypassed or replaced with new metadata.
57
+ Restore the original metadata or create a fresh export from a training checkpoint.
58
+
59
+ No generated snapshots, metadata or transcripts are committed to Git.
60
+
61
+ ## Controls and one-shot probes
62
+
63
+ - `/new`: clear conversation context.
64
+ - `/raw`: clear context and switch to plain text completion.
65
+ - `/chat`: clear context and restore the Q/A wrapper.
66
+ - `/info`: show the exact snapshot boundary and metadata.
67
+ - `/help`, `/exit` (or Ctrl-D): help / quit.
68
+ - Ctrl-C during generation cancels that reply, not training.
69
+
70
+ Each turn replays the retained prompt with one independent generation cache per
71
+ logical layer. A completed tile updates fast/slow memory only after producing
72
+ its own outputs. The total context budget defaults to 2048 tokens, including
73
+ the requested new tokens; an oversized prompt drops its oldest tokens. Long
74
+ histories therefore have substantial CPU prefill latency.
75
+
76
+ ```bash
77
+ # A short reproducible greedy probe; no interactive session.
78
+ bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/model.pt \
79
+ --prompt 'What is Facebook?' --temperature 0 --max-new-tokens 32
80
+
81
+ # Sampling settings are explicit and do not affect training RNG.
82
+ bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/model.pt \
83
+ --threads 16 --temperature 0.8 --top-k 40 --top-p 0.9 --seed 1337 \
84
+ --max-new-tokens 128
85
+ ```
86
+
87
+ For a separate terminal, attach to the prepared session using
88
+ `tmux attach -t warpstate-chat`. Detach with Ctrl-b, then d. Chatting in that
89
+ session does not attach to or control the training console.
90
+
91
+ ## Repetition protection
92
+
93
+ Chat and raw completion now default to `--repetition-penalty 1.2` and
94
+ `--no-repeat-ngram-size 4`. Already generated tokens receive a lower score,
95
+ and a token cannot complete a four-token sequence already emitted in the
96
+ same reply. These controls run before greedy selection, top-k and top-p;
97
+ EOS can still end a reply normally. The prompt and earlier turns are excluded
98
+ from the restrictions, so the model can reuse names and phrases from a question.
99
+ Protection resets for each reply. The active settings are printed at startup.
100
+
101
+ This prevents exact token loops of that size and discourages shorter repeats;
102
+ it does not guarantee distinct meanings or turn a pretrained model into an
103
+ instruction-following assistant. Strict n-gram blocking can also restrict
104
+ intentional repetition in code, quotations or lists. For an unfiltered baseline,
105
+ pass `--repetition-penalty 1 --no-repeat-ngram-size 0`. Both options affect only
106
+ inference; no weights or training settings change.
107
+
108
+ After updating a running chat, use `/exit` and relaunch it with the same
109
+ `--snapshot` path. This loads the new defaults and clears old repetitive
110
+ conversation context. Use `/new` to clear context later without reloading weights.
111
+
112
+ ## Verification
113
+
114
+ The CPU test suite covers strict state loading, tokenizer identity, independent
115
+ depth caches, greedy/seeded sampling, repetition penalties for both logit signs,
116
+ phrase-loop blocking, per-reply reset, EOS, nonfinite logits, and snapshot
117
+ independence when `latest.pt` is atomically replaced during loading. It also
118
+ checks the CLI with `--snapshot latest.pt`, sealed snapshot reuse and rejection
119
+ of missing or invalid export metadata. It compares
120
+ incremental logits against full forward across three tile boundaries, with
121
+ both sequential and parallel memory. Agreement is tolerance-based: their
122
+ floating-point reduction orders are different, and CPU FP32 inference is not
123
+ claimed bitwise-equivalent to BF16 training.
124
+
125
+ ```bash
126
+ cd /workspace/WarpState
127
+ source rtx6000_40b/ops/training-env.sh
128
+ CUDA_VISIBLE_DEVICES='' rtx6000_40b/.server-venv/bin/python -B -m pytest \
129
+ rtx6000_40b/test_chat.py -q --basetemp=/workspace/WarpState/tmp/pytest-chat
130
+ ```
131
+
132
+ The initial real-model smoke probe loaded an independent step-3250 snapshot
133
+ (425,984,000 training tokens) and produced finite greedy output on CPU. This is
134
+ a functionality check, not a new validation benchmark or a quality guarantee.
ops/COMMUNICATION_EXPLORATION_20260912.md ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Communication scheduling exploration — September 12, 2026
2
+
3
+ These experiments use disposable copies of the step-40,081 full checkpoint.
4
+ Production training remains paused. The model, corpus cursor, 2,048-token
5
+ context, 131,072-token update, BF16 computation, FP32 gradient communication,
6
+ and full FP32 AdamW state are held fixed. The production resume preset retains
7
+ the previously selected conservative communication scheduler.
8
+
9
+ ## Explicit scheduler controls
10
+
11
+ `run_40b.py` and `ops/benchmark_training.py` expose three opt-in controls for
12
+ `--gradient-mode compiled-autograd-overlap`:
13
+
14
+ | Flag | Default | Meaning |
15
+ | --- | --- | --- |
16
+ | `--overlap-swap-collectives` | Off | Move a wait and its dependent copies past later collective launches while preserving collective launch order. |
17
+ | `--overlap-runtime-estimates` | Off | Use measured GEMM estimates, aligned across distributed ranks, to guide wait placement. |
18
+ | `--overlap-memory-budget FRACTION` | `0.0` | Permit a fractional increase in the scheduler's estimated peak memory, bounded to 0–0.25. This does not increase the CUDA allocator cap. |
19
+
20
+ The helper always disables unsafe collective reordering. Settings are immutable,
21
+ validated before compiler configuration, and recorded in execution metadata.
22
+ The runner records all three controls in checkpoint signatures and requires
23
+ explicit `--reconfigure` for changes. Legacy signatures normalize to the same
24
+ False/False/0 defaults, preserving existing resume behavior.
25
+
26
+ Runtime estimation uses installed experimental PyTorch compiler APIs. It adds
27
+ GEMM benchmarking during compilation and distributed estimate alignment; this
28
+ is separate from the timed training updates. Configurations use fresh processes
29
+ because compiler/reducer settings are process-wide. Both ranks must compile the
30
+ same collective-bearing graphs.
31
+
32
+ The installed FX cache explicitly includes `config_comms` in its key, and the
33
+ AOT cache uses the same payload. Actual cache-key checks distinguish these
34
+ options; they cannot silently reuse a differently configured scheduler merely
35
+ because the forward graph is unchanged. The runtime-estimate screen also
36
+ records an additional pre-update AllGather consistent with rank alignment.
37
+ `TORCH_LOGS=overlap` alone does not print the detailed swap decisions: the
38
+ installed pass additionally requires `reorder_sink_verbose_logging=True`.
39
+
40
+ ## Complete-model screens
41
+
42
+ All rows use the same source versions and checkpoint. Changes are relative to
43
+ the fresh 128 MiB conservative control, not the earlier eager-gradient baseline.
44
+
45
+ | Configuration / artifact directory | Tokens/s | Change | Reserved GiB/rank |
46
+ | --- | ---: | ---: | ---: |
47
+ | Conservative, 128 MiB (`control-128`) | 31,174.13 | — | 66.98 |
48
+ | Waits past launches, 128 MiB (`swap-128`) | 31,201.81 | +0.09% | 66.98 |
49
+ | Waits past launches, 64 MiB (`swap-64`) | 30,824.54 | -1.12% | 66.98 |
50
+ | Waits past launches, 256 MiB (`swap-256`) | 30,385.00 | -2.53% | 66.98 |
51
+ | Waits past launches + runtime estimates, 128 MiB (`swap-runtime-128`) | 31,117.39 | -0.18% | 66.46 / 66.96 |
52
+ | Same conservative control repeated (`control-repeat`) | 31,070.03 | -0.33% | 66.98 |
53
+
54
+ The wait-only and runtime-estimate results are effectively tied with control;
55
+ neither establishes a throughput gain. The runtime-estimate choice reduces
56
+ allocator reservations by approximately 0.5 GiB on rank zero; rank one is
57
+ essentially unchanged. Peak allocated memory remains 65.71188 GiB on both
58
+ ranks, so this does not demonstrate a smaller model working set.
59
+ Smaller and larger buckets both lose throughput here. None of these results
60
+ justifies changing the production preset.
61
+
62
+ Repeating the unchanged control moves throughput by -0.33%, larger than either
63
+ apparent wait-only or runtime-estimate effect. Its maximum loss difference is
64
+ 0.00007796 and maximum relative gradient-norm difference is 0.0836%. Thus even
65
+ the unchanged full model does not give a bitwise scalar trajectory in these
66
+ screens. The other configurations' norm differences can be larger; a pair of
67
+ scalar traces is not proof of per-parameter equivalence or long-run quality.
68
+
69
+ Control and wait-only runs both slow by about 1.14% from their first five to
70
+ last five measured updates. Telemetry shows similar temperatures and clocks
71
+ across the control, wait-only, and 64 MiB runs; their mean GPU temperatures are
72
+ approximately 52 C and 55 C by rank. One-second clock samples mix GEMM and
73
+ non-GEMM phases and are not measurements of sustained GEMM clock alone.
74
+
75
+ `ops/compare_communication_benchmarks.py` validates checkpoint/data identities,
76
+ batch geometry, update cursors, recorded precision, and rank consistency before
77
+ comparing complete unprofiled windows. Its report preserves paired loss/norm
78
+ differences, timing variation, per-rank memory peaks, and source/input hashes.
79
+ These screens' largest loss difference is 0.00018311 and largest relative
80
+ gradient-norm difference is 0.366%; those are observed differences, not general
81
+ acceptance thresholds or evidence of improved model quality.
82
+
83
+ Generated conservative and wait-only synchronization graphs have identical
84
+ Triton arithmetic bodies, GEMM/attention order, collective launch order, and
85
+ statement multisets; only wait/copy placement changes. Existing embedding
86
+ backward uses relaxed FP32 atomic additions to repeated-token rows. Small
87
+ cross-run differences are therefore consistent with existing nondeterministic
88
+ reductions amplified by subsequent BF16 rounding, although this observation
89
+ alone does not prove their cause. Both ranks agree on gradient norms in each
90
+ run. Same-mode repetition and explicit continuation tests are separate checks.
91
+
92
+ The paired report is
93
+ `work/validation/communication-exploration/comparison-final.json`. It can be
94
+ rebuilt from the saved benchmark summaries and rank logs with:
95
+
96
+ ```bash
97
+ .server-venv/bin/python ops/compare_communication_benchmarks.py \
98
+ --control work/benchmark-communication-20260912/control-128 \
99
+ --candidate work/benchmark-communication-20260912/swap-128 \
100
+ work/benchmark-communication-20260912/swap-64 \
101
+ work/benchmark-communication-20260912/swap-256 \
102
+ work/benchmark-communication-20260912/swap-runtime-128 \
103
+ work/benchmark-communication-20260912/control-repeat \
104
+ --output work/validation/communication-exploration/comparison-rebuilt.json
105
+ ```
106
+
107
+ ## Method
108
+
109
+ ### Fresh trace comparison
110
+
111
+ Separate profiles use eight warmup updates and one instrumented update, with
112
+ identical checkpoint/corpus offsets. Their throughput is not included in the
113
+ unprofiled comparison above.
114
+
115
+ | CUDA interval budget | Conservative rank 0 / rank 1 | Wait swap rank 0 / rank 1 |
116
+ | --- | ---: | ---: |
117
+ | NCCL overlapping other CUDA work | 191.805 / 191.069 ms | 193.185 / 191.983 ms |
118
+ | NCCL with no other CUDA work | 153.136 / 266.595 ms | 152.460 / 274.337 ms |
119
+
120
+ Profiled update wall time is 4.17505 seconds for control and 4.18611 for swap.
121
+ Both contain 59 collective calls including scalar metrics, with identical CPU
122
+ operator/shape/type submission fingerprints. The structural wait movement does
123
+ not create a material increase in observed overlap on this model.
124
+
125
+ Rank one's larger exposed interval includes waiting for its peer: its
126
+ noncommunication CUDA union is approximately 3.833 seconds, versus 3.946 on
127
+ rank zero in the control. Exposed intervals must not be added across ranks or
128
+ treated as wholly recoverable communication overhead.
129
+
130
+ The report is `work/validation/communication-exploration/profile-comparison.json`.
131
+ Rebuild it with `ops/analyze_communication_profile.py`, the four
132
+ `profile-{control,swap}/profile-rank-{0,1}-trace.json` files under the benchmark
133
+ directory, and a fresh `--output` JSON path under `work/`.
134
+
135
+ ### Next scheduling work
136
+
137
+ The installed compiler already enables `simple_overlap` after the iterative
138
+ wait pass. Enabling it again is not a new optimization. Its adjacent launch
139
+ moves stop at a gradient producer, an earlier collective, or estimated memory
140
+ growth. The next useful experiment would move the producer work with the
141
+ collective, making a bucket ready sooner rather than only delaying its wait.
142
+
143
+ `reorder_communication_preserving_peak_memory` can group packing/casting
144
+ producers with collectives. A future screen must explicitly set
145
+ `reorder_iterative_unsafe_collectives_reorder=False` and
146
+ `reorder_iterative_peak_memory_budget=0.0`; the installed defaults are True and
147
+ 0.2. Its non-estimated mode stops at GEMM producers; aligned runtime estimates
148
+ allow considering larger producer groups. Another option, `raise_comms`,
149
+ prioritizes the next collective's producer ancestors but has no peak-memory
150
+ guard. Neither pass was implemented or benchmarked by this exploration.
151
+
152
+ The nonzero memory-allowance option is exercised by the continuation fixture;
153
+ nonzero allowances were not included in the complete-model throughput sweep.
154
+ Bucket readiness still depends on every member gradient, and the final
155
+ reductions must complete before clipping and AdamW. These are real limits on
156
+ how much overlap any scheduler can obtain.
157
+
158
+ ### Timing procedure
159
+
160
+ The unprofiled comparisons use eight complete warmup updates followed by twenty
161
+ timed updates, with microbatch four per GPU and eight accumulation microbatches.
162
+ Each starts at the same checkpoint and token offset. The benchmark neither
163
+ modifies nor writes production checkpoints. Both GPUs retain their 300 W limits
164
+ and an 88 GiB PyTorch allocator cap. Matrix recipes, external weight caching,
165
+ and standalone activation replacements are off.
166
+
167
+ NVIDIA-SMI samples clocks, power, temperature, and memory once per second outside
168
+ the timed Python regions. Initial diagnostic comparisons use identical NCCL
169
+ logging settings. A single sub-percent timing difference is insufficient to
170
+ select a default, particularly when sustained clock or thermal drift is similar
171
+ in size. Loss and gradient-norm sequences must be compared at identical update
172
+ indices, including warmups.
173
+
174
+ Profiles are separate diagnostics. CUDA intervals are unioned per rank/device;
175
+ overlapping streams and annotation ranges are not summed as elapsed time.
176
+ Communication-only time is the union of NCCL intervals outside other CUDA
177
+ activity. It is an exposed-work budget, not a guarantee that every millisecond
178
+ can be hidden. Ranks' times must not be added together.
179
+
180
+ `ops/analyze_communication_profile.py` produces reproducible overlap reports
181
+ from the benchmark's Chrome traces. Collective payload metadata describes
182
+ logical input bytes, not measured physical link traffic. Its CPU collective
183
+ sequence fingerprint covers operation names and shapes; it is supporting
184
+ evidence and does not establish tensor identity or numerical correctness.
185
+
186
+ ## Transport evidence
187
+
188
+ The fresh control's NCCL initialization confirms direct `P2P/CUMEM` transport
189
+ in both directions and two collective channels. Its tuning messages identify
190
+ large all-reduces as RING/SIMPLE. Network/socket initialization messages are not
191
+ evidence that gradient traffic uses sockets; kernel symbol suffixes alone also
192
+ do not establish the selected protocol. The GPUs have a SYS path across CPU
193
+ NUMA nodes, so direct peer access still traverses the system interconnect.
194
+
195
+ The benchmark records NCCL version, relevant environment overrides, and CPU
196
+ affinity. Raw logs and telemetry are under
197
+ `work/benchmark-communication-20260912/` and remain ignored.
198
+
199
+ ## Reproduction
200
+
201
+ Reserve both GPUs with production training paused, then use a fresh output
202
+ directory for each process:
203
+
204
+ ```bash
205
+ cd /workspace/WarpState/rtx6000_40b
206
+ source ops/training-env.sh
207
+ CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO \
208
+ NCCL_DEBUG_SUBSYS=INIT,GRAPH,TUNING,ENV \
209
+ .server-venv/bin/torchrun --standalone --nproc-per-node=2 \
210
+ ops/benchmark_training.py \
211
+ --checkpoint work/run/latest.pt --data work/data/train.bin \
212
+ --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
213
+ --no-activation-checkpointing --autocast-cache \
214
+ --compile-cores --compile-mode default --loss-mode compiled-full \
215
+ --bf16-residuals --gradient-mode compiled-autograd-overlap \
216
+ --optimizer-state resume --vram-limit-gib 88 \
217
+ --warmup-updates 8 --measured-updates 20 \
218
+ --output-dir work/benchmark-communication-reproduction/control-128
219
+ ```
220
+
221
+ Append `--overlap-swap-collectives` for wait movement. Vary
222
+ `--ddp-bucket-cap-mb 64|128|256` in separate runs. The Python reducer groups
223
+ whole parameter gradients; a nominal bucket capacity does not split a larger
224
+ individual parameter. Runtime estimates and an estimated memory allowance
225
+ must be identified separately in results.
226
+
227
+ The backward memory-traffic prototype is a separate experiment; see
228
+ [BACKWARD_LAYOUT_EXPERIMENT.md](BACKWARD_LAYOUT_EXPERIMENT.md). Isolated kernel
229
+ speedups cannot be added to complete-model throughput results.
230
+
231
+ ## Validation and retained state
232
+
233
+ The final CPU suite passed **214 tests** with 17 CUDA checks skipped in that
234
+ CPU-only invocation. It covers option validation, legacy signatures, explicit
235
+ execution reconfiguration, gradient accumulation, rank-aligned estimates,
236
+ trace interval accounting, paired-run consistency, and backward-layout math.
237
+ The separate backward-layout CUDA comparisons are documented in its report.
238
+
239
+ Both selected two-GPU continuation fixtures passed in 136.90 seconds:
240
+ wait movement with a 2% estimated memory allowance and 64 MiB buckets, and
241
+ wait movement with aligned runtime estimates and 128 MiB buckets. Each compares
242
+ continuous training with an asynchronous save/stop/resume, including a final
243
+ update with an entirely masked rank. Model tensors, complete FP32 optimizer
244
+ state, and both ranks' RNG state match bitwise within the same execution mode.
245
+ These small-model continuation checks are distinct from the full-model
246
+ cross-configuration numerical differences above.
247
+
248
+ Validation logs and reports are under
249
+ `work/validation/communication-exploration/`, including `pytest-final-cpu.log`,
250
+ `pytest-resume.log`, `comparison-final.json`, and `profile-comparison.json`.
251
+ The retained step-40,081 checkpoint passes its before/after SHA256 check
252
+ (`checkpoint-before.sha256`, `checkpoint-after.log`). All package manifest
253
+ entries also verify. Production remains paused, and no new scheduler setting
254
+ or backward-layout adapter is selected in the production preset.
ops/DATA_PREPARATION.md ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WarpState training data preparation
2
+
3
+ The managed job prepares 40,000,000,000 training tokens and 1,048,576
4
+ validation tokens from `HuggingFaceFW/fineweb-edu`, configuration
5
+ `sample-100BT`, using the package's unchanged SentencePiece tokenizer.
6
+ The upstream commit is pinned in the preparation state and final manifest.
7
+
8
+ All job files, dependencies, caches, logs, and temporary files are below
9
+ `/workspace/WarpState`. This job uses CPU and network resources, with 16
10
+ tokenizer threads and a lower CPU scheduling priority. It does not train a
11
+ model. The package's original Python files are unchanged.
12
+
13
+ The launcher uses `ops/prepare_data.py` to avoid a reproduced PyArrow/Python
14
+ shutdown crash on this host. It waits for the original pipeline to return
15
+ (after token/state files are flushed and committed), flushes console output,
16
+ then exits without interpreter finalization. Exceptions are logged and keep
17
+ a nonzero exit code, so Supervisor can restart from committed progress.
18
+ Garbage collection alone did not eliminate the crash. This follows the
19
+ same process-exit approach documented in NVIDIA's
20
+ [Cosmos launcher](https://github.com/NVIDIA/cosmos-framework/blob/main/cosmos_framework/scripts/train.py).
21
+
22
+ The launcher was checked on a real FineWeb-Edu stream with 1,000,000 train
23
+ tokens and 64 validation tokens. A SIGTERM stop committed its partial batch
24
+ and exited successfully. Resuming produced byte-identical files to an
25
+ uninterrupted run, and both completion manifests matched the computed
26
+ SHA256 checksums. Changed data settings were rejected with a nonzero exit
27
+ code. The results are in
28
+ `work/validation/live-resume-exitguard/validation.json`.
29
+
30
+ ## Paths
31
+
32
+ - Python environment: `rtx6000_40b/.data-venv`
33
+ - Resolved Python dependencies: `rtx6000_40b/ops/data-requirements-resolved.txt`
34
+ - Training tokens: `rtx6000_40b/work/data/train.bin`
35
+ - Validation tokens: `rtx6000_40b/work/data/val.bin`
36
+ - Resumable state: `rtx6000_40b/work/data/prepare-state.pkl`
37
+ - Completion manifest and SHA256 checksums: `rtx6000_40b/work/data/manifest.json`
38
+ - Preparation log: `rtx6000_40b/work/logs/prepare.log`
39
+ - Hugging Face cache: `rtx6000_40b/work/hf-cache`
40
+ - Package download cache: `.cache/uv`
41
+ - Temporary files: `tmp`
42
+
43
+ The final token files contain little-endian uint16 IDs and occupy
44
+ 80,002,097,152 bytes together. An incomplete `train.bin` is not a completed
45
+ dataset. Completion requires `manifest.json` with `complete: true`; the job
46
+ computes both file checksums before writing that manifest.
47
+
48
+ ## Monitor and control
49
+
50
+ Run from `/workspace/WarpState/rtx6000_40b`:
51
+
52
+ ```bash
53
+ ops/datactl.sh status
54
+ tail -f work/logs/prepare.log
55
+
56
+ # Gracefully stop, committing the current preparation batch:
57
+ ops/datactl.sh stop warpstate-prepare
58
+
59
+ # Resume from the saved iterator and committed token counts:
60
+ ops/datactl.sh start warpstate-prepare
61
+ ```
62
+
63
+ Supervisor runs independently of the terminal, restarts the job after an
64
+ unexpected exit, and leaves it stopped after successful completion. Its
65
+ configuration, Unix socket, PID, and logs are local to this project. It does
66
+ not change the instance's system Supervisor configuration or expose a port.
67
+
68
+ If this project's Supervisor itself is no longer running, start it with:
69
+
70
+ ```bash
71
+ TMPDIR=/workspace/WarpState/tmp PYTHONDONTWRITEBYTECODE=1 \
72
+ /usr/local/bin/supervisord -c ops/supervisord.conf
73
+ ```
74
+
75
+ The prepared data can later be used with the existing trainer's
76
+ `--stage train --work-dir /workspace/WarpState/rtx6000_40b/work` and its
77
+ default dataset, revision, tokenizer, and token-count arguments. Training
78
+ also requires a CUDA PyTorch environment and compatible initial weights.
ops/DPO_CHAT.md ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Final DPO archive and chat
2
+
3
+ DPO finished on 2026-09-15 at 01:05:06.882 UTC, with the guardian's final
4
+ verification complete at 01:05:42 UTC. It completed 2,334 updates over 74,661
5
+ preference pairs. Training remains paused.
6
+
7
+ The historical archive is `work/archives/step-00076294-dpo-final/` relative to
8
+ `rtx6000_40b`. It preserves the full FP32 model/Adam/RNG checkpoint and a
9
+ tensor-exact inference export, plus the tokenizer, evaluation, recipes,
10
+ reference scores, and provenance. Files are sealed read-only and checksummed.
11
+ Existing raw and SFT archives are preserved.
12
+
13
+ From `/workspace/WarpState`, create or verify the archive and prepare the
14
+ independent chat copy:
15
+
16
+ ```bash
17
+ source rtx6000_40b/ops/training-env.sh
18
+ rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/archive_dpo.py
19
+ rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/prepare_dpo_chat.py
20
+ ```
21
+
22
+ The chat copy is `rtx6000_40b/work/chat/dpo-final/`: separate physical files,
23
+ with hashes equal to the archive. The chat launcher reads this copy by default.
24
+ Neither chat nor preparation modifies archived weights.
25
+
26
+ ```bash
27
+ rtx6000_40b/ops/chat-dpo.sh
28
+ # Or attach to the already-open interface:
29
+ tmux attach -t warpstate-chat
30
+ ```
31
+
32
+ Type a message at `You:`. Ctrl-C discards unfinished input or cancels the current
33
+ reply and returns to `You:`, preserving completed conversation turns.
34
+ `/reset` clears conversation history; `/quit` exits.
35
+ Detach tmux with Ctrl-B followed by D. The interface uses GPU 0, BF16 compute,
36
+ original FP32 master weights, a 2,048-token context, and greedy decoding.
37
+ Responses print when complete. Alternate CUDA devices can be selected with
38
+ `--device cuda:1`; `--temperature 0.7` enables sampling.
39
+
40
+ Prompt serialization matches the segmented role headers, BOS and assistant EOS
41
+ used during posttraining. Cached inference follows the model's parallel chunk
42
+ memory; shape-dependent BF16 rounding can differ from full-sequence execution.
43
+
44
+ Run reproducible sample prompts, choosing a new output filename each time:
45
+
46
+ ```bash
47
+ rtx6000_40b/ops/chat-dpo.sh --prompts-json rtx6000_40b/configs/posttrain_smoke_prompts.json --max-new-tokens 128 --output-json rtx6000_40b/work/validation/dpo-smoke.json
48
+ ```
49
+
50
+ These few prompts are sanity checks, not a capability benchmark. DPO's final
51
+ held-out preference loss was 0.5790; 245 of 339 pairs had positive relative
52
+ reward margins. This 72.27% is not a generation win rate or answer accuracy.
53
+ Exponentiating DPO loss does not produce language-model perplexity.
54
+
55
+ For a matched full SFT holdout evaluation, `ops/evaluate_sft_snapshot.py`
56
+ accepts `--snapshot`, `--device`, and a new `--output` path. Reports and generated
57
+ responses stay under ignored `work/`; source scripts and prompts are tracked.
ops/DPO_HANDOFF.md ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Archive completed SFT and continue into DPO
2
+
3
+ The user authorized this sequence on 2026-09-14:
4
+
5
+ **Finish SFT → archive its full state → independent initialization → DPO → pause.**
6
+
7
+ The separate `warpstate-dpo` Supervisor job waits while the existing SFT job
8
+ continues unchanged. The original SFT plan still says pause because it is part
9
+ of that run's immutable resume signature; its guardian must complete and exit
10
+ before this newly authorized continuation can start. The active DPO plan is
11
+ [`../configs/dpo_10b.json`](../configs/dpo_10b.json).
12
+
13
+ ## Completion and archival gate
14
+
15
+ The DPO guardian requires all of the following before any GPU work:
16
+
17
+ - SFT Supervisor state `EXITED`, exit status 0, and its guardian's durable state
18
+ `complete`.
19
+ - Exclusive SFT guardian, base launcher and pipeline locks, then recheck.
20
+ - Exactly 10,312 SFT updates, 329,973 samples, 212,321,372 shifted input tokens,
21
+ and 143,695,039 supervised assistant tokens.
22
+ - A completed full SFT checkpoint with Adam and both ranks' RNG state, and its
23
+ matching final evaluation of all 1,603 held-out conversations.
24
+ - Unchanged parent SFT plan, tokenizer, prepared-data manifest and source pins.
25
+
26
+ The archive is `rtx6000_40b/work/archives/step-00076294-sft-final/`. It contains:
27
+
28
+ | File | Purpose |
29
+ | --- | --- |
30
+ | `checkpoint.pt` | Physical byte-for-byte copy of the full SFT checkpoint, including optimizer/RNG/cursor/signatures |
31
+ | `model.pt`, `snapshot.json` | Original FP32 model-only export, with every tensor compared exactly to the full checkpoint |
32
+ | `tokenizer.model` | Exact original SentencePiece tokenizer |
33
+ | `evaluation.json`, `completion.json`, `settings.json`, `metrics.jsonl` | Original final evaluation, completion, execution settings and complete SFT history |
34
+ | `sft-execution.json`, `sft-handoff.json`, `PROVENANCE.json` | Source/data identities and both raw-pretraining and SFT stage counters |
35
+ | `ARCHIVE.md`, `SHA256SUMS` | Retention instructions and integrity checks |
36
+
37
+ All files are sealed read-only before the complete directory is atomically
38
+ published without replacement. Existing archives are validated and reused, or
39
+ rejected without alteration. Interrupted hidden staging directories remain for
40
+ diagnosis. The original SFT resume checkpoint and all raw historical archives
41
+ remain in place.
42
+
43
+ DPO reads a physically independent copy of `model.pt`, `snapshot.json` and the
44
+ tokenizer under `work/posttrain/step-00076294-dpo/initialization/`. It does not
45
+ inherit SFT Adam state or write into either SFT archive or SFT run directory.
46
+ The full SFT archive is roughly 32 GB decimal; its counters describe SFT after
47
+ the raw step-76,294 model, not an additional 10B-token pretraining run.
48
+
49
+ ## Preference objective and execution
50
+
51
+ This first DPO experiment uses:
52
+
53
+ | Setting | Value |
54
+ | --- | --- |
55
+ | Data | 74,661 training pairs and 339 held-out pairs already prepared |
56
+ | Objective | Length-normalized DPO, beta 5 |
57
+ | Epochs | 1; 2,334 optimizer updates |
58
+ | Global batch | 32 preference pairs; 2 pairs per rank × 2 ranks × 8 accumulation microbatches |
59
+ | Context | 2,048 tokens per chosen/rejected branch; no truncation or cross-conversation packing |
60
+ | Learning rate | Peak `5e-7`, 10% warmup, linear decay to zero |
61
+ | Optimizer | Fresh FP32 AdamW, betas 0.9/0.95, zero weight decay, clipping at 1 |
62
+ | Precision | BF16 compute, original FP32 parameters/gradients/Adam; no FP8 |
63
+ | Evaluation/save | Every 100 DPO updates; full holdout at start and completion |
64
+
65
+ For each branch, sum the log probabilities of **only the final assistant
66
+ response and its actual EOS token**. System/user/tool messages, earlier
67
+ assistant turns, role headers and padding are conditioning or masked. Divide
68
+ each policy-minus-reference log-probability sum by that branch's response-token
69
+ count, subtract rejected from chosen, multiply by beta, and apply negative
70
+ log-sigmoid. Average over actual preference pairs across ranks; short final
71
+ batches and empty rank slots do not change the normalization.
72
+
73
+ This uses the length-normalized objective and conservative hyperparameters
74
+ described in the [Tülu 3 paper, equation 6 and table 20](https://arxiv.org/html/2411.15124v3).
75
+ It is a pilot for this 2B/10B base, not a reproduced or tuned Tülu result.
76
+ Ordinary summed-log-probability DPO is a separate supported objective and its
77
+ usual beta cannot be substituted for beta 5 here without changing the recipe.
78
+
79
+ Before policy updates, score the immutable SFT model in BF16 on all selected
80
+ chosen/rejected branches. Each GPU saves resumable reference-score shards; the
81
+ final cache is bound to SFT weights, data, tokenizer, masks, execution settings,
82
+ library version and source hashes. Reusing or rebuilding it must preserve that
83
+ identity. The reference remains the completed SFT model after every policy
84
+ checkpoint/resume. Once scores are cached, release the reference model before
85
+ allocating policy optimizer state, avoiding a second resident full model.
86
+
87
+ The output head uses checkpointed token chunks to avoid retaining full-sequence
88
+ vocabulary logits for both responses. A tiny two-GPU BF16 preflight checks the
89
+ actual DPO/reference/continuation path after SFT releases the GPUs. CPU setup
90
+ tests do not claim that this deferred GPU gate or production-size startup has
91
+ already run. Any failed gate stops automatic progression with the archive kept.
92
+
93
+ ## State, monitoring and controls
94
+
95
+ Paths below are relative to `rtx6000_40b/work/posttrain/step-00076294-dpo/`.
96
+
97
+ | Path | Purpose |
98
+ | --- | --- |
99
+ | `control/handoff.json`, `control/execution.json` | Guardian phase and frozen execution provenance |
100
+ | `initialization/` | Independent immutable SFT model copy |
101
+ | `preflight/` | Tiny GPU validation and its acceptance report |
102
+ | `reference-cache/` | Fixed SFT reference log probabilities and resumable scoring shards |
103
+ | `run/latest.pt` | Independent full DPO resume checkpoint, format `warpstate-dpo-v1` |
104
+ | `run/metrics.jsonl`, `run/evaluation.json`, `run/completion.json` | Training history, evaluation and durable completion |
105
+
106
+ From `/workspace/WarpState`:
107
+
108
+ ```bash
109
+ rtx6000_40b/ops/datactl.sh status warpstate-posttrain warpstate-dpo
110
+ rtx6000_40b/ops/dpo-handoff.sh --check
111
+ tmux attach -t warpstate-dpo
112
+ tail -f rtx6000_40b/work/logs/dpo.log
113
+ ```
114
+
115
+ To cancel automatic DPO while SFT continues, or gracefully stop active DPO:
116
+
117
+ ```bash
118
+ rtx6000_40b/ops/datactl.sh stop warpstate-dpo
119
+ ```
120
+
121
+ To explicitly resume the unchanged guardian/reference preparation/DPO state:
122
+
123
+ ```bash
124
+ rtx6000_40b/ops/datactl.sh start warpstate-dpo
125
+ ```
126
+
127
+ No restart of SFT is needed. The new guardian preserves the SFT guardian and
128
+ base-launch exclusion locks throughout DPO. Every worker holds a shared base
129
+ pipeline lock, so surviving workers block accidental base/SFT/DPO relaunches
130
+ even if their launcher dies. Stop files and lock files are never shared with
131
+ the live SFT stop request. The program has `autostart=false` and
132
+ `autorestart=false`; a whole-machine/Supervisor restart requires explicitly
133
+ restarting the appropriate jobs. A completed DPO run stays paused when its
134
+ guardian is started again.
135
+
136
+ On normal completion, validate final counters, all 339 held-out pairs, fixed
137
+ reference-cache identity and full archive/init hashes before recording success.
138
+ No additional training stage starts automatically. Next compare the archived
139
+ SFT model with DPO on instruction following and tool use. DPO loss, reward
140
+ margin and preference accuracy describe this objective; exponentiating DPO
141
+ loss does not produce language-model perplexity.
142
+
143
+ All generated data, scores, archives, checkpoints, logs and environments remain
144
+ ignored under `work/`. Only source/configuration/tests/documentation are tracked.
ops/LOCAL_RESUME.md ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Resume the B200 checkpoint on the RTX workstation
2
+
3
+ This configuration continues the B200 training run on the local RTX PRO 6000
4
+ workstation.
5
+ The source checkpoint was saved on the B200 at **step 680 / 89,128,960
6
+ tokens**, with validation loss **4.39469** and SHA256:
7
+
8
+ ```text
9
+ bf689a10a030842699bfa3a25ff62e488cfa2d5b022b2b6e3bfc860359eb8b4f
10
+ ```
11
+
12
+ After the verified import, `work/run/latest.pt` is the local resume checkpoint.
13
+ Later local saves replace it atomically; the checksum above identifies the
14
+ imported step-680 checkpoint. [B200_HANDOFF.md](B200_HANDOFF.md) and the B200
15
+ performance documents record the remote run and its historical controls.
16
+ Use the workstation controls below on this host.
17
+ The B200 source history was imported through commit `752563e`. The generated
18
+ local migration record is `work/validation/local-resume/import.json`, relative
19
+ to `rtx6000_40b`; it records the verified checkpoint and continuation state.
20
+
21
+ [`transfer_artifacts.py`](transfer_artifacts.py) accepts `--pull` to retrieve
22
+ a remote source into a local destination with parallel SSH connections.
23
+ Use `--expected-sha256` with the checkpoint hash above, keep the private key
24
+ on the local host, and use the existing verified host-key file. The download
25
+ uses resumable chunks and verifies the entire file before publishing its
26
+ final name. No second download of the unchanged 40B-token corpus is needed.
27
+
28
+ The prepared 40,000,000,000 training tokens and 1,048,576 validation tokens
29
+ are unchanged. Their immutable upstream revision, tokenizer hash, split rule,
30
+ and preparation dependencies remain in
31
+ [`../configs/data_40b.json`](../configs/data_40b.json). The existing data
32
+ signature retains the original requested revision `main`, which was resolved
33
+ to the pinned upstream commit. Preserve that signature when resuming.
34
+
35
+ ## Local execution
36
+
37
+ [`train-local-resume.sh`](train-local-resume.sh) requires an existing full
38
+ checkpoint and selects both physical workstation GPUs by UUID. Both GPUs
39
+ became available before the local resume. This launcher uses the measured
40
+ BF16 configuration and continues the existing run. Check both GPUs'
41
+ availability before every start:
42
+
43
+ | Setting | Value |
44
+ | --- | --- |
45
+ | Sequence length | 2,048 |
46
+ | Microbatch per GPU / gradient accumulation | 4 / 8 |
47
+ | GPUs / global tokens per optimizer update | 2 / 131,072 |
48
+ | Parameters, gradients, full AdamW moments | FP32 on the GPU |
49
+ | Forward/backward computation / residual activations | BF16 autocast / BF16 |
50
+ | Core activation checkpointing | Disabled |
51
+ | Core compilation / autocast weight cache | Enabled, default mode / enabled |
52
+ | Vocabulary loss | Compiled full loss |
53
+ | Gradient execution / communication | Compiled autograd with delayed waits/copies; FP32 reduction, 128 MiB buckets |
54
+ | Allocator cap / extra startup headroom per GPU | 88 GiB / 3 GiB |
55
+ | Checkpoint writing | Synchronous immutable CPU snapshot; background serialization and atomic replacement |
56
+ | Save / evaluation interval | Every 100 / 100 updates |
57
+ | Validation display | Fresh results only, labeled with their evaluation step |
58
+
59
+ The current preset includes gradient conversion/accumulation fusion and moves
60
+ collective waits with their dependent copies past independent backward GEMMs.
61
+ The repeated 20-update comparison measured **31,045 tokens/s** versus
62
+ **30,578 tokens/s** without those changes (**1.53% faster**), with **66.98 GiB
63
+ reserved per GPU**. Both GPU ranks remain data-parallel; communication overlaps
64
+ their local computation. See [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md)
65
+ for the profile, numerical comparisons and exact two-GPU checkpoint/resume tests.
66
+ Shape-specific BF16 matrix plans are available through `--matmul-plan`. The
67
+ experimental fused-epilogue/TMA recipe measured 31,156 versus 31,022 tokens/s
68
+ with the same overlap mode in one comparison (+0.43%); that additional gain is
69
+ not yet confirmed across repeated windows, so the launcher keeps its current
70
+ matrix implementation. See [MATMUL_TUNING.md](MATMUL_TUNING.md) for the recipe,
71
+ tile tuning, persistent kernels, precision checks, and full-model measurements.
72
+ The earlier loss/residual changes are documented in
73
+ [RTX_PROFILE_20260910.md](RTX_PROFILE_20260910.md), and asynchronous checkpoints
74
+ in [RTX_ASYNC_TRAINING.md](RTX_ASYNC_TRAINING.md).
75
+
76
+ These rates exclude evaluation, checkpoint pauses and compilation. Compiled
77
+ autograd can spend approximately three minutes compiling its first full update;
78
+ even with cached kernels, each new process has graph-capture/setup costs.
79
+ The allocator cap excludes CUDA context/library allocations. Startup requires
80
+ 91 GiB free on each GPU and does not reserve memory for other jobs.
81
+ Keep at least **53 GiB of free SSD space** beyond existing files for full
82
+ checkpoint writing and replacement.
83
+
84
+ Asynchronous saves use about **22.4 GiB of additional host memory** for one
85
+ full model/optimizer snapshot. GPU-to-CPU staging still pauses both workers;
86
+ serialization, file synchronization and publication overlap later updates.
87
+ There is at most one outstanding snapshot, and a new save waits for the
88
+ previous write if necessary. A logged `Staged step` is not yet durable: use
89
+ the corresponding `Saved ... (step ..., ... background write ...)` message.
90
+ Normal completion and graceful stop wait for the final write. A save failure
91
+ is reported to both workers at the next coordinated check, save or close and
92
+ stops training; a failure during an update can allow that update to finish.
93
+
94
+ Training step lines display `val ... (step ...)` only when that step has a
95
+ fresh validation result. Every step still records `val_loss`, `val_step` and
96
+ `val_tokens` in `metrics.jsonl`; these fields retain the latest measured
97
+ result between evaluations. Checkpoints retain the same validation metadata.
98
+ This adds no per-step evaluation work.
99
+ Startup still evaluates the resumed weights, and graceful stop/final completion
100
+ still evaluates and saves even when the step is not a multiple of 100.
101
+
102
+ `--reconfigure` preserves the model, full optimizer, token cursor, objective,
103
+ and learning-rate schedule while recording execution changes in the checkpoint.
104
+ The original B200 import restored rank-zero random states and seeded the new
105
+ rank reproducibly. Subsequent two-GPU resumes restore both ranks' states.
106
+ BF16 residual activations retain FP32 master parameters, gradients and Adam
107
+ moments, but change activation rounding. A small validation change at the
108
+ same saved weights is not training progress. Disabling them requires
109
+ `--no-bf16-residuals` with an explicit reconfiguration.
110
+
111
+ ## Controls and current progress
112
+
113
+ Run from `/workspace/WarpState`:
114
+
115
+ ```bash
116
+ rtx6000_40b/ops/datactl.sh status
117
+ nvidia-smi --query-gpu=index,memory.used,memory.free,utilization.gpu --format=csv
118
+ rtx6000_40b/ops/datactl.sh start warpstate-train
119
+ tail -n 10 rtx6000_40b/work/logs/train.log
120
+ tail -n 3 rtx6000_40b/work/run/metrics.jsonl
121
+ ```
122
+
123
+ The status command and latest metrics identify the actual running state and
124
+ current cursor. This document records the import and launcher configuration;
125
+ it does not imply that a job is running. Preparation and training both have
126
+ automatic startup and restart disabled. After changing Supervisor
127
+ configuration, use `datactl.sh reread` and `datactl.sh update` before an
128
+ explicit start.
129
+
130
+ Stop with `rtx6000_40b/ops/datactl.sh stop warpstate-train`. The launcher
131
+ requests a stop through its stop file, waits for the current update and full
132
+ checkpoint, and has a 900-second Supervisor stop allowance. Check the saved
133
+ step in the log before treating an in-memory update as durable.
134
+
135
+ ## Resuming after a deliberate pause
136
+
137
+ Keep a deliberately paused run stopped until the user requests resumption.
138
+ As requested on 2026-09-14, the workstation Supervisor command includes
139
+ `--stop-at-step 76294`: the trainer finishes optimizer update 76,294, evaluates,
140
+ and publishes the full checkpoint before exiting. This boundary corresponds to
141
+ 10,000,007,168 consumed tokens. No update 76,295 runs. Async checkpoint writing
142
+ is flushed before exit, and Supervisor has automatic restart disabled.
143
+
144
+ `--stop-at-step` is an absolute limit, so an intervening restart cannot move
145
+ the boundary. `--max-steps` still limits additional updates in one invocation;
146
+ if both are supplied, the earlier boundary wins. Neither option changes the
147
+ 40B-token learning-rate schedule or the checkpoint's training signature.
148
+ The active boundary is also recorded in `work/run/settings.json` under
149
+ `runtime.stop_at_step` and `runtime.end_step`.
150
+
151
+ When the user requests training beyond this boundary, first remove or increase
152
+ `--stop-at-step` in the `warpstate-train` command in `ops/supervisord.conf`, then
153
+ run `ops/datactl.sh reread` and `ops/datactl.sh update warpstate-train` from the
154
+ package directory before starting the job. A restart retaining a boundary at
155
+ or below the loaded checkpoint step performs no further optimizer updates.
156
+
157
+ The generated `work/validation/paused-run.json` records the last verified
158
+ pause: checkpoint path, SHA256, step, tokens consumed, validation loss,
159
+ optimizer/RNG checks, execution settings, source commit and next update.
160
+ `work/validation/paused-run.sha256` provides a directly checkable checksum.
161
+ Both records are excluded from Git. Subsequent training replaces `latest.pt`,
162
+ so these records identify the last deliberate pause rather than live progress.
163
+
164
+ Before resuming a stopped run, inspect its saved state and available hardware:
165
+
166
+ ```bash
167
+ cd /workspace/WarpState
168
+ rtx6000_40b/ops/datactl.sh status
169
+ cat rtx6000_40b/work/validation/paused-run.json
170
+ sha256sum -c rtx6000_40b/work/validation/paused-run.sha256
171
+ nvidia-smi --query-gpu=index,memory.used,memory.free,utilization.gpu --format=csv
172
+ ```
173
+
174
+ Use the existing full checkpoint at `work/run/latest.pt`, which contains
175
+ model weights, full AdamW state and both ranks' RNG states. Preserve
176
+ `work/data/`, `work/run/`, the `.server-venv` environment, tokenizer and
177
+ existing launchers. No data preparation or checkpoint conversion is needed.
178
+ Both workstation GPUs must have at least 91 GiB free before launch. The
179
+ launcher retains BF16 compute, the 131,072-token global batch, the 40B-token
180
+ schedule, asynchronous saves, and 100-step save/evaluation intervals.
181
+
182
+ When resumption is requested:
183
+
184
+ ```bash
185
+ cd /workspace/WarpState
186
+ rtx6000_40b/ops/datactl.sh start warpstate-train
187
+ tmux attach -t warpstate-train
188
+ ```
189
+
190
+ The launcher clears the previous stop request under its process lock. If the
191
+ host has rebooted and Supervisor is absent, initialize it once with
192
+ `/usr/local/bin/supervisord -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf`
193
+ before using the controls. Its training and preparation jobs start paused.
194
+ If the tmux monitor was lost during reboot, recreate it with:
195
+
196
+ ```bash
197
+ tmux new-session -d -s warpstate-train -n training -c /workspace/WarpState \
198
+ 'tail -n 30 -F /workspace/WarpState/rtx6000_40b/work/logs/train.log'
199
+ ```
200
+
201
+ Keep the active token files, their manifest/preparation state, the latest full
202
+ checkpoint, active environment, and controls required to continue training.
203
+ Git retains source, pinned dependency specifications, tokenizer, and input
204
+ provenance for rebuilding. Obsolete checkpoints, experimental outputs,
205
+ download staging, caches, and disposable test artifacts can be removed once
206
+ their replacement or source recipe is verified; generated runtime files stay
207
+ out of Git.
ops/LOSS_FORECAST_20260913.md ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WarpState validation-loss forecasting, 2026-09-13
2
+
3
+ **Neither proposed token budget currently guarantees validation loss below 2.0.**
4
+ The observed curve supports continued improvement, but the crossing estimate is
5
+ extremely sensitive to the assumed eventual loss floor. A power-law fit to the
6
+ 1–7B-token history crosses near 1.07T tokens; excluding the first billion tokens
7
+ of that fit moves its extrapolated floor above 2.0. This is uncertainty about the
8
+ forecast, not evidence of a demonstrated architecture limit.
9
+
10
+ This research did not change training, checkpoints, the dataset, or the schedule.
11
+ Generated snapshots, numerical results, and figures belong under
12
+ `work/validation/loss-forecast-20260913/` and remain excluded from Git.
13
+
14
+ ## Current measurements and interpretation of the budgets
15
+
16
+ The frozen validation history ends at step 54,300: **7,117,209,600 tokens,
17
+ validation loss 2.42521**, with 555 evaluation records. Recent values fluctuate
18
+ around 2.41–2.43. The complete 6.75–7.00B bin averages 2.41219. Loss 2.0 corresponds
19
+ to perplexity `exp(2) = 7.389` on this tokenizer and evaluation distribution.
20
+
21
+ Interpret `8000*256` and `8000*1024` as **total optimizer steps**, not additional
22
+ steps or individual tokens. The active configuration processes
23
+ `2 GPUs * 4 sequences * 2048 tokens * 8 accumulation microsteps = 131072`
24
+ tokens per optimizer update.
25
+
26
+ | Budget | Optimizer steps | Total training tokens | Tokens per model parameter | Passes over the existing 40B-token corpus | Remaining wall time |
27
+ | --- | ---: | ---: | ---: | ---: | ---: |
28
+ | `8000*256` | 2,048,000 | 268,435,456,000 | 133.88 | 6.711 | 100.5 days |
29
+ | `8000*1024` | 8,192,000 | 1,073,741,824,000 | 535.50 | 26.844 | 410.3 days |
30
+
31
+ The model has 2,005,111,424 parameters. Wall-time arithmetic uses the live metrics
32
+ at step 54,360 on 2026-09-13 11:41:14 UTC: the preceding 3,000 optimizer steps
33
+ averaged **4.35589249 seconds/update**, or **30,090.7 effective tokens/second**,
34
+ including periodic evaluation and checkpoint overhead. Durations assume the
35
+ current two-GPU workstation, unchanged batch size and throughput, and uninterrupted
36
+ operation. They estimate the time to consume a budget, not the time to attain a
37
+ particular loss. From zero, the budgets would take 103.3 and 413.0 days.
38
+
39
+ The configured objective and prepared token file are still **40B tokens**.
40
+ The larger budgets require an explicit continuation schedule and data plan.
41
+ If the question meant additional steps, add the current consumed tokens and
42
+ about 2.74 days to the corresponding remaining-duration arithmetic.
43
+
44
+ ## What the primary research establishes
45
+
46
+ Language-model learning curves often admit power-law approximations after an
47
+ initial transient. These describe diminishing returns: each additional billion
48
+ tokens generally produces a smaller gain. Published coefficients depend on the
49
+ model, tokenization, data, and experimental setup.
50
+ [Kaplan et al., Sections 4–5](https://arxiv.org/pdf/2001.08361).
51
+
52
+ Chinchilla fits endpoint loss as
53
+ `L(N,D) = E + A/N^alpha + B/D^beta`. At fixed model size this has the form
54
+ `L(D) = F + B/D^beta`, where `F` includes the model-size contribution as well as
55
+ irreducible uncertainty. Its compute-optimal parameter/data allocation does not
56
+ define a stopping point for a fixed model. Appendix B also shows that the cosine
57
+ schedule duration must be calibrated to the intended training duration.
58
+ [Hoffmann et al., Section 3.3 and Appendix B](https://arxiv.org/pdf/2203.15556).
59
+
60
+ Schedule-aware work separates continuing optimization from the extra improvement
61
+ caused by lowering the learning rate. Tissue et al. model validation loss using
62
+ both accumulated learning rate and an annealing-history term; their Appendix C.3
63
+ uses shorter constant and cosine runs to forecast a 1.7B model trained on 1.4T
64
+ tokens. Such calibrated experiments support forecasting, but do not establish
65
+ that one early training trace can identify a 150-fold extension.
66
+ [Scaling Law with Learning Rate Annealing](https://arxiv.org/html/2408.11029).
67
+
68
+ Constant-rate training followed by a terminal cooldown provides a way to compare
69
+ different training budgets while reusing checkpoints. A loss reached after
70
+ cooldown need not have the same subsequent slope as a loss reached at a high
71
+ learning rate.
72
+ [Hägele et al., Sections 2–3](https://arxiv.org/html/2405.18392v3).
73
+
74
+ Long training of small models can remain useful. Sardana et al. study a 2.46B
75
+ model up to 500 tokens/parameter, close to the larger proposed budget here. Their
76
+ study also warns that fits from shorter training can overstate the benefits of
77
+ much longer training. Its reported final training loss and unique-data setup
78
+ cannot be converted into WarpState validation loss 2.0.
79
+ [Sardana et al., Sections 3–4 and Appendix C](https://arxiv.org/html/2401.00448).
80
+ Gadre et al. successfully forecast a 1.4B/900B-token model using calibrated
81
+ experiments, further supporting the feasibility of forecasting with the right
82
+ measurements.
83
+ [ICLR 2025 paper](https://proceedings.iclr.cc/paper_files/paper/2025/hash/a91869936a63d814971b6423990ecf6e-Abstract-Conference.html).
84
+
85
+ Repeated tokens have diminishing value compared with fresh tokens. Muennighoff
86
+ et al. find small penalties through roughly four epochs in their experiments,
87
+ with progressively smaller returns from further repetition. That finding is not
88
+ a universal epoch cutoff. Repeating our corpus 6.7 or 26.8 times is a different
89
+ experiment from acquiring 268B or 1.07T comparable fresh tokens.
90
+ [Scaling Data-Constrained Language Models](https://arxiv.org/html/2305.16264v3).
91
+
92
+ ## Fits to WarpState: scenarios, not confidence intervals
93
+
94
+ Use actual evaluations from `work/run/validation.jsonl`, rather than the latest
95
+ validation value repeated in every training-metric row. Verify strictly increasing,
96
+ unique token counts, start at 1B tokens to exclude early execution changes, and
97
+ average complete 0.25B-token bins. Each bin receives equal weight, at its actual
98
+ mean token position. The frozen snapshot contains 24 complete bins from 1–7B;
99
+ the partial final bin is excluded from fitting.
100
+
101
+ Fit in loss space, using `t = tokens / 1e9`:
102
+
103
+ - Logarithmic: `L(t) = a + b*ln(t)`.
104
+ - Power law with a floor: `L(t) = F + A*t^(-beta)`, with `A >= 0`, `F >= 0`,
105
+ and positive `beta`.
106
+ - Fixed-floor sensitivity: choose an illustrative `F` and fit `A` and `beta`.
107
+ These chosen floors are assumptions, not measured properties of WarpState.
108
+
109
+ For the power-law scenario, a finite crossing exists only if `F < 2`, and its
110
+ location is `t = (A/(2-F))^(1/beta)`. A small uncertainty in `F` close to 2 thus
111
+ creates a very large uncertainty in the crossing time.
112
+
113
+ | Fit | Loss at 268.435B | Loss at 1.073742T | Crossing below 2.0 |
114
+ | --- | ---: | ---: | --- |
115
+ | Logarithmic, starts at 3/4/5B and ends at 7B | 1.68–1.85 | 1.40–1.64 | 53–103B |
116
+ | Power law, free floor, 1–7B history | 2.052 | approximately 2.000 | approximately 1.07T |
117
+ | Power law, free floor, 2–7B history | 2.096 | 2.058 | No finite crossing in this fitted curve |
118
+ | Power law, **assumed floor 1.5**, 1–7B history | 1.905 | 1.799 | 103B |
119
+ | Power law, **assumed floor 1.8**, 1–7B history | 2.001 | 1.932 | 272B |
120
+ | Power law, **assumed floor 1.9**, 1–7B history | 2.043 | 1.989 | 762B |
121
+
122
+ The 1–7B free-floor fit is approximately
123
+ `L(t) = 1.918124 + 0.975783*t^(-0.355255)`.
124
+ The 2–7B fit has floor 2.009366 and exponent 0.419261. Neither fitted floor is a
125
+ measured asymptotic limit. Shifting the start to 3, 4, or 5B yields floors around
126
+ 2.19–2.33, showing how poorly the eventual floor is identified by this history.
127
+ The numerical 1.07T crossing should not be interpreted as a precise prediction
128
+ or as evidence that the larger budget was chosen correctly.
129
+
130
+ The logarithmic fit eventually predicts unboundedly negative loss, so it is a
131
+ local trend approximation. It is included because it illustrates an optimistic
132
+ extrapolation, not because it is a suitable global asymptotic law. Small residuals
133
+ on fitted data also do not establish long-range predictive accuracy.
134
+
135
+ Historical forward checks fit an earlier interval and predict the next complete
136
+ billion tokens. The table shows mean predicted minus observed loss; negative
137
+ values mean the forecast was too optimistic.
138
+
139
+ | Fit interval → held-out interval | Logarithmic bias | Free-floor power-law bias |
140
+ | --- | ---: | ---: |
141
+ | 2–4B → 4–5B | -0.00197 | +0.00695 |
142
+ | 2–5B → 5–6B | -0.00604 | -0.00194 |
143
+ | 2–6B → 6–7B | -0.01665 | -0.01023 |
144
+
145
+ Neither family wins every historical comparison. Both overpredicted the latest
146
+ interval's improvement, particularly the logarithmic fit. These checks use
147
+ 0.25B-bin means, so their residuals should not be confused with errors on raw
148
+ individual validation points.
149
+
150
+ ## Why there is no dependable crossing date yet
151
+
152
+ The observations extend to about 7.1B tokens, whereas the requested budgets are
153
+ roughly **38 and 151 times** that amount. Historical one-billion-token holdouts
154
+ can reject some overly optimistic local trends; they cannot validate such a
155
+ large extrapolation. Successive measurements are correlated observations of one
156
+ training run, not independent experimental replications.
157
+
158
+ The run still uses a learning rate near `1.40e-4`, about 93% of its `1.50e-4`
159
+ peak, with a planned 40B-token cosine decay toward `1.50e-5`. The simple fits
160
+ above do not model that future annealing. Cooldown may improve loss beyond their
161
+ predictions; a changed continuation schedule or repeated data may produce a
162
+ different curve. There is no fitted magnitude for those effects in WarpState.
163
+
164
+ Current evaluation samples eight fixed sequences of length 2,048: **16,384
165
+ target tokens**, drawn at fixed spaced positions from the 1,048,576-token
166
+ held-out file. A single reported value below 2 would be weaker evidence than a
167
+ sustained crossing and confirmation on more held-out tokens. Absolute loss 2
168
+ also cannot be compared directly with another paper's tokenizer or validation
169
+ distribution, and does not by itself determine downstream task performance.
170
+
171
+ The useful next forecast experiment is a matched control and cooldown branch
172
+ from the same full checkpoint, retaining optimizer state, data order, batch
173
+ geometry, BF16 computation, and the same evaluation metric. Evaluate additional
174
+ held-out tokens separately, preserving the historical metric for comparison.
175
+ Combining those measurements with later 10–15B-token checkpoints would help
176
+ separate continued-learning gains from annealing gains and allow an actual
177
+ out-of-sample test of a schedule-aware model. This document recommends that
178
+ experiment; it does not launch it.
179
+
180
+ For planning, treat sub-2.0 loss as an unresolved target. **268B tokens might be
181
+ enough, and 1.07T provides much more training, but the present evidence does not
182
+ justify promising either budget will cross the threshold.**
183
+
184
+ ## Reproduction
185
+
186
+ The input snapshot SHA256 is
187
+ `13f7fa766a59b884d376515132e5f41fe07b771438e7e66ed294dce030e9f38e`.
188
+ The analysis program is `ops/loss_curve_forecast.py`. It requires NumPy;
189
+ Matplotlib is optional for the figure. The generated JSON records dependency
190
+ versions and hashes of both the snapshot and analysis source. The isolated
191
+ `work/analysis-venv` keeps analysis dependencies separate from training.
192
+ This analysis used Python 3.12.3, NumPy 2.5.3, and Matplotlib 3.11.2.
193
+
194
+ From `/workspace/WarpState/rtx6000_40b`:
195
+
196
+ ```bash
197
+ source ops/training-env.sh
198
+ python3.12 -m venv work/analysis-venv # Only when rebuilding an absent environment.
199
+ work/analysis-venv/bin/python -m pip install 'numpy==2.5.3' 'matplotlib==3.11.2'
200
+ work/analysis-venv/bin/python -B ops/loss_curve_forecast.py \
201
+ --input work/validation/loss-forecast-20260913/validation-snapshot.jsonl \
202
+ --output work/validation/loss-forecast-20260913/forecast.json \
203
+ --plot work/validation/loss-forecast-20260913/loss-scenarios.png
204
+ ```
205
+
206
+ The same command with an `.svg` plot extension generates a vector figure.
207
+ Use a new output directory when analyzing a later frozen snapshot so that the
208
+ original forecast remains available for comparison with subsequent observations.
ops/MATMUL_PROFILE_20260912.md ADDED
@@ -0,0 +1,316 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WarpState matrix multiplication profile, 2026-09-12
2
+
3
+ The largest dense BF16 products already sustain approximately **243–263 TFLOPS
4
+ per GPU** in the refreshed isolated test. Their observed clocks and 300 W board
5
+ power explain much of the gap from the boost-clock specification. The best next
6
+ experiments are selective algorithm/tile selection and small, measured GEMM
7
+ fusions. A global backend replacement has not demonstrated a useful improvement.
8
+
9
+ This analysis uses the refreshed production-configuration profile and new
10
+ isolated GEMM timings. Production remains paused at step 40,081 with its full
11
+ resume state preserved. All step-8,000, step-16,000, and step-40,000 archives are
12
+ unchanged. No matrix experiment in this report writes production checkpoints.
13
+
14
+ ## Profile and attribution
15
+
16
+ The configuration is the two RTX PRO 6000 Blackwell Max-Q GPUs, microbatch 4 per
17
+ rank, sequence length 2,048, eight accumulation microbatches, 131,072 global
18
+ tokens/update, compiled cores in default mode, compiled full loss, BF16 residual
19
+ activations, and the existing FP32 fused AdamW state. The unprofiled 20-update
20
+ baseline is **30,799 tokens/s during optimizer updates**; the separate profiled
21
+ update took 4.1781 seconds after two warmups.
22
+
23
+ Each GPU kernel's `External id` is joined to its `aten::mm`, `aten::bmm`, or
24
+ `aten::addmm` CPU event. Dimensions, strides, and dtypes come from that event.
25
+ The square output projection's forward and input-gradient shapes are identical;
26
+ the RHS transpose stride distinguishes them. This avoids assigning the same
27
+ square GEMM to both directions.
28
+
29
+ For `A[M,K] @ B[K,N]`, arithmetic is counted as `2*M*N*K` FLOPs, multiplied by
30
+ batch and call count. Achieved TFLOPS divide these FLOPs by matched GPU kernel
31
+ time, not CPU launch time or the entire training update. CPU operator totals can
32
+ also include associated fills/copies; the tables below use kernel durations.
33
+
34
+ All **5,400 matrix kernels per rank** are accounted for:
35
+
36
+ | Profile | Rank 0 matrix time | Rank 1 matrix time | FLOPs per rank/update |
37
+ | --- | ---: | ---: | ---: |
38
+ | Current baseline | 3,166.449 ms | 3,049.219 ms | 791.597 trillion |
39
+ | Compiled-autograd candidate | 3,122.129 ms | 3,005.404 ms | 791.597 trillion |
40
+
41
+ Baseline matrix kernels represent 71.19% and 68.62% of summed GPU activity on
42
+ ranks 0 and 1. Streams overlap with communication and other work, so these
43
+ percentages are not partitions of elapsed update time. The matrix operations
44
+ themselves execute on the main compute stream in this trace. The compiled
45
+ autograd comparison changes surrounding execution and communication overlap;
46
+ its slightly shorter matrix times do not establish a faster full training run.
47
+
48
+ ## Which products dominate
49
+
50
+ For a linear layer, forward computes `Y = X @ W.T`, the input derivative
51
+ computes `dX = dY @ W`, and the weight derivative computes `dW = dY.T @ X`.
52
+ Every core projection executes 224 times per direction: 28 cores times eight
53
+ microbatches. Backward therefore includes two dense products for each forward.
54
+
55
+ | Projection family | Rank 0 forward | Rank 0 dX | Rank 0 dW | Total | Share of matrix time |
56
+ | --- | ---: | ---: | ---: | ---: | ---: |
57
+ | FFN up and gate, 2,048 → 14,336 | 405.939 ms | 433.643 ms | 427.218 ms | 1,266.800 ms | 40.01% |
58
+ | Q/K/V/gate/memory input, 2,048 → 10,240 | 284.852 ms | 312.126 ms | 305.255 ms | 902.232 ms | 28.49% |
59
+ | FFN down, 7,168 → 2,048 | 218.817 ms | 209.116 ms | 198.243 ms | 626.176 ms | 19.78% |
60
+ | Core output, 2,048 → 2,048 | 57.486 ms | 56.861 ms | 57.828 ms | 172.176 ms | 5.44% |
61
+ | Vocabulary head, 2,048 → 32,768 | 33.869 ms | 34.131 ms | 35.123 ms | 103.124 ms | 3.26% |
62
+ | State-memory batched products | — | — | — | 95.942 ms | 3.03% |
63
+
64
+ FFN up/gate, input projection, and FFN down together consume **88.28% of matrix
65
+ time**. Dense linear products perform 788.350 trillion FLOPs per rank/update;
66
+ state-memory batched products account for the remaining 3.247 trillion.
67
+
68
+ The vocabulary head now uses eight complete 8,192-token products per direction.
69
+ The September 10 profile used chunked head loss and repeated head forward
70
+ products during backward. Its larger forward-head count should not be reused
71
+ to estimate the current configuration.
72
+
73
+ The dominant shapes and their rank-0 kernel performance are:
74
+
75
+ | Product | M | N | K | Mean kernel time | Achieved TFLOPS |
76
+ | --- | ---: | ---: | ---: | ---: | ---: |
77
+ | FFN up/gate forward | 8,192 | 14,336 | 2,048 | 1.8122 ms | 265.4 |
78
+ | FFN up/gate dX | 8,192 | 2,048 | 14,336 | 1.9359 ms | 248.5 |
79
+ | FFN up/gate dW | 14,336 | 2,048 | 8,192 | 1.9072 ms | 252.2 |
80
+ | Input projection forward | 8,192 | 10,240 | 2,048 | 1.2717 ms | 270.2 |
81
+ | Input projection dX | 8,192 | 2,048 | 10,240 | 1.3934 ms | 246.6 |
82
+ | Input projection dW | 10,240 | 2,048 | 8,192 | 1.3627 ms | 252.1 |
83
+ | FFN down forward | 8,192 | 2,048 | 7,168 | 0.9769 ms | 246.2 |
84
+ | FFN down dX | 8,192 | 7,168 | 2,048 | 0.9336 ms | 257.6 |
85
+ | FFN down dW | 2,048 | 7,168 | 8,192 | 0.8850 ms | 271.8 |
86
+
87
+ These are timings within the full-model profile. They can differ from isolated
88
+ tests because clocks, cache state, memory pressure, and communication differ.
89
+ The isolated tests below use longer warmup and repeated timing trials.
90
+
91
+ ## What executes inside the dense kernels
92
+
93
+ Most dense kernels use symbols beginning
94
+ `cutlass_80_tensorop_bf16_s16816gemm_relu_bf16`. Forward commonly selects
95
+ `128x256_32x3_tn_align8`; dX uses `256x128_32x3_nn_align8`; dW uses
96
+ `128x256_32x3_nt_align8`. The head forward uses a `256x128` TN variant.
97
+
98
+ These are tiled tensor-core kernels: tiles load operands, reuse them while
99
+ advancing through K in pipelined stages, accumulate partial products, then store
100
+ the output. The symbols encode tile/pipeline/operand variants. The embedded
101
+ `80` and `relu` text does not establish obsolete hardware execution or an actual
102
+ ReLU in this model; semantic attribution comes from the originating operators.
103
+
104
+ Observed launches use 256 threads, approximately 218–232 registers per thread,
105
+ and 73,728 bytes of dynamic shared memory. Several have a grid Z dimension of
106
+ 2 or 4. That is consistent with additional partitioned work, but launch metadata
107
+ alone does not prove the precise split-K/reduction implementation. A proposed
108
+ split-K experiment must first identify what the selected library algorithm
109
+ already does.
110
+
111
+ Profiler occupancy estimates report zero for some kernels that demonstrably
112
+ execute. They are not valid measured occupancy. The previous hardware-counter
113
+ attempt was denied by the driver's `ERR_NVGPUCTRPERM`; this analysis makes no
114
+ new occupancy, tensor-pipe-utilization, or bandwidth-counter claim.
115
+
116
+ The large dense shapes are multiples of tensor-core alignment and ordinary
117
+ tile sizes. Their transpose operands are views with unit stride in one
118
+ dimension, not evidence of a full physical transpose before every GEMM.
119
+ Blindly adding `.contiguous()` would create extra traffic; any alternative
120
+ layout must include its preparation cost in the benchmark.
121
+
122
+ ## Where parallel execution already happens
123
+
124
+ Both GPUs already train concurrently using data parallelism. Each holds the
125
+ model and processes **65,536 of the 131,072 global tokens per update**: eight
126
+ 8,192-token microbatches per rank. There is no intentional schedule that runs
127
+ GPU 0's training first and GPU 1's afterward. Gradient synchronization combines
128
+ their contributions to the same optimizer update.
129
+
130
+ The original compiled-autograd candidate lost overlap between computation and
131
+ communication **within each GPU's schedule**. That is different from serializing
132
+ the two GPUs: both ranks still compute concurrently, but local waits block
133
+ independent backward GEMMs while collectives finish. Recovering that overlap
134
+ is the immediate parallelization target identified by the trace.
135
+
136
+ The refreshed `nvidia-smi topo -m` reports the GPU-to-GPU path as `SYS`, meaning
137
+ PCIe plus an inter-NUMA CPU interconnect, with no NVLink entry. Reported CPU
138
+ affinities are `0-95,192-287` for GPU 0 and `96-191,288-383` for GPU 1. Topology
139
+ identifies the route; it does not measure achieved communication bandwidth.
140
+ No CPU/GPU affinity or power settings were changed for this analysis.
141
+
142
+ Some dX and dW products are mathematically independent and could use separate
143
+ CUDA streams. However, the large GEMMs already consume much of each GPU's
144
+ compute and power capacity; concurrent launches share those resources rather
145
+ than doubling them. A focused stream experiment could test thin or final-tail
146
+ products, including their synchronization cost, before changing the full backward
147
+ schedule. Model or pipeline parallelism would add activation transfers and
148
+ potential pipeline bubbles over this topology, with additional implementation
149
+ complexity. It is not an automatic improvement over the existing two-GPU data
150
+ parallel execution.
151
+
152
+ ## Refreshed isolated performance and compute headroom
153
+
154
+ The existing `benchmark_gemm.py` tested all 12 core shape/direction pairs on
155
+ GPU 0 using deterministic BF16 operands with production transpose strides.
156
+ Each case had at least two seconds of burn-in and five CUDA-event timing trials
157
+ of 30 products. The existing BF16 intermediate-reduction allowance was preserved;
158
+ no lower-precision model or gradient representation was introduced.
159
+
160
+ NVIDIA specifies 438.9 dense BF16 TFLOPS with FP32 accumulation at a 2,280 MHz
161
+ boost clock, 300 W board power, and 1,792 GB/s memory bandwidth for this Max-Q
162
+ GPU. Source: Appendix A of the
163
+ [RTX Blackwell architecture whitepaper](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46).
164
+
165
+ The final column below divides achieved performance by
166
+ `438.9 * observed_SM_clock_MHz / 2280`. It is an illustrative clock-scaled
167
+ comparison, not a hardware-counter utilization measurement. Clocks are telemetry
168
+ snapshots after each timed case, not time-integrated clock measurements.
169
+
170
+ | Product | Median time | TFLOPS | Observed SM clock | Clock-scaled comparison |
171
+ | --- | ---: | ---: | ---: | ---: |
172
+ | FFN up/gate forward | 1.8473 ms | 260.4 | 1,492 MHz | 90.7% |
173
+ | FFN up/gate dX | 1.9148 ms | 251.2 | 1,402 MHz | 93.1% |
174
+ | FFN up/gate dW | 1.8925 ms | 254.2 | 1,425 MHz | 92.7% |
175
+ | Input projection forward | 1.3337 ms | 257.6 | 1,485 MHz | 90.1% |
176
+ | Input projection dX | 1.3958 ms | 246.2 | 1,417 MHz | 90.2% |
177
+ | Input projection dW | 1.3531 ms | 253.9 | 1,447 MHz | 91.2% |
178
+ | FFN down forward | 0.9879 ms | 243.5 | 1,440 MHz | 87.8% |
179
+ | FFN down dX | 0.9162 ms | 262.5 | 1,530 MHz | 89.1% |
180
+ | FFN down dW | 0.9399 ms | 255.9 | 1,485 MHz | 89.5% |
181
+ | Core output forward | 0.2656 ms | 258.8 | 1,612 MHz | 83.4% |
182
+ | Core output dX | 0.2625 ms | 261.8 | 1,620 MHz | 83.9% |
183
+ | Core output dW | 0.2578 ms | 266.5 | 1,725 MHz | 80.3% |
184
+
185
+ Every timed case's final power sample was approximately 300 W. The dominant
186
+ families remain close to the earlier measured range and approximately 88–93%
187
+ of this clock-scaled estimate. The smaller output projection has more apparent
188
+ relative headroom, but represents only 5.44% of matrix time. Neither comparison
189
+ implies all remaining theoretical capacity can be recovered by software.
190
+
191
+ ## Compute versus operand traffic
192
+
193
+ For one idealized BF16 read of A/B and one write of C, arithmetic intensity is
194
+ `2*M*N*K / (2*(M*K + K*N + M*N))`. It is about 1,470 FLOPs/byte for the FFN
195
+ up/gate products, 1,412 for the input projection, 1,334 for FFN down, and 910
196
+ for the square output projection. Actual traffic includes reloads, cache effects,
197
+ and possible intermediate reductions; these are operand-footprint estimates.
198
+
199
+ This supports a compute-efficiency focus for the large dense GEMMs. It does not
200
+ prove that every tile load is perfectly hidden. A persistent/TMA implementation
201
+ can improve operand delivery or scheduling even when the overall dense product
202
+ is predominantly compute-limited.
203
+
204
+ State-memory products are different. Shapes with batch 128 and `M=16, N=4096,
205
+ K=16` have only about 8 FLOPs/byte under that estimate; the `128x64x64` and
206
+ `64x64x128` batched products have about 25.6. Observed rates are roughly
207
+ 10–61 TFLOPS, with 16x16/32x32 WMMA kernel tiles. They merit smaller-shape or
208
+ producer/consumer fusion experiments, but their entire current kernel budget
209
+ is only 95.94 ms, approximately 2.3% of the profiled update.
210
+
211
+ ## Prior experiments and specific next tests
212
+
213
+ The September 10 experiments already covered eager cuBLAS, cuBLASLt preference,
214
+ ATEN-only compilation, default Triton autotuning, and persistent TMA. See
215
+ [`RTX_PROFILE_20260910.md`](RTX_PROFILE_20260910.md).
216
+
217
+ | FFN up/gate product | Prior cuBLAS | Prior default Triton | Prior persistent TMA | Current cuBLAS |
218
+ | --- | ---: | ---: | ---: | ---: |
219
+ | Forward | 1.828 ms | 1.981 ms | 1.798 ms | 1.847 ms |
220
+ | dX | 1.885 ms | 2.057 ms | 1.890 ms | 1.915 ms |
221
+ | dW | 1.871 ms | 1.869 ms | 1.810 ms | 1.892 ms |
222
+
223
+ These are separate runs with different observed clocks, not a controlled
224
+ same-clock speedup claim. Default Triton usually lost to cuBLAS; persistent TMA
225
+ closed much of that gap and won some shapes modestly. cuBLASLt preference was
226
+ mixed, and its earlier up/gate dW result coincided with a higher clock. Repeating
227
+ a global backend toggle is therefore a weak next experiment.
228
+
229
+ The following targeted experiments have a better basis:
230
+
231
+ 1. **Select algorithms per dominant shape/direction.** Retest persistent TMA
232
+ against ATEN/cuBLAS for up/gate forward and dW first, with alternating warmed
233
+ trials and clock samples. Keep cuBLAS for shapes where it wins. Measure an
234
+ entire layer forward/backward and then the full two-GPU update so template
235
+ fusion and DDP overlap are included. Preserve the BF16 dW rounding before
236
+ FP32 accumulation.
237
+ 2. **Test a GEMM epilogue that performs gradient conversion and accumulation.**
238
+ The current conversion/addition budget is 129.076 ms/update. A fused kernel
239
+ must retain the BF16 GEMM result's rounding before its FP32 addition and keep
240
+ autograd/DDP ownership correct. Direct FP32-output `addmm` was already tested:
241
+ it changes rounding and was not consistently better. The present compiled
242
+ autograd candidate also demonstrates why local fusion needs full-update
243
+ communication validation.
244
+ 3. **Explore a paired up/gate epilogue only if its GEMM remains competitive.**
245
+ The combined projection is already one GEMM, with up and gate columns 7,168
246
+ apart. A tile/layout that places matching pairs together could compute SwiGLU
247
+ before writing its activated output. Backward still needs the projected up
248
+ and gate values, so their storage cannot simply disappear. The removable
249
+ standalone forward budget is about 41.16 ms/update; recomputing the projection
250
+ would cost about 405.94 ms and would defeat that purpose. Repacking must use
251
+ temporary views/copies and retain canonical archive/checkpoint layout.
252
+ 4. **Explore the small state-memory GEMMs separately.** Their low intensity and
253
+ small dimensions make tiled producer/consumer fusion more plausible than
254
+ expecting dense-projection TFLOPS. Account for both gradient products and
255
+ layout preparation; a low TFLOPS number on its own does not establish waste.
256
+
257
+ The installed compiler already has epilogue/prologue fusion enabled for suitable
258
+ templates. Its `kernel/mm.py` adds persistent TMA choices only through the
259
+ max-autotune template route, gated by `ENABLE_PERSISTENT_TMA_MATMUL`. Enabling
260
+ that variable alone does not replace every existing library GEMM.
261
+
262
+ The distinction between GPU architectures matters: this workstation is SM120.
263
+ The installed `use_triton_blackwell_tma_template()` additionally requires
264
+ `is_datacenter_blackwell_arch()`, whose predicate is `100 <= arch < 110`.
265
+ The named datacenter-Blackwell warp-specialized template and the CUTEDSL route
266
+ therefore are not automatic options for this RTX GPU; generic persistent TMA
267
+ is the relevant already-tested path. The environment also lacks the `cutlass`
268
+ and `cutlass_library` packages and the configured CUTLASS source directory, so
269
+ CUTLASS/NVGEMM are not available simply by changing the backend flag.
270
+
271
+ As a conditional scale check, retaining the measured 3.1664 seconds of matrix
272
+ time and deleting every other cost would give about **41.4K tokens/s**. This is
273
+ not a hardware limit or an achievable configuration. Reaching 50K with all
274
+ other costs zero would still require about 17.2% less matrix time. The isolated
275
+ results give no evidence for a software-only reduction of that scale across all
276
+ products at the present power limit.
277
+
278
+ ## Evidence and reproduction
279
+
280
+ Generated evidence remains ignored in Git:
281
+
282
+ - `work/validation/rtx-opt-20260912/matmul/matmul-attribution.json`: both ranks,
283
+ both profiles, shapes/strides/dtypes, call counts, exact kernel names and launch
284
+ metadata, FLOPs, operand intensities, and current clock-scaled comparisons.
285
+ - `work/benchmark-opt-20260912/profile-baseline/`: source traces, kernel TSVs,
286
+ operator tables, and category summaries.
287
+ - `work/benchmark-opt-20260912/profile-compiled-autograd/`: corresponding
288
+ candidate traces. Include its eight BF16 `aten::addmm` head-dW calls when
289
+ counting arithmetic; otherwise the head's weight-gradient FLOPs are omitted.
290
+ - `work/benchmark-gemm-20260912-current/results.json`: refreshed isolated trials
291
+ with telemetry for each product.
292
+ - `work/benchmark-gemm-20260910/`, `work/benchmark-gemm-cublaslt-20260910/`, and
293
+ `work/benchmark-gemm-tma-20260910/`: earlier backend comparisons.
294
+
295
+ Installed compiler source evidence is under
296
+ `.server-venv/lib/python3.12/site-packages/torch/_inductor/`: `kernel/mm.py`,
297
+ `config.py`, `utils.py`, and `codegen/cuda/cuda_env.py`. These observations apply
298
+ to the installed `2.14.0+cu130` build and should be rechecked after an upgrade.
299
+
300
+ To repeat the isolated sweep in an exclusive GPU window, with a new output
301
+ directory:
302
+
303
+ ```bash
304
+ cd /workspace/WarpState/rtx6000_40b
305
+ source ops/training-env.sh
306
+ CUDA_VISIBLE_DEVICES=0 .server-venv/bin/python -B ops/benchmark_gemm.py \
307
+ --families ffn_up_gate in_proj ffn_down out_proj \
308
+ --operations fwd dx dw --rows 8192 --backends eager \
309
+ --burnin-seconds 2 --repeats 30 --trials 5 \
310
+ --output-dir work/benchmark-gemm-20260912-recheck
311
+ ```
312
+
313
+ The full-model profiling command and checkpoint protections are documented in
314
+ [`OPTIMIZATION_20260912.md`](OPTIMIZATION_20260912.md). Compilation and benchmark
315
+ outputs must stay under the workspace, and a new experiment must not overwrite
316
+ the retained reports or archived weights.
ops/MATMUL_TUNING.md ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ WarpState's matrix experiments select BF16 kernels by operation, exact matrix
2
+ dimensions, and operand strides. They are opt-in through `--matmul-plan` in
3
+ `run_40b.py` and `ops/benchmark_training.py`. The workstation preset selects
4
+ compiled autograd with communication overlap. Matrix kernels remain explicit
5
+ experiments; isolated GEMM improvements alone do not select them for production.
6
+ The complete two-GPU comparisons and validation are recorded below.
7
+
8
+ The implementation is split across these source files:
9
+
10
+ | File | Responsibility |
11
+ | --- | --- |
12
+ | [matmul_kernels.py](matmul_kernels.py) | BF16 ATen, tiled, persistent, and persistent TMA matrix launches; tile validation and shape/stride signatures. |
13
+ | [matmul_epilogue.py](matmul_epilogue.py) | Paired up/gate matrix multiplication with a SwiGLU epilogue; returns both packed projection and activation. |
14
+ | [matmul_execution.py](matmul_execution.py) | Strict plan loading, registered autograd, and selected core-linear/SwiGLU training adapters. |
15
+ | [tune_matmul.py](tune_matmul.py) | Disposable ordinary/fused FFN matrix tuning with warmed controls and generated candidate recipes. |
16
+ | [activation_execution.py](activation_execution.py) | Existing BF16 SwiGLU derivative used by the fused epilogue's backward. |
17
+
18
+ The adapter is applied before core compilation and DDP construction. Original
19
+ FP32 parameters remain the autograd leaves. Their differentiable BF16 casts stay
20
+ inside the visible forward graph; the adapter does not retain external casts
21
+ across updates or write leaf gradients through hooks. Matrix operands, returned
22
+ matrix gradients, projections, and activations are BF16. Custom dot products
23
+ accumulate in FP32. Parameter-gradient accumulation, clipping, DDP reduction,
24
+ FP32 master parameters, and AdamW state remain under the existing training
25
+ framework. The token cursor, objective, optimizer schedule, and checkpoint keys
26
+ retain their existing meanings. CPU and evaluation model execution use the
27
+ original module paths.
28
+
29
+ Selected kernel launches use opaque registered custom operations with fake
30
+ output metadata, surrounded by the adapter's registered autograd. This is
31
+ necessary because installed PyTorch 2.14 compiled autograd cannot retrace AOT's
32
+ functionalized Triton launch operator. Unselected ATen products remain visible
33
+ to the compiler. The CUDA suite exercises this second backward capture as well
34
+ as ordinary eager/AOT execution; a plain forward compilation test would miss it.
35
+
36
+ Each JSON plan has format `warpstate-matmul-plan-v1`, an optional description,
37
+ and an `entries` array. For `A[M,K] @ B[K,N]`, `m`, `n`, and `k` describe that
38
+ operation; strides are in elements. They are essential: a transposed weight and
39
+ a contiguous matrix with the same dimensions can require different kernels.
40
+ The complete dispatch key is `(kind, m, n, k, a_stride, b_stride)`. Missing entries
41
+ use ATen. An empty plan selects no custom kernels. Duplicate JSON keys, duplicate
42
+ dispatch entries, unknown fields, invalid tiles, and non-BF16 entries are rejected.
43
+
44
+ This is a **candidate example**, not a selected production recipe:
45
+
46
+ ```json
47
+ {
48
+ "format": "warpstate-matmul-plan-v1",
49
+ "description": "Candidate paired FFN epilogue; requires complete-model validation.",
50
+ "entries": [
51
+ {
52
+ "kind": "swiglu",
53
+ "name": "ffn_up_gate.fwd",
54
+ "m": 8192,
55
+ "n": 14336,
56
+ "k": 2048,
57
+ "a_stride": [2048, 1],
58
+ "b_stride": [1, 2048],
59
+ "dtype": "bfloat16",
60
+ "algorithm": "tiled",
61
+ "tile": {
62
+ "block_m": 128,
63
+ "block_n": 128,
64
+ "block_k": 32,
65
+ "num_warps": 8,
66
+ "num_stages": 3
67
+ }
68
+ }
69
+ ]
70
+ }
71
+ ```
72
+
73
+ `kind: "mm"` selects an ordinary matrix product. Forward and backward products
74
+ have independent entries. The FFN up/gate forward uses `(M,N,K) =
75
+ (8192,14336,2048)` with strides `[2048,1]` and `[1,2048]`. Its weight-gradient
76
+ product uses `(14336,2048,8192)` with strides `[1,14336]` and `[2048,1]`, matching
77
+ the original `dY.T @ X` orientation. A forward winner therefore does not select
78
+ that weight-gradient kernel automatically.
79
+
80
+ `kind: "swiglu"` selects the fused forward. Here `N=2H` is the complete packed
81
+ projection width, while `block_n` counts columns in **each** half. Backward uses
82
+ the saved BF16 packed projection, the existing SwiGLU derivative, and independently
83
+ selected `mm` entries for `dX` and `dW`.
84
+
85
+ | Algorithm | Execution | Supported plan kinds |
86
+ | --- | --- | --- |
87
+ | `aten` | Calls the installed PyTorch matrix implementation with its existing process settings. | `mm`, `swiglu` |
88
+ | `tiled` | Launches one program for each output tile, with grouped row tiles for weight reuse. | `mm`, `swiglu` |
89
+ | `persistent` | Launches at most one program per SM; each program strides over several output tiles using ordinary loads. | `mm`, `swiglu` |
90
+ | `persistent_tma` | Persistent scheduling with Triton tensor descriptors and hardware TMA operand loads. | `mm` |
91
+
92
+ Persistent scheduling alone does not imply TMA or warp specialization. The TMA
93
+ implementation uses an opaque registered custom-op boundary because the installed
94
+ PyTorch 2.14 AOT path does not preserve the host descriptor metadata through a
95
+ plain `triton_op` launch. It does not secretly substitute another algorithm.
96
+ TMA operands need an inner stride of one and 16-byte-aligned rows and storage
97
+ offsets. The training selector falls back to ATen for an incompatible layout;
98
+ the explicit kernel launcher reports unsupported requests. The paired SwiGLU
99
+ kernel does not currently support TMA.
100
+
101
+ Tiles specify `(block_m, block_n, block_k, num_warps, num_stages)`. Block sizes
102
+ must be powers of two from 16 through 256, with 4 or 8 warps and 1 through 4
103
+ pipeline stages. A conservative SM120 operand-pipeline estimate rejects ordinary
104
+ tiles above `(BM+BN)*BK*2*stages = 96 KiB` and paired tiles above
105
+ `(BM+2*BN)*BK*2*stages = 96 KiB`. This is an estimate, not a guarantee about
106
+ compiler register/shared-memory allocation. The tuner records compilation failures
107
+ without promoting those candidates.
108
+
109
+ The paired epilogue arranges adjacent logical columns as up/gate pairs, using
110
+ their physical packed offsets `j` and `j+H`. One wider tensor-core dot reuses the
111
+ input tile, then splits the accumulator in registers. Both FP32 projection halves
112
+ are rounded to BF16 **before** the activation. The kernel stores that rounded
113
+ packed projection for backward and also writes the BF16 activation. It saves a
114
+ subsequent packed-projection read and activation launch; it does not remove the
115
+ projection storage required by backward.
116
+
117
+ The `eager_rounding` argument additionally rounds the SiLU result to BF16 before
118
+ multiplication when following eager execution. Compiled execution follows the
119
+ existing compiler's fused SiLU/product arithmetic, while still rounding the GEMM
120
+ projection first. No FP8, FP16, quantization, split-K atomics, or reduced-precision
121
+ gradient communication is introduced. Exact equality on measured aligned shapes
122
+ does not promise bitwise equality for every GEMM layout or compiler version.
123
+ For example, the tested ragged `K=193` cuBLAS reference used a different reduction
124
+ result: its packed relative-L2 difference was 0.269%, while the custom result
125
+ differed by 0.000394% from an FP32-dot reference. Tests separately check projection
126
+ accuracy and that activation consumes the saved rounded BF16 projection.
127
+
128
+ On GPU 1 of the RTX PRO 6000 Blackwell Max-Q workstation, the final paired kernel
129
+ was compared with `torch.compile(fullgraph=True)` around the complete BF16
130
+ `mm -> split -> SiLU -> multiply` forward, returning both packed and activation
131
+ outputs. Inputs had shape `[8192,2048]` and transposed weights `[2048,14336]`.
132
+ With seed 563, 400 warmup calls per variant, and 15 alternating-order batches of
133
+ 40 CUDA-event-timed calls, the selected microbenchmark tile measured:
134
+
135
+ | Forward implementation | Median milliseconds |
136
+ | --- | ---: |
137
+ | Compiled ATen matrix multiplication plus SwiGLU | 1.950789 |
138
+ | Paired single-dot epilogue, tiled `(128,128,32,8,3)` | 1.905561 |
139
+
140
+ This is **2.37% greater primitive throughput**, or approximately 45 microseconds
141
+ less per call; paired batch gains ranged from 2.08% to 2.75%. Packed projections
142
+ and activations matched the compiled reference bitwise for two production-size
143
+ random input seeds. The final epilogue suite passed 33 tests, covering both
144
+ schedules, eager/compiled rounding, ragged dimensions, and the default/best tiles.
145
+ A subsequent CPU-only invocation passed 9 and skipped 24 CUDA fixtures.
146
+
147
+ Applying the measured 45-microsecond difference to 28 cores and eight microbatches
148
+ gives about 10 milliseconds per update before interaction with backward,
149
+ communication, compiler scheduling, and allocator behavior. That arithmetic is
150
+ not a measured complete-training gain. The earlier two-dot implementation lost
151
+ to the control; it is superseded by the single-dot source.
152
+
153
+ Generated evidence remains under
154
+ `work/validation/rtx-opt-20260912/epilogue/`:
155
+
156
+ - `paired-swiglu-confirm.json`: final steady-state comparison and second input seed.
157
+ - `paired-swiglu-single-dot.json`: final single-dot implementation's tile/schedule screen.
158
+ - `paired-swiglu.json` and `paired-swiglu-more-tiles.json`: superseded two-dot screens.
159
+
160
+ The ordinary GEMM screen tested three algorithms and five tiles for each FFN
161
+ up/gate forward and weight-gradient shape: 30 candidates total. Every candidate
162
+ matched the ATen BF16 output exactly. All forward candidates lost. The selected
163
+ weight-gradient candidate used persistent TMA with tile `(256,128,32,8,3)`:
164
+ median candidate time 1.811495 ms and median paired throughput gain 3.39%.
165
+ Its weakest of three paired trials gained 1.30%. The first trial's two controls
166
+ drifted from 1.589741 to 1.765839 ms, so this short screen alone is insufficient
167
+ to select a production configuration. The complete generated sweep and candidate
168
+ recipe are in `work/benchmark-matmul-tiles-20260912/`.
169
+
170
+ To reproduce the ordinary FFN forward/weight-gradient tuning, reserve an idle
171
+ GPU first. From the repository root, the following command uses physical GPU 0
172
+ and keeps outputs and compiler caches inside the workspace. The output directory
173
+ must be a new `work/benchmark-*` directory; existing directories are rejected.
174
+
175
+ ```bash
176
+ cd /workspace/WarpState/rtx6000_40b
177
+ source ops/training-env.sh
178
+ CUDA_VISIBLE_DEVICES=0 .server-venv/bin/python -B ops/tune_matmul.py \
179
+ --kind mm --operations fwd dw --algorithms tiled persistent persistent_tma \
180
+ --rows 8192 --seed 1337 --warmup 10 --burnin-seconds 1 \
181
+ --repeats 30 --trials 3 --minimum-gain 0.01 \
182
+ --maximum-relative-rms 0.003 \
183
+ --output-dir work/benchmark-matmul-tuning-reproduction
184
+ ```
185
+
186
+ To recheck the measured fused tile with the same dimensions, seed, and operand
187
+ scale, use the fused mode on an idle GPU:
188
+
189
+ ```bash
190
+ CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B ops/tune_matmul.py \
191
+ --kind swiglu --operations fwd --algorithms tiled \
192
+ --tile 128,128,32,8,3 --rows 8192 --seed 563 --input-scale 0.1 \
193
+ --warmup 400 --burnin-seconds 1 --repeats 40 --trials 15 \
194
+ --minimum-gain 0.01 --maximum-relative-rms 0.003 \
195
+ --output-dir work/benchmark-matmul-swiglu-reproduction
196
+ ```
197
+
198
+ This uses the tuner's bracketed control/candidate/control protocol. The historical
199
+ 15-batch report used alternating pairs, so compare the resulting measurements
200
+ as independent checks rather than expecting identical timing samples. Fused
201
+ mode compiles the complete matrix-plus-SwiGLU reference and validates both packed
202
+ projection and activation separately. It supports only `--operations fwd` with
203
+ `tiled` or `persistent`. Omit `--tile` to use that mode's bounded tile list, or
204
+ repeat it to specify several candidates. `--input-scale` defaults to 1.0 for both
205
+ operands; the recorded epilogue comparisons used 0.1.
206
+
207
+ `--list-cases` prints the requested dimensions and strides without using CUDA.
208
+ The tuner generates deterministic BF16 random operands from the supplied seed;
209
+ it does not open a model, checkpoint, or token database. In this disposable process
210
+ it disables BF16 reduced-precision reduction and TF32 for the ATen reference.
211
+ Every timed candidate has preceding and following warmed controls. Eligibility
212
+ requires finite results, the configured relative RMS tolerance, and at least
213
+ the configured gain in **every** trial. Compilation failures remain in
214
+ `results.json`. The best eligible entry for each tested operation is written to
215
+ `candidate-policy.json`; an empty recipe means no candidate passed selection.
216
+ Results include source hashes, software/device metadata, input signatures,
217
+ timing triples, per-output numerical error, and GPU telemetry. Generated reports and
218
+ candidate recipes remain ignored under `work/`; a recipe selected after complete
219
+ validation should be copied into tracked `configs/` with its provenance.
220
+
221
+ Use the same checkpoint, input offsets, microbatch, loss/gradient modes, and
222
+ warmup/measurement counts to compare a candidate with a control. For example,
223
+ after reserving both workstation GPUs, this disposable command tests the generated
224
+ recipe against the retained full checkpoint:
225
+
226
+ ```bash
227
+ CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/torchrun --standalone --nproc-per-node=2 \
228
+ ops/benchmark_training.py --checkpoint work/run/latest.pt \
229
+ --data work/data/train.bin --micro-batch 4 --seq-len 2048 \
230
+ --global-tokens-per-update 131072 --no-activation-checkpointing \
231
+ --compile-cores --compile-mode default --loss-mode compiled-full \
232
+ --bf16-residuals --gradient-mode compiled-autograd-overlap \
233
+ --optimizer-state resume --warmup-updates 8 --measured-updates 20 \
234
+ --matmul-plan work/benchmark-matmul-tuning-reproduction/candidate-policy.json \
235
+ --output-dir work/benchmark-matmul-training-candidate
236
+ ```
237
+
238
+ Run a matching control with `--matmul-plan` omitted and a different new output
239
+ directory. Neither benchmark advances the production checkpoint. Run numerical
240
+ backward/optimizer and checkpoint/resume tests before selecting a training recipe.
241
+ The production runner accepts the same `--matmul-plan PATH` option alongside its
242
+ existing resume arguments; it is not an instruction to start a new run or change
243
+ the established global token batch. External core-weight cast caching and the
244
+ `reduce-overhead` CUDA-graph mode are explicitly incompatible with this adapter.
245
+ Standalone `--activation-mode swiglu|rmsnorm|all` kernels require eager gradient
246
+ execution; their transparent backward launches have the same compiled-autograd
247
+ limitation described above. Both CLIs reject that unsupported combination.
248
+ Use `--activation-mode baseline` with compiled autograd; matrix-plan fused
249
+ epilogues still work through their separate opaque boundaries.
250
+ Combining a matrix plan and compiled autograd also requires activation
251
+ checkpointing off. The installed compiler cannot trace the saved-tensor unpack
252
+ hook when the runner wraps an already compiled core in an eager checkpoint.
253
+ Recomputation captured inside AOT passes the small kernel tests, but does not
254
+ establish support for that different runner nesting.
255
+
256
+ Plan loading happens once, before execution. It computes a canonical recipe
257
+ SHA256 and implementation SHA256 values for `matmul_execution.py` and
258
+ `matmul_kernels.py`; a fused plan also includes `matmul_epilogue.py` and
259
+ `activation_execution.py`. The runner records this metadata in settings and the
260
+ checkpoint training signature, and the benchmark records it with its source
261
+ provenance. A changed recipe or implementation therefore changes the execution
262
+ signature. Legacy checkpoints normalize to no matrix plan. Resuming with an
263
+ execution change requires the existing explicit `--reconfigure` mechanism,
264
+ which still protects the data, context, global token batch, and optimizer schedule.
265
+ Reformatting JSON without changing its parsed canonical content leaves the recipe
266
+ identity unchanged; changing a description or an entry changes it.
267
+
268
+ The complete-model comparisons resume the same step-40,081 checkpoint into
269
+ disposable GPU model/AdamW copies. Each uses eight warmup updates and 20 timed
270
+ updates, BF16 residuals, compiled full loss, the same 131,072-token global batch,
271
+ and compiled autograd with delayed communication waits. They exclude compilation,
272
+ evaluation and checkpoint pauses. The generated rank logs and summaries are
273
+ under `work/benchmark-opt-20260912/`.
274
+
275
+ | Configuration | Timed tokens/s | Change from control | Reserved GiB/rank |
276
+ | --- | ---: | ---: | ---: |
277
+ | Communication-overlap control (`matmul-control`) | 31,022.34 | — | 66.98 |
278
+ | Persistent TMA dW only (`matmul-dw`) | 31,063.57 | +0.13% | 66.99 |
279
+ | Fused forward + TMA dW, initial boundary (`matmul-combined`) | 31,002.83 | -0.06% | 66.54 |
280
+ | Fused forward + TMA dW, shared weight buffer (`matmul-combined-native`) | 31,156.20 | +0.43% | 66.98 |
281
+
282
+ These first comparisons are effectively tied. Over all 28 updates, the initial
283
+ combined candidate's maximum loss difference from the fresh control was
284
+ 0.00008535, and the maximum relative gradient-norm difference was 0.124%.
285
+ Small BF16 reduction/fusion differences do not imply training progress.
286
+
287
+ After the weight-layout correction, the candidate averaged 4.20693 seconds per
288
+ update versus 4.22508 for the control. Its maximum loss difference was 0.00010061
289
+ and its maximum relative gradient-norm difference was below 0.114%; all observed losses
290
+ and norms were finite. This single comparison shows a small +0.43% gain, which
291
+ has not been confirmed across repeated timing windows. The matrix recipe remains
292
+ opt-in. The resume launcher retains the repeatedly verified communication-overlap
293
+ configuration from [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md), which
294
+ measured a 1.53% improvement over eager gradients in the repeated comparison.
295
+ These percentages come from separate experiments and must not be added into a
296
+ claimed combined speedup. The generated numerical/timing comparison is
297
+ `work/validation/rtx-opt-20260912/matmul/full-model-comparison.json`.
298
+
299
+ Generated-code inspection verified the selected forward and dW routes, with
300
+ 224 uses each per rank/update inferred from the compiled graphs and loop counts.
301
+ It also found an avoidable cost in the initial fused boundary: passing `W.T`
302
+ into an opaque operation made Inductor store two identical BF16 weight buffers,
303
+ while the control shared one between forward and backward. The final adapter
304
+ passes native-layout BF16 `W` and forms its transpose inside the launch. The
305
+ actual-size compiled core now shares one allocation and one cast with backward,
306
+ removing 58,720,256 extra bytes of stores per core call, or 13,153,337,344 bytes
307
+ per rank/update. This is generated-code evidence, not a measured bandwidth
308
+ counter or a guaranteed wall-time saving. The TMA dW route already reuses its
309
+ gradient buffer and required no corresponding change.
310
+
311
+ The tracked
312
+ [`matmul_rtx6000_bf16_experimental.json`](../configs/matmul_rtx6000_bf16_experimental.json)
313
+ contains both exact candidate shapes and tiles. The final native-weight boundary
314
+ comparison uses this recipe directly. Static dispatch and buffer evidence is
315
+ recorded in `work/validation/rtx-opt-20260912/matmul/compiled-dispatch-evidence.json`
316
+ and `dw-dispatch-evidence.json`. Production remains paused; these experiments
317
+ leave the step-40,081 checkpoint and archived weights unchanged.
318
+
319
+ The integration validation passed 208 CPU tests, with 83 CUDA fixtures skipped
320
+ and three existing compiler warnings. Separate reserved-GPU runs passed the
321
+ 33-test epilogue suite and 35 adapter tests, including tiled, persistent, and
322
+ TMA forward/dX/dW, FP32 gradient accumulation and AdamW, ordinary AOT, and compiled
323
+ autograd. The adapter suite includes two alignment regressions: shifted inputs
324
+ use exact ATen fallback in eager mode and match a compiled ATen control when
325
+ compiled. Inductor can realign an input before the opaque boundary, making
326
+ TMA valid; the original caller's offset need not survive compilation.
327
+
328
+ A two-GPU runner test combined the fused persistent epilogue, persistent TMA
329
+ weight gradients, and delayed communication waits. Continuous execution matched
330
+ asynchronous save/stop/resume exactly for model weights, full FP32 AdamW state,
331
+ both ranks' RNG state, losses, and token cursors. Its final partial update also
332
+ covered a rank with entirely masked targets. The final implementation passed in
333
+ 73.63 seconds; its log is
334
+ `work/validation/rtx-opt-20260912/pytest-matmul-resume-final.log`.
335
+ The CPU and alignment regression logs are `pytest-matmul-final-cpu.log` and
336
+ `pytest-offset-guard.log` in the same validation directory. These checks prove
337
+ continuation consistency within the selected execution mode, not bitwise
338
+ equivalence to a different BF16 reduction/fusion strategy.
339
+ The final CLI compatibility/signature checks passed 103 tests with eight GPU
340
+ fixtures skipped (`pytest-matmul-cli-final.log`). The complete package manifest
341
+ was verified after updating the runner's SHA256/byte count. After the final
342
+ benchmark, the production checkpoint's SHA256 still matched the paused record.
ops/OPTIMIZATION_20260912.md ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # RTX training optimization, 2026-09-12
2
+
3
+ The refreshed baseline delivers **30,799 tokens/s during optimizer updates** on
4
+ the two RTX PRO 6000 Blackwell Max-Q GPUs. The experiments below address
5
+ conversion and gradient accumulation, DDP communication, SwiGLU/RMSNorm, and
6
+ repeated core weight casts. The communication-overlap candidate beats the baseline in two full-model
7
+ comparisons. The workstation resume preset selects this mode; isolated kernel
8
+ improvements are not evidence of the same gain in complete training. See
9
+ [MATMUL_PROFILE_20260912.md](MATMUL_PROFILE_20260912.md) for the matrix breakdown.
10
+
11
+ ## Protected training state
12
+
13
+ Production training was gracefully paused at step **40,081**, after
14
+ **5,253,496,832 tokens**. The next update is step 40,082. Its full resume checkpoint
15
+ is `work/run/latest.pt`, containing FP32 model parameters, AdamW moments, optimizer
16
+ step 40,081, and both ranks' RNG state. All inspected model and optimizer tensors
17
+ were finite. The model has 2,005,111,424 parameters.
18
+
19
+ The checkpoint is 24,061,689,997 bytes; its SHA256 at the pause was
20
+ `9745d93345dff7c389ac6fa65948dc78ec021c90f56b40e9caed8ddcd84e8c54`.
21
+ The generated verification record is
22
+ [`work/validation/rtx-opt-20260912/paused-checkpoint.json`](../work/validation/rtx-opt-20260912/paused-checkpoint.json).
23
+
24
+ The step-8,000, step-16,000, and step-40,000 archives under `work/archives/` are
25
+ immutable inputs. These experiments do not replace, re-export, or edit them.
26
+ The benchmark reads the production checkpoint through a read-only CPU mapping,
27
+ loads independent CUDA model/optimizer tensors, and writes no checkpoints.
28
+ Benchmark updates do not advance the production token cursor or RNG state.
29
+
30
+ Production remains paused while disposable experiments run. A later production
31
+ resume must use the retained full checkpoint and record any selected execution
32
+ changes through the runner's explicit reconfiguration mechanism.
33
+
34
+ ## Baseline and profile method
35
+
36
+ The baseline resumes the full AdamW state from step 40,081 and uses deterministic
37
+ training-token offsets. It measures 20 updates after three warmup updates:
38
+
39
+ | Setting | Baseline |
40
+ | --- | --- |
41
+ | PyTorch / CUDA | `2.14.0+cu130` / `13.0` |
42
+ | Devices | Two RTX PRO 6000 Blackwell Max-Q GPUs |
43
+ | Sequence length / per-rank microbatch | 2,048 / 4 |
44
+ | Accumulation / global tokens per update | 8 / 131,072 |
45
+ | Core execution | `torch.compile`, default mode |
46
+ | Loss / residual activations | Compiled full-vocabulary loss / BF16 |
47
+ | Activation checkpointing | Off |
48
+ | DDP | FP32 gradients, 128 MiB buckets, bucket views |
49
+ | Optimizer | Fused AdamW, original FP32 state |
50
+ | Measured throughput | 30,799.17 tokens/s |
51
+ | Peak allocated / reserved memory, rank 0 | 65.49 / 66.44 GiB |
52
+
53
+ This throughput includes gradient zeroing, forward/backward, accumulation,
54
+ gradient clipping, and optimizer updates. It excludes periodic production
55
+ evaluation/checkpoint pauses and initial compilation. It must not be presented
56
+ as sustained wall-clock throughput over a long training run.
57
+
58
+ The separate profile records one complete update after two warmups. That
59
+ instrumented update took **4.1781 seconds**. Use the unprofiled 20-update baseline
60
+ for speed comparisons; the shorter profiled run has a different warmup history
61
+ and instrumentation overhead.
62
+
63
+ | GPU activity | Rank 0 summed milliseconds | Rank 1 summed milliseconds |
64
+ | --- | ---: | ---: |
65
+ | Matrix multiplication | 3,166.45 | 3,049.22 |
66
+ | Compiled elementwise operations and reductions | 512.12 | 515.05 |
67
+ | NCCL communication | 422.56 | 532.72 |
68
+ | Other tensor kernels | 206.16 | 206.27 |
69
+ | Local attention | 99.41 | 98.82 |
70
+ | Fused AdamW | 38.50 | 38.46 |
71
+ | Explicit copies and fills | 2.96 | 3.08 |
72
+
73
+ Matrix multiplication accounts for 71.19% and 68.62% of summed GPU activity.
74
+ The totals exceed wall time because streams overlap. Rank 0's GPU activity union
75
+ is 4.1257 seconds; its summed durations are 4.4482 seconds. Name-based kernel
76
+ categories are attribution aids, not hardware utilization or bandwidth counters.
77
+ Operator and kernel tables describe the same work and must not be added together.
78
+
79
+ The explicit copy category excludes memory traffic inside arithmetic kernels.
80
+ It therefore does not measure total VRAM bandwidth consumption. This profile
81
+ does not establish a hardware bandwidth ceiling.
82
+
83
+ ## Work outside matrix multiplication
84
+
85
+ The following rank-0 costs are subsets of the preceding categories, measured
86
+ over all eight accumulation microbatches and 28 cores:
87
+
88
+ | Candidate work | Summed milliseconds per update |
89
+ | --- | ---: |
90
+ | SwiGLU forward / backward | 41.156 / 87.166 |
91
+ | Core gradient BF16-to-FP32 conversion / FP32 addition | 39.749 / 89.327 |
92
+ | Repeated core weight casts | 54.006 |
93
+ | Normalization forward / backward | 12.087 / 50.377 |
94
+ | Two groups of backward attention/projection transforms | 82.969 / 32.730 |
95
+ | BF16 attention layout copy | 14.036 |
96
+ | DDP parameter/bucket scaling, 256 calls | Approximately 7.884 |
97
+
98
+ The conversion-plus-accumulation budget is **129.076 ms/update**. Its attribution
99
+ uses CUDA external IDs mapped to CPU operator input dimensions to distinguish
100
+ parameter gradients from activation gradients. Weight casts are the three
101
+ compiled transpose/copy families with durations 16.138, 14.107, and 23.761 ms.
102
+
103
+ NCCL overlaps substantial computation. Intersecting communication intervals with
104
+ the union of other GPU activity leaves approximately **100.05 ms on rank 0** and
105
+ **215.80 ms on rank 1** containing only communication. These exposed intervals
106
+ are not a guaranteed speedup: moving bucket boundaries can alter overlap and
107
+ contention elsewhere in the update.
108
+
109
+ The current compiler already fuses many operations. A custom normalization
110
+ kernel may save local kernel time while preventing a larger surrounding fusion.
111
+ The candidate budgets therefore cannot be summed into a projected speedup.
112
+
113
+ ## Implemented execution options
114
+
115
+ The runner and disposable benchmark expose the following options. Defaults
116
+ retain the prior execution, and the runner's resume signature includes the new
117
+ settings. Legacy checkpoints normalize to these defaults; a selected change
118
+ requires explicit resume reconfiguration while preserving the objective, global
119
+ token batch, training cursor, optimizer, and schedule.
120
+
121
+ | Option | Default | Implementation and scope |
122
+ | --- | --- | --- |
123
+ | `--gradient-mode eager\|compiled-autograd\|compiled-autograd-overlap` | `eager` | Captures backward through leaf accumulation so Inductor can fuse BF16 gradient conversion with FP32 addition. |
124
+ | `--activation-mode baseline\|swiglu\|rmsnorm\|all` | `baseline` | Optional Triton activation/normalization forwards and registered autograd backwards. |
125
+ | `--cache-core-weight-casts` | Off | Reuses differentiable BF16 core linear-weight views for one accumulation update. |
126
+ | `--ddp-bucket-cap-mb` | `128` | Changes communication grouping while retaining FP32 reduction. |
127
+ | `--ddp-batched-grad-copy` | Off | Batches gradient copying/scaling in the C++ DDP reducer. |
128
+
129
+ Compiled autograd uses the Python DDP reducer and cannot be combined with the
130
+ C++ reducer's batched gradient copy option. The installed PyTorch can mishandle
131
+ a saved CPU scalar in compiled CUDA backward; `scale_loss()` materializes that
132
+ same scaling coefficient on the loss device for this mode. Unsupported requested
133
+ DDP options fail explicitly. The helper also supports capacity lists for
134
+ controlled programmatic experiments; the current CLI exposes a uniform capacity.
135
+
136
+ The weight cast cache creates fresh views at the start of each optimizer update
137
+ and clears them on normal or exceptional exit, before clipping/optimizer,
138
+ evaluation, or checkpoint operations. FP32 master identities and version counters
139
+ are checked. Its current scope requires one use of each core per forward,
140
+ activation checkpointing off, and unaliased trainable linear weights. It does not
141
+ cache the tied embedding/output weight. Extra BF16 views consume VRAM; a complete
142
+ model measurement must establish their memory cost and benefit.
143
+
144
+ These options retain BF16 tensor-core computation and BF16 activations. FP32
145
+ master parameters, parameter gradients, normalization reductions, DDP gradient
146
+ communication, and AdamW moments retain their original representation. No FP8,
147
+ FP16, quantized weights, or reduced-precision gradient communication is introduced.
148
+ Canonical checkpoint keys and the default CPU chat/evaluation paths are unchanged.
149
+
150
+ ## Numerical checks and isolated measurements
151
+
152
+ The activation suite passed **22 tests**, including eager and compiled CUDA
153
+ forward/backward comparisons, BF16 residual integration, FP32 AdamW, canonical
154
+ state reload, and unchanged CPU/evaluation behavior. Its final log is
155
+ `work/validation/rtx-opt-20260912/activation-tests-final.log`.
156
+
157
+ The activation implementation distinguishes eager rounding from current compiled
158
+ arithmetic. Inductor can eliminate intermediate BF16 cast pairs; the compiled
159
+ variant follows that behavior instead of forcing additional eager truncations.
160
+ The comparison does not promise bitwise equality for every reduction shape or
161
+ future compiler configuration.
162
+
163
+ On GPU 1, five timing trials of 20 iterations after five warmup iterations used
164
+ the actual activation shapes, seed 951, BF16 random upstream gradients, and
165
+ compiled forward plus backward. Compilation used `fullgraph=True, dynamic=False`;
166
+ CUDA events timed each trial:
167
+
168
+ | Primitive | Compiled baseline | Custom | Numerical comparison |
169
+ | --- | ---: | ---: | --- |
170
+ | SwiGLU, packed `8192 x 14336` | 0.61335 ms | 0.59965 ms | Outputs and input gradients bitwise equal in this fixture. |
171
+ | RMSNorm, `8192 x 2048` | 0.43256 ms | 0.39248 ms | Relative RMS error: output `1.64e-6`, input gradient `1.17e-5`, FP32 weight gradient `0.001604`. |
172
+
173
+ These are approximately 2.23% and 9.27% primitive improvements, not complete
174
+ training gains. RMSNorm's large-shape baseline can retain a final BF16 rounding
175
+ at a compiler reduction boundary; the custom compiled reduction retains FP32.
176
+ That explains the approximately 0.16% weight-gradient discrepancy and requires
177
+ whole-model numerical screening before selection.
178
+
179
+ A separate large-linear gradient fixture verifies that compiled autograd can
180
+ replace repeated conversion/addition work with fused kernels while preserving
181
+ the accumulated gradient exactly in that fixture. Its measured cycle was
182
+ 44.668 ms eager versus 41.013 ms compiled autograd. The fixture is not a complete
183
+ training update. Small-model comparisons show some changed BF16 rounding in
184
+ embedding, depth modulation, and final normalization, so exact linear gradients
185
+ do not establish bitwise equality for the whole model.
186
+
187
+ DDP tests exercise repeated eight-microbatch accumulation and optimizer updates,
188
+ including multiple buckets. They preserve FP32 reduction and optimizer state.
189
+ Full-model two-GPU tests remain the criterion for selecting an execution change.
190
+
191
+ ## Whole-model candidate results
192
+
193
+ All screens start from the same step-40,081 full checkpoint and token cursor,
194
+ with three warmups and ten measured updates. The initial baseline measures 20
195
+ updates. The benchmark holds the checkpoint's learning rate fixed for every
196
+ candidate; production continues its existing schedule. The measurements below
197
+ exclude compilation, evaluation and saving.
198
+
199
+ | Candidate / artifact directory | Tokens/s | Peak reserved/GPU | Maximum absolute loss difference from baseline |
200
+ | --- | ---: | ---: | ---: |
201
+ | Baseline, 20 updates (`baseline`) | 30,799.17 | 66.44 GiB | 0 |
202
+ | Batched DDP copies, 64 MiB (`ddp-copy64`) | 30,896.39 | 67.42 GiB | 0 |
203
+ | Compiled autograd (`compiled-autograd`) | 30,498.20 | 66.45 GiB | 0.00009394 |
204
+ | Cached core casts (`cache`) | 29,956.94 | 66.44 GiB | 0.00008345 |
205
+ | Custom SwiGLU (`swiglu`) | 30,749.24 | 66.44 GiB | 0 |
206
+ | Custom RMSNorm (`rmsnorm`) | 30,737.22 | 66.14 GiB | 0.00012708 |
207
+ | Compiled autograd with wait sinking (`ca-overlap`) | **31,331.80** | 66.98 GiB | 0.00016069 |
208
+
209
+ A second pair of complete, unprofiled **20-update** measurements confirms the
210
+ communication fix: `ca-overlap-confirm` reaches **31,045.42 tokens/s**, versus
211
+ **30,578.17 tokens/s** for the subsequent `baseline-confirm` (**1.53% faster**).
212
+ The initial ten-update screen was 1.73% above its baseline. These are modest,
213
+ repeatable gains, not a claim of 50K throughput or long-run model quality.
214
+
215
+ Custom activation/normalization kernels and cached casts remain opt-in
216
+ experiments. No complete-model gain justifies selecting them in this preset.
217
+ The cache's BF16 views also stop DDP's compiler partitioner from counting those
218
+ inputs as `nn.Parameter`s. Generated layouts match baseline, but whole-core
219
+ backward regions can delay leaf-gradient readiness and change fusion boundaries.
220
+ Do not attribute that regression to a confirmed matrix-transpose problem.
221
+
222
+ ### Why fusion alone lost, and the communication fix
223
+
224
+ Compiled autograd removes nearly all native gradient-add kernels. Rank 0's
225
+ other-kernel category falls from 206.17 to 51.36 ms while compiled elementwise
226
+ work rises from 512.12 to 585.32 ms, saving roughly 82 ms across those categories.
227
+ However, its default schedule puts each wait and gradient copy immediately after
228
+ the collective, before independent backward GEMMs. The 57 gradient collectives
229
+ carry the same 8,020,445,696 FP32 bytes.
230
+
231
+ | Communication time/update | Baseline | Compiled autograd | With wait sinking |
232
+ | --- | ---: | ---: | ---: |
233
+ | Overlapping compute, rank 0 | 322.51 ms | 0 ms | **190.74 ms** |
234
+ | Overlapping compute, rank 1 | 316.91 ms | 0 ms | **190.58 ms** |
235
+ | Communication only, rank 0 | 100.05 ms | 285.17 ms | **166.64 ms** |
236
+ | Communication only, rank 1 | 215.80 ms | 407.35 ms | **267.61 ms** |
237
+
238
+ `--gradient-mode compiled-autograd-overlap` enables Inductor's
239
+ `sink_waits_iterative` scheduler. It moves waits together with their dependent
240
+ copies past independent computation, retaining collective launch order and
241
+ requiring no increase in the scheduler's **estimated** peak memory. It does not
242
+ permit unsafe collective reordering or moves past subsequent collectives.
243
+ Actual allocator peaks remain measured separately.
244
+
245
+ The two CA profiles contain identical kernel counts: 5,400 matmuls, 59
246
+ communication events, 9,577 compiled pointwise/reduction kernels and 1,531 other
247
+ kernels per rank. Profiled update time changes from 4.25901 to 4.12683 seconds;
248
+ the gain is restored overlap, not less arithmetic. Compared with baseline,
249
+ 52–67 ms more communication remains exposed. That is potential scheduling work,
250
+ not a guaranteed further saving.
251
+
252
+ Cold first-update compilation took approximately 195–196 seconds in these
253
+ full-model CA screens. A cached second process still incurs graph capture and
254
+ compiler setup; restarts can spend minutes before regular updates appear.
255
+
256
+ Whole-backward compilation can fuse a tied-head BF16 matrix product and gradient
257
+ addition into BF16 `addmm`, changing an intermediate rounding boundary. Original
258
+ FP32 master weights, gradients, AdamW state and communication remain FP32; GEMM
259
+ operands and outputs remain BF16. Short loss/gradient comparisons use explicit
260
+ numerical tolerances rather than claiming a bitwise baseline trajectory.
261
+
262
+ The CLI rejects weight cast caching with either compiled-autograd mode, and
263
+ rejects either feature with `--compile-mode reduce-overhead`. Cache plus
264
+ activation checkpointing is rejected before loading; it requires
265
+ `--no-activation-checkpointing`. Batched DDP copying uses the C++ reducer and is
266
+ incompatible with both compiled-autograd modes' Python reducer.
267
+
268
+ ### Validation and selected preset
269
+
270
+ The completed CPU suite passed **159 tests** (28 CUDA tests skipped in that
271
+ CPU-only invocation). Separate activation, gradient and cast-cache CUDA checks
272
+ passed. The final two-GPU runner suite passed **both** continuation cases in
273
+ 136.79 seconds: combined custom activations/cached casts/batched DDP copies, and
274
+ compiled autograd with communication overlap. Each compares uninterrupted
275
+ training against an asynchronous stop/resume, including an entirely masked
276
+ final rank. Model tensors, full FP32 optimizer state and both ranks' RNG states
277
+ match **bitwise** between continuous and resumed execution of the same mode.
278
+ This is separate from the bounded numerical differences versus baseline.
279
+
280
+ `train-local-resume.sh` selects `--gradient-mode compiled-autograd-overlap`.
281
+ Activation kernels, external weight-cast caching and batched DDP copies remain
282
+ at their baseline settings. BF16 compute, FP32 state, the global token batch,
283
+ context, data, optimizer schedule and save/evaluation intervals are preserved.
284
+ Production remains paused at step 40,081 until resumption is requested.
285
+
286
+ Follow-up scheduling controls and paired measurements are documented in
287
+ [COMMUNICATION_EXPLORATION_20260912.md](COMMUNICATION_EXPLORATION_20260912.md).
288
+ The controls are opt-in; the preset above keeps its conservative defaults.
289
+ The separate [backward layout experiment](BACKWARD_LAYOUT_EXPERIMENT.md)
290
+ measures a fused rotary-gradient and projection-packing region without changing
291
+ production model execution.
292
+
293
+ Test logs are `work/validation/rtx-opt-20260912/pytest-communication-final.log`
294
+ and `work/validation/rtx-opt-20260912/pytest-optimized-resume.log`.
295
+
296
+ ## Reproducing and locating results
297
+
298
+ All paths above are relative to `rtx6000_40b/`. Source the workspace environment
299
+ before any test or benchmark. GPU experiments require training paused and an
300
+ exclusive device window. Run each candidate in a fresh process and a new output
301
+ directory because compiler/reducer settings are process-wide.
302
+
303
+ The baseline can be reproduced with:
304
+
305
+ ```bash
306
+ cd /workspace/WarpState/rtx6000_40b
307
+ source ops/training-env.sh
308
+ .server-venv/bin/torchrun --standalone --nproc-per-node=2 \
309
+ ops/benchmark_training.py \
310
+ --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
311
+ --no-activation-checkpointing --autocast-cache \
312
+ --compile-cores --compile-mode default --loss-mode compiled-full \
313
+ --bf16-residuals --optimizer-state resume --vram-limit-gib 88 \
314
+ --warmup-updates 3 --measured-updates 20 \
315
+ --output-dir work/benchmark-opt-20260912/baseline-recheck
316
+ ```
317
+
318
+ Use a new output directory and append the candidate flags for each comparison.
319
+ For the profile, use two warmup updates, one measured update, and `--profile`.
320
+ The original run artifacts are:
321
+
322
+ - `work/benchmark-opt-20260912/baseline/summary.json`: configuration, per-rank
323
+ update measurements, source hashes, checkpoint identity, and memory peaks.
324
+ - `work/benchmark-opt-20260912/profile-baseline/`: both ranks' Chrome traces,
325
+ operator tables, kernel TSV files, and summary/category JSON.
326
+ - `work/validation/rtx-opt-20260912/activation/actual-shape-compiled-arithmetic.json`:
327
+ final primitive timings and random-gradient differences. The earlier
328
+ `actual-shape-kernels.json` tested eager rounding against compiled math and is
329
+ superseded for comparing the current implementation.
330
+ - `work/validation/rtx-opt-20260912/gradient/cuda-linear-proof.json`: isolated
331
+ accumulation timing, gradient equality, and kernel trace counts.
332
+ - `work/validation/rtx-opt-20260912/gradient/cuda-gradient-deltas.json`: initial
333
+ small-model per-parameter differences for compiled autograd.
334
+
335
+ The source implementations are `activation_execution.py`,
336
+ `gradient_execution.py`, `weight_cast_execution.py`, and `distributed_execution.py`
337
+ in this directory. Their focused tests live at the package root. Generated
338
+ benchmarks, traces, validation outputs, data, and checkpoints remain ignored in
339
+ Git; this document records methods and findings for source tracking.
340
+
341
+ The subsequent paired attention-backward integration and separate gradient
342
+ producer-scheduling experiment are documented in
343
+ [BACKWARD_LAYOUT_INTEGRATION.md](BACKWARD_LAYOUT_INTEGRATION.md), with gradient
344
+ and checkpoint-continuation checks in
345
+ [BACKWARD_LAYOUT_TRAINING_VALIDATION.md](BACKWARD_LAYOUT_TRAINING_VALIDATION.md).
346
+ The completed integration selects `--backward-layout paired` in
347
+ `train-local-resume.sh`: repeated update throughput improves from 31,001.57 to
348
+ 31,199.03 tokens/second (+0.637%). The separate producer scheduler remains
349
+ disabled after a smaller, less consistent gain. Production remains paused.
ops/OPTIMIZED_TRAINING.md ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training with both GPUs available
2
+
3
+ **Currently paused at the user's request.** Training and benchmarks are
4
+ stopped so another session can resume VORTEX. The latest full checkpoint is
5
+ step 448 / 58,720,256 tokens. Start commands below are for a future
6
+ user-requested resume after checking GPU availability.
7
+
8
+ The selected configuration measured **29,490 training tokens/second** on two
9
+ NVIDIA RTX PRO 6000 Blackwell Max-Q GPUs, with approximately **68.8 GiB reserved
10
+ and 25 GiB free per GPU**. The measurements below were taken on September 9,
11
+ 2026, after VORTEX and the previous WarpState training process had stopped.
12
+
13
+ See [PERFORMANCE.md](PERFORMANCE.md) for the subsequent bottleneck analysis,
14
+ GPU profiling method, and the compute requirements of a 100K-token/second
15
+ target.
16
+
17
+ ## Selected configuration
18
+
19
+ [`train-fast.sh`](train-fast.sh) launches two workers through
20
+ [`launch_training.py`](launch_training.py). Each worker holds the complete
21
+ 2,005,111,424-parameter model and AdamW state; DDP synchronizes gradients.
22
+
23
+ | Setting | Value |
24
+ | --- | --- |
25
+ | GPUs / workers | 2, one worker per GPU |
26
+ | Sequence length | 2,048 tokens |
27
+ | Microbatch per GPU | 4 sequences |
28
+ | Gradient accumulation | 8 microbatches per GPU |
29
+ | Global tokens per optimizer update | **2 × 4 × 2,048 × 8 = 131,072** |
30
+ | Parameters, gradients, AdamW moments | FP32 on each GPU |
31
+ | Matrix computation | BF16 autocast, weight cache enabled |
32
+ | Compilation | Each core's training forward, full graph, fixed shapes |
33
+ | Core activation checkpointing | Disabled |
34
+ | Loss projection chunk | 1,024 tokens; loss projection recomputation remains enabled |
35
+ | DDP gradient buckets | 128 MiB target, gradients use bucket views |
36
+ | PyTorch allocator cap | 88 GiB per GPU |
37
+ | Additional free memory required at startup | 3 GiB per GPU |
38
+ | Validation | 8 batches of 1 sequence, initially and every 100 updates |
39
+ | Checkpoint | Every 50 updates, on graceful stop, and at completion |
40
+
41
+ The global batch, sequence length, data cursor, optimizer state, and learning
42
+ rate schedule are preserved when moving from the previous shared-GPU run.
43
+ The launcher's `--reconfigure` allows execution changes while the runner
44
+ checks those invariants and records the transition in `execution_history`.
45
+ Compilation and changed batch grouping can change floating-point rounding;
46
+ this is not a promise of bitwise equivalence across configurations.
47
+
48
+ The allocator cap excludes CUDA context and library allocations. The startup
49
+ headroom check does not reserve memory for another process. This configuration
50
+ assumes VORTEX remains paused and both GPUs are available.
51
+
52
+ ## Measured configuration grid
53
+
54
+ All rows use sequence length 2,048, 131,072 global tokens per update, BF16
55
+ autocast with its cache enabled, FP32 parameters and full fused AdamW state.
56
+ Memory is the largest steady reserved allocation on any participating GPU.
57
+ The single-GPU trials used a 90 GiB allocator cap; DDP trials used 88 GiB.
58
+
59
+ | GPUs | Microbatch/GPU | Core execution | Core checkpointing | Loss chunk | Tokens/s | Reserved/GPU |
60
+ | ---: | ---: | --- | --- | ---: | ---: | ---: |
61
+ | 1 | 2 | Eager | Off | 1,024 | 12,180 | 51.72 GiB |
62
+ | 1 | 4 | Eager | Off | 1,024 | 12,252 | 68.83 GiB |
63
+ | 1 | 8 | Eager | Off | 1,024 | Out of memory | Exceeded 90 GiB cap |
64
+ | 1 | 8 | Eager | On | 1,024 | 9,249 | 45.13 GiB |
65
+ | 1 | 4 | Compiled cores | Off | 1,024 | 15,142 | 60.44 GiB |
66
+ | 1 | 8 | Compiled cores | Off | 1,024 | 15,014 | 86.08 GiB |
67
+ | 1 | 4 | Compiled cores | Off | 8,192 | 15,321 | 62.89 GiB |
68
+ | **2** | **4** | **Compiled cores** | **Off** | **1,024** | **29,490** | **68.76 GiB** |
69
+ | 2 | 2 | Compiled cores | Off | 1,024 | 28,140 | 55.33 GiB |
70
+
71
+ Compiled cores improved single-GPU throughput by about 24% over eager
72
+ microbatch 4 and reduced reserved memory by about 8.4 GiB. Increasing the
73
+ compiled microbatch to 8 consumed another 25.6 GiB without improving speed.
74
+ Activation checkpointing saved memory but repeated enough computation to
75
+ make the tested eager microbatch-8 configuration slower.
76
+
77
+ The 8,192-token loss chunk improved single-GPU throughput by only 1.2%, used
78
+ another 2.5 GiB, and produced slightly different gradient rounding. Retaining
79
+ 1,024 keeps the existing loss computation closer without a material measured
80
+ speed penalty. Among the two-GPU trials, microbatch 4 was 4.8% faster than
81
+ microbatch 2 and still left substantial memory available.
82
+
83
+ `nvidia-smi topo -m` reported a **SYS** connection between these GPUs: traffic
84
+ crosses PCIe and the CPU interconnect between NUMA nodes. There is no NVLink
85
+ connection between this pair. Despite that topology, the measured compiled
86
+ microbatch-4 DDP run achieved **1.95×** the corresponding single-GPU throughput.
87
+
88
+ ## Measurement method and limits
89
+
90
+ [`benchmark_training.py`](benchmark_training.py) loaded the trusted local
91
+ checkpoint `work/run/before-optimization-step-237.pt`, an immutable hardlink
92
+ to the checkpoint at step 237 and 31,064,064 training tokens. It mmap-loaded
93
+ the checkpoint on the CPU, copied model parameters and every optimizer
94
+ tensor into separate CUDA storage, and optimized those disposable copies.
95
+ It read the prepared corpus in place and never saved a training checkpoint.
96
+
97
+ Each case performed complete forward/backward, gradient clipping, and fused
98
+ AdamW updates over 131,072 global tokens. Timings include data batching and
99
+ device transfer. DDP timings include gradient communication and use the
100
+ slowest rank's elapsed time. Compilation and the first update were excluded
101
+ as warmup; DDP excluded two warmup updates to allow gradient bucket rebuilding.
102
+ Every successful result aggregates **three measured updates**. The full
103
+ 14.94 GiB of FP32 Adam moments was present on each GPU, and all measured
104
+ losses and gradient norms were finite.
105
+
106
+ These are short steady-compute samples, not long-run throughput guarantees.
107
+ They exclude checkpoint loading/saving, validation, process startup, and
108
+ initial compilation. At an unchanged 29,490 tokens/second, 40B tokens take
109
+ about 15.7 days of training compute; operational overhead adds time.
110
+
111
+ Generated evidence remains ignored by Git:
112
+
113
+ - `work/benchmark-grid-summary.json`: all nine cases and throughput ranges.
114
+ - `work/benchmark-*/summary.json`: exact arguments, source hashes, checkpoint
115
+ identity, per-update times, and first-update/steady memory peaks.
116
+ - `work/benchmark-*/rank-*.jsonl`: individual worker measurements.
117
+ - `work/benchmark-eager-mb8/failure.json`: the bounded allocation failure.
118
+
119
+ The source hashes in each result identify the files present for that run.
120
+ Runner source evolved during this optimization task; the full-model benchmark
121
+ used its own update loop and the same batching/model computation throughout.
122
+
123
+ ## Live continuation and validation
124
+
125
+ The production run resumed at step 237 with 31,064,064 tokens consumed and
126
+ completed twelve updates using the selected configuration. Steady updates
127
+ ran at approximately 29.0–29.7K tokens/second, with 67.75 GiB allocated,
128
+ 68.64 GiB reserved, and 25.31 GiB physically free per GPU. The previous
129
+ shared settings measured 8.5K tokens/second after VORTEX stopped and
130
+ 5.5–6K while VORTEX was training.
131
+
132
+ A real `datactl.sh stop warpstate-train` completed the current update and
133
+ saved step 249 at 32,636,928 tokens. Both workers exited and released GPU
134
+ memory. Inspection of the 22.41 GiB checkpoint confirmed all 256 canonical
135
+ model tensors, full FP32 Adam moments and step counters, both rank random
136
+ states, the original initialization metadata, and the execution transition
137
+ at step 237. Restart restored validation loss 5.56880 from that checkpoint.
138
+ The original step-237 checkpoint remains at
139
+ `work/run/before-optimization-step-237.pt`.
140
+
141
+ Verification also passed 64 CPU tests and four CUDA integration tests
142
+ (two original single-GPU cases and two distributed cases). These cover
143
+ exact resume within one configuration, one-to-two-GPU continuation,
144
+ the final partial update with an empty rank, and graceful/forced worker
145
+ cleanup. A full-model eager-versus-compiled comparison had a maximum
146
+ loss difference of 0.000012 and relative gradient-norm difference below
147
+ 0.072% across three updates. Initial validation across the two execution
148
+ configurations was 5.63291 versus 5.63279, consistent with small rounding
149
+ differences.
150
+
151
+ The generated live checkpoint verification is recorded in
152
+ `work/optimization-resume-validation.json`.
153
+
154
+ ## Production controls
155
+
156
+ In optimized mode, the `warpstate-train` program in
157
+ [`supervisord.conf`](supervisord.conf) must point to `ops/train-fast.sh`.
158
+ Use the project-private Supervisor through `datactl.sh`:
159
+
160
+ ```bash
161
+ cd /workspace/WarpState
162
+ rtx6000_40b/ops/datactl.sh status
163
+ rtx6000_40b/ops/datactl.sh start warpstate-train
164
+ tail -n 10 rtx6000_40b/work/logs/train.log
165
+ tail -n 3 rtx6000_40b/work/run/metrics.jsonl
166
+ ```
167
+
168
+ To stop and preserve progress:
169
+
170
+ ```bash
171
+ rtx6000_40b/ops/datactl.sh stop warpstate-train
172
+ ```
173
+
174
+ The launcher receives the stop signal and creates
175
+ `work/control/training-stop-request`. Both workers finish the current update;
176
+ the runner saves the model, full optimizer state, progress, and rank random
177
+ states before exiting. The launcher allows up to 820 seconds for this,
178
+ followed by 45 seconds for termination if workers fail to exit. Supervisor's
179
+ stop timeout is 900 seconds. Use the launcher/Supervisor control rather than
180
+ signalling an individual DDP worker. Starting the same program resumes
181
+ `work/run/latest.pt`; do not delete the checkpoint to change execution options.
182
+
183
+ Keep at least 53 GiB free for a full checkpoint and atomic replacement.
184
+ Training artifacts, caches, and runtime control files remain inside
185
+ `/workspace/WarpState` and are excluded from Git.
186
+
187
+ ## Reproducing benchmarks
188
+
189
+ **Stop training first and wait until both GPUs are free.** A benchmark uses
190
+ disposable model state, but it still competes for GPU memory and computation.
191
+ VORTEX must also remain paused. From the repository root:
192
+
193
+ ```bash
194
+ rtx6000_40b/ops/datactl.sh stop warpstate-train
195
+ rtx6000_40b/ops/datactl.sh status
196
+ nvidia-smi
197
+ rtx6000_40b/ops/datactl.sh start warpstate-benchmark
198
+ tail -n 10 rtx6000_40b/work/logs/benchmark.log
199
+ ```
200
+
201
+ [`benchmark-supervisord.conf`](benchmark-supervisord.conf) specifies the
202
+ winning two-GPU configuration and sets `autostart=false` and
203
+ `autorestart=false`. Its command, expressed with environment variables, is:
204
+
205
+ ```bash
206
+ cd /workspace/WarpState/rtx6000_40b
207
+ WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 \
208
+ TORCHINDUCTOR_COMPILE_THREADS=8 ops/benchmark.sh \
209
+ --checkpoint /workspace/WarpState/rtx6000_40b/work/run/before-optimization-step-237.pt \
210
+ --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
211
+ --no-activation-checkpointing --compile-cores --autocast-cache \
212
+ --loss-chunk-size 1024 --vram-limit-gib 88 \
213
+ --warmup-updates 2 --measured-updates 3
214
+ ```
215
+
216
+ Use Supervisor for routine runs; the command above documents its effective
217
+ arguments. `benchmark.sh` sources `training-env.sh` and launches `torchrun`.
218
+ Each invocation automatically receives a fresh
219
+ `work/benchmark-<TORCHELASTIC_RUN_ID>` directory. An explicit `--output-dir`
220
+ must name a new directory under `work/benchmark-*`; existing rank log files
221
+ are not overwritten. If reproducing against another trusted checkpoint,
222
+ change the configured checkpoint path and record which checkpoint was used.
223
+
224
+ For a different candidate, edit only the benchmark program's command or
225
+ environment, then apply it with `ops/datactl.sh reread` and
226
+ `ops/datactl.sh update warpstate-benchmark` from the package directory.
227
+ Starting that program runs the selected finite benchmark. Once it has exited,
228
+ check `nvidia-smi` and restart `warpstate-train` to resume production.
ops/PERFORMANCE.md ADDED
@@ -0,0 +1,236 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WarpState training bottlenecks
2
+
3
+ For the current single-B200 measurements and selected preset, see
4
+ [B200_PERFORMANCE.md](B200_PERFORMANCE.md). This document records the original
5
+ workstation experiments.
6
+
7
+ **Paused at the user's request:** production and benchmarks are stopped,
8
+ with the full production checkpoint saved at step 448 / 58,720,256 tokens.
9
+ Another session will resume VORTEX. Do not start the reproduction commands
10
+ until the user requests further WarpState work and the GPUs are available.
11
+
12
+ Measurements and analysis from September 9, 2026, using the configuration in
13
+ [OPTIMIZED_TRAINING.md](OPTIMIZED_TRAINING.md): two RTX PRO 6000 Blackwell
14
+ Max-Q GPUs, 300 W each, compiled cores, BF16 matrix computation, FP32 AdamW,
15
+ microbatch 4 per GPU, accumulation 8, sequence length 2,048.
16
+
17
+ ## Sustained throughput
18
+
19
+ Production updates 251–315 averaged 28,928 training tokens/second inside
20
+ the timed update loop. Their median update took 4.544 seconds for 131,072
21
+ global tokens. The interval between completed updates averaged 26,711
22
+ tokens/second including the validation/checkpoint pause after step 300.
23
+ That pause produced a 28.31-second gap between updates 300 and 301.
24
+
25
+ The GPUs had about 68.64 GiB reserved and 25.31 GiB physically free each.
26
+ They were busy at the 300 W power limit, with power-cap limiting observed
27
+ and no thermal slowdown. Available VRAM does not imply available compute.
28
+ The driver reported a maximum configurable power limit of 325 W; that is
29
+ only 8.3% above the current power allowance, and does not imply an equal
30
+ throughput gain. Hardware settings were left unchanged.
31
+
32
+ ## GPU profile
33
+
34
+ One full update was profiled after two warmup updates on disposable copies
35
+ of checkpoint step 357. The update took 4.452 seconds. Rank 0 showed:
36
+
37
+ | GPU activity | Summed activity time |
38
+ | --- | ---: |
39
+ | Matrix multiplication | 3.215 seconds |
40
+ | Compiled pointwise functions and reductions | 0.561 seconds |
41
+ | NCCL communication | 0.439 seconds |
42
+ | Local attention | 0.100 seconds |
43
+ | Fused AdamW kernels | 0.039 seconds |
44
+ | Other kernels and copies | Approximately 0.382 seconds |
45
+
46
+ These are actual CUDA kernel/copy activities, excluding nested GPU annotation
47
+ ranges. They sum to more than elapsed time because streams overlap. Across
48
+ the two ranks, matrix multiplication was approximately 65–68% of summed
49
+ activity time and attention approximately 2.1%. Operator shapes attribute
50
+ 1.888 seconds on rank 0 to FFN matrix multiplication, 0.919 seconds to the
51
+ input projection, and 0.141 seconds to the full-vocabulary head's matrix
52
+ multiplications. Normalization, gating, state operations, casts, gradient
53
+ accumulation, and other memory traffic account for much of the remaining
54
+ work. GPU utilization near 100% does not mean peak tensor arithmetic
55
+ throughput.
56
+
57
+ Communication overlaps computation, so its summed time is not the amount
58
+ recoverable by removing it. The previous two-GPU benchmark scaled 1.95 times
59
+ the equivalent single-GPU configuration. Neither more data-loader workers
60
+ nor a different attention implementation addresses the dominant matrix cost.
61
+
62
+ The raw evidence is under `work/benchmark-profile-20260909`. Profiler
63
+ instrumentation was confined to the benchmark; production resumed from
64
+ the unchanged step-357 checkpoint afterward.
65
+
66
+ ## Compute required for 100K tokens/second
67
+
68
+ The following counts use two FLOPs per multiply-add. Let `d=2048`,
69
+ `f=7168`, `L=28`, and `V=32768`. Each core has an input projection with
70
+ `5*d*d` weights, an output projection with `d*d` weights, and SwiGLU
71
+ matrices with `3*d*f` weights. Forward and backward together require
72
+ approximately six FLOPs per dense matrix weight per token.
73
+
74
+ | Matrix computation | Billion FLOPs / training token |
75
+ | --- | ---: |
76
+ | FFNs: `6*L*3*d*f` | 7.398752 |
77
+ | Core input/output projections: `6*L*6*d*d` | 4.227858 |
78
+ | Tied output head with forward recomputation: `8*d*V` | 0.536871 |
79
+ | Parallel tensor-memory operations | 0.049545 |
80
+ | Local causal attention, including backward | Approximately 0.044–0.103 |
81
+ | **Total matrix computation** | **Approximately 12.26–12.32** |
82
+
83
+ The attention range reflects causal triangular arithmetic versus tiled work
84
+ and backward recomputation. These counts exclude normalization, nonlinear
85
+ functions, casts, optimizer work, and communication. Core activation
86
+ checkpointing is disabled; the chunked loss projection still recomputes
87
+ its forward during backward.
88
+
89
+ At 100,000 tokens/second, the matrix operations alone require approximately
90
+ 1.23 PFLOPS. NVIDIA specifies 438.9 TFLOPS of dense BF16 with FP32
91
+ accumulation per Max-Q GPU, or 877.8 TFLOPS for this pair. Thus the
92
+ advertised ideal ceiling is about 71K tokens/second before other work.
93
+ Actual clocks, kernel efficiency, and memory traffic reduce that ceiling.
94
+ This rules out 100K through ordinary BF16 configuration tuning for the
95
+ unchanged dense computation. NVIDIA's 3,511 AI TOPS figure refers to sparse
96
+ FP4 and does not describe this BF16 training. See the
97
+ [NVIDIA architecture whitepaper, Appendix A, pages 45–46](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1.0.pdf#page=45).
98
+
99
+ ## What to investigate next
100
+
101
+ Completed BF16 experiments used disposable copies of the same step-448
102
+ checkpoint, two warmup updates and three measured updates per case:
103
+
104
+ | Configuration | Tokens/second | Peak reserved/GPU |
105
+ | --- | ---: | ---: |
106
+ | Existing compiled-core baseline | 29,396 | 68.64 GiB |
107
+ | Full loss without recomputation | 29,576 | 72.03 GiB |
108
+ | Compiled full loss | 30,315 | 68.99 GiB |
109
+ | Compiled full loss + cuBLASLt / 32 MiB workspace | 30,329 | 68.99 GiB |
110
+ | Compiled full loss + matrix autotuning | 30,403 | 68.99 GiB |
111
+ | Compiled full loss + BF16 residual activations, microbatch 4 | 31,111 | 66.42 GiB |
112
+
113
+ The 50K target was not reached. None of these experimental configurations
114
+ was promoted to production. The microbatch-8 residual trial has no completed
115
+ summary and must not be treated as a measured result. All benchmark losses
116
+ and gradient norms in the completed cases were finite. The compiled-full
117
+ cases differed from baseline by up to 0.00078 loss and 2.47% gradient norm
118
+ after several updates; execution changes are not bitwise equivalent.
119
+
120
+ The added loss and residual helpers passed CPU numerical/parameter-identity
121
+ checks, and the full-model benchmarks exercised their GPU execution.
122
+ The new compiled-full two-GPU checkpoint/resume regression and a production
123
+ continuation/validation trial remain outstanding. Complete those before
124
+ promoting a new execution configuration. The production launcher retains
125
+ its previously validated BF16 settings. No FP8 GPU test or training ran.
126
+ At pause handoff, the combined CPU suite passed 84 tests; five CUDA cases
127
+ were skipped to leave the GPUs available for VORTEX.
128
+
129
+ The next requested target is 50K tokens/second, about 1.73 times the current
130
+ steady rate. It is below the advertised ideal BF16 ceiling, but attaining it
131
+ in BF16 would require unusually high sustained efficiency at this power cap.
132
+ As a rough illustration, halving the observed 3.215 seconds of matrix work
133
+ while leaving everything else fixed would reduce a 4.452-second update to
134
+ about 2.845 seconds, or 46K tokens/second. Overlap and power behavior make
135
+ this an illustration, not a measured prediction. Reaching 50K likely needs
136
+ both faster matrix computation and reductions in the remaining costs.
137
+
138
+ The requested constraint is BF16 matrix computation with FP32 master
139
+ weights and optimizer state. The experiments therefore focus on selecting
140
+ faster BF16 matrix kernels, using the spare memory to remove loss
141
+ recomputation, and reducing activation-memory traffic. Lower-precision
142
+ matrix training is outside this task's scope.
143
+
144
+ A fused linear cross-entropy implementation could reduce repeated output
145
+ projection and intermediate-memory traffic. The current 32,768-way head
146
+ is only 4.4% of the counted dense linear FLOPs, although its memory costs
147
+ can consume a larger fraction of execution time. An implementation such as
148
+ [Cut Cross Entropy](https://github.com/apple/ml-cross-entropy) needs a
149
+ numerical comparison and an end-to-end benchmark; its default gradient
150
+ filtering should not be silently introduced into this pretraining run.
151
+
152
+ Less frequent full checkpoints can recover some wall-clock throughput.
153
+ The current interval is 50 updates, about four minutes of compute. Raising
154
+ that interval increases the amount of work lost after an abrupt failure.
155
+ Checkpoint frequency does not change tokens/second inside the update loop.
156
+
157
+ Reaching 100K could instead use more aggregate GPU compute, or a smaller,
158
+ low-rank, or conditionally activated model. Those architecture changes
159
+ alter the experiment and require their own training/quality comparison.
160
+ VORTEX uses low-rank projections, conditional parameter pages, and a
161
+ hierarchical output head. Its total parameter count does not imply the
162
+ same computation per token as WarpState's 28 dense cores.
163
+
164
+ ## Planning for one B200 GPU
165
+
166
+ The user specified one B200 GPU and BF16 training. NVIDIA lists 180 GB
167
+ HBM3e, approximately 7.7 TB/s memory bandwidth, and 2.25 PFLOPS dense
168
+ BF16 for one HGX B200 GPU. The larger 4.5 PFLOPS BF16 number uses
169
+ sparsity. These specifications differ from eight-GPU DGX totals. Sources:
170
+ [NVIDIA Blackwell datasheet](https://dam-cdn.nvd.orangelogic.com/AssetLink/y441155802qub41q118b2852i557jem5.pdf),
171
+ [NVIDIA performance reference](https://github.com/NVIDIA/exemplar-performance/blob/main/README.md#peak-theoretical-throughput).
172
+
173
+ For this custom model, 60–90K BF16 training tokens/second is an engineering
174
+ planning estimate, not a measured range or guarantee. The 75K midpoint
175
+ roughly preserves the current fraction of advertised compute throughput;
176
+ bandwidth, larger useful microbatches, B200 kernel selection, and removal
177
+ of two-GPU gradient synchronization can change the result. The theoretical
178
+ matrix-only ceiling is approximately 183K tokens/second. Reaching 100K
179
+ would need roughly 55% of advertised dense BF16 compute before accounting
180
+ for non-matrix work, so it is a plausible tuning target on B200 that still
181
+ requires an actual benchmark.
182
+
183
+ | Assumed update throughput | Training tokens/day | Compute time for 40B |
184
+ | ---: | ---: | ---: |
185
+ | 60K/s | 5.18 billion | 7.7 days |
186
+ | 75K/s | 6.48 billion | 6.2 days |
187
+ | 90K/s | 7.78 billion | 5.1 days |
188
+ | 100K/s | 8.64 billion | 4.6 days |
189
+
190
+ Checkpointing and validation add elapsed time to these compute estimates.
191
+ The current 50-update checkpoint interval becomes relatively more
192
+ expensive as updates get faster.
193
+
194
+ Storage is independent of GPU choice. In decimal GB, the prepared corpus
195
+ uses 80.002 GB, a complete model/Adam checkpoint uses 24.062 GB, and atomic
196
+ replacement needs another 24.062 GB temporarily. The current Python
197
+ environment is approximately 5.7 GB. Extra retained checkpoints each add
198
+ 24.062 GB. A deployment with token data, one checkpoint, and its replacement
199
+ therefore needs roughly 134 GB before OS, cache, and reserve space. The
200
+ runner also checks for about 53 GiB free at startup for checkpoint writing.
201
+
202
+ Allocate **250 GB as a practical minimum**, or **500 GB NVMe SSD for
203
+ comfortable checkpoint retention and working space**. This assumes copying
204
+ the already prepared tokens, not retaining raw source downloads. A 1 TB
205
+ SSD is not required for this 40B-token run. SSD capacity does not cap the
206
+ lifetime number of training tokens: further data can be prepared or
207
+ streamed after old data is released.
208
+
209
+ ## Reproducing the GPU profile
210
+
211
+ Stop production gracefully and wait for both GPUs to be free before running
212
+ this finite benchmark. It reads the latest checkpoint into disposable CUDA
213
+ copies and writes no training checkpoint. VORTEX must remain paused.
214
+
215
+ ```bash
216
+ cd /workspace/WarpState/rtx6000_40b
217
+ ops/datactl.sh stop warpstate-train
218
+ nvidia-smi
219
+ WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 \
220
+ TORCHINDUCTOR_COMPILE_THREADS=8 ops/benchmark.sh \
221
+ --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt \
222
+ --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
223
+ --no-activation-checkpointing --compile-cores --autocast-cache \
224
+ --loss-chunk-size 1024 --vram-limit-gib 88 \
225
+ --warmup-updates 2 --measured-updates 1 --profile
226
+ ops/datactl.sh start warpstate-train
227
+ ```
228
+
229
+ Only the first measured update is profiled. Each rank exports a Chrome
230
+ trace, an operator table grouped by shape, a kernel table, and a JSON
231
+ summary under its fresh `work/benchmark-*` directory. Profiling collects
232
+ CPU and CUDA activity with shapes, without Python stacks or memory
233
+ tracking. Use unprofiled runs to measure throughput: recording adds
234
+ overhead. CUDA kernels on different streams can overlap, so summed kernel
235
+ time is not the same as elapsed time. Operator and kernel tables describe
236
+ the same work and must not be added together.
ops/POINTWISE_HEADROOM_20260913.md ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Remaining pointwise work and GEMM saturation — September 13, 2026
2
+
3
+ This analysis uses the saved September 12 full-model profile of the selected
4
+ paired backward layout. It does not launch a new GPU profile, pause production,
5
+ or change the training recipe. At the initial status check, production was
6
+ running past step 53,854, at approximately 31.2K tokens/second during updates.
7
+
8
+ ## Timing scope
9
+
10
+ The profile is one update after eight warmups, with two RTX PRO 6000 Blackwell
11
+ Max-Q GPUs, microbatch four per rank, sequence 2,048, eight accumulation
12
+ microbatches, and 131,072 global tokens. Model computation uses BF16; master
13
+ parameters, accumulated gradients, communication, and AdamW state use FP32.
14
+ The communication preset uses conservative wait sinking and 128 MiB buckets.
15
+
16
+ The profiled update lasts 4.16463 seconds. Repeated unprofiled measurements of
17
+ this preset averaged 4.20116 seconds, or 31,199.03 tokens/second. Profiles include
18
+ instrumentation; neither number includes periodic validation/checkpoint pauses.
19
+ Kernel durations from separate streams can overlap and must not be summed into
20
+ elapsed time without interval accounting. Individual operator totals and their
21
+ kernel totals describe the same work.
22
+
23
+ ## Remaining pointwise and reduction work
24
+
25
+ The name-based compiled-pointwise/reduction category contains 475.795 ms on
26
+ rank zero and 479.331 ms on rank one. This includes arithmetic, conversions,
27
+ copies, and reductions fused into generated kernels. It excludes the selected
28
+ custom paired layout, whose two kernels take approximately 78.96 ms per rank.
29
+ Consequently, neither the small explicit memcpy category nor the 476 ms category
30
+ alone describes all memory traffic or all work outside matrix multiplication.
31
+
32
+ The largest individually attributed groups in that category are:
33
+
34
+ | Work | Rank 0 ms/update | Rank 1 ms/update | Current boundary |
35
+ | --- | ---: | ---: | --- |
36
+ | SwiGLU backward | 85.70 | 85.96 | Packed activation-gradient output |
37
+ | Parameter gradient conversion/accumulation | 69.55 | 70.29 | Includes already fused FP32 accumulation |
38
+ | Core normalization/residual backward | 68.42 | 68.10 | Several operations share each kernel |
39
+ | Attention/memory forward transforms | 66.84 | 69.35 | Rotary, gating, and layout work |
40
+ | Core FP32-to-BF16 weight casts | 49.59 | 49.54 | Four weight families, eight microbatches |
41
+ | SwiGLU forward | 49.20 | 49.13 | Already fused SiLU and multiplication |
42
+ | Core normalization forward | 11.20 | 11.18 | Includes fused surrounding work |
43
+
44
+ These are disjoint groups within the compiled category, not the full category.
45
+ Attention/memory backward transforms, residuals, bucket operations, loss, and small
46
+ parameter reductions account for the remainder. Attribution groups profiler
47
+ operators by complete input shapes and checked generated-code bodies. The same
48
+ generic kernel number can represent different operations in different graphs;
49
+ names alone do not establish the mapping. The generated report is
50
+ `work/validation/pointwise-headroom-20260913/semantic-families.json` and retains
51
+ the explicit per-operator mapping, input shapes, source paths, and provenance.
52
+
53
+ Cutting rank zero's 475.795 ms budget by 25% would remove about 119 ms. If every
54
+ saved millisecond reduced elapsed time and other execution stayed unchanged,
55
+ 4.20116 seconds would become 4.08221 seconds: approximately 2.9% more throughput.
56
+ This is conditional arithmetic, not a measured candidate or a promised gain.
57
+
58
+ Prior experiments already tested standalone SwiGLU and RMSNorm replacements.
59
+ SwiGLU forward/backward improved from 0.61335 to 0.59965 ms in its isolated
60
+ production-shaped fixture; RMSNorm improved from 0.43256 to 0.39248 ms. Neither
61
+ standalone mode improved the full-model screen. External weight-cast caching
62
+ also regressed overall and changed compiler/DDP partition boundaries. Their
63
+ switches are not supported with the selected compiled-autograd configuration.
64
+ See [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md).
65
+
66
+ The earlier fused SwiGLU-forward/TMA-weight-gradient recipe produced one +0.43%
67
+ full-model comparison after correcting duplicated weight storage. It lacks a
68
+ repeated gain and has not been combined with the selected paired layout.
69
+ The isolated epilogue's 45-microsecond gain predicts only about 10 ms per update
70
+ before interactions. See [MATMUL_TUNING.md](MATMUL_TUNING.md).
71
+
72
+ ### Best focused next experiment
73
+
74
+ Start with the **85.7 ms SwiGLU backward region**, using a replacement inside
75
+ the compiled backward graph rather than the previously screened standalone
76
+ activation mode. Its generated kernel traverses a packed `8192 x 14336` output
77
+ and selects between the two derivative halves. The source has five masked
78
+ loads across those branches. A paired-output implementation can load the
79
+ up value, gate value, and upstream derivative once for each logical pair, then
80
+ write both derivative halves. This targets repeated operand access and branch
81
+ work while retaining native packed storage and the existing GEMMs.
82
+ The existing `activation_execution.py` already contains a paired backward
83
+ kernel. First compare that arithmetic and layout directly against this actual
84
+ compiled region; a new primitive implementation is not the starting requirement.
85
+ The integration route would preserve the surrounding compiled-autograd graph.
86
+
87
+ Arithmetic must follow the compiled reference. The up derivative uses
88
+ `g / (1 + libdevice.exp(-g))`, while the gate derivative uses `tl.sigmoid(g)`.
89
+ Sharing one sigmoid value by algebraic rewriting is not automatically equivalent
90
+ to those two implementations. Preserve their evaluation and BF16 output rounding
91
+ unless a separately declared numerical comparison justifies a change.
92
+
93
+ The relevant body is `triton_poi_fused_5` near line 436 of
94
+ `.cache/torchinductor/cw/ccwrp762vinzoemcnamsrl2wt54aoanpxjxxqobh7e7oxxutputc.py`.
95
+ Its synchronized variant is classified separately by matching operand shapes.
96
+ The source suggests an opportunity; it does not prove an instruction-count,
97
+ physical-bandwidth, or elapsed-time saving. A 25% reduction of this region
98
+ would save about 21.4 ms per update, approximately 0.5% more throughput if fully
99
+ realized. No new candidate kernel was timed or selected in this analysis.
100
+
101
+ The next candidate is tuning the **68.4 ms normalization/residual backward
102
+ group as a whole**. The existing kernels already fuse FP32 row reductions,
103
+ parameter-gradient partials, residual additions, and BF16 output stores.
104
+ Tune tiling and intermediate-buffer traffic while preserving those combinations;
105
+ the entire group is not removable normalization overhead.
106
+ One specific subregion accounts for approximately 24.5 ms and loops twice over
107
+ 2,048 channels, rereading inputs for the normalization input derivative and
108
+ residual output. A persistent row kernel could retain loaded intermediates,
109
+ but increased register pressure could offset the saved reads. This is a narrower
110
+ test than replacing every normalization operation.
111
+
112
+ Gradient conversion/accumulation is already fused after many matrix products.
113
+ Further integration into a GEMM epilogue must preserve the BF16 product's
114
+ rounding before FP32 accumulation and keep gradient readiness/communication
115
+ ordering. Hoisting weight casts across accumulation microbatches could reduce
116
+ the 49.6 ms cast budget, but must solve the prior autograd/DDP-boundary regression.
117
+ These are more involved follow-ups than the isolated SwiGLU backward replacement.
118
+
119
+ Use the same sequence as the selected layout integration: exact-shape primitive
120
+ comparison, guarded graph matching and runtime dispatch evidence, full gradient/
121
+ optimizer/resume fixtures, then repeated complete-model timing. An isolated
122
+ kernel gain alone is insufficient to change the training preset.
123
+
124
+ ## Where matrix time is spent
125
+
126
+ Every one of the 5,400 matrix kernels per rank is joined by its CUDA External ID
127
+ to the originating `aten::mm`, `aten::bmm`, or `aten::addmm` event. Exact operand
128
+ shapes and strides identify families and forward/input-gradient/weight-gradient
129
+ directions. This includes all eight vocabulary-head weight-gradient ADDMM calls.
130
+ The calculation counts 791.597 trillion matrix FLOPs per rank per update.
131
+ All matrix kernels use the same compute stream in these traces.
132
+
133
+ | Matrix family | Rank 0 ms | Rank 1 ms | Rank 0 share of matrix time |
134
+ | --- | ---: | ---: | ---: |
135
+ | FFN up/gate | 1,281.23 | 1,225.96 | 40.05% |
136
+ | Attention/memory input projection | 907.66 | 862.34 | 28.38% |
137
+ | FFN down | 632.02 | 602.74 | 19.76% |
138
+ | Core output projection | 174.37 | 171.46 | 5.45% |
139
+ | Vocabulary head | 103.37 | 98.64 | 3.23% |
140
+ | Small state-memory batched products | 100.09 | 96.61 | 3.13% |
141
+ | Total | 3,198.75 | 3,057.76 | 100% |
142
+
143
+ The first three families account for **88.19% / 88.01% of matrix time** and
144
+ **67.74% / 64.62% of the profiled update** on ranks zero/one. All matrix kernels
145
+ occupy 76.81% / 73.42% of that update. These are duration shares, not measures
146
+ of compute saturation. The generated attribution is
147
+ `work/validation/pointwise-headroom-20260913/paired-matmul-attribution.json`.
148
+ Its source profiles and reviewed shape/stride mapping are hashed in the report.
149
+
150
+ ## What the saturation evidence establishes
151
+
152
+ The dominant dense families sustain approximately 252–256 TFLOPS on rank zero
153
+ and 264–268 TFLOPS on rank one in this stored full-model profile. Earlier warmed
154
+ isolated tests measured approximately 243–263 TFLOPS for their individual shapes.
155
+ Relative to an illustrative peak adjusted for their observed clocks, those
156
+ isolated results reached approximately **88–93%**. That estimate scales the
157
+ published 438.9 dense BF16 TFLOPS by observed MHz / 2,280 MHz. The specification
158
+ uses boost clocks; the denominator is not a measured sustained hardware limit.
159
+ [NVIDIA architecture specifications](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46).
160
+
161
+ Those isolated clock samples are snapshots, not time-integrated clocks for the
162
+ new profile. The results support a compute-efficiency focus for large dense
163
+ GEMMs, while small state-memory products have much lower arithmetic intensity.
164
+ They do **not** mean that 88% of matrix time is fully saturated, or that each
165
+ dense kernel spends 88–93% of its lifetime at full utilization. Operand loads,
166
+ Tensor Core instructions, and output work overlap within a pipelined GEMM;
167
+ kernel duration alone cannot partition that execution into saturated and
168
+ unsaturated intervals.
169
+
170
+ The exact requested fraction remains unmeasured. A prior Nsight Compute attempt
171
+ failed with `ERR_NVGPUCTRPERM`; the read-only driver check still reports
172
+ `RmProfilingAdminOnly=1`. No counter-access policy was changed for this analysis.
173
+ The session's effective `CAP_SYS_ADMIN` and `CAP_PERFMON` capabilities are both
174
+ absent. The read-only policy/capability check does not collect GPU counters.
175
+ The prior failure is saved in
176
+ `work/validation/rtx-profile-20260910/ncu-metrics.txt`.
177
+ NVIDIA documents this as restricted access to GPU performance counters.
178
+ [Counter permission reference](https://developer.nvidia.com/nvidia-development-tools-solutions-err_nvgpuctrperm-permission-issue-performance-counters).
179
+
180
+ After counter access is available, a bounded measurement should use disposable,
181
+ warmed copies of the dominant GEMM shapes, with the same BF16 operands, strides,
182
+ power limits, and relevant cache conditions. Collect Tensor-pipeline throughput,
183
+ SM activity, memory throughput, and scheduler statistics. Prefer elapsed-cycle
184
+ metrics when assessing whole-device utilization: active-cycle metrics exclude
185
+ idle cycles. Query the installed tool's supported metrics rather than assuming
186
+ a metric name from another architecture.
187
+
188
+ To answer a literal fraction-of-time question, collect supported PM-sampling
189
+ timelines and declare a near-saturation threshold, for example 90% of the
190
+ appropriate Tensor-pipeline peak. Correlate above-threshold sample intervals
191
+ with the GEMM execution intervals, and report sampling resolution and coverage.
192
+ Per-kernel average utilization alone cannot determine that time fraction.
193
+ Nsight Compute documents both active/elapsed normalization and time-resolved PM
194
+ sampling, including examples of operand-delivery stalls and underfilled tails.
195
+ [Profiling guide](https://docs.nvidia.com/nsight-compute/ProfilingGuide/#pm-sampling).
ops/POSTTRAIN.md ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Step 76,294 archive and supervised fine-tuning
2
+
3
+ The SFT stage runs from
4
+ [`../configs/posttrain_10b.json`](../configs/posttrain_10b.json). The user later
5
+ authorized **archiving the full completed SFT state, then automatically running
6
+ DPO**. A separate `warpstate-dpo` Supervisor job implements that continuation;
7
+ see [DPO_HANDOFF.md](DPO_HANDOFF.md). The active SFT plan/source bytes stay fixed
8
+ to preserve their checkpoint/resume signatures. Its original guardian still
9
+ finishes and exits cleanly before the DPO guardian can take over.
10
+
11
+ `prepare_posttrain_data.py` prepares conversations and preference pairs. The
12
+ separate `posttrain.py` runner implements assistant-only supervised training.
13
+
14
+ ## Automatic sequence
15
+
16
+ 1. `warpstate-posttrain-data` prepares the pinned dataset recipe on CPU now.
17
+ 2. `warpstate-posttrain` waits for `warpstate-train` to exit successfully. It
18
+ never changes the base run or resumes an earlier stopped base checkpoint.
19
+ 3. Under the base launch/pipeline locks, require the full checkpoint to contain
20
+ step **76,294**, **10,000,007,168 tokens**, and final validation at that same
21
+ boundary. A checkpoint before the boundary remains waiting; an overshoot,
22
+ stale validation, or mismatched tokenizer fails without starting SFT.
23
+ 4. Export the original FP32 model using the same `chat.capture_latest` method as
24
+ the 8K, 16K and 40K archives. Compare every exported tensor exactly against the
25
+ stopped checkpoint; copy the exact tokenizer; write provenance/checksums;
26
+ seal files read-only; publish the complete archive atomically without
27
+ replacing an existing directory.
28
+ 5. Make a physical, independent copy of the archive as SFT initialization. The
29
+ full pretraining checkpoint, corpus, historical archives and chat selection
30
+ are retained. SFT starts with a fresh AdamW optimizer on its first launch.
31
+ 6. Verify prepared SFT data, run a tiny two-GPU BF16 training/resume preflight,
32
+ then start the full SFT model. Failed preflight or data checks stop the
33
+ handoff and retain the raw archive for diagnosis.
34
+ 7. Save final SFT state and evaluation results, verify the raw archive and
35
+ initialization remain unchanged, then exit. Neither program automatically
36
+ restarts training after completion or failure. The separately armed DPO job
37
+ then archives this full SFT state and continues according to DPO_HANDOFF.md.
38
+
39
+ The boundary triggers archival, checksum verification, compilation/preflight
40
+ and initialization before the first SFT update; allow several minutes for this
41
+ handoff. GPU preflight is deliberately deferred until base training releases
42
+ the GPUs. Its result is not claimed by the CPU-only setup tests.
43
+
44
+ ## Paths
45
+
46
+ All paths below are relative to `/workspace/WarpState/rtx6000_40b`.
47
+
48
+ | Purpose | Path |
49
+ | --- | --- |
50
+ | Untouched full base resume checkpoint | `work/run/latest.pt` |
51
+ | Raw historical model archive | `work/archives/step-00076294/` |
52
+ | Independent SFT initialization | `work/posttrain/step-00076294/initialization/` |
53
+ | Prepared SFT/DPO data and final manifest | `work/posttrain/data/` |
54
+ | SFT checkpoints, settings and metrics | `work/posttrain/step-00076294/run/` |
55
+ | Final evaluation / completion | `run/evaluation.json`, `run/completion.json` under the SFT directory |
56
+ | Handoff state and frozen execution provenance | `work/posttrain/step-00076294/control/` |
57
+ | GPU preflight artifacts | `work/posttrain/step-00076294/preflight/` |
58
+ | Handoff and training log | `work/logs/posttrain.log` |
59
+ | CPU preparation log | `work/logs/posttrain-data.log` |
60
+
61
+ Archives contain model weights only, without Adam or training RNG state. SFT's
62
+ `latest.pt` is a separate full `warpstate-posttrain-v1` checkpoint with optimizer,
63
+ per-rank RNG, deterministic sample cursor, source/data identities and base
64
+ provenance. It is not interchangeable with the base runner's checkpoint.
65
+
66
+ ## SFT recipe and validation
67
+
68
+ This first run uses one epoch, two GPUs, sequence length 2,048, microbatch 4 per
69
+ rank and four accumulation microbatches: **32 conversations per update**. There
70
+ is no cross-conversation packing. All parameters train with BF16 compute,
71
+ original FP32 parameters/gradients and fresh FP32 AdamW state; no FP8 is used.
72
+ The learning rate peaks at `5e-6`, with 3% warmup and cosine decay to zero,
73
+ zero weight decay and gradient clipping at 1. Evaluation/checkpoints occur
74
+ every 100 SFT updates and at the end. Initial and final validation use the full
75
+ SFT holdout; intermediate checks use the same fixed 128-example subset.
76
+ Settings and effective epoch length are
77
+ recorded in the SFT run's `settings.json` once data is loaded.
78
+
79
+ The learning rate is a conservative experimental starting point, not a
80
+ validated optimum for this 2B model. This is not a reproduction of Tülu 3's
81
+ SFT schedule: its published learning rate was calibrated using summed token
82
+ loss, and its models/context/batch sizes differ. See the
83
+ [Tülu 3 paper, section 4.3.2](https://arxiv.org/html/2411.15124v3).
84
+
85
+ Role headers use ordinary tokens from the existing SentencePiece vocabulary.
86
+ Only assistant content and the actual EOS ID contribute to the loss. Prompt,
87
+ role-header and padding labels are masked; DDP gradients are normalized by the
88
+ global count of supervised tokens, including uneven final batches. The shared
89
+ renderer is `ops/posttrain_data.py`; its hash is pinned in the data manifest.
90
+
91
+ SFT metrics report actual nonpadding input tokens and supervised assistant
92
+ tokens separately. SFT validation loss/perplexity uses the assistant-token
93
+ objective on the prepared holdout; it is **not directly comparable** with the
94
+ base model's all-token pretraining validation loss. Evaluation reports include
95
+ the exact evaluated sample/token counts, dataset and base-weight identities.
96
+ Behavioral instruction-following and tool-use evaluation is the next manual
97
+ decision after the full authorized SFT/DPO sequence pauses; a lower SFT loss
98
+ alone does not establish those capabilities. The final SFT archive preserves
99
+ the exact model and its evaluation for comparison with DPO.
100
+
101
+ ## Monitor, pause and resume
102
+
103
+ From the repository root:
104
+
105
+ ```bash
106
+ rtx6000_40b/ops/datactl.sh status warpstate-train warpstate-posttrain-data warpstate-posttrain
107
+ rtx6000_40b/ops/posttrain-handoff.sh --check
108
+ tmux attach -t warpstate-posttrain
109
+ tail -f rtx6000_40b/work/logs/posttrain.log
110
+ ```
111
+
112
+ The tmux session displays the durable Supervisor log. Detaching tmux does not
113
+ stop the handoff or training. `RUNNING` on the posttraining program can mean
114
+ waiting for the base boundary; inspect `--check` or its phase log.
115
+
116
+ To pause the handoff or gracefully checkpoint an active SFT run:
117
+
118
+ ```bash
119
+ rtx6000_40b/ops/datactl.sh stop warpstate-posttrain
120
+ ```
121
+
122
+ To explicitly re-enable the unchanged handoff or resume an interrupted SFT run:
123
+
124
+ ```bash
125
+ rtx6000_40b/ops/datactl.sh start warpstate-posttrain
126
+ ```
127
+
128
+ The saved plan/source identity must still match. A completed run stays paused
129
+ even if the program is started again. Do not remove its completion state to
130
+ repeat SFT. Use a separately reviewed plan and output directory for a new
131
+ experiment. Do not clear lock files or launch the base trainer manually while
132
+ SFT workers are alive: every SFT rank holds a shared base pipeline lock, and
133
+ the base/handoff require its exclusive lock. The launcher also retains its
134
+ exclusion locks across child launch and unexpected watcher death.
135
+
136
+ SFT Supervisor includes are in `ops/posttrain-supervisord.conf`; both programs
137
+ have `autostart=false` and `autorestart=false`. Setup explicitly starts the
138
+ authorized jobs. A whole-machine/Supervisor restart requires an explicit
139
+ restart of the appropriate jobs; there is no hidden reboot training launch.
140
+
141
+ To cancel automatic continuation into DPO while allowing SFT to finish, stop
142
+ the separate waiting job with `rtx6000_40b/ops/datactl.sh stop warpstate-dpo`.
143
+ Stopping SFT itself leaves the DPO guardian waiting for verified completion.
144
+
145
+ ## Rebuild the prepared inputs
146
+
147
+ The recipe in `configs/posttrain_data.json` pins immutable dataset revisions,
148
+ the tokenizer identity, sample counts/seed, split rule, renderer and required
149
+ preparation packages. Preparation records downloaded file hashes, schemas,
150
+ source/licensing metadata, row/token counts, rejections and output hashes.
151
+ Prompt groups use the same deterministic partition across SFT and DPO; exact
152
+ duplicate conversations are excluded. Overlength examples and DPO branches
153
+ are rejected rather than truncated. Gated xLAM data is recorded as skipped
154
+ without automatically accepting terms or requiring interactive input.
155
+
156
+ To rebuild the isolated CPU environment, source `ops/training-env.sh`, create
157
+ `work/posttrain-data-venv` with Python 3.12, then install
158
+ `ops/requirements-posttrain-data.lock` into that environment. Do not change the
159
+ active training environment. `ops/prepare-posttrain-data.sh` runs the recipe;
160
+ do not regenerate/replace data while SFT is running. The completed manifest is
161
+ published only after every selected output has been written and hashed.
162
+
163
+ Source/configuration/tests/dependency metadata belong in Git. Downloaded data,
164
+ token caches, generated manifests/reports, checkpoints, archives, environments
165
+ and logs under `work/` remain ignored.
ops/RESTART_HANDOFF.md ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # RTX workstation restart handoff
2
+
3
+ Prepared on 2026-09-15. All paths below are relative to `/workspace/WarpState`.
4
+ Training, posttraining, and benchmarks have finished; leave them stopped.
5
+ Use the existing environments and prepared data. BF16 compute remains required.
6
+
7
+ ## Saved model state
8
+
9
+ | Stage | Final step | Full resumable checkpoint | Immutable model archive |
10
+ | --- | ---: | --- | --- |
11
+ | Raw pretraining | 76,294 | `rtx6000_40b/work/run/latest.pt` | `rtx6000_40b/work/archives/step-00076294/` |
12
+ | SFT | 10,312 | `rtx6000_40b/work/archives/step-00076294-sft-final/checkpoint.pt` | `rtx6000_40b/work/archives/step-00076294-sft-final/` |
13
+ | DPO | 2,334 | `rtx6000_40b/work/archives/step-00076294-dpo-final/checkpoint.pt` | `rtx6000_40b/work/archives/step-00076294-dpo-final/` |
14
+
15
+ Raw training reached 10,000,007,168 tokens with validation loss 2.3496886492.
16
+ SFT and DPO checkpoints both record `completed=true`. Their validation losses
17
+ are 1.1157395900 and 0.5790014537 respectively, for different objectives.
18
+ The earlier step 8,000, 16,000, and 40,000 archives are also retained.
19
+ Do not modify archived files. Chat uses an independent copy in
20
+ `rtx6000_40b/work/chat/dpo-final/`.
21
+
22
+ With user approval, five byte-identical working weight files were removed
23
+ after comparison with their retained archives, reclaiming 72,184,948,679 bytes.
24
+ These are the SFT and DPO working `run/latest.pt` and `initialization/model.pt`
25
+ files, plus `work/chat/step-8000-s536yem8/model.pt` (under `rtx6000_40b/`).
26
+ All archives and the independent current chat copy remain intact.
27
+
28
+ Before any future posttraining continuation or rerun of completion checks,
29
+ restore independent working files from the following sources. Paths in this
30
+ table are relative to `rtx6000_40b/work/`:
31
+
32
+ | Archived source | Working destination |
33
+ | --- | --- |
34
+ | `archives/step-00076294/model.pt` | `posttrain/step-00076294/initialization/model.pt` |
35
+ | `archives/step-00076294-sft-final/checkpoint.pt` | `posttrain/step-00076294/run/latest.pt` |
36
+ | `archives/step-00076294-sft-final/model.pt` | `posttrain/step-00076294-dpo/initialization/model.pt` |
37
+ | `archives/step-00076294-dpo-final/checkpoint.pt` | `posttrain/step-00076294-dpo/run/latest.pt` |
38
+
39
+ Verify the archive's `SHA256SUMS`, copy only to an absent destination using
40
+ `cp --reflink=auto --no-clobber`, and compare the copy with its source before
41
+ use. Never train against an archive path or a hardlink to archived weights.
42
+ Restoring these files does not authorize starting training.
43
+
44
+ Prepared pretraining tokens remain in `rtx6000_40b/work/data/`; posttraining
45
+ data remain in `rtx6000_40b/work/posttrain/data/`. Benchmark results are in
46
+ `rtx6000_40b/work/validation/nord-dpo/` and
47
+ `rtx6000_40b/work/validation/nord-stage-comparison/COMPARISON.md`.
48
+
49
+ ## After reboot
50
+
51
+ Start in `/workspace/WarpState` and read `AGENTS.md` and this handoff.
52
+ Check `nvidia-smi` and `git status --short --branch`.
53
+ To restore post-DPO chat in its own tmux session:
54
+
55
+ ```bash
56
+ cd /workspace/WarpState
57
+ tmux has-session -t warpstate-chat 2>/dev/null || tmux new-session -d -s warpstate-chat -n dpo-chat -c /workspace/WarpState 'rtx6000_40b/ops/chat-dpo.sh'
58
+ tmux attach -t warpstate-chat
59
+ ```
60
+
61
+ Ctrl-C cancels typing or generation and returns to the next input. `/reset`
62
+ clears conversation history, `/quit` exits, and Ctrl-B then D detaches tmux.
63
+ Tmux sessions and in-memory chat history do not survive reboot; captured
64
+ terminal history is available in the restart record below.
65
+
66
+ All workspace Supervisor jobs have `autostart=false` and `autorestart=false`.
67
+ If job management is needed, check `rtx6000_40b/ops/datactl.sh status` first.
68
+ Initialize the workspace Supervisor only when it is absent:
69
+
70
+ ```bash
71
+ /usr/local/bin/supervisord -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf
72
+ ```
73
+
74
+ This starts no training. The raw launcher still stops at step 76,294, and
75
+ posttraining has completed. Further training requires a new user request and
76
+ an explicitly chosen continuation configuration; do not simply restart jobs.
77
+ Do not use the B200 launcher on this RTX workstation.
78
+
79
+ Jupyter currently uses HTTPS port 8080 with the existing authentication.
80
+ After reboot, check the platform-managed Jupyter service before launching
81
+ another server. Runtime files and logs from the last standalone launch are
82
+ under `rtx6000_40b/work/control/jupyter/`; treat them as private because they
83
+ can contain authentication data. Never commit them or expose their contents.
84
+ The raw model download path on the Jupyter host is
85
+ `/files/workspace/WarpState/rtx6000_40b/work/archives/step-00076294/model.pt?download=1`.
86
+ For post-DPO weights, substitute `step-00076294-dpo-final`.
87
+
88
+ ## Restart record
89
+
90
+ `rtx6000_40b/work/restart/LATEST` identifies the private local restart record.
91
+ It contains available tmux history, checkpoint metadata, file sizes, archive
92
+ checksum results, job state, and a Git bundle of committed sources.
93
+ The earlier checkpoint inventory is a pre-cleanup snapshot; the current paths
94
+ are listed above. `rtx6000_40b/work/restart/duplicate-weight-deletion.json`
95
+ records the approved removals and their retained counterparts.
96
+ Artifacts and this bundle are on the same machine, not an off-machine backup.
97
+ Keep datasets, weights, runtime files, and this generated record out of Git.
ops/RTX_ASYNC_TRAINING.md ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # RTX BF16 execution and overlapping checkpoints
2
+
3
+ The local launcher selects the fastest complete-update experiment from
4
+ [RTX_PROFILE_20260910.md](RTX_PROFILE_20260910.md): compiled cores in default
5
+ mode, compiled full vocabulary loss, BF16 residual activations, microbatch 4
6
+ per GPU, and accumulation 8 across two RTX PRO 6000 GPUs. The measured update
7
+ rate was 30,707 tokens/second versus 29,081 for the previous configuration,
8
+ a 5.59% increase. These measurements exclude checkpoint and evaluation pauses.
9
+ The larger microbatch experiments and standalone GEMM/gradient prototypes
10
+ are not enabled in the production launcher.
11
+
12
+ The model, tokenizer, token corpus, 131,072-token global batch, context length,
13
+ optimizer and learning-rate schedule are preserved. Forward/backward compute
14
+ uses BF16. Master parameters, gradients and Adam moments remain FP32 on the
15
+ GPU. BF16 residual execution changes activation rounding; the first validation
16
+ at an unchanged checkpoint measures that numerical difference, not learning.
17
+
18
+ ## Checkpoint lifecycle
19
+
20
+ `--async-checkpoint` works with one GPU or DDP. Every save captures random
21
+ states from every rank at a completed optimizer update. Only rank zero
22
+ constructs the checkpoint and owns the background writer.
23
+
24
+ 1. Finish the previous write, if it is still pending.
25
+ 2. Copy all checkpoint tensors and mutable metadata into an immutable CPU
26
+ snapshot. Both ranks wait; later updates cannot mutate this snapshot.
27
+ 3. Submit serialization to one CPU thread and let both GPUs resume training.
28
+ 4. Write `latest.pt.tmp`, synchronize the file, atomically replace `latest.pt`,
29
+ and synchronize its directory. The previous published checkpoint stays
30
+ available until replacement.
31
+
32
+ One full snapshot costs approximately 22.4 GiB of host RAM for this model and
33
+ optimizer. There is no additional full checkpoint clone on either GPU, and
34
+ at most one snapshot/write is outstanding. The local launcher now saves and
35
+ evaluates every 100 updates. The initial promotion used a 50-update save
36
+ interval; increasing it to 100 reduces time spent staging checkpoints.
37
+
38
+ Step lines display validation loss only when that step has a fresh result,
39
+ together with its evaluation step. Metrics and checkpoints still record
40
+ `val_step` and `val_tokens` alongside `val_loss`, retaining the latest measured
41
+ values between evaluations. This avoids repeated validation values in the
42
+ terminal and adds no GPU work. Startup evaluation and final evaluation and
43
+ saving on a graceful stop remain enabled.
44
+
45
+ The main training threads coordinate initialization, construction, staging,
46
+ completed background errors and final close. The writer thread performs no
47
+ CUDA or distributed operations. Every update first checks the prior writer's
48
+ status on both ranks. A failure observed there is raised on both ranks before
49
+ the next update; a failure occurring during an update is observed at the next
50
+ check or save. Normal completion and graceful stop join the final write.
51
+ Exceptional exit joins only the local writer, avoiding extra collectives
52
+ when another worker may already have failed.
53
+
54
+ `Staged step ...` identifies a frozen snapshot, not durable progress. A later
55
+ `Saved ... (step ..., tokens=...; background write ...s)` identifies completed
56
+ publication. GPU-to-CPU staging and any wait for a previous save remain
57
+ synchronous; only disk serialization and publication overlap training.
58
+
59
+ ## Validation and reproduction
60
+
61
+ The promotion checks passed 112 CPU tests and two CUDA continuation tests
62
+ (one GPU and two GPUs). The full-model canary started from the verified
63
+ production checkpoint at step 5,618 / 736,362,496 tokens. Its initial
64
+ validation remained 2.97925 to five decimal places. After startup, the first
65
+ three measured updates ran at 31,637, 31,482 and 31,450 tokens/second with
66
+ 66.57 GiB reserved per GPU. This short canary validates operation; the
67
+ 20-update comparison above remains the controlled performance experiment.
68
+
69
+ The canary staged step 5,620 in 11.41 seconds, then completed updates 5,621
70
+ and 5,622 before that checkpoint's 10.62-second background write finished.
71
+ It published the final step-5,622 full checkpoint before process exit.
72
+ This confirms real overlap and also shows that the CPU snapshot remains a
73
+ material pause. The initial production save interval was 50 updates; the
74
+ canary used two updates to exercise overlap and queue backpressure quickly.
75
+
76
+ Reloading that full checkpoint completed two further updates and published
77
+ step 5,624 / 737,148,928 tokens. These six canary updates belong only to the
78
+ disposable run; production resumes its original step-5,618 checkpoint. The
79
+ canary's logs, exact launcher arguments and integrity report remain in
80
+ `work/validation/async-promotion/`.
81
+
82
+ The distributed writer tests inject failures during state construction, CPU
83
+ snapshot allocation, previous-save waits, background checks, flush and close.
84
+ Two CPU/Gloo processes must both report each failure without hanging. Tests
85
+ also verify immutable snapshots, rank-zero-only writing, file and directory
86
+ synchronization, and cleanup without collectives on exceptional exit.
87
+
88
+ The CUDA execution regression compares continuous synchronous training with
89
+ an asynchronous split/resume run using compiled full loss and BF16 residuals.
90
+ Model tensors, Adam state, RNG state, validation loss and progress must match
91
+ exactly. It exercises periodic saves, final flush and a partially filled last
92
+ update where one rank has no valid tokens. This verifies exact continuation
93
+ for the test model and selected execution configuration; it does not establish
94
+ bitwise equivalence to the previous FP32-residual configuration.
95
+
96
+ Run from `rtx6000_40b` with both GPUs available:
97
+
98
+ ```bash
99
+ source ops/training-env.sh
100
+ mkdir -p work/validation/async-promotion
101
+ .server-venv/bin/python -m pytest -q \
102
+ test_checkpoint_writer.py test_distributed_checkpoint_writer.py \
103
+ --basetemp="$PWD/work/validation/async-promotion/pytest-writer"
104
+ .server-venv/bin/python -m pytest -q test_training_execution.py \
105
+ -k 'two_gpu_bf16_residual_async_full_state_resume_and_empty_final_rank or max-autotune-no-cudagraphs-async' \
106
+ --basetemp="$PWD/work/validation/async-promotion/pytest-gpu"
107
+ ```
108
+
109
+ Full-model validation uses a separate work directory with read-only access to
110
+ the prepared data and an initial hardlink to the paused checkpoint. Saving
111
+ atomically replaces only the disposable checkpoint path. Source checkpoint
112
+ integrity, canary logs and generated verification reports are retained under
113
+ `work/validation/async-promotion/`, outside Git. Production controls and status
114
+ commands are in [LOCAL_RESUME.md](LOCAL_RESUME.md).
ops/RTX_PROFILE_20260910.md ADDED
@@ -0,0 +1,453 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # RTX training profile, 2026-09-10
2
+
3
+ This report records the experiments before production promotion. The selected
4
+ BF16 configuration and subsequent checkpoint-overlap implementation are
5
+ documented in [RTX_ASYNC_TRAINING.md](RTX_ASYNC_TRAINING.md); use
6
+ [LOCAL_RESUME.md](LOCAL_RESUME.md) for the current launcher and status controls.
7
+
8
+ The production run was gracefully paused to measure training bottlenecks and
9
+ compare disposable execution experiments. Production remains paused while the
10
+ experiments run. Benchmark updates do not advance the saved training run.
11
+
12
+ ## Preserved training state
13
+
14
+ The completed checkpoint was verified at 18:12:57 UTC:
15
+
16
+ | Field | Value |
17
+ | --- | --- |
18
+ | Checkpoint | `work/run/latest.pt` |
19
+ | Step | 5,618 |
20
+ | Tokens consumed | 736,362,496 |
21
+ | Validation loss at pause | 2.979250341653824 |
22
+ | Bytes | 24,061,700,237 (22.409 GiB) |
23
+ | Saved optimizer entries | 256 |
24
+ | Saved RNG ranks | 2 |
25
+ | SHA256 | `0f2e2a47f13b9d91d0049a0e99109c289399fe593c244bab046f120c1f7ea835` |
26
+
27
+ The benchmark loads this full checkpoint, including optimizer state, into
28
+ disposable processes. It does not write production checkpoints. Its configuration
29
+ and final summary record the checkpoint file identity before and after the run.
30
+ The verification record is generated at
31
+ `work/validation/rtx-profile-20260910/paused-checkpoint.json`.
32
+
33
+ ## Configuration and measurement method
34
+
35
+ The hardware is two NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition
36
+ GPUs, each with approximately 96 GB VRAM. The 2,005,111,424-parameter model uses
37
+ BF16 autocast with FP32 master parameters, gradients, and full fused AdamW state.
38
+ No FP8 or reduced-precision optimizer state is used.
39
+
40
+ The production configuration uses sequence length 2,048, microbatch 4 per GPU,
41
+ and eight accumulation microsteps. The global batch is therefore 131,072 tokens
42
+ per optimizer update. It compiles the model cores in `default` mode, uses the
43
+ checkpointed vocabulary loss with 1,024-token chunks, enables the autocast cache,
44
+ and disables activation checkpointing for the model blocks. Production saves
45
+ every 50 updates and evaluates every 100 updates.
46
+
47
+ The unprofiled baseline restores the checkpoint's optimizer state and runs two
48
+ warmup updates followed by 20 measured updates. Its aggregate throughput was
49
+ **29,081.0 tokens/s**, with a median of **28,996.8 tokens/s**. Peak allocated VRAM
50
+ was **67.747 GiB** and reserved VRAM **68.643 GiB** per GPU, with a minimum of
51
+ **25.311 GiB** free. The short baseline excludes evaluation and checkpoint writes
52
+ and has a transient speed change as the GPUs warm up. Sustained production
53
+ measurements below provide the wall throughput reference.
54
+
55
+ Profiler runs identify operators and timing relationships. Their instrumented
56
+ update durations must not be presented as unprofiled training throughput.
57
+ Summed CUDA operator durations may overlap; they do not automatically partition
58
+ wall time. Candidate speed comparisons must use unprofiled measurements with
59
+ the same global token batch and full optimizer state.
60
+
61
+ Baseline artifacts are in `work/benchmark-rtx-20260910/baseline/`. Telemetry is
62
+ in `work/validation/rtx-profile-20260910/baseline-gpu-telemetry.json`.
63
+
64
+ ## Production overhead
65
+
66
+ The last 200 completed production intervals end at step 5,618 and start just
67
+ after step 5,418. This interval stays within the restart segment and excludes
68
+ the shutdown gap. It includes four periodic checkpoints and two evaluations;
69
+ the final pause-time evaluation and save occur after its last update timestamp.
70
+
71
+ | Measurement | Result |
72
+ | --- | ---: |
73
+ | Tokens processed | 26,214,400 |
74
+ | Sum of measured optimizer-update time | 909.490 s |
75
+ | Elapsed time between metric timestamps | 1,000.906 s |
76
+ | Throughput within updates | **28,823.2 tokens/s** |
77
+ | Throughput including recurring overhead | **26,190.7 tokens/s** |
78
+ | Wall time outside updates | **9.133%** |
79
+ | Median ordinary inter-update overhead | **2.10 ms** |
80
+
81
+ For each consecutive metric pair, outside-update time is the difference between
82
+ record timestamps minus the later record's update duration. Metrics are emitted
83
+ before periodic evaluation and saving, so the gap following a save step captures
84
+ that save's stall.
85
+
86
+ | After step | Operations before next update | Outside-update time |
87
+ | --- | --- | ---: |
88
+ | 5,450 | Checkpoint | 22.730 s |
89
+ | 5,500 | Validation and checkpoint | 23.026 s |
90
+ | 5,550 | Checkpoint | 22.505 s |
91
+ | 5,600 | Validation and checkpoint | 22.743 s |
92
+
93
+ Checkpoint-only gaps average 22.618 s. The difference between grouped means
94
+ suggests approximately 0.27 s for validation in this window, but validation was
95
+ not separately timed and checkpoint duration also varies. The wider restart
96
+ segment gives approximately 22.49 s per checkpoint and a 0.35 s grouped
97
+ validation difference. Both windows show that synchronous checkpoint saving
98
+ dominates the approximately 9% operational overhead.
99
+
100
+ These gaps combine GPU-to-host staging, serialization, filesystem writes,
101
+ durability synchronization, and waiting. Dividing checkpoint bytes by the pause
102
+ gives approximately 1.06 GB/s for the combined path; this is not an isolated SSD
103
+ bandwidth measurement. Raw interval calculations are recorded in
104
+ `work/validation/rtx-profile-20260910/production-overhead.json`.
105
+
106
+ ### Checkpoint cadence model
107
+
108
+ Holding compute time and evaluation cadence fixed, amortizing approximately
109
+ 22.6 s per checkpoint gives the following estimates. These are predictions,
110
+ not measured candidate runs; production cadence has not been changed.
111
+
112
+ | Save every | Estimated wall throughput | Approximate gain | Maximum replay after a crash |
113
+ | --- | ---: | ---: | --- |
114
+ | 50 updates, current | 26.2K tokens/s | Reference | 6.55M tokens, 4.2 min |
115
+ | 100 updates | 27.4K tokens/s | 4.7% | 13.11M tokens, 8.0 min |
116
+ | 200 updates | 28.1K tokens/s | 7.3% | 26.21M tokens, 15.5 min |
117
+ | 500 updates | 28.5K tokens/s | 8.9% | 65.54M tokens, 38.4 min |
118
+
119
+ Eliminating all measured outside-update time would raise this production window
120
+ from 26.19K to 28.82K tokens/s: an approximately **10.1% upper bound** for
121
+ operational improvements alone. Substantially higher throughput requires faster
122
+ optimizer updates as well.
123
+
124
+ `run_40b.py` currently saves model and optimizer GPU tensors synchronously on
125
+ rank 0 and then waits at a distributed barrier. Its `--async-checkpoint` option
126
+ explicitly rejects a world size other than one. The existing
127
+ `ops/checkpoint_writer.py` stages an immutable CPU snapshot before overlapping
128
+ serialization and durable publication with training. Adapting this path to DDP
129
+ requires coordinated error handling and validation; the flag cannot simply be
130
+ enabled for the current two-GPU run. CPU snapshot staging would still block,
131
+ and approximately one full checkpoint of additional host memory is required.
132
+
133
+ ## GPU profile
134
+
135
+ One complete update, after two warmup updates, took 4.4745 seconds with profiler
136
+ instrumentation. Actual CUDA activity occupied approximately 98.8% of that
137
+ interval. This is the union of kernel/copy intervals, not tensor-core utilization.
138
+
139
+ | GPU activity | Rank 0 summed time | Rank 1 summed time |
140
+ | --- | ---: | ---: |
141
+ | Matrix multiplication | 3.231 s | 3.122 s |
142
+ | Compiled pointwise functions and reductions | 0.561 s | 0.563 s |
143
+ | NCCL communication | 0.448 s | 0.545 s |
144
+ | Other kernels | 0.378 s | 0.378 s |
145
+ | Local attention | 0.100 s | 0.099 s |
146
+ | Fused AdamW | 0.0385 s | 0.0384 s |
147
+ | CUDA copies and fills | 0.0033 s | 0.0031 s |
148
+
149
+ Matrices represent 65.7–67.9% of summed GPU activity. Streams overlap, so this
150
+ table must not be added up as a partition of elapsed time. Rank 0 communication
151
+ overlaps other activity for 337 ms; its communication-only intervals total
152
+ 111 ms. Rank 1 has 215 ms of communication-only activity. Those intervals are
153
+ not guaranteed recoverable by removing communication.
154
+
155
+ The eight data batches require only 5.65/5.12 ms of host time for construction,
156
+ pinning and transfer submission. Increasing loader workers does not address a
157
+ measured bottleneck. Gradient clipping's 280/380 ms host range mostly waits for
158
+ previously queued GPU work during the nonfinite check. Actual clipping-related
159
+ CUDA work is approximately 16 ms. Optimizer and attention replacements likewise
160
+ have small direct budgets compared with dense projections.
161
+
162
+ Raw traces and operator/kernel tables are under
163
+ `work/benchmark-rtx-20260910/profile/`. The generated interpretation is
164
+ `work/validation/rtx-profile-20260910/gpu-attribution.json`.
165
+
166
+ ## What the matrix multiplication contains
167
+
168
+ For a linear projection with input X and weight W, training computes three
169
+ matrix products: `Y = X @ W.T`, `dX = dY @ W`, and `dW = dY.T @ X`. Each product
170
+ costs approximately `2*M*N*K` FLOPs. Thus backward contributes approximately
171
+ twice the dense arithmetic of forward, before other backward operations.
172
+
173
+ Each rank processes 65,536 tokens per update as eight 8,192-token microbatches.
174
+ Each core projection therefore executes 224 times: 28 cores times eight
175
+ microbatches. The following rank-0 times sum all calls within the profiled update.
176
+
177
+ | Projection | Forward input/output width | Forward | Input gradient | Weight gradient |
178
+ | --- | --- | ---: | ---: | ---: |
179
+ | FFN up and gate | 2,048 → 14,336 | 407.4 ms | 437.1 ms | 431.5 ms |
180
+ | FFN down | 7,168 → 2,048 | 222.2 ms | 210.5 ms | 195.5 ms |
181
+ | Q/K/V/gate/memory input | 2,048 → 10,240 | 291.0 ms | 314.9 ms | 309.1 ms |
182
+ | Core output | 2,048 → 2,048 | 57.8 ms | 57.4 ms | 57.5 ms |
183
+ | Vocabulary head | 2,048 → 32,768 | 71.2 ms | 39.8 ms | 31.2 ms |
184
+
185
+ Head forward includes recomputation: 64 original and 64 recomputed projections
186
+ of 1,024 tokens, with 64 calls for each derivative. Core activation checkpointing
187
+ is disabled; head checkpointing still repeats the head's forward projection.
188
+ State-memory batched matrix operations add another 96.5 ms on rank 0.
189
+
190
+ The dense linear products alone account for 797.146 trillion FLOPs per rank
191
+ per update, or 12.1635 billion FLOPs per training token. Their summed execution
192
+ time is 3.1342 s, approximately 254.3 TFLOPS during those operations. FFNs alone
193
+ take 1.905 s, and the five-way input projection takes 0.916 s. These are the
194
+ primary matrix optimization targets.
195
+
196
+ The large forward GEMMs have ideal one-pass BF16 operand arithmetic intensities
197
+ of roughly 910–1,470 FLOPs/byte. This is an operand-footprint calculation, not
198
+ measured device-memory traffic: tile reloads, caches, casts and accumulation
199
+ change actual traffic. It supports investigating compute efficiency before
200
+ assuming that the large GEMMs are starved for external memory bandwidth.
201
+
202
+ Shapes, strides, dtype, counts, FLOPs and matching kernel symbols are recorded
203
+ in `work/validation/rtx-profile-20260910/matmul-detail.json`. The dominant kernel
204
+ names begin `cutlass_80_tensorop_bf16`, with 128×256 or 256×128 tiles, K32 and
205
+ three pipeline stages. The symbol identifies an implementation family; it does
206
+ not establish that the GPU is executing in an obsolete architecture mode.
207
+
208
+ ## Compute ceiling at the measured power limit
209
+
210
+ NVIDIA specifies **438.9 TFLOPS of dense BF16 with FP32 accumulation per Max-Q
211
+ GPU**, based on a **2,280 MHz boost clock**, with **300 W board power** and
212
+ **1,792 GB/s memory bandwidth**. The larger sparsity figure is not applicable
213
+ to this dense model. See Appendix A of the
214
+ [NVIDIA RTX Blackwell architecture whitepaper](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46).
215
+
216
+ During loaded baseline samples, median SM clocks were approximately 1,537 and
217
+ 1,571 MHz. The software power cap was active; thermal throttling was not.
218
+ Scaling the advertised rate by those clocks gives roughly 296–302 TFLOPS per
219
+ GPU as an illustrative instantaneous ceiling, substantially below boost-rated
220
+ peak. This calculation assumes arithmetic capacity scales linearly with SM
221
+ clock; it is not a hardware-counter utilization measurement.
222
+
223
+ The isolated GEMM tests below sustain approximately 244–263 TFLOPS with
224
+ observed clocks of 1,417–1,537 MHz and approximately 300 W. Relative to the
225
+ clock-scaled specification, these correspond to approximately **88–94%**.
226
+ Telemetry is sampled after each timed case and clocks vary within a case, so
227
+ this percentage is approximate. Nevertheless, the evidence indicates that the
228
+ large library GEMMs already use much of the compute available at this power
229
+ limit. Whole-update efficiency is lower because normalization, activations,
230
+ gradient handling, memory operations and communication still consume time.
231
+
232
+ Nsight Compute 2025.1.1 is installed, but its hardware-counter query was rejected
233
+ with `ERR_NVGPUCTRPERM` by the host driver. No driver permissions or power settings
234
+ were changed. Profiler occupancy fields reporting zero for active kernels were
235
+ not treated as measured occupancy. The conclusion above uses CUDA timings,
236
+ shape-derived FLOPs and clock observations, with those limits made explicit.
237
+
238
+ ## Isolated GEMM algorithm experiments
239
+
240
+ [`benchmark_gemm.py`](benchmark_gemm.py) generates deterministic BF16 operands
241
+ with the projection's actual transpose strides. It preserves the production
242
+ BF16 reduction flag, excludes compilation, warms each case for at least two
243
+ seconds, then measures five CUDA-event trials of 30 products. The inputs are
244
+ reused and no autograd, communication, optimizer or training checkpoint is
245
+ involved. These are kernel comparisons, not training throughput measurements.
246
+
247
+ | Product | cuBLAS | cuBLASLt preference | Default Triton-only autotune |
248
+ | --- | ---: | ---: | ---: |
249
+ | FFN up/gate forward | 1.828 ms | 1.838 ms | 1.981 ms |
250
+ | FFN up/gate dX | 1.885 ms | 1.885 ms | 2.057 ms |
251
+ | FFN up/gate dW | 1.871 ms | 1.777 ms | 1.869 ms |
252
+ | Input projection forward | 1.326 ms | 1.314 ms | 1.441 ms |
253
+ | Input projection dX | 1.389 ms | 1.376 ms | 1.474 ms |
254
+ | Input projection dW | 1.351 ms | 1.336 ms | 1.350 ms |
255
+ | FFN down forward | 0.984 ms | 0.965 ms | 1.076 ms |
256
+ | FFN down dX | 0.914 ms | 0.902 ms | 0.993 ms |
257
+ | FFN down dW | 0.935 ms | 0.955 ms | 0.993 ms |
258
+
259
+ ATEN-only compilation was essentially equal to eager cuBLAS. Default
260
+ Triton-only selection was usually slower; swapping backends globally is not an
261
+ established improvement. cuBLASLt gave mixed shape-specific results. Its 5.3%
262
+ up/gate dW gain coincided with a higher observed clock, so it must not be
263
+ attributed solely to a better algorithm. Some Triton input-gradient cases
264
+ differed from the BF16 cuBLAS reference by about 0.30% relative RMS; finite
265
+ outputs do not establish numerical equivalence.
266
+
267
+ Enabling `ENABLE_PERSISTENT_TMA_MATMUL=1` added persistent TMA candidates and
268
+ selected them for the three up/gate products. Measured times were 1.798 ms
269
+ forward, 1.890 ms dX and 1.810 ms dW: approximately 268, 255 and 266 TFLOPS.
270
+ This closes the default Triton gap and gives small shape-specific improvements
271
+ over cuBLAS, rather than a large untapped acceleration. Autotuner trial times
272
+ were faster than the subsequent warmed measurements; the latter are reported.
273
+
274
+ Evidence is under `work/benchmark-gemm-20260910/`,
275
+ `work/benchmark-gemm-cublaslt-20260910/`, and
276
+ `work/benchmark-gemm-tma-20260910/`.
277
+
278
+ ## Fusion opportunities
279
+
280
+ The two FFN input projections already share one GEMM. Q/K/V/gate/memory likewise
281
+ share one projection. The compiled core already fuses residual addition,
282
+ RMSNorm and BF16 conversion, as well as SwiGLU split/SiLU/multiply and several
283
+ attention transforms. Generated wrappers still use `extern_kernels.mm` between
284
+ these fused regions.
285
+
286
+ The FFN computes `(X @ W_up * silu(X @ W_gate)) @ W_down`. The nonlinearity
287
+ prevents collapsing both projection stages into a single ordinary linear map.
288
+ A custom fused implementation still performs both matrix stages.
289
+
290
+ | Remaining target | Rank-0 measured kernel-time budget | Practical constraint |
291
+ | --- | ---: | --- |
292
+ | Weight-gradient conversion and FP32 accumulation | 130.5 ms | Preserve BF16 rounding and DDP gradient hooks |
293
+ | SwiGLU backward | 88.0 ms | Paired-element derivative can reuse loads and sigmoid |
294
+ | Repeated core weight casts | 56.4 ms | Cached BF16 copies need optimizer-step invalidation and correct autograd |
295
+ | SwiGLU forward | 41.1 ms | Up/gate columns are 7,168 apart; backward still needs saved values |
296
+ | Forward norms | 29.6 ms | Row reductions complicate fusion into tiled GEMMs |
297
+ | Forward Q/K/V/U transforms | 43.9 ms | Multiple attention and memory consumers need compatible layouts |
298
+
299
+ These are budgets, not promised savings, and are not additive end-to-end
300
+ speedups. A slower custom GEMM can erase the benefit of its fused epilogue.
301
+ The complete SwiGLU forward/backward pointwise cost is only 129 ms, about 2.9%
302
+ of the profiled update. Recomputing the up/gate projection to avoid saving its
303
+ activations would add approximately 407 ms, exceeding its forward fusion budget.
304
+
305
+ [`benchmark_gradient_accumulation.py`](benchmark_gradient_accumulation.py)
306
+ compares existing BF16 dW followed by FP32 accumulation, fused conversion/add,
307
+ and direct FP32-output `addmm`. The direct variant changes the original
308
+ BF16-round-before-add semantics. Neither helper implements production autograd
309
+ or DDP integration, and neither has been enabled in the training run.
310
+
311
+ The final accumulation test uses eight distinct deterministic input pairs. It
312
+ creates the first gradient and then performs seven additions, matching the
313
+ production accumulation order. Each case uses two seconds of burn-in and five
314
+ timing trials, each containing ten eight-gradient cycles.
315
+
316
+ | Weight-gradient family | Separate BF16 GEMM/cast/add | Fused cast/add | Direct FP32 GEMM accumulation |
317
+ | --- | ---: | ---: | ---: |
318
+ | FFN up/gate | 16.928 ms | 15.850 ms | 15.438 ms |
319
+ | Input projection | 11.804 ms | 11.308 ms | 11.620 ms |
320
+ | FFN down | 8.141 ms | 7.802 ms | 7.873 ms |
321
+
322
+ Fused conversion/add improves these complete gradient cycles by **4.3–6.8%**,
323
+ with zero output difference on all three tested fixtures. Scaling the measured
324
+ cycle differences over 28 blocks suggests approximately 54 ms per update,
325
+ around **1.2%** of profiled wall time. This is a hypothesis for integration,
326
+ not a measured whole-model improvement; autograd, DDP hooks and overlap remain
327
+ to be tested. Direct FP32 accumulation is not consistently faster than the
328
+ rounding-preserving fusion and differs by approximately 0.166–0.203% relative
329
+ RMS from the baseline on these synthetic gradients.
330
+
331
+ Final evidence is in `work/benchmark-gradient-distinct-20260910/`. The earlier
332
+ `work/benchmark-gradient-20260910/` experiment used eight identical inputs and
333
+ a zero-reset-plus-eight-add cycle; it is retained as exploratory evidence and
334
+ is not used for the table or integration estimate above.
335
+
336
+ ## Complete training-update experiments
337
+
338
+ Each successful case uses the same checkpoint and token cursor, two warmup
339
+ updates and 20 measured updates, full FP32 optimizer state and BF16 computation.
340
+ Global batch and context are unchanged. The table excludes validation and saves.
341
+
342
+ | Execution | Tokens/s | Gain over baseline | Peak reserved/GPU |
343
+ | --- | ---: | ---: | ---: |
344
+ | Production baseline, microbatch 4 | 29,081 | Reference | 68.64 GiB |
345
+ | Compiled full vocabulary loss, microbatch 4 | 30,050 | 3.3% | 68.99 GiB |
346
+ | Compiled full loss + BF16 residuals, microbatch 4 | 30,707 | 5.6% | 66.42 GiB |
347
+ | Chunked loss + BF16 residuals, microbatch 8 | 30,394 | 4.5% | 89.49 GiB |
348
+ | Compiled full loss + BF16 residuals, microbatch 8 | Out of memory | — | Exceeded 91 GiB cap |
349
+ | Baseline repeated after the other experiments | 29,073 | -0.03% | 68.64 GiB |
350
+
351
+ The repeated baseline also measured 20 updates and returned within 0.03% of
352
+ the initial aggregate throughput, supporting the small measured candidate gains.
353
+
354
+ The full-loss microbatch-8 trial failed while allocating its 1 GiB logits
355
+ buffer, before completing a warmup update. Retaining chunked loss made that
356
+ microbatch fit, but it was slower than microbatch-4 compiled-full with BF16
357
+ residuals. Available VRAM did not translate into a larger useful speed gain.
358
+
359
+ Against baseline's corresponding 20 updates, maximum absolute loss differences
360
+ were 0.000210 for compiled-full, 0.000235 for compiled-full plus BF16 residuals,
361
+ and 0.000146 for microbatch-8 chunked residuals. Maximum relative gradient-norm
362
+ differences were 0.236%, 0.384% and 0.362%, respectively. All completed updates
363
+ had finite losses and norms. These short comparisons do not establish long-run
364
+ quality or distributed checkpoint/resume equivalence for an experimental path.
365
+
366
+ The immediate measured compute improvement is approximately 3–6%. A further
367
+ implementation effort should prioritize gradient conversion/accumulation,
368
+ selective GEMM tuning, and checkpoint staging/overlap. Preserving the current
369
+ save frequency is especially useful after the recent abrupt shutdown. Production
370
+ remains paused at step 5,618; no experiment has been promoted to its launcher.
371
+
372
+ ## Reproduction and paused-state controls
373
+
374
+ These commands target the two-GPU workstation. The dedicated B200 host instead
375
+ uses `ops/b200ctl.sh` and its own handoff instructions.
376
+
377
+ Start at the repository root and inspect the production job. If it is running,
378
+ request a graceful stop and wait for `STOPPED` and the final checkpoint save
379
+ before starting a benchmark. A stopped job may make the status command return a
380
+ nonzero shell exit status; inspect its output.
381
+
382
+ ```bash
383
+ cd /workspace/WarpState
384
+ source rtx6000_40b/ops/training-env.sh
385
+ rtx6000_40b/ops/datactl.sh status warpstate-train
386
+ # Only when the production job is running:
387
+ rtx6000_40b/ops/datactl.sh stop warpstate-train
388
+ rtx6000_40b/ops/datactl.sh status warpstate-train
389
+ tail -n 12 rtx6000_40b/work/logs/train.log
390
+ ```
391
+
392
+ Use a fresh output directory for every disposable run. The benchmark restores
393
+ full optimizer state, preserves the global batch, and writes no checkpoint:
394
+
395
+ ```bash
396
+ cd /workspace/WarpState
397
+ source rtx6000_40b/ops/training-env.sh
398
+ rtx_profile_run=$(date -u +%Y%m%dT%H%M%S)
399
+ WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 TORCHINDUCTOR_COMPILE_THREADS=8 \
400
+ rtx6000_40b/ops/benchmark.sh \
401
+ --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt \
402
+ --data /workspace/WarpState/rtx6000_40b/work/data/train.bin \
403
+ --output-dir "/workspace/WarpState/rtx6000_40b/work/benchmark-rtx-${rtx_profile_run}/baseline" \
404
+ --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
405
+ --compile-cores --compile-mode default \
406
+ --no-activation-checkpointing --autocast-cache \
407
+ --loss-mode checkpointed --loss-chunk-size 1024 \
408
+ --optimizer-state resume --warmup-updates 2 --measured-updates 20 \
409
+ --vram-limit-gib 88 --threads 4 --ddp-bucket-cap-mb 128
410
+ ```
411
+
412
+ For the operator trace, use another fresh output directory, add `--profile`, and
413
+ `--diagnose-stalls`, and use `--measured-updates 1`. Only the first measured update is traced. Inspect
414
+ the output summary and both ranks' profile artifacts, then compare throughput
415
+ using separate unprofiled runs. Leave production paused after the experiments.
416
+
417
+ Candidate overrides, each with a new output directory, are `--loss-mode
418
+ compiled-full`; then additionally `--bf16-residuals`; and, for the larger batch,
419
+ `--micro-batch 8 --vram-limit-gib 91`. The successful larger-batch case retains
420
+ `--loss-mode checkpointed`. Do not run these GPU experiments concurrently.
421
+
422
+ The isolated matrix and gradient comparisons use no training data or checkpoint:
423
+
424
+ ```bash
425
+ cd /workspace/WarpState
426
+ source rtx6000_40b/ops/training-env.sh
427
+ export CUDA_VISIBLE_DEVICES=0
428
+ export TORCHINDUCTOR_COMPILE_THREADS=8
429
+ rtx6000_40b/.server-venv/bin/python rtx6000_40b/ops/benchmark_gemm.py \
430
+ --backends eager aten triton --warmup 10 --burnin-seconds 2 --repeats 30 --trials 5
431
+ rtx6000_40b/.server-venv/bin/python rtx6000_40b/ops/benchmark_gemm.py \
432
+ --backends eager --blas-backend cublaslt
433
+ ENABLE_PERSISTENT_TMA_MATMUL=1 rtx6000_40b/.server-venv/bin/python \
434
+ rtx6000_40b/ops/benchmark_gemm.py --families ffn_up_gate --backends triton
435
+ rtx6000_40b/.server-venv/bin/python \
436
+ rtx6000_40b/ops/benchmark_gradient_accumulation.py
437
+ ```
438
+
439
+ Each invocation defaults to a fresh timestamped output directory and refuses
440
+ to overwrite an existing directory. `--list-shapes` validates and lists the
441
+ selected layouts on CPU. GPU comparisons save source hashes, arguments,
442
+ precision settings, numerical errors and timing samples. The two new helpers
443
+ passed CPU syntax, layout/FLOP and accumulation-control checks, and actual BF16
444
+ GPU comparisons; they do not alter the production model or launcher.
445
+
446
+ When the user requests production resume, use the existing verified checkpoint
447
+ and `rtx6000_40b/ops/datactl.sh start warpstate-train`. The tmux session named
448
+ `warpstate-train` is the monitoring session; Supervisor controls the training
449
+ process. Never use a from-scratch launcher to resume this run.
450
+
451
+ Raw profiles, benchmark summaries, telemetry, and verification reports remain
452
+ ignored under `work/`. This source document records the recipe and interpretation
453
+ without adding generated artifacts or checkpoints to Git.
ops/TRAINING.md ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training environment and shared GPU configuration
2
+
3
+ This document records the original two-GPU workstation configuration. Its
4
+ Supervisor and launchers are not the controls for the single-B200 host.
5
+ Current B200 settings, checkpoint and pause/resume controls are in
6
+ [B200_HANDOFF.md](B200_HANDOFF.md). For workstation history, see
7
+ [OPTIMIZED_TRAINING.md](OPTIMIZED_TRAINING.md); the shared GPU settings below
8
+ preceded that workstation preset.
9
+
10
+ The completed corpus in `work/data` contains exactly 40,000,000,000 training
11
+ tokens and 1,048,576 validation tokens. Its generated manifest records the
12
+ SHA256 checksums. The model has 2,005,111,424 parameters; 40B is the token
13
+ budget, not the parameter count.
14
+
15
+ No initial WarpState checkpoint was supplied. The shared-GPU launcher uses
16
+ `--from-scratch` with seed 1337 for its first launch. Once `work/run/latest.pt`
17
+ exists, subsequent launches restore its model, full AdamW state, token cursor,
18
+ learning-rate schedule, and random states. `--from-scratch` never overwrites
19
+ an existing run. To initialize another run from trusted weights, invoke
20
+ `run_40b.py --weights ...` with a separate work directory instead.
21
+
22
+ ## Shared GPU settings
23
+
24
+ `train-shared.sh` selects physical GPU 1 by its UUID. VORTEX uses both GPUs,
25
+ so concurrent training shares GPU compute and can slow both jobs.
26
+ On another machine, set `WARPSTATE_CUDA_DEVICE` to the intended GPU UUID or
27
+ index before launching.
28
+
29
+ | Setting | Value |
30
+ | --- | --- |
31
+ | Sequence length | 2,048 tokens |
32
+ | Microbatch | 1 sequence |
33
+ | Gradient accumulation | 64 microbatches |
34
+ | Tokens per optimizer update | 131,072 |
35
+ | Parameters, gradients, AdamW moments | FP32 on the GPU |
36
+ | Forward/backward matrix operations | BF16 autocast |
37
+ | Activation checkpointing | Enabled |
38
+ | Autocast weight cache | Disabled to reduce retained weight copies |
39
+ | Loss projection chunks | 256 tokens |
40
+ | PyTorch allocator limit | 33.5 GiB |
41
+ | Required extra free memory at startup | 3 GiB |
42
+ | Validation | 8 batches initially and every 100 updates |
43
+ | Checkpoint | Every 50 updates, on a graceful stop, and at completion |
44
+
45
+ The September 9, 2026 live trial completed two full updates, saved a
46
+ 22.41 GiB checkpoint, and resumed through step 5. Peak memory was 31.54 GiB
47
+ allocated and 32.02 GiB reserved, leaving about 4.7 GiB physically free.
48
+ Observed training throughput was approximately 5,500–6,000 tokens/second.
49
+ At that unchanged rate, 40B tokens would take roughly 77–84 days before
50
+ checkpoint/evaluation overhead; this is an early measurement, not an ETA
51
+ that accounts for future GPU availability. VORTEX continued advancing,
52
+ with overlapping updates around 68–69K tokens/second versus roughly 100K
53
+ before the trial.
54
+
55
+ The fixed FP32 parameters, gradients, and two AdamW moments alone occupy
56
+ about 29.88 GiB. Activations and allocator overhead must also fit the cap.
57
+ The launcher refuses to start if the cap plus startup headroom exceeds
58
+ available memory. This is a check, not a reservation for VORTEX.
59
+
60
+ PyTorch's [allocator memory limit](https://docs.pytorch.org/docs/2.14/generated/torch.cuda.memory.set_per_process_memory_fraction.html)
61
+ excludes CUDA context and library allocations. Compare both the allocated
62
+ and reserved peaks in `work/run/metrics.jsonl` with physical memory in
63
+ `nvidia-smi`. A different VORTEX workload may need different headroom.
64
+
65
+ Each full checkpoint is approximately 22.4 GiB. Keep at least 53 GiB free
66
+ for checkpoint writing and atomic replacement. Checkpoints, metrics, data,
67
+ and environments stay inside this project and are ignored by Git.
68
+
69
+ ## Environment
70
+
71
+ The CUDA training environment is separate from the CPU data-preparation
72
+ environment and from VORTEX's environment. Recreate it with Python 3.12:
73
+
74
+ ```bash
75
+ cd /workspace/WarpState
76
+ source rtx6000_40b/ops/training-env.sh
77
+ python3 -m venv rtx6000_40b/.server-venv
78
+ uv pip install --python rtx6000_40b/.server-venv/bin/python \
79
+ -r rtx6000_40b/ops/training-requirements-resolved.txt \
80
+ --extra-index-url https://download.pytorch.org/whl/cu130 \
81
+ --index-strategy unsafe-best-match
82
+ uv pip check --python rtx6000_40b/.server-venv/bin/python
83
+ ```
84
+
85
+ The lock pins PyTorch 2.14.0+cu130 and its installed dependencies. Always
86
+ source `training-env.sh` when testing or provisioning so caches and temporary
87
+ files remain under `/workspace/WarpState`. The launcher sets these paths
88
+ itself and uses `--no-bootstrap`; the minimal GPU environment does not
89
+ contain the separate data-preparation dependencies.
90
+
91
+ ## Operation
92
+
93
+ The existing project-private Supervisor manages both preparation and training.
94
+ Its training command is now `train-fast.sh`, which needs both GPUs available.
95
+ `datactl.sh` is its control wrapper, including for the training job:
96
+
97
+ ```bash
98
+ cd /workspace/WarpState
99
+ rtx6000_40b/ops/datactl.sh status
100
+ rtx6000_40b/ops/datactl.sh start warpstate-train
101
+ tail -n 10 rtx6000_40b/work/logs/train.log
102
+ tail -n 3 rtx6000_40b/work/run/metrics.jsonl
103
+ nvidia-smi
104
+ ```
105
+
106
+ To stop, use `rtx6000_40b/ops/datactl.sh stop warpstate-train`. The runner
107
+ finishes its current optimizer update and saves before exiting. Supervisor
108
+ allows up to 15 minutes for that. Starting the same job resumes its latest
109
+ checkpoint. `--reconfigure` permits changes to execution settings while keeping
110
+ the sequence length, global tokens per update, data, and optimizer schedule
111
+ unchanged. Changes to accumulation or GPU count can change floating-point
112
+ rounding; the model and full optimizer state are retained.
113
+
114
+ Training is started explicitly and does not automatically restart after a
115
+ failure, avoiding repeated allocation attempts on a shared GPU. Inspect the
116
+ log and available memory before restarting a failed job. A container restart
117
+ also requires starting the project Supervisor and the training job again.
118
+
119
+ For a bounded trial of the current configuration, run
120
+ `ops/train-fast.sh --max-steps 2` from the package directory while the
121
+ supervised training job is stopped. This saves after two
122
+ additional updates while preserving the full 40B schedule. Only one runner
123
+ can hold `work/pipeline.lock` at a time.
124
+
125
+ To return to the original shared GPU configuration, stop the supervised job
126
+ and run `ops/train-shared.sh --reconfigure` with enough free memory on GPU 1.
127
+ For persistent supervision, update the program command to that launcher plus
128
+ `--reconfigure` and use `stopsignal=TERM` before `datactl.sh reread` and
129
+ `datactl.sh update warpstate-train`. The shared launcher runs the worker
130
+ directly; the optimized launcher relays stop requests to distributed workers.
131
+ Do not run the benchmark or another launcher alongside the production job.
132
+
133
+ ## Checks
134
+
135
+ The tests exercise interrupted data preparation, exact final partial updates,
136
+ the memory-headroom refusal, and exact CUDA checkpoint/resume behavior for
137
+ both imported weights and fresh initialization. The CUDA tests use a tiny
138
+ model and a 0.5 GiB allocator cap:
139
+
140
+ ```bash
141
+ cd /workspace/WarpState
142
+ source rtx6000_40b/ops/training-env.sh
143
+ CUDA_VISIBLE_DEVICES=1 rtx6000_40b/.server-venv/bin/python \
144
+ -m pytest rtx6000_40b/test_server.py -q
145
+ ```
146
+
147
+ Use `CUDA_VISIBLE_DEVICES=''` to skip CUDA tests when there is insufficient
148
+ free memory. Test artifacts use the project-local `TMPDIR`.
ops/activation_execution.py ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Opt-in BF16 SwiGLU and RMSNorm execution without replacing parameters.
2
+
3
+ Eager execution retains the model's BF16 rounding boundaries. Compiled execution
4
+ uses the current Inductor arithmetic, eliding intermediate BF16 casts while
5
+ retaining BF16 kernel inputs/outputs. Normalization statistics and master
6
+ parameter gradients remain FP32. FP32 arithmetic reductions can differ
7
+ slightly from Inductor's reduction order; this is an execution experiment, not a
8
+ claim of bitwise equivalence. CPU, evaluation and non-BF16 paths stay original.
9
+
10
+ Apply before compiling cores. Checkpoints and inference archives are unchanged.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import torch
16
+ import triton
17
+ import triton.language as tl
18
+ from triton.language.extra.cuda import libdevice
19
+ from torch.library import triton_op, wrap_triton
20
+
21
+
22
+ @triton.jit
23
+ def _swiglu_forward_kernel(X, Y, N: tl.constexpr, H: tl.constexpr,
24
+ BLOCK: tl.constexpr, ROUND: tl.constexpr):
25
+ i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
26
+ mask = i < N
27
+ packed = (i // H) * (2 * H) + i % H
28
+ up = tl.load(X + packed, mask, 0).to(tl.float32)
29
+ gate = tl.load(X + packed + H, mask, 0).to(tl.float32)
30
+ silu = gate / (1.0 + libdevice.exp(-gate))
31
+ if ROUND:
32
+ silu = silu.to(tl.bfloat16).to(tl.float32)
33
+ tl.store(Y + i, silu * up, mask)
34
+
35
+
36
+ @triton.jit
37
+ def _swiglu_backward_kernel(X, DY, DX, N: tl.constexpr, H: tl.constexpr,
38
+ BLOCK: tl.constexpr, ROUND: tl.constexpr):
39
+ i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
40
+ mask = i < N
41
+ packed = (i // H) * (2 * H) + i % H
42
+ up = tl.load(X + packed, mask, 0).to(tl.float32)
43
+ gate = tl.load(X + packed + H, mask, 0).to(tl.float32)
44
+ dy = tl.load(DY + i, mask, 0).to(tl.float32)
45
+ sigmoid = 1.0 / (1.0 + tl.exp(-gate))
46
+ silu = gate / (1.0 + libdevice.exp(-gate))
47
+ if ROUND:
48
+ silu = silu.to(tl.bfloat16).to(tl.float32)
49
+ # aten's multiply backward rounds before aten's silu backward executes.
50
+ dsilu = dy * up
51
+ if ROUND:
52
+ dsilu = dsilu.to(tl.bfloat16).to(tl.float32)
53
+ dgate = (dsilu * sigmoid) * (1.0 + gate * (1.0 - sigmoid))
54
+ tl.store(DX + packed, dy * silu, mask)
55
+ tl.store(DX + packed + H, dgate, mask)
56
+
57
+
58
+ @triton_op("warpstate::bf16_swiglu", mutates_args={})
59
+ def bf16_swiglu(packed: torch.Tensor, eager_rounding: bool = True) -> torch.Tensor:
60
+ """SwiGLU of a contiguous BF16 CUDA [*, 2*hidden] projection.
61
+
62
+ Set ``eager_rounding=False`` to match the current compiled core arithmetic.
63
+ """
64
+ hidden = packed.shape[-1] // 2
65
+ output = torch.empty((*packed.shape[:-1], hidden), device=packed.device,
66
+ dtype=packed.dtype)
67
+ wrap_triton(_swiglu_forward_kernel)[(triton.cdiv(output.numel(), 1024),)](
68
+ packed, output, output.numel(), hidden, 1024, eager_rounding,
69
+ enable_fp_fusion=not eager_rounding)
70
+ return output
71
+
72
+
73
+ @triton_op("warpstate::bf16_swiglu_backward", mutates_args={})
74
+ def _swiglu_backward(packed: torch.Tensor, grad: torch.Tensor,
75
+ eager_rounding: bool) -> torch.Tensor:
76
+ grad = grad.contiguous()
77
+ output = torch.empty_like(packed)
78
+ wrap_triton(_swiglu_backward_kernel)[(triton.cdiv(grad.numel(), 1024),)](
79
+ packed, grad, output, grad.numel(), packed.shape[-1] // 2, 1024, eager_rounding,
80
+ enable_fp_fusion=not eager_rounding)
81
+ return output
82
+
83
+
84
+ def _swiglu_setup(ctx, inputs, output):
85
+ ctx.save_for_backward(inputs[0])
86
+ ctx.eager_rounding = inputs[1]
87
+
88
+
89
+ def _swiglu_autograd(ctx, grad):
90
+ return _swiglu_backward(ctx.saved_tensors[0], grad, ctx.eager_rounding), None
91
+
92
+
93
+ bf16_swiglu.register_autograd(_swiglu_autograd, setup_context=_swiglu_setup)
94
+
95
+
96
+ @triton.jit
97
+ def _rms_forward_kernel(X, W, Y, R, D: tl.constexpr, EPS: tl.constexpr,
98
+ BLOCK: tl.constexpr, ROUND: tl.constexpr):
99
+ row = tl.program_id(0)
100
+ col = tl.arange(0, BLOCK)
101
+ x = tl.load(X + row * D + col, col < D, 0).to(tl.float32)
102
+ w = tl.load(W + col, col < D, 0)
103
+ if ROUND:
104
+ w = w.to(tl.bfloat16).to(tl.float32)
105
+ r = tl.rsqrt(tl.sum(x * x, 0) / D + EPS)
106
+ z = x * r
107
+ if ROUND:
108
+ z = z.to(tl.bfloat16).to(tl.float32)
109
+ tl.store(Y + row * D + col, z * w, col < D)
110
+ tl.store(R + row, r)
111
+
112
+
113
+ @triton.jit
114
+ def _rms_backward_kernel(X, W, R, DY, DX, DW_PART,
115
+ M: tl.constexpr, D: tl.constexpr,
116
+ ROWS: tl.constexpr, BLOCK: tl.constexpr, ROUND: tl.constexpr):
117
+ group = tl.program_id(0)
118
+ rows = group * ROWS + tl.arange(0, ROWS)
119
+ cols = tl.arange(0, BLOCK)
120
+ mask = (rows[:, None] < M) & (cols[None, :] < D)
121
+ x = tl.load(X + rows[:, None] * D + cols[None, :], mask, 0).to(tl.float32)
122
+ dy = tl.load(DY + rows[:, None] * D + cols[None, :], mask, 0).to(tl.float32)
123
+ w = tl.load(W + cols, cols < D, 0)
124
+ if ROUND:
125
+ w = w.to(tl.bfloat16).to(tl.float32)
126
+ r = tl.load(R + rows, rows < M, 0)
127
+ z = x * r[:, None]
128
+ dz = dy * w[None, :]
129
+ if ROUND:
130
+ z = z.to(tl.bfloat16).to(tl.float32)
131
+ dz = dz.to(tl.bfloat16).to(tl.float32)
132
+ dr = tl.sum(dz * x, 1)
133
+ # Keep the same explicit derivative chain as rsqrt(mean(x**2) + eps).
134
+ dsquare = (-0.5 * dr * (r * r * r)) / D
135
+ dx = dz * r[:, None] + dsquare[:, None] * (2.0 * x)
136
+ dw = dy * z
137
+ if ROUND:
138
+ dw = dw.to(tl.bfloat16).to(tl.float32)
139
+ partial = tl.sum(dw, 0)
140
+ tl.store(DX + rows[:, None] * D + cols[None, :], dx, mask)
141
+ tl.store(DW_PART + group * D + cols, partial, cols < D)
142
+
143
+
144
+ @triton.jit
145
+ def _rms_weight_reduce_kernel(PART, DW, D: tl.constexpr, GROUPS: tl.constexpr,
146
+ BLOCK_G: tl.constexpr, BLOCK_D: tl.constexpr, ROUND: tl.constexpr):
147
+ cols = tl.program_id(0) * BLOCK_D + tl.arange(0, BLOCK_D)
148
+ groups = tl.arange(0, BLOCK_G)
149
+ partial = tl.load(PART + groups[:, None] * D + cols[None, :],
150
+ (groups[:, None] < GROUPS) & (cols[None, :] < D), 0)
151
+ # Eager weight.to(BF16) makes sum-to-size return BF16 before its gradient is
152
+ # cast back to FP32. The compiled path retains the FP32 reduction; Inductor's
153
+ # own partitioning can retain a final BF16 cast at some large shapes, so this
154
+ # comparison is numerically bounded rather than universally bitwise equal.
155
+ total = tl.sum(partial, 0)
156
+ if ROUND:
157
+ total = total.to(tl.bfloat16).to(tl.float32)
158
+ tl.store(DW + cols, total, cols < D)
159
+
160
+
161
+ @triton_op("warpstate::bf16_rmsnorm", mutates_args={})
162
+ def bf16_rmsnorm(x: torch.Tensor, weight: torch.Tensor,
163
+ eps: float, eager_rounding: bool = True) -> tuple[torch.Tensor, torch.Tensor]:
164
+ """BF16 CUDA RMSNorm with FP32 statistics; also return saved row rsqrts."""
165
+ width = x.shape[-1]
166
+ rows = x.numel() // width
167
+ output = torch.empty_like(x)
168
+ rstd = torch.empty((rows,), device=x.device, dtype=torch.float32)
169
+ block = triton.next_power_of_2(width)
170
+ wrap_triton(_rms_forward_kernel)[(rows,)](
171
+ x, weight, output, rstd, width, eps, block, eager_rounding,
172
+ num_warps=4 if block <= 4096 else 8, enable_fp_fusion=not eager_rounding)
173
+ return output, rstd
174
+
175
+
176
+ @triton_op("warpstate::bf16_rmsnorm_backward", mutates_args={})
177
+ def _rms_backward(x: torch.Tensor, weight: torch.Tensor, rstd: torch.Tensor,
178
+ grad: torch.Tensor, eager_rounding: bool) -> tuple[torch.Tensor, torch.Tensor]:
179
+ grad = grad.contiguous()
180
+ width = x.shape[-1]
181
+ rows = x.numel() // width
182
+ group_rows = 4
183
+ groups = triton.cdiv(rows, group_rows)
184
+ output = torch.empty_like(x)
185
+ partial = torch.empty((groups, width), device=x.device, dtype=torch.float32)
186
+ weight_grad = torch.empty_like(weight)
187
+ block = triton.next_power_of_2(width)
188
+ wrap_triton(_rms_backward_kernel)[(groups,)](
189
+ x, weight, rstd, grad, output, partial, rows, width, group_rows, block, eager_rounding,
190
+ num_warps=4 if block <= 2048 else 8, enable_fp_fusion=not eager_rounding)
191
+ wrap_triton(_rms_weight_reduce_kernel)[(triton.cdiv(width, 8),)](
192
+ partial, weight_grad, width, groups, triton.next_power_of_2(groups), 8, eager_rounding,
193
+ num_warps=8, enable_fp_fusion=False)
194
+ return output, weight_grad
195
+
196
+
197
+ def _rms_setup(ctx, inputs, output):
198
+ ctx.save_for_backward(inputs[0], inputs[1], output[1])
199
+ ctx.mark_non_differentiable(output[1])
200
+ ctx.eager_rounding = inputs[3]
201
+
202
+
203
+ def _rms_autograd(ctx, grad, _grad_rstd):
204
+ x, weight, rstd = ctx.saved_tensors
205
+ dx, dw = _rms_backward(x, weight, rstd, grad, ctx.eager_rounding)
206
+ return dx, dw, None, None
207
+
208
+
209
+ bf16_rmsnorm.register_autograd(_rms_autograd, setup_context=_rms_setup)
210
+
211
+
212
+ def apply_activation_execution(model: torch.nn.Module, mode: str = "baseline") -> None:
213
+ """Patch training forwards only, retaining canonical FP32 parameter objects.
214
+
215
+ Modes select fused ``swiglu``, ``rmsnorm``, or ``all``; ``baseline`` is a no-op.
216
+ Inputs outside the supported contiguous BF16 CUDA shapes use eager model
217
+ code. Evaluation retains the original path, avoiding validation graph churn.
218
+ Eager invocation preserves eager casts; core graph capture selects the
219
+ current Inductor arithmetic with elided intermediate casts.
220
+ """
221
+ from warpstate.model import RMSNorm, SwiGLU
222
+
223
+ if mode not in {"baseline", "swiglu", "rmsnorm", "all"}:
224
+ raise ValueError(f"Unknown activation execution mode: {mode}")
225
+ if getattr(model, "_activation_execution_mode", "baseline") != "baseline":
226
+ raise ValueError("Activation execution has already been applied")
227
+ if mode == "baseline":
228
+ return
229
+ if any(p.dtype != torch.float32 for p in model.parameters()):
230
+ raise ValueError("Activation execution requires FP32 master parameters")
231
+
232
+ for module in model.modules():
233
+ original = module.forward
234
+ if isinstance(module, SwiGLU) and mode in {"swiglu", "all"}:
235
+ def swiglu_forward(x, _module=module, _original=original):
236
+ if (_module.training and torch.is_grad_enabled() and x.is_cuda
237
+ and torch.is_autocast_enabled("cuda")
238
+ and torch.get_autocast_dtype("cuda") == torch.bfloat16):
239
+ packed = _module.up_gate(x)
240
+ return _module.down(bf16_swiglu(packed.contiguous(),
241
+ eager_rounding=not torch.compiler.is_compiling()))
242
+ return _original(x)
243
+ module.forward = swiglu_forward
244
+ elif isinstance(module, RMSNorm) and mode in {"rmsnorm", "all"}:
245
+ def rms_forward(x, _module=module, _original=original):
246
+ if (_module.training and torch.is_grad_enabled() and x.is_cuda
247
+ and x.dtype == torch.bfloat16 and x.is_contiguous()
248
+ and 0 < x.shape[-1] <= 8192 and x.numel() > 0):
249
+ return bf16_rmsnorm(x, _module.weight, _module.eps,
250
+ eager_rounding=not torch.compiler.is_compiling())[0]
251
+ return _original(x)
252
+ module.forward = rms_forward
253
+ model._activation_execution_mode = mode
ops/analyze_communication_profile.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Summarize compute/communication overlap from saved Chrome traces, without CUDA.
3
+
4
+ Each rank and device is reported separately. All time values use microseconds.
5
+ Noncommunication activity includes kernels, copies, and fills. Logical NCCL
6
+ input payload bytes are distinct from bytes physically transported over a link.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ from collections import Counter
13
+ import hashlib
14
+ import json
15
+ import math
16
+ from pathlib import Path
17
+ import re
18
+
19
+
20
+ PACKAGE = Path(__file__).resolve().parents[1]
21
+ WORKSPACE = PACKAGE.parent
22
+ ACTIVITY_CATEGORIES = {"kernel", "gpu_memcpy", "gpu_memset"}
23
+ COLLECTIVE_OPERATORS = {
24
+ "c10d::allreduce_", "c10d::allreduce_coalesced_", "c10d::allgather_",
25
+ "c10d::_allgather_base_", "c10d::allgather_into_tensor_coalesced_",
26
+ "c10d::reduce_scatter_", "c10d::_reduce_scatter_base_",
27
+ "c10d::reduce_scatter_tensor_coalesced_", "c10d::broadcast_", "c10d::reduce_",
28
+ "c10d::alltoall_", "c10d::alltoall_base_", "c10d::gather_", "c10d::scatter_",
29
+ "c10d::barrier",
30
+ }
31
+ DTYPE_BYTES = {"float": 4, "float32": 4, "double": 8, "float64": 8,
32
+ "half": 2, "float16": 2, "bfloat16": 2, "long": 8,
33
+ "int64": 8, "int": 4, "int32": 4, "short": 2,
34
+ "int16": 2, "char": 1, "int8": 1, "byte": 1, "uint8": 1,
35
+ "bool": 1}
36
+
37
+
38
+ def merge_intervals(intervals):
39
+ """Return disjoint intervals, counting overlapping streams only once."""
40
+ merged = []
41
+ for begin, end in sorted(intervals):
42
+ if not math.isfinite(begin) or not math.isfinite(end) or end < begin:
43
+ raise ValueError("Activity intervals must have finite, ordered endpoints")
44
+ if begin == end:
45
+ continue
46
+ if merged and begin <= merged[-1][1]:
47
+ merged[-1] = (merged[-1][0], max(end, merged[-1][1]))
48
+ else:
49
+ merged.append((begin, end))
50
+ return merged
51
+
52
+
53
+ def interval_duration(intervals):
54
+ return sum(end - begin for begin, end in intervals)
55
+
56
+
57
+ def intersection_duration(left, right):
58
+ """Measure intersection of two disjoint, ordered interval lists."""
59
+ total = 0.0
60
+ i = j = 0
61
+ while i < len(left) and j < len(right):
62
+ total += max(0.0, min(left[i][1], right[j][1]) - max(left[i][0], right[j][0]))
63
+ if left[i][1] <= right[j][1]:
64
+ i += 1
65
+ else:
66
+ j += 1
67
+ return total
68
+
69
+
70
+ def collective_metadata(kernels):
71
+ """Deduplicate channel kernels using process-group name and sequence number.
72
+
73
+ Unknown/inconsistent metadata yields null totals instead of guessed bytes or
74
+ a kernel count mislabeled as a collective count. Scalar metric collectives
75
+ are included; the returned sequence permits separating them from gradients.
76
+ """
77
+ calls = {}
78
+ missing_identity = 0
79
+ for event in sorted(kernels, key=lambda item: item["ts"]):
80
+ args = event.get("args", {})
81
+ sequence_id = args.get("Seq")
82
+ if (args.get("Process Group Name") is None or not args.get("Collective name")
83
+ or type(sequence_id) not in (int, str) or not str(sequence_id).isdigit()):
84
+ missing_identity += 1
85
+ continue
86
+ key = (str(args["Process Group Name"]), str(args["Seq"]))
87
+ count = args.get("In msg nelems")
88
+ width = DTYPE_BYTES.get(str(args.get("dtype", "")).lower())
89
+ byte_count = count * width if type(count) is int and count >= 0 and width else None
90
+ row = {"process_group": key[0], "sequence": key[1],
91
+ "collective": args["Collective name"], "dtype": args.get("dtype"),
92
+ "input_elements": count, "input_payload_bytes": byte_count}
93
+ previous = calls.get(key)
94
+ if previous is None:
95
+ calls[key] = {**row, "kernel_count": 1, "metadata_consistent": True}
96
+ else:
97
+ previous["kernel_count"] += 1
98
+ if any(previous[name] != value for name, value in row.items()):
99
+ previous["metadata_consistent"] = False
100
+ previous["input_payload_bytes"] = None
101
+ sequence = list(calls.values())
102
+ known_bytes = sum(row["input_payload_bytes"] or 0 for row in sequence)
103
+ complete = not missing_identity and all(row["metadata_consistent"] and
104
+ row["input_payload_bytes"] is not None
105
+ for row in sequence)
106
+ return {"nccl_kernel_count": len(kernels),
107
+ "collective_call_count": len(sequence) if not missing_identity else None,
108
+ "known_collective_call_count": len(sequence),
109
+ "kernels_missing_collective_identity": missing_identity,
110
+ "logical_input_payload_bytes": known_bytes if complete else None,
111
+ "known_logical_input_payload_bytes": known_bytes,
112
+ "collectives": sequence}
113
+
114
+
115
+ def cpu_collective_fingerprint(events):
116
+ """Structural CPU submission order, never double-counting NCCL annotations.
117
+
118
+ Shapes/types do not identify tensor contents. This can detect many bucket
119
+ order changes but cannot prove equal tensor order for equal-shaped buckets.
120
+ """
121
+ selected = sorted((event for event in events if event.get("ph") == "X"
122
+ and event.get("cat") == "cpu_op"
123
+ and event.get("name") in COLLECTIVE_OPERATORS),
124
+ key=lambda event: event["ts"])
125
+ sequence = [{"operator": event["name"],
126
+ "input_dims": event.get("args", {}).get("Input Dims"),
127
+ "input_types": event.get("args", {}).get("Input type")}
128
+ for event in selected]
129
+ payload = json.dumps(sequence, sort_keys=True, separators=(",", ":"))
130
+ return {"call_count": len(sequence), "sequence": sequence,
131
+ "sha256": hashlib.sha256(payload.encode()).hexdigest() if sequence else None,
132
+ "all_input_shapes_recorded": all(row["input_dims"] is not None for row in sequence)}
133
+
134
+
135
+ def summarize_trace(trace):
136
+ events = trace.get("traceEvents", [])
137
+ devices = {}
138
+ excluded = 0
139
+ for event in events:
140
+ if event.get("ph") != "X":
141
+ continue
142
+ category = event.get("cat")
143
+ if category == "gpu_user_annotation":
144
+ excluded += 1
145
+ if category not in ACTIVITY_CATEGORIES:
146
+ continue
147
+ begin, duration = event.get("ts"), event.get("dur")
148
+ if (not isinstance(begin, (int, float)) or not isinstance(duration, (int, float))
149
+ or not math.isfinite(begin) or not math.isfinite(duration) or duration < 0):
150
+ raise ValueError("CUDA activities require finite timestamps and nonnegative durations")
151
+ device = str(event.get("args", {}).get("device", event.get("pid", "unknown")))
152
+ group = devices.setdefault(device, {"communication": [], "compute": [], "kernels": []})
153
+ communication = category == "kernel" and "nccl" in event.get("name", "").lower()
154
+ group["communication" if communication else "compute"].append((begin, begin + duration))
155
+ if communication:
156
+ group["kernels"].append(event)
157
+ summaries = []
158
+ for device, group in sorted(devices.items()):
159
+ compute = merge_intervals(group["compute"])
160
+ communication = merge_intervals(group["communication"])
161
+ active = merge_intervals([*compute, *communication])
162
+ overlap = intersection_duration(compute, communication)
163
+ compute_us = interval_duration(compute)
164
+ communication_us = interval_duration(communication)
165
+ span = active[-1][1] - active[0][0] if active else 0.0
166
+ active_us = interval_duration(active)
167
+ summaries.append({"device": device, "activity_span_us": span,
168
+ "active_union_us": active_us, "idle_within_activity_span_us": span - active_us,
169
+ "noncommunication_union_us": compute_us,
170
+ "communication_union_us": communication_us,
171
+ "compute_only_us": max(0.0, compute_us - overlap),
172
+ "compute_communication_overlap_us": overlap,
173
+ "communication_only_us": max(0.0, communication_us - overlap),
174
+ "communication_overlap_fraction": overlap / communication_us if communication_us else None,
175
+ "summed_noncommunication_us": interval_duration(group["compute"]),
176
+ "summed_communication_us": interval_duration(group["communication"]),
177
+ "nccl_kernel_names": dict(Counter(event["name"] for event in group["kernels"])),
178
+ **collective_metadata(group["kernels"])})
179
+ return {"rank": trace.get("distributedInfo", {}).get("rank"),
180
+ "distributed_info": trace.get("distributedInfo"), "devices": summaries,
181
+ "excluded_gpu_annotation_ranges": excluded,
182
+ "cpu_collective_fingerprint": cpu_collective_fingerprint(events)}
183
+
184
+
185
+ def main(argv=None):
186
+ parser = argparse.ArgumentParser(description=__doc__)
187
+ parser.add_argument("traces", nargs="+", type=Path)
188
+ parser.add_argument("--output", required=True, type=Path)
189
+ args = parser.parse_args(argv)
190
+ args.traces = [path.expanduser().resolve() for path in args.traces]
191
+ args.output = args.output.expanduser().resolve()
192
+ if any(not path.is_relative_to(WORKSPACE) for path in args.traces):
193
+ parser.error("Trace inputs must stay inside the WarpState workspace")
194
+ if not args.output.is_relative_to(PACKAGE / "work") or args.output.suffix != ".json":
195
+ parser.error("Output must be a JSON file under rtx6000_40b/work")
196
+ if args.output.exists():
197
+ parser.error("Output already exists; select a new report path")
198
+ report = {"format": "warpstate-communication-profile-v1",
199
+ "analyzer_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
200
+ "ranks": [], "limitations": [
201
+ "Per-rank/device budgets; never sum ranks or devices into a wall-time estimate.",
202
+ "Compute includes all non-NCCL kernels, memory copies, and fills.",
203
+ "GPU annotation ranges are excluded; real CUDA activities alone define intervals.",
204
+ "Communication-only time is an exposed interval, not a guaranteed removable delay.",
205
+ "Payload bytes describe logical collective inputs, not physical transport traffic.",
206
+ "Scalar benchmark metric collectives remain included in counts and bytes.",
207
+ "CPU fingerprints compare operator/shape/type submission sequences, not tensor identities.",
208
+ "Activity spans exclude untraced startup/tail time; profiling adds instrumentation overhead."]}
209
+ for path in args.traces:
210
+ payload = path.read_bytes()
211
+ summary = summarize_trace(json.loads(payload))
212
+ if summary["rank"] is None:
213
+ match = re.search(r"profile-rank-(\d+)", path.name)
214
+ if match:
215
+ summary["rank"] = int(match.group(1))
216
+ report["ranks"].append({"trace": str(path), "trace_sha256": hashlib.sha256(payload).hexdigest(),
217
+ **summary})
218
+ args.output.parent.mkdir(parents=True, exist_ok=True)
219
+ temporary = args.output.with_name(args.output.name + ".partial")
220
+ with temporary.open("x") as output:
221
+ output.write(json.dumps(report, indent=2, allow_nan=False) + "\n")
222
+ temporary.replace(args.output)
223
+ print(json.dumps({"output": str(args.output), "traces": len(report["ranks"])}))
224
+
225
+
226
+ if __name__ == "__main__":
227
+ main()
ops/archive_dpo.py ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Seal a completed DPO run as independent, read-only full-state/model archives.
3
+
4
+ Run from /workspace/WarpState after the DPO guardian exits successfully. The
5
+ default destination is work/archives/step-00076294-dpo-final. Existing archives
6
+ are verified and reused; they are never replaced or modified.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import contextlib
13
+ import json
14
+ import math
15
+ import os
16
+ from pathlib import Path
17
+ import stat
18
+ import subprocess
19
+ import sys
20
+ import tempfile
21
+
22
+ PACKAGE = Path(__file__).resolve().parents[1]
23
+ WORKSPACE = PACKAGE.parent
24
+ sys.path.insert(0, str(PACKAGE))
25
+
26
+ import numpy as np
27
+ import torch
28
+
29
+ from dpo import DEFAULTS, identity_digest
30
+ from ops.dpo_handoff import load_plan, read_json, seal, verify_files, verify_dpo_completion
31
+ from ops.launch_training import workspace_path
32
+ from ops.posttrain_handoff import (atomic_json, clean_exit, copy_regular, exclusive_lock,
33
+ process_status, publish_directory, sha256, tensors_equal, utc_now)
34
+
35
+ FILES = ('checkpoint.pt', 'model.pt', 'snapshot.json', 'tokenizer.model', 'plan.json',
36
+ 'settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl',
37
+ 'dpo-execution.json', 'dpo-handoff.json', 'gpu-preflight.json',
38
+ 'data-manifest.json', 'reference-manifest.json', 'reference-train.npy',
39
+ 'reference-val.npy', 'PROVENANCE.json', 'ARCHIVE.md')
40
+
41
+
42
+ def validate_state(checkpoint, evaluation, completion, settings_report, data, reference, plan):
43
+ """Validate an archive using its own preserved records, without a live run."""
44
+ settings = DEFAULTS | plan['training']
45
+ processes = plan['handoff']['processes']
46
+ train, val = (data['outputs'][name] for name in ('dpo_train.jsonl', 'dpo_val.jsonl'))
47
+ total_steps = math.ceil(train['rows'] / (settings['micro_batch'] * settings['grad_accum'] * processes)) * settings['epochs']
48
+ if settings['max_steps'] is not None:
49
+ raise ValueError('Archive requires a completed whole DPO epoch, without a plan step cap')
50
+ expected = dict(step=total_steps, total_steps=total_steps,
51
+ sample_cursor=train['rows'] * settings['epochs'],
52
+ tokens_seen=(train['tokens'] - 2 * train['rows']) * settings['epochs'],
53
+ response_tokens_seen=train['supervised_tokens'] * settings['epochs'], val_step=total_steps)
54
+ if (checkpoint.get('format') != 'warpstate-dpo-v1' or checkpoint.get('completed') is not True or
55
+ any(checkpoint.get(key) != value for key, value in expected.items()) or
56
+ not math.isfinite(checkpoint.get('val_loss', float('nan')))):
57
+ raise ValueError('DPO checkpoint is not the fully completed, finally evaluated state')
58
+ signature = checkpoint['signature']
59
+ if (signature.get('plan') != plan or signature.get('settings') != settings or
60
+ signature.get('world_size') != processes or
61
+ signature.get('tokenizer_sha256') != plan['sft']['tokenizer_sha256'] or
62
+ signature.get('data_sha256') != plan['handoff']['data_manifest_sha256'] or
63
+ settings_report.get('signature') != signature or
64
+ settings_report.get('total_steps') != total_steps or
65
+ settings_report.get('train_rows') != train['rows'] or
66
+ settings_report.get('val_rows') != val['rows']):
67
+ raise ValueError('DPO checkpoint/settings signatures or dataset counts differ')
68
+ if (data.get('format') != 'warpstate-posttrain-data-v1' or data.get('complete') is not True or
69
+ data.get('tokenizer_sha256') != signature['tokenizer_sha256'] or
70
+ data.get('max_length') != settings['sequence_length']):
71
+ raise ValueError('DPO data provenance differs from its signature')
72
+ origin = checkpoint['origin']
73
+ if (origin.get('stage') != 'sft' or origin.get('step') != plan['sft']['step'] or
74
+ origin.get('weights_sha256') != signature['base_sha256'] or
75
+ origin.get('tokens_seen') != plan['sft']['tokens_seen'] or
76
+ origin.get('origin', {}).get('step') != plan['sft']['base_step'] or
77
+ origin.get('origin', {}).get('tokens_seen') != plan['sft']['base_tokens_seen']):
78
+ raise ValueError('DPO SFT/raw ancestry differs from the authorized plan')
79
+ if (not checkpoint.get('optimizer', {}).get('state') or
80
+ len(checkpoint.get('rng_by_rank', [])) != processes or
81
+ any('torch' not in value or 'cuda' not in value for value in checkpoint['rng_by_rank'])):
82
+ raise ValueError('DPO optimizer or per-rank RNG state is missing')
83
+ for value in checkpoint['optimizer']['state'].values():
84
+ if float(value['step']) != total_steps:
85
+ raise ValueError('DPO Adam step differs from the final checkpoint')
86
+ for name in ('exp_avg', 'exp_avg_sq'):
87
+ if value[name].dtype != torch.float32:
88
+ raise ValueError('DPO Adam state must retain its original FP32 tensors')
89
+ if not checkpoint.get('model'):
90
+ raise ValueError('DPO model tensors are missing')
91
+ for name, tensor in checkpoint['model'].items():
92
+ if tensor.is_floating_point() and tensor.dtype != torch.float32:
93
+ raise ValueError(f'DPO model tensor is not original FP32: {name}')
94
+ counters = ('step', 'total_steps', 'sample_cursor', 'tokens_seen', 'response_tokens_seen', 'val_loss')
95
+ if (completion.get('format') != 'warpstate-dpo-completion-v1' or completion.get('stage') != 'dpo' or
96
+ completion.get('completed') is not True or completion.get('paused_for_evaluation') is not True or
97
+ any(completion.get(key) != checkpoint[key] for key in counters)):
98
+ raise ValueError('DPO completion report differs from its checkpoint')
99
+ if (evaluation.get('stage') != 'dpo' or evaluation.get('step') != total_steps or
100
+ evaluation.get('full_holdout') is not True or evaluation.get('evaluated_samples') != val['rows'] or
101
+ evaluation.get('available_validation_samples') != val['rows'] or
102
+ evaluation.get('objective') != settings['objective'] or
103
+ any(evaluation.get(key) != checkpoint['validation'].get(key) for key in
104
+ ('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll', 'evaluated_samples')) or
105
+ any(evaluation.get(key) != signature[key] for key in
106
+ ('base_sha256', 'data_sha256', 'tokenizer_sha256', 'reference_sha256')) or
107
+ not all(math.isfinite(evaluation.get(key, float('nan'))) for key in
108
+ ('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'))):
109
+ raise ValueError('DPO final full-holdout evaluation differs from its checkpoint')
110
+ if not 0 <= evaluation['reward_accuracy'] <= 1 or evaluation['chosen_nll'] < 0:
111
+ raise ValueError('DPO evaluation contains impossible accuracy/NLL')
112
+ identity = reference.get('identity', {})
113
+ if (reference.get('format') != 'warpstate-dpo-reference-v1' or reference.get('complete') is not True or
114
+ reference.get('identity_sha256') != identity_digest(identity) or
115
+ identity.get('weights_sha256') != signature['base_sha256'] or
116
+ identity.get('data_sha256') != signature['data_sha256'] or
117
+ identity.get('tokenizer_sha256') != signature['tokenizer_sha256'] or
118
+ identity.get('sources') != signature['sources'] or
119
+ identity.get('renderer_sha256') != data.get('renderer_sha256') or
120
+ identity.get('mask_format') != signature['mask'] or
121
+ identity.get('device_type') != signature['device_type'] or
122
+ identity.get('world_size') != processes or
123
+ identity.get('torch_version') != signature['torch_version'] or
124
+ any(value != settings.get(key) for key, value in identity.get('settings', {}).items()) or
125
+ completion.get('reference_sha256') != signature['reference_sha256']):
126
+ raise ValueError('DPO fixed-reference identity differs from its final checkpoint')
127
+ return expected
128
+
129
+
130
+ def verify_dpo_archive(folder, *, expected_checkpoint_hash=None):
131
+ folder = Path(folder)
132
+ entries = verify_files(folder, FILES)
133
+ if stat.S_IMODE(folder.stat().st_mode) & 0o222 or {path.name for path in folder.iterdir()} != set(FILES) | {'SHA256SUMS'}:
134
+ raise ValueError('DPO archive directory is not sealed or contains unexpected files')
135
+ if expected_checkpoint_hash and entries['checkpoint.pt'] != expected_checkpoint_hash:
136
+ raise ValueError('Archived DPO state differs from the source checkpoint')
137
+ checkpoint = torch.load(folder / 'checkpoint.pt', map_location='cpu', weights_only=False, mmap=True)
138
+ plan, data = read_json(folder / 'plan.json'), read_json(folder / 'data-manifest.json')
139
+ reference = read_json(folder / 'reference-manifest.json')
140
+ validate_state(checkpoint, read_json(folder / 'evaluation.json'), read_json(folder / 'completion.json'),
141
+ read_json(folder / 'settings.json'), data, reference, plan)
142
+ if (entries['data-manifest.json'] != checkpoint['signature']['data_sha256'] or
143
+ entries['reference-manifest.json'] != checkpoint['signature']['reference_sha256'] or
144
+ entries['tokenizer.model'] != checkpoint['signature']['tokenizer_sha256']):
145
+ raise ValueError('Archived data/reference/tokenizer hashes differ from checkpoint signatures')
146
+ for split in ('train', 'val'):
147
+ entry = reference['files'][split + '.npy']
148
+ name = f'reference-{split}.npy'
149
+ rows = data['outputs'][f'dpo_{split}.jsonl']['rows']
150
+ values = np.load(folder / name, allow_pickle=False, mmap_mode='r')
151
+ if (entries[name] != entry['sha256'] or entry['rows'] != rows or
152
+ values.shape != (rows, 2) or values.dtype != np.float64 or
153
+ not np.isfinite(values).all() or not (values <= 0).all()):
154
+ raise ValueError('Archived fixed-reference score array differs')
155
+ metadata = read_json(folder / 'snapshot.json')
156
+ if (metadata.get('format') != 'warpstate-chat-v1' or metadata.get('inference_only') is not True or
157
+ metadata.get('stage') != 'dpo' or metadata.get('config') != checkpoint['config'] or
158
+ any(metadata.get(key) != checkpoint[key] for key in
159
+ ('step', 'tokens_seen', 'response_tokens_seen', 'val_loss', 'origin')) or
160
+ metadata.get('tokenizer_sha256') != entries['tokenizer.model'] or
161
+ metadata.get('snapshot_sha256') != entries['model.pt'] or
162
+ metadata.get('snapshot_bytes') != (folder / 'model.pt').stat().st_size or
163
+ metadata.get('source', {}).get('sha256') != entries['checkpoint.pt']):
164
+ raise ValueError('DPO model export metadata differs from the full archive')
165
+ exported = torch.load(folder / 'model.pt', map_location='cpu', weights_only=False, mmap=True)
166
+ if {key: value for key, value in exported.items() if key != 'model'} != {
167
+ key: value for key, value in metadata.items() if key not in ('snapshot_sha256', 'snapshot_bytes')}:
168
+ raise ValueError('DPO exported model metadata differs from snapshot.json')
169
+ tensors_equal(checkpoint['model'], exported['model'])
170
+ return metadata, entries
171
+
172
+
173
+ def archive_dpo(work, tokenizer, destination, plan, revision):
174
+ """Caller owns all training exclusion locks; no source files are changed."""
175
+ work, tokenizer, destination = map(Path, (work, tokenizer, destination))
176
+ run, reference_dir = work / 'run', work / 'reference-cache'
177
+ source = run / 'latest.pt'
178
+ if source.is_symlink() or not source.is_file():
179
+ raise ValueError('DPO checkpoint must be a regular completed file')
180
+ before = source.stat()
181
+ checkpoint = torch.load(source, map_location='cpu', weights_only=False, mmap=True)
182
+ data_path = workspace_path(WORKSPACE / plan['handoff']['data_dir']) / 'manifest.json'
183
+ validate_state(checkpoint, read_json(run / 'evaluation.json'), read_json(run / 'completion.json'),
184
+ read_json(run / 'settings.json'), read_json(data_path), read_json(reference_dir / 'manifest.json'), plan)
185
+ full_hash = sha256(source)
186
+ if destination.exists() or destination.is_symlink():
187
+ return verify_dpo_archive(destination, expected_checkpoint_hash=full_hash)
188
+ destination.parent.mkdir(parents=True, exist_ok=True)
189
+ staging = Path(tempfile.mkdtemp(prefix='.dpo-final-', dir=destination.parent))
190
+ copy_regular(source, staging / 'checkpoint.pt')
191
+ if sha256(staging / 'checkpoint.pt') != full_hash:
192
+ raise ValueError('DPO full-state archive copy differs from its source')
193
+ metadata = dict(format='warpstate-chat-v1', inference_only=True, stage='dpo',
194
+ config=checkpoint['config'], step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'],
195
+ response_tokens_seen=checkpoint['response_tokens_seen'], val_loss=checkpoint['val_loss'],
196
+ tokenizer_sha256=checkpoint['signature']['tokenizer_sha256'], origin=checkpoint['origin'],
197
+ captured_at_utc=utc_now(), source=dict(path=str(source), device=before.st_dev,
198
+ inode=before.st_ino, size=before.st_size, mtime_ns=before.st_mtime_ns, sha256=full_hash))
199
+ with (staging / 'model.pt').open('xb') as stream:
200
+ torch.save(dict(metadata, model=checkpoint['model']), stream)
201
+ stream.flush()
202
+ os.fsync(stream.fileno())
203
+ metadata.update(snapshot_sha256=sha256(staging / 'model.pt'),
204
+ snapshot_bytes=(staging / 'model.pt').stat().st_size)
205
+ atomic_json(staging / 'snapshot.json', metadata)
206
+ atomic_json(staging / 'plan.json', plan)
207
+ copies = {name: run / name for name in ('settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl')}
208
+ copies.update({'tokenizer.model': tokenizer, 'dpo-execution.json': work / 'control/execution.json',
209
+ 'dpo-handoff.json': work / 'control/handoff.json', 'gpu-preflight.json': work / 'preflight/report.json',
210
+ 'data-manifest.json': data_path, 'reference-manifest.json': reference_dir / 'manifest.json',
211
+ 'reference-train.npy': reference_dir / 'train.npy', 'reference-val.npy': reference_dir / 'val.npy'})
212
+ for name, path in copies.items():
213
+ copy_regular(path, staging / name)
214
+ atomic_json(staging / 'PROVENANCE.json', dict(format='warpstate-dpo-archive-v1',
215
+ repository_revision=revision, archive_command_sha256=sha256(Path(__file__)),
216
+ source_checkpoint_sha256=full_hash, model_sha256=metadata['snapshot_sha256'],
217
+ dpo_step=checkpoint['step'], dpo_input_tokens=checkpoint['tokens_seen'],
218
+ dpo_response_tokens=checkpoint['response_tokens_seen'], dpo_pairs=checkpoint['sample_cursor'],
219
+ sft_step=plan['sft']['step'], sft_model_sha256=checkpoint['signature']['base_sha256'],
220
+ raw_base_step=plan['sft']['base_step'], raw_base_tokens=plan['sft']['base_tokens_seen'],
221
+ signature=checkpoint['signature']))
222
+ note = f'''# KEEP: completed DPO after raw step {plan['sft']['base_step']}
223
+
224
+ User-requested historical archive created {utc_now()}. Preserve during routine
225
+ cleanup, further training and chat refreshes. Delete only on explicit request.
226
+
227
+ - DPO update {checkpoint['step']:,}; {checkpoint['sample_cursor']:,} preference pairs.
228
+ - DPO input tokens {checkpoint['tokens_seen']:,}; response targets {checkpoint['response_tokens_seen']:,}.
229
+ - Parent SFT update {plan['sft']['step']:,}; raw base update {plan['sft']['base_step']:,}.
230
+ - Final full-holdout DPO objective loss: {checkpoint['val_loss']:.12f}.
231
+ - `checkpoint.pt`: byte-identical full checkpoint, including FP32 model and Adam,
232
+ per-rank RNG, sample cursor, model configuration and immutable signatures.
233
+ - `model.pt`: tensor-exact FP32 inference export with the original tokenizer.
234
+ - Plan, data/reference manifests and reference scores, original settings,
235
+ evaluation, completion, metrics, preflight and execution provenance are included.
236
+ - Stage counters are distinct. DPO objective loss is not language-model perplexity.
237
+ - All files are independent physical copies, read-only, without hardlinks/reflinks.
238
+ - Existing raw/SFT archives and the original DPO run remain unchanged.
239
+ - Repository revision at archival time: `{revision}`.
240
+
241
+ Verify with `sha256sum -c SHA256SUMS`, or run `archive_dpo.py --verify-only`
242
+ with this directory as `--destination`. This is an on-device archive.
243
+ '''
244
+ with (staging / 'ARCHIVE.md').open('x') as stream:
245
+ stream.write(note)
246
+ stream.flush()
247
+ os.fsync(stream.fileno())
248
+ after = source.stat()
249
+ if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
250
+ after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns):
251
+ raise ValueError('DPO checkpoint changed during archival')
252
+ seal(staging, FILES)
253
+ result = verify_dpo_archive(staging, expected_checkpoint_hash=full_hash)
254
+ publish_directory(staging, destination)
255
+ return result
256
+
257
+
258
+ def run(config, destination=None, *, program='warpstate-dpo', verify_only=False):
259
+ config = workspace_path(config)
260
+ plan, paths = load_plan(config)
261
+ work, base = paths['work_dir'], paths['base_work_dir']
262
+ destination = workspace_path(destination or (base / 'archives' / f"step-{plan['sft']['base_step']:08d}-dpo-final"))
263
+ if (destination.parent != base / 'archives' or
264
+ destination == paths['archive_dir']):
265
+ raise ValueError('DPO archive must use a distinct direct child of work/archives')
266
+ if verify_only:
267
+ metadata, checksums = verify_dpo_archive(destination)
268
+ else:
269
+ with contextlib.ExitStack() as locks:
270
+ for path in (work / 'control/archive.lock', work / 'control/handoff.lock',
271
+ paths['sft_work_dir'] / 'control/handoff.lock',
272
+ base / 'control/training-launcher.lock', base / 'pipeline.lock'):
273
+ locks.enter_context(exclusive_lock(path))
274
+ if not clean_exit(process_status(paths['supervisor_socket'], program)):
275
+ raise ValueError('DPO guardian must have exited successfully before archival')
276
+ if read_json(work / 'control/handoff.json').get('phase') != 'complete':
277
+ raise ValueError('DPO guardian has not completed its final integrity checks')
278
+ model_hash = sha256(work / 'initialization/model.pt')
279
+ if model_hash != sha256(paths['archive_dir'] / 'model.pt'):
280
+ raise ValueError('Original SFT archive and DPO initialization differ')
281
+ verify_dpo_completion(work / 'run', plan, model_hash)
282
+ revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=WORKSPACE, text=True).strip()
283
+ metadata, checksums = archive_dpo(work, paths['tokenizer'], destination, plan, revision)
284
+ print(json.dumps(dict(archive=str(destination), step=metadata['step'], stage='dpo',
285
+ checkpoint_sha256=checksums['checkpoint.pt'], model_sha256=checksums['model.pt'],
286
+ tokens_seen=metadata['tokens_seen'], response_tokens_seen=metadata['response_tokens_seen']), indent=2))
287
+ return metadata, checksums
288
+
289
+
290
+ def main():
291
+ parser = argparse.ArgumentParser(description=__doc__)
292
+ parser.add_argument('--config', type=Path, default=PACKAGE / 'configs/dpo_10b.json')
293
+ parser.add_argument('--destination', type=Path)
294
+ parser.add_argument('--program', default='warpstate-dpo')
295
+ parser.add_argument('--verify-only', action='store_true')
296
+ args = parser.parse_args()
297
+ torch.set_num_threads(8)
298
+ run(args.config, args.destination, program=args.program, verify_only=args.verify_only)
299
+
300
+
301
+ if __name__ == '__main__':
302
+ main()
ops/b200-requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ -r training-requirements-resolved.txt
2
+ supervisor==4.3.0
ops/b200-supervisord.conf ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [unix_http_server]
2
+ file=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisor.sock
3
+ chmod=0700
4
+
5
+ [supervisord]
6
+ logfile=/workspace/WarpState/rtx6000_40b/work/logs/b200-supervisor.log
7
+ logfile_maxbytes=10MB
8
+ logfile_backups=3
9
+ pidfile=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisord.pid
10
+ childlogdir=/workspace/WarpState/rtx6000_40b/work/logs
11
+ directory=/workspace/WarpState/rtx6000_40b
12
+ umask=077
13
+
14
+ [rpcinterface:supervisor]
15
+ supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
16
+
17
+ [supervisorctl]
18
+ serverurl=unix:///workspace/WarpState/rtx6000_40b/work/control/b200-supervisor.sock
19
+ history_file=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisorctl-history
20
+
21
+ [program:warpstate-train]
22
+ command=/workspace/WarpState/rtx6000_40b/ops/train-b200.sh
23
+ directory=/workspace/WarpState/rtx6000_40b
24
+ autostart=false
25
+ autorestart=false
26
+ startsecs=5
27
+ startretries=0
28
+ stopsignal=USR1
29
+ stopwaitsecs=900
30
+ stopasgroup=true
31
+ killasgroup=true
32
+ stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/b200-train.log
33
+ stdout_logfile_maxbytes=20MB
34
+ stdout_logfile_backups=5
35
+ redirect_stderr=true
ops/b200ctl.sh ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ source /workspace/WarpState/rtx6000_40b/ops/training-env.sh
4
+ exec /workspace/WarpState/rtx6000_40b/.server-venv/bin/supervisorctl \
5
+ -c /workspace/WarpState/rtx6000_40b/ops/b200-supervisord.conf "$@"
ops/backward_layout.py ADDED
@@ -0,0 +1,283 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Isolated BF16 attention-backward layout experiment, not a training adapter.
2
+
3
+ The reference boundary is the September 12 compiled backward's rotary kernel
4
+ followed by Q/K/V/gate/U packing. Both BF16 intermediate roundings are retained.
5
+ No reductions, GEMMs, parameter gradients or optimizer state are changed here.
6
+ All saved tensors use [batch,chunks,heads,chunk_size,head_dim] contiguous layout,
7
+ except transposed K-write gradients and token-major output gradients.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import NamedTuple
13
+
14
+ import torch
15
+ import triton
16
+ import triton.language as tl
17
+
18
+
19
+ PROFILE_SHAPE = (4, 16, 32, 128, 64)
20
+ ALGORITHMS = ("profile", "paired", "single")
21
+
22
+
23
+ class LayoutInputs(NamedTuple):
24
+ q_memory: torch.Tensor
25
+ q_local: torch.Tensor
26
+ sin: torch.Tensor
27
+ cos: torch.Tensor
28
+ k_write: torch.Tensor
29
+ tanh_k: torch.Tensor
30
+ k_local: torch.Tensor
31
+ v_local: torch.Tensor
32
+ grad_y: torch.Tensor
33
+ memory: torch.Tensor
34
+ local: torch.Tensor
35
+ gate: torch.Tensor
36
+ u_write: torch.Tensor
37
+ tanh_u: torch.Tensor
38
+
39
+
40
+ def validate_inputs(inputs: LayoutInputs, *, cuda=False):
41
+ if not isinstance(inputs, LayoutInputs):
42
+ raise TypeError("Expected named LayoutInputs")
43
+ shape = tuple(inputs.q_memory.shape)
44
+ if len(shape) != 5 or min(shape) < 1 or shape[-1] % 2:
45
+ raise ValueError("Expected positive [B,N,H,C,D] with an even head dimension")
46
+ b, n, h, c, d = shape
47
+ device = inputs.q_memory.device
48
+ if cuda and device.type != "cuda":
49
+ raise ValueError("Triton layout kernels require CUDA")
50
+ for name, tensor in zip(inputs._fields, inputs, strict=True):
51
+ wanted = shape
52
+ dtype = torch.bfloat16
53
+ if name in ("sin", "cos"):
54
+ wanted, dtype = (c, d), torch.float32
55
+ elif name == "k_write":
56
+ wanted = (b, n, h, d, c)
57
+ elif name == "grad_y":
58
+ wanted = (b, n * c, h * d)
59
+ if tuple(tensor.shape) != wanted or tensor.dtype != dtype:
60
+ raise ValueError(f"{name} must have shape {wanted} and dtype {dtype}")
61
+ if tensor.device != device or not tensor.is_contiguous():
62
+ raise ValueError(f"{name} must be contiguous on {device}")
63
+ return shape
64
+
65
+
66
+ def make_inputs(shape=PROFILE_SHAPE, *, device="cpu", seed=817) -> LayoutInputs:
67
+ """Deterministic synthetic saved activations/gradients, no model/data reads."""
68
+ b, n, h, c, d = shape
69
+ generator = torch.Generator(device=device).manual_seed(seed)
70
+
71
+ def random(dims=shape, scale=0.1):
72
+ return torch.randn(dims, device=device, dtype=torch.bfloat16,
73
+ generator=generator) * scale
74
+
75
+ from warpstate.model import rope_cos_sin
76
+ cos, sin = rope_cos_sin(c, d, torch.device(device), torch.float32)
77
+ return LayoutInputs(random(), random(), sin, cos, random((b, n, h, d, c)),
78
+ random(scale=1).float().tanh().bfloat16(), random(), random(),
79
+ random((b, n * c, h * d)), random(), random(),
80
+ random(scale=1).float().sigmoid().bfloat16(), random(),
81
+ random(scale=1).float().tanh().bfloat16())
82
+
83
+
84
+ def reference_layout(inputs: LayoutInputs) -> torch.Tensor:
85
+ """Independent tensor expression for both stages, including BF16 boundaries.
86
+
87
+ FP32 operation contraction may differ between eager PyTorch and Triton;
88
+ use the tracked profile kernels as the exact GPU rounding oracle.
89
+ """
90
+ b, n, h, c, d = validate_inputs(inputs)
91
+ q = inputs.q_memory.float() + inputs.q_local.float()
92
+ k = (inputs.k_write.transpose(-2, -1).float()
93
+ * (1 - inputs.tanh_k.float().square()) + inputs.k_local.float())
94
+
95
+ def inverse_rotate(value):
96
+ return torch.stack((value[..., 1::2], -value[..., ::2]), dim=-1).flatten(-2)
97
+
98
+ q_rounded = (inverse_rotate(q * inputs.sin) + q * inputs.cos).bfloat16()
99
+ k_rotated_rounded = inverse_rotate(k * inputs.sin).bfloat16()
100
+ k_packed = (k_rotated_rounded.float() + k * inputs.cos).bfloat16()
101
+ dy = inputs.grad_y.view(b, n, c, h, d).permute(0, 1, 3, 2, 4).float()
102
+ gate = inputs.gate.float()
103
+ dg = ((-(dy * (inputs.memory.float() * d**-0.5))
104
+ + dy * inputs.local.float()) * (gate * (1 - gate))).bfloat16()
105
+ du = (inputs.u_write.float() * (1 - inputs.tanh_u.float().square())).bfloat16()
106
+
107
+ def token_major(value):
108
+ return value.permute(0, 1, 3, 2, 4).reshape(b, n * c, h * d)
109
+
110
+ return torch.cat([token_major(value) for value in
111
+ (q_rounded, k_packed, inputs.v_local, dg, du)], dim=-1)
112
+
113
+
114
+ @triton.jit
115
+ def _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT,
116
+ ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
117
+ D: tl.constexpr, BLOCK: tl.constexpr, program):
118
+ pairs = program * BLOCK + tl.arange(0, BLOCK)
119
+ even = pairs * 2
120
+ odd = even + 1
121
+ mask = even < ELEMENTS
122
+ token = (even // D) % C
123
+ channel = even % D
124
+ group = even // (C * D)
125
+ rope = token * D + channel
126
+ s0 = tl.load(SIN + rope, mask, 0)
127
+ s1 = tl.load(SIN + rope + 1, mask, 0)
128
+ c0 = tl.load(COS + rope, mask, 0)
129
+ c1 = tl.load(COS + rope + 1, mask, 0)
130
+ q0 = (tl.load(QM + even, mask, 0).to(tl.float32)
131
+ + tl.load(QL + even, mask, 0).to(tl.float32))
132
+ q1 = (tl.load(QM + odd, mask, 0).to(tl.float32)
133
+ + tl.load(QL + odd, mask, 0).to(tl.float32))
134
+ # The profiled interleave sums each active rotary lane with an inactive +0.
135
+ # Retain that IEEE signed-zero boundary as well as its BF16 store below.
136
+ q_even_rot = q1 * s1 + 0.0
137
+ q_odd_rot = (-q0 * s0) + 0.0
138
+ q_even = (q_even_rot + q0 * c0).to(tl.bfloat16)
139
+ q_odd = (q_odd_rot + q1 * c1).to(tl.bfloat16)
140
+ kw_offset = group * C * D + channel * C + token
141
+ kw0 = tl.load(KW + kw_offset, mask, 0).to(tl.float32)
142
+ kw1 = tl.load(KW + kw_offset + C, mask, 0).to(tl.float32)
143
+ tk0 = tl.load(TK + even, mask, 0).to(tl.float32)
144
+ tk1 = tl.load(TK + odd, mask, 0).to(tl.float32)
145
+ k0 = kw0 * (1.0 - tk0 * tk0) + tl.load(KL + even, mask, 0).to(tl.float32)
146
+ k1 = kw1 * (1.0 - tk1 * tk1) + tl.load(KL + odd, mask, 0).to(tl.float32)
147
+ # Stage 17 stores these separately, before adding the cosine term in 18.
148
+ # Retain the truncation even though the fused implementation has no scratch.
149
+ k_even_rot = (k1 * s1 + 0.0).to(tl.bfloat16).to(tl.float32)
150
+ k_odd_rot = ((-k0 * s0) + 0.0).to(tl.bfloat16).to(tl.float32)
151
+ k_even = k_even_rot + k0 * c0
152
+ k_odd = k_odd_rot + k1 * c1
153
+ out = ((group // H) * C + token) * (5 * H * D) + (group % H) * D + channel
154
+ tl.store(OUT + out, q_even, mask)
155
+ tl.store(OUT + out + 1, q_odd, mask)
156
+ tl.store(OUT + out + H * D, k_even, mask)
157
+ tl.store(OUT + out + H * D + 1, k_odd, mask)
158
+
159
+
160
+ @triton.jit
161
+ def _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
162
+ ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
163
+ D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr, program):
164
+ index = program * BLOCK + tl.arange(0, BLOCK)
165
+ mask = index < ELEMENTS
166
+ token = (index // D) % C
167
+ channel = index % D
168
+ group = index // (C * D)
169
+ token_row = (group // H) * C + token
170
+ feature = (group % H) * D + channel
171
+ dy = tl.load(DY + token_row * H * D + feature, mask, 0).to(tl.float32)
172
+ memory = tl.load(MEMORY + index, mask, 0).to(tl.float32)
173
+ local = tl.load(LOCAL + index, mask, 0).to(tl.float32)
174
+ gate = tl.load(GATE + index, mask, 0).to(tl.float32)
175
+ # Keep the original chain rather than algebraically factoring dy.
176
+ dgate = (-(dy * (memory * SCALE)) + dy * local) * (gate * (1.0 - gate))
177
+ uw = tl.load(UW + index, mask, 0).to(tl.float32)
178
+ tu = tl.load(TU + index, mask, 0).to(tl.float32)
179
+ du = uw * (1.0 - tu * tu)
180
+ out = token_row * 5 * H * D + feature
181
+ tl.store(OUT + out + 2 * H * D, tl.load(VL + index, mask, 0), mask)
182
+ tl.store(OUT + out + 3 * H * D, dgate, mask)
183
+ tl.store(OUT + out + 4 * H * D, du, mask)
184
+
185
+
186
+ @triton.jit
187
+ def _paired_kernel(QM, QL, SIN, COS, KW, TK, KL, VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
188
+ ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
189
+ D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr,
190
+ ROTARY: tl.constexpr):
191
+ if ROTARY:
192
+ _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT,
193
+ ELEMENTS, H, C, D, BLOCK, tl.program_id(0))
194
+ else:
195
+ _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
196
+ ELEMENTS, H, C, D, SCALE, BLOCK, tl.program_id(0))
197
+
198
+
199
+ @triton.jit
200
+ def _single_kernel(QM, QL, SIN, COS, KW, TK, KL, VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
201
+ ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
202
+ D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr):
203
+ rotary_programs = tl.cdiv(ELEMENTS // 2, BLOCK)
204
+ program = tl.program_id(0)
205
+ if program < rotary_programs:
206
+ _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT,
207
+ ELEMENTS, H, C, D, BLOCK, program)
208
+ else:
209
+ _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
210
+ ELEMENTS, H, C, D, SCALE, BLOCK, program - rotary_programs)
211
+
212
+
213
+ def prepare_layout(inputs: LayoutInputs, algorithm="paired", *, block=512, warps=4,
214
+ output=None, scratch=None):
215
+ """Validate once and return an allocation-free launch callable.
216
+
217
+ Callers must keep the inputs/output/scratch metadata and storage stable.
218
+ No GPU kernel executes until the returned callable is invoked.
219
+ """
220
+ b, n, h, c, d = validate_inputs(inputs, cuda=True)
221
+ if algorithm not in ALGORITHMS:
222
+ raise ValueError(f"Unknown backward layout algorithm: {algorithm}")
223
+ if block not in (128, 256, 512, 1024) or warps not in (4, 8):
224
+ raise ValueError("Use block 128/256/512/1024 with 4/8 warps")
225
+ shape = (b, n * c, 5 * h * d)
226
+ if output is None:
227
+ output = torch.empty(shape, dtype=torch.bfloat16, device=inputs.q_memory.device)
228
+ if (tuple(output.shape) != shape or output.dtype != torch.bfloat16
229
+ or output.device != inputs.q_memory.device or not output.is_contiguous()):
230
+ raise ValueError("Output must be a contiguous BF16 packed gradient on the input device")
231
+ if any(output.untyped_storage().data_ptr() == value.untyped_storage().data_ptr()
232
+ for value in inputs):
233
+ raise ValueError("Output may not alias a saved input")
234
+ if algorithm == "profile":
235
+ if (b, n, h, c, d) != PROFILE_SHAPE:
236
+ raise ValueError("The exact profiled baseline supports PROFILE_SHAPE only")
237
+ from ops.backward_layout_baseline import _profile_pack, _profile_rotary
238
+ if scratch is None:
239
+ scratch = (torch.empty_like(inputs.q_memory), torch.empty_like(inputs.q_memory))
240
+ if (len(scratch) != 2 or any(value.shape != inputs.q_memory.shape
241
+ or value.dtype != torch.bfloat16 or value.device != output.device
242
+ or not value.is_contiguous() for value in scratch)):
243
+ raise ValueError("The profile baseline needs two contiguous BF16 scratch tensors")
244
+ q, kr = scratch
245
+ storages = [value.untyped_storage().data_ptr() for value in (*inputs, output)]
246
+ if (q.untyped_storage().data_ptr() in storages
247
+ or kr.untyped_storage().data_ptr() in storages
248
+ or q.untyped_storage().data_ptr() == kr.untyped_storage().data_ptr()):
249
+ raise ValueError("Scratch tensors may not alias inputs, output, or one another")
250
+
251
+ def operation():
252
+ _profile_rotary[(1, 16384)](
253
+ inputs.q_memory, inputs.q_local, inputs.sin, inputs.cos,
254
+ inputs.k_write, inputs.tanh_k, inputs.k_local, q, kr,
255
+ 262144, 64, YBLOCK=16, XBLOCK=64, num_warps=4, enable_fp_fusion=True)
256
+ _profile_pack[(160, 128)](
257
+ q, kr, inputs.k_write, inputs.tanh_k, inputs.k_local, inputs.cos,
258
+ inputs.v_local, inputs.grad_y, inputs.memory, inputs.local, inputs.gate,
259
+ inputs.u_write, inputs.tanh_u, output, 8192, 10240,
260
+ YBLOCK=64, XBLOCK=64, num_warps=8, enable_fp_fusion=True)
261
+ return output
262
+ else:
263
+ elements = inputs.q_memory.numel()
264
+ args = (*inputs, output, elements, h, c, d, d**-0.5, block)
265
+
266
+ def operation():
267
+ if algorithm == "paired":
268
+ _paired_kernel[(triton.cdiv(elements // 2, block),)](
269
+ *args, True, num_warps=warps, enable_fp_fusion=True)
270
+ _paired_kernel[(triton.cdiv(elements, block),)](
271
+ *args, False, num_warps=warps, enable_fp_fusion=True)
272
+ else:
273
+ _single_kernel[(triton.cdiv(elements // 2, block) + triton.cdiv(elements, block),)](
274
+ *args, num_warps=warps, enable_fp_fusion=True)
275
+ return output
276
+ return operation
277
+
278
+
279
+ def launch_layout(inputs: LayoutInputs, algorithm="paired", *, block=512, warps=4,
280
+ output=None, scratch=None) -> torch.Tensor:
281
+ """One checked launch; timed loops should use prepare_layout once instead."""
282
+ return prepare_layout(inputs, algorithm, block=block, warps=warps,
283
+ output=output, scratch=scratch)()
ops/backward_layout_baseline.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tracked reference kernels from the September 12 CA-overlap backward profile.
2
+
3
+ The bodies below reproduce the exact profiled arithmetic/indexing at shape
4
+ B=4, chunks=16, heads=32, chunk_size=128, head_dim=64. They are kept as source,
5
+ not imported from generated compiler caches. Names 17/18 in that trace consumed
6
+ 31.907/77.576 ms per rank-0 update. Their measured launch choices were
7
+ XBLOCK=64/YBLOCK=16/4 warps and XBLOCK=64/YBLOCK=64/8 warps respectively.
8
+
9
+ Provenance: ops/model sources and profile-ca-overlap trace map these bodies to
10
+ cache hashes cq665jkyfy6uac7wrzvwa5aigwids6rzt4n7vzip6xvufai4ajjw and
11
+ cbr6gfx3vjhnzjbykfch3pg5peunfivhgdrfvl5hnzolhvfjfgdy. The isolated launcher
12
+ preallocates both scratch tensors; production reused a dead buffer for one of
13
+ them. Compare kernel time and traffic, not an inferred peak-allocation saving.
14
+ """
15
+
16
+ import triton
17
+ import triton.language as tl
18
+ from torch._inductor.runtime import triton_helpers
19
+
20
+
21
+ @triton.jit
22
+ def _profile_rotary(in_ptr0, in_ptr1, in_ptr2, in_ptr3, in_ptr4, in_ptr5, in_ptr6, out_ptr0, out_ptr1, ynumel, xnumel, YBLOCK : tl.constexpr, XBLOCK : tl.constexpr):
23
+ ynumel = 262144
24
+ xnumel = 64
25
+ yoffset = (tl.program_id(1) + tl.program_id(2) * tl.num_programs(1)) * YBLOCK
26
+ yindex = yoffset + tl.arange(0, YBLOCK)[:, None]
27
+ ymask = yindex < ynumel
28
+ xoffset = tl.program_id(0) * XBLOCK
29
+ xindex = xoffset + tl.arange(0, XBLOCK)[None, :]
30
+ xmask = xindex < xnumel
31
+ x2 = xindex
32
+ y3 = yindex
33
+ y0 = (yindex % 128)
34
+ y1 = yindex // 128
35
+ tmp33 = tl.load(in_ptr0 + (x2 + 64*y3), xmask & ymask).to(tl.float32)
36
+ tmp34 = tl.load(in_ptr1 + (x2 + 64*y3), xmask & ymask).to(tl.float32)
37
+ tmp36 = tl.load(in_ptr3 + (x2 + 64*y0), xmask & ymask, eviction_policy='evict_last')
38
+ tmp0 = (x2).to(tl.int64)
39
+ tmp1 = (tmp0).to(tl.int64)
40
+ tmp2 = tl.full([1, 1], 1, tl.int64)
41
+ tmp3 = tmp1 >= tmp2
42
+ tmp4 = ((((-1) + x2) % 2)).to(tl.int64)
43
+ tmp5 = (tmp4).to(tl.int64)
44
+ tmp6 = tl.full([1, 1], 0, tl.int64)
45
+ tmp7 = tmp5 == tmp6
46
+ tmp8 = tmp3 & tmp7
47
+ tmp9 = tl.load(in_ptr0 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
48
+ tmp10 = tl.load(in_ptr1 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
49
+ tmp11 = tmp9 + tmp10
50
+ tmp12 = tl.load(in_ptr2 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y0), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0)
51
+ tmp13 = tmp12.to(tl.float32)
52
+ tmp14 = tmp11 * tmp13
53
+ tmp15 = -tmp14
54
+ tmp16 = tl.full(tmp15.shape, 0.0, tmp15.dtype)
55
+ tmp17 = tl.where(tmp8, tmp15, tmp16)
56
+ tmp18 = tl.full([1, 1], 0.0, tl.float32)
57
+ tmp19 = tl.where(tmp8, tmp17, tmp18)
58
+ tmp20 = ((x2 % 2)).to(tl.int64)
59
+ tmp21 = (tmp20).to(tl.int64)
60
+ tmp22 = tmp21 == tmp6
61
+ tmp23 = tl.load(in_ptr0 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
62
+ tmp24 = tl.load(in_ptr1 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
63
+ tmp25 = tmp23 + tmp24
64
+ tmp26 = tl.load(in_ptr2 + (1 + 2*(x2 // 2) + 64*y0), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0)
65
+ tmp27 = tmp26.to(tl.float32)
66
+ tmp28 = tmp25 * tmp27
67
+ tmp29 = tl.full(tmp28.shape, 0.0, tmp28.dtype)
68
+ tmp30 = tl.where(tmp22, tmp28, tmp29)
69
+ tmp31 = tl.where(tmp22, tmp30, tmp18)
70
+ tmp32 = tmp19 + tmp31
71
+ tmp35 = tmp33 + tmp34
72
+ tmp37 = tmp36.to(tl.float32)
73
+ tmp38 = tmp35 * tmp37
74
+ tmp39 = tmp32 + tmp38
75
+ tmp40 = tl.load(in_ptr4 + (y0 + 256*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 8192*y1), tmp8 & xmask & ymask, other=0.0).to(tl.float32)
76
+ tmp41 = tmp40.to(tl.float32)
77
+ tmp42 = tl.load(in_ptr5 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
78
+ tmp43 = tmp42.to(tl.float32)
79
+ tmp44 = tmp43 * tmp43
80
+ tmp45 = tl.full([1, 1], 1.0, tl.float32)
81
+ tmp46 = tmp45 - tmp44
82
+ tmp47 = tmp41 * tmp46
83
+ tmp48 = tmp47.to(tl.float32)
84
+ tmp49 = tl.load(in_ptr6 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
85
+ tmp50 = tmp48 + tmp49
86
+ tmp51 = tmp50 * tmp13
87
+ tmp52 = -tmp51
88
+ tmp53 = tl.full(tmp52.shape, 0.0, tmp52.dtype)
89
+ tmp54 = tl.where(tmp8, tmp52, tmp53)
90
+ tmp55 = tl.where(tmp8, tmp54, tmp18)
91
+ tmp56 = tl.load(in_ptr4 + (128 + y0 + 256*(x2 // 2) + 8192*y1), tmp22 & xmask & ymask, other=0.0).to(tl.float32)
92
+ tmp57 = tmp56.to(tl.float32)
93
+ tmp58 = tl.load(in_ptr5 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
94
+ tmp59 = tmp58.to(tl.float32)
95
+ tmp60 = tmp59 * tmp59
96
+ tmp61 = tl.full([1, 1], 1.0, tl.float32)
97
+ tmp62 = tmp61 - tmp60
98
+ tmp63 = tmp57 * tmp62
99
+ tmp64 = tmp63.to(tl.float32)
100
+ tmp65 = tl.load(in_ptr6 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
101
+ tmp66 = tmp64 + tmp65
102
+ tmp67 = tmp66 * tmp27
103
+ tmp68 = tl.full(tmp67.shape, 0.0, tmp67.dtype)
104
+ tmp69 = tl.where(tmp22, tmp67, tmp68)
105
+ tmp70 = tl.where(tmp22, tmp69, tmp18)
106
+ tmp71 = tmp55 + tmp70
107
+ tl.store(out_ptr0 + (x2 + 64*y3), tmp39, xmask & ymask)
108
+ tl.store(out_ptr1 + (x2 + 64*y3), tmp71, xmask & ymask)
109
+
110
+
111
+ @triton.jit
112
+ def _profile_pack(in_ptr0, in_ptr1, in_ptr2, in_ptr3, in_ptr4, in_ptr5, in_ptr6, in_ptr7, in_ptr8, in_ptr9, in_ptr10, in_ptr11, in_ptr12, out_ptr0, ynumel, xnumel, YBLOCK : tl.constexpr, XBLOCK : tl.constexpr):
113
+ ynumel = 8192
114
+ xnumel = 10240
115
+ yoffset = tl.program_id(1) * YBLOCK
116
+ yindex = yoffset + tl.arange(0, YBLOCK)[:, None]
117
+ ymask = tl.full([YBLOCK], True, tl.int1)[:, None]
118
+ xoffset = tl.program_id(0) * XBLOCK
119
+ xindex = xoffset + tl.arange(0, XBLOCK)[None, :]
120
+ xmask = xindex < xnumel
121
+ x2 = xindex
122
+ y0 = (yindex % 2048)
123
+ y1 = yindex // 2048
124
+ y3 = yindex
125
+ tmp0 = (x2).to(tl.int32)
126
+ tmp1 = tl.full([1, 1], 0, tl.int64)
127
+ tmp2 = tmp0 >= tmp1
128
+ tmp3 = (x2).to(tl.int64)
129
+ tmp4 = (tmp3).to(tl.int64)
130
+ tmp5 = tl.full([1, 1], 2048, tl.int64)
131
+ tmp6 = tmp4 < tmp5
132
+ tmp7 = tl.load(in_ptr0 + (64*((y0 % 128)) + 8192*((((x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + (((x2) % 64))), tmp6 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
133
+ tmp8 = tmp4 >= tmp5
134
+ tmp9 = tl.full([1, 1], 4096, tl.int64)
135
+ tmp10 = tmp4 < tmp9
136
+ tmp11 = tmp8 & tmp10
137
+ tmp12 = tl.load(in_ptr1 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
138
+ tmp13 = tl.load(in_ptr2 + (128*((((-2048) + x2) % 2048)) + 262144*(y0 // 128) + 4194304*y1 + ((y0 % 128))), tmp11 & xmask, other=0.0).to(tl.float32)
139
+ tmp14 = tmp13.to(tl.float32)
140
+ tmp15 = tl.load(in_ptr3 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
141
+ tmp16 = tmp15.to(tl.float32)
142
+ tmp17 = tmp16 * tmp16
143
+ tmp18 = tl.full([1, 1], 1.0, tl.float32)
144
+ tmp19 = tmp18 - tmp17
145
+ tmp20 = tmp14 * tmp19
146
+ tmp21 = tmp20.to(tl.float32)
147
+ tmp22 = tl.load(in_ptr4 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
148
+ tmp23 = tmp21 + tmp22
149
+ tmp24 = tl.load(in_ptr5 + (64*((y0 % 128)) + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0)
150
+ tmp25 = tmp24.to(tl.float32)
151
+ tmp26 = tmp23 * tmp25
152
+ tmp27 = tmp12 + tmp26
153
+ tmp28 = tl.full(tmp27.shape, 0.0, tmp27.dtype)
154
+ tmp29 = tl.where(tmp11, tmp27, tmp28)
155
+ tmp30 = tmp4 >= tmp9
156
+ tmp31 = tl.full([1, 1], 6144, tl.int64)
157
+ tmp32 = tmp4 < tmp31
158
+ tmp33 = tmp30 & tmp32
159
+ tmp34 = tl.load(in_ptr6 + (64*((y0 % 128)) + 8192*(((((-4096) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-4096) + x2) % 64))), tmp33 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
160
+ tmp35 = tmp4 >= tmp31
161
+ tmp36 = tl.full([1, 1], 8192, tl.int64)
162
+ tmp37 = tmp4 < tmp36
163
+ tmp38 = tmp35 & tmp37
164
+ tmp39 = tl.load(in_ptr7 + (2048*y3 + ((((-6144) + x2) % 2048))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
165
+ tmp40 = tl.load(in_ptr8 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
166
+ tmp41 = tl.full([1, 1], 0.125, tl.float32)
167
+ tmp42 = tmp40 * tmp41
168
+ tmp43 = tmp39 * tmp42
169
+ tmp44 = -tmp43
170
+ tmp45 = tl.load(in_ptr9 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
171
+ tmp46 = tmp39 * tmp45
172
+ tmp47 = tmp44 + tmp46
173
+ tmp48 = tmp47.to(tl.float32)
174
+ tmp49 = tl.load(in_ptr10 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
175
+ tmp50 = tmp49.to(tl.float32)
176
+ tmp51 = tl.full([1, 1], 1.0, tl.float32)
177
+ tmp52 = tmp51 - tmp50
178
+ tmp53 = tmp50 * tmp52
179
+ tmp54 = tmp48 * tmp53
180
+ tmp55 = tmp54.to(tl.float32)
181
+ tmp56 = tl.full(tmp55.shape, 0.0, tmp55.dtype)
182
+ tmp57 = tl.where(tmp38, tmp55, tmp56)
183
+ tmp58 = tmp0 >= tmp36
184
+ tmp59 = tl.full([1, 1], 10240, tl.int64)
185
+ tmp60 = tmp0 < tmp59
186
+ tmp61 = tl.load(in_ptr11 + (64*((y0 % 128)) + 8192*(((((-8192) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-8192) + x2) % 64))), tmp58 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
187
+ tmp62 = tmp61.to(tl.float32)
188
+ tmp63 = tl.load(in_ptr12 + (64*((y0 % 128)) + 8192*(((((-8192) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-8192) + x2) % 64))), tmp58 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
189
+ tmp64 = tmp63.to(tl.float32)
190
+ tmp65 = tmp64 * tmp64
191
+ tmp66 = tl.full([1, 1], 1.0, tl.float32)
192
+ tmp67 = tmp66 - tmp65
193
+ tmp68 = tmp62 * tmp67
194
+ tmp69 = tmp68.to(tl.float32)
195
+ tmp70 = tl.full(tmp69.shape, 0.0, tmp69.dtype)
196
+ tmp71 = tl.where(tmp58, tmp69, tmp70)
197
+ tmp72 = tl.where(tmp38, tmp57, tmp71)
198
+ tmp73 = tl.where(tmp33, tmp34, tmp72)
199
+ tmp74 = tl.where(tmp11, tmp29, tmp73)
200
+ tmp75 = tl.where(tmp6, tmp7, tmp74)
201
+ tl.store(out_ptr0 + (x2 + 10240*y3), tmp75, xmask)
ops/backward_layout_benchmark.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Reproduce isolated BF16 rotary/packed-gradient traffic measurements on one GPU."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import datetime as dt
8
+ import hashlib
9
+ import json
10
+ import math
11
+ import os
12
+ from pathlib import Path
13
+ import statistics
14
+ import sys
15
+ import time
16
+
17
+ PACKAGE = Path(__file__).resolve().parents[1]
18
+ WORKSPACE = PACKAGE.parent
19
+ sys.path.insert(0, str(PACKAGE))
20
+
21
+
22
+ def arguments(argv=None):
23
+ parser = argparse.ArgumentParser(description=__doc__)
24
+ parser.add_argument("--algorithms", nargs="+", choices=["paired", "single"], default=["paired", "single"])
25
+ parser.add_argument("--blocks", nargs="+", type=int, choices=[128, 256, 512, 1024], default=[256, 512, 1024])
26
+ parser.add_argument("--warps", nargs="+", type=int, choices=[4, 8], default=[4, 8])
27
+ parser.add_argument("--trials", type=int, default=5)
28
+ parser.add_argument("--repeats", type=int, default=40)
29
+ parser.add_argument("--warmup", type=int, default=20)
30
+ parser.add_argument("--burnin-seconds", type=float, default=0.5)
31
+ parser.add_argument("--seed", type=int, default=817)
32
+ parser.add_argument("--output-dir", type=Path, required=True)
33
+ args = parser.parse_args(argv)
34
+ for name in ("trials", "repeats", "warmup"):
35
+ if getattr(args, name) < 1:
36
+ parser.error(name + " must be positive")
37
+ if not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0:
38
+ parser.error("burnin-seconds must be finite and nonnegative")
39
+ args.output_dir = args.output_dir.expanduser().resolve()
40
+ work = PACKAGE / "work"
41
+ if (not args.output_dir.is_relative_to(work) or args.output_dir == work
42
+ or not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")):
43
+ parser.error("output-dir must be below rtx6000_40b/work/benchmark-*")
44
+ for name in ("TMPDIR", "XDG_CACHE_HOME", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"):
45
+ if not os.environ.get(name) or not Path(os.environ[name]).resolve().is_relative_to(WORKSPACE):
46
+ parser.error("Source ops/training-env.sh; " + name + " must stay inside the workspace")
47
+ return args
48
+
49
+
50
+ def output_errors(actual, expected):
51
+ """Check each projection independently so a bad derivative cannot hide."""
52
+ import torch
53
+ result = {}
54
+ for name, value, wanted in zip(("q", "k", "v", "gate", "u"),
55
+ actual.chunk(5, -1), expected.chunk(5, -1), strict=True):
56
+ delta = value.float() - wanted.float()
57
+ finite = bool(torch.isfinite(value).all())
58
+ result[name] = {"finite": finite,
59
+ "unequal": int((value != wanted).sum()),
60
+ "bitwise_unequal": int((value.view(torch.int16) != wanted.view(torch.int16)).sum()),
61
+ "relative_rms": (float(delta.norm() / wanted.float().norm().clamp_min(1e-30))
62
+ if finite else None),
63
+ "max_abs": float(delta.abs().max()) if finite else None}
64
+ return result
65
+
66
+
67
+ def main(argv=None):
68
+ args = arguments(argv)
69
+ import torch
70
+ from ops.backward_layout import PROFILE_SHAPE, make_inputs, prepare_layout, reference_layout
71
+ if not torch.cuda.is_available() or torch.cuda.device_count() != 1:
72
+ raise RuntimeError("Reserve exactly one GPU with CUDA_VISIBLE_DEVICES")
73
+ torch.cuda.set_device(0)
74
+ inputs = make_inputs(PROFILE_SHAPE, device="cuda", seed=args.seed)
75
+ baseline_output = torch.empty((4, 2048, 10240), device="cuda", dtype=torch.bfloat16)
76
+ candidate_output = torch.empty_like(baseline_output)
77
+ scratch = tuple(torch.empty_like(inputs.q_memory) for _ in range(2))
78
+ baseline = prepare_layout(inputs, "profile", output=baseline_output, scratch=scratch)
79
+ baseline()
80
+ independent = reference_layout(inputs)
81
+ baseline_errors = output_errors(baseline_output, independent)
82
+ del independent
83
+ if any(not v["finite"] or v["relative_rms"] > 0.002 for v in baseline_errors.values()):
84
+ raise RuntimeError("Tracked profile arithmetic disagrees with independent tensor reference")
85
+ args.output_dir.mkdir(parents=True, exist_ok=False)
86
+ sources = ["ops/backward_layout.py", "ops/backward_layout_baseline.py",
87
+ "ops/backward_layout_benchmark.py", "warpstate/model.py"]
88
+ report = {"format": "warpstate-backward-layout-v1", "started_utc": dt.datetime.now(dt.UTC).isoformat(),
89
+ "arguments": {k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()},
90
+ "shape_b_n_h_c_d": PROFILE_SHAPE, "dtype": "bfloat16", "rope_dtype": "float32",
91
+ "torch": torch.__version__, "cuda": torch.version.cuda,
92
+ "device": torch.cuda.get_device_name(0), "cuda_visible_devices": os.environ["CUDA_VISIBLE_DEVICES"],
93
+ "source_sha256": {name: hashlib.sha256((PACKAGE / name).read_bytes()).hexdigest() for name in sources},
94
+ "baseline_reference_errors": baseline_errors,
95
+ "static_removed_scratch_store_and_read_bytes_per_call": inputs.q_memory.numel() * 2 * 4,
96
+ "limitations": ["Isolated backward region, not a measured training speedup.",
97
+ "Preallocated outputs and scratch; CUDA event times include launch scheduling.",
98
+ "Synthetic saved inputs are reused; no model/checkpoint/token files are read.",
99
+ "Static removed scratch traffic is not a measured bandwidth counter.",
100
+ "Production reuses one dead buffer; this is not a claim of two saved allocations.",
101
+ "Integration must preserve separate local/memory gradients and rounding joins."], "results": []}
102
+
103
+ def save():
104
+ path = args.output_dir / "results.json"
105
+ temporary = path.with_suffix(".json.partial")
106
+ temporary.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
107
+ temporary.replace(path)
108
+
109
+ def elapsed_ms(operation):
110
+ begin, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
111
+ begin.record()
112
+ for _ in range(args.repeats):
113
+ operation()
114
+ end.record()
115
+ end.synchronize()
116
+ return begin.elapsed_time(end) / args.repeats
117
+
118
+ for algorithm in args.algorithms:
119
+ for block in args.blocks:
120
+ for warps in args.warps:
121
+ row = {"algorithm": algorithm, "block": block, "warps": warps}
122
+ try:
123
+ candidate = prepare_layout(inputs, algorithm, block=block, warps=warps,
124
+ output=candidate_output)
125
+ candidate()
126
+ row["errors"] = output_errors(candidate_output, baseline_output)
127
+ if any(not v["finite"] for v in row["errors"].values()):
128
+ raise RuntimeError("Nonfinite candidate output")
129
+ row["bitwise_equal"] = all(v["bitwise_unequal"] == 0 for v in row["errors"].values())
130
+ for _ in range(args.warmup):
131
+ baseline()
132
+ candidate()
133
+ torch.cuda.synchronize()
134
+ begin = time.monotonic()
135
+ while time.monotonic() - begin < args.burnin_seconds:
136
+ for _ in range(10):
137
+ baseline()
138
+ candidate()
139
+ torch.cuda.synchronize()
140
+ trials = []
141
+ for _ in range(args.trials):
142
+ trials.append({"baseline_before_ms": elapsed_ms(baseline),
143
+ "candidate_ms": elapsed_ms(candidate),
144
+ "baseline_after_ms": elapsed_ms(baseline)})
145
+ gains = [((t["baseline_before_ms"] + t["baseline_after_ms"]) / 2
146
+ / t["candidate_ms"] - 1) * 100 for t in trials]
147
+ row.update(status="ok", trials=trials,
148
+ median_candidate_ms=statistics.median(t["candidate_ms"] for t in trials),
149
+ median_baseline_ms=statistics.median((t["baseline_before_ms"]
150
+ + t["baseline_after_ms"]) / 2 for t in trials),
151
+ median_gain_percent=statistics.median(gains), minimum_gain_percent=min(gains))
152
+ except Exception as error:
153
+ row.update(status="error", error=type(error).__name__ + ": " + str(error)[:4000])
154
+ report["results"].append(row)
155
+ save()
156
+ print(json.dumps(row), flush=True)
157
+ report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat()
158
+ save()
159
+ if not any(row["status"] == "ok" for row in report["results"]):
160
+ raise RuntimeError("No candidate completed")
161
+
162
+
163
+ if __name__ == "__main__":
164
+ main()
ops/backward_layout_execution.py ADDED
@@ -0,0 +1,355 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Opt-in replacement of the measured, normalized attention backward subgraph.
2
+
3
+ The boundary is the existing compiled BF16 kernel arithmetic, not eager BF16
4
+ ATen arithmetic: Inductor normally retains FP32 intermediates within a fused
5
+ kernel. Forward/autograd ownership, leaf accumulation and communication remain
6
+ unchanged. Unsupported graphs are retained; callers must require actual runtime
7
+ dispatches in opt-in validation. Configure once before compiling in a fresh process.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections import Counter
13
+ from functools import lru_cache
14
+ import hashlib
15
+ import json
16
+ import operator
17
+ from pathlib import Path
18
+
19
+ import torch
20
+ from torch._inductor.custom_graph_pass import CustomGraphPass
21
+ from torch.fx import Node
22
+ from torch.library import custom_op
23
+
24
+
25
+ BACKWARD_LAYOUT_MODES = ("baseline", "inspect", "paired")
26
+ _SHAPE = (4, 16, 32, 128, 64)
27
+ _NAMES = ("q_memory", "q_local", "sin", "cos", "k_write", "tanh_k", "k_local",
28
+ "v_local", "grad_y", "memory", "local", "gate", "u_write", "tanh_u")
29
+ _NATIVE_SHAPES = (
30
+ (2048, 128, 64), (64, 32, 128, 64), (128, 64), (128, 64),
31
+ (2048, 64, 128), _SHAPE, (64, 32, 128, 64), (64, 32, 128, 64),
32
+ (8192, 2048), (2048, 128, 64), (64, 32, 128, 64), _SHAPE,
33
+ (2048, 128, 64), _SHAPE,
34
+ )
35
+ _COUNTS = Counter()
36
+ _MISSES = Counter()
37
+ _ACTIVE_MODE = "baseline"
38
+ _ACTIVE_PASS = None
39
+
40
+
41
+ def _aten_boundary(qm, ql, sin, cos, kw, tk, kl, vl, dy, memory, local, gate, uw, tu):
42
+ """Exact normalized ATen dataflow of the real production backward boundary.
43
+
44
+ This recipe is traced on fake tensors once to construct a structural pattern.
45
+ Explicit operations retain every original cast, view, projection position,
46
+ and zero-filled scatter; arbitrary algebraic equivalents are not accepted.
47
+ It is never called to execute training or to provide a numeric GPU oracle.
48
+ """
49
+ a = torch.ops.aten
50
+ cvt = torch.ops.prims.convert_element_type.default
51
+ reshape = a.reshape.default
52
+ head = lambda x: reshape(x, list(_SHAPE))
53
+ sin = reshape(cvt(sin, torch.bfloat16), [1, 1, 1, 128, 64])
54
+ cos = reshape(cvt(cos, torch.bfloat16), [1, 1, 1, 128, 64])
55
+ zero = a.full.default(list(_SHAPE), 0, dtype=torch.bfloat16,
56
+ layout=torch.strided, device=qm.device, pin_memory=False)
57
+
58
+ def rotary(value):
59
+ lanes = reshape(a.mul.Tensor(value, sin), [4, 16, 32, 128, 32, 2])
60
+ odd = a.slice_scatter.default(zero, a.neg.default(a.select.int(lanes, -1, 0)),
61
+ 4, 1, 9223372036854775807, 2)
62
+ even = a.slice_scatter.default(zero, a.select.int(lanes, -1, 1),
63
+ 4, 0, 9223372036854775807, 2)
64
+ return a.add.Tensor(a.add.Tensor(odd, even), a.mul.Tensor(value, cos))
65
+
66
+ def tanh_backward(gradient, saved):
67
+ gradient = cvt(gradient, torch.float32)
68
+ saved = cvt(saved, torch.float32)
69
+ return cvt(a.mul.Tensor(gradient, a.sub.Tensor(1, a.mul.Tensor(saved, saved))),
70
+ torch.bfloat16)
71
+
72
+ q = rotary(a.add.Tensor(head(qm), head(ql)))
73
+ kw = a.permute.default(reshape(kw, [4, 16, 32, 64, 128]), [0, 1, 2, 4, 3])
74
+ k = rotary(a.add.Tensor(tanh_backward(kw, tk), head(kl)))
75
+ v = head(vl)
76
+ dy = a.permute.default(reshape(reshape(dy, [4, 2048, 2048]),
77
+ [4, 16, 128, 32, 64]), [0, 1, 3, 2, 4])
78
+ dg = a.add.Tensor(a.neg.default(a.mul.Tensor(dy, a.div.Tensor(head(memory), 8.0))),
79
+ a.mul.Tensor(dy, head(local)))
80
+ gf = cvt(gate, torch.float32)
81
+ dg = cvt(a.mul.Tensor(cvt(dg, torch.float32),
82
+ a.mul.Tensor(gf, a.sub.Tensor(1, gf))), torch.bfloat16)
83
+ u = tanh_backward(head(uw), tu)
84
+
85
+ def pack(value, clone=True):
86
+ value = a.permute.default(value, [0, 1, 3, 2, 4])
87
+ if clone:
88
+ value = a.clone.default(value, memory_format=torch.contiguous_format)
89
+ return reshape(reshape(value, [4, 2048, 32, 64]), [4, 2048, 2048])
90
+
91
+ return a.cat.default([pack(q), pack(k), pack(v), pack(dg, False), pack(u)], 2)
92
+
93
+
94
+ @lru_cache(maxsize=1)
95
+ def _pattern():
96
+ from torch._subclasses.fake_tensor import FakeTensorMode
97
+ from torch.fx.experimental.proxy_tensor import make_fx
98
+
99
+ with FakeTensorMode():
100
+ inputs = [torch.empty(shape, dtype=torch.float32 if name in ("sin", "cos")
101
+ else torch.bfloat16, device="cuda:0")
102
+ for name, shape in zip(_NAMES, _NATIVE_SHAPES, strict=True)]
103
+ gm = make_fx(_aten_boundary, tracing_mode="fake")(*inputs)
104
+ # make_fx normalizes explicit reshape to view; this hook runs after
105
+ # Inductor's view-to-reshape normalization.
106
+ for node in gm.graph.nodes:
107
+ if node.target == torch.ops.aten.view.default:
108
+ node.target = torch.ops.aten.reshape.default
109
+ return gm
110
+
111
+
112
+ def _metadata(value):
113
+ if not isinstance(value, torch.Tensor):
114
+ return None
115
+ # Symbolic/unsupported dimensions are rejected, not specialized implicitly.
116
+ if any(type(x) is not int for x in (*value.shape, *value.stride())):
117
+ return None
118
+ return (tuple(value.shape), tuple(value.stride()), value.dtype, value.device.type)
119
+
120
+
121
+ class _Mismatch(Exception):
122
+ pass
123
+
124
+
125
+ def _match(candidate, pattern):
126
+ binding, reverse = {}, {}
127
+
128
+ def walk(actual, expected):
129
+ if isinstance(expected, Node):
130
+ if not isinstance(actual, Node):
131
+ raise _Mismatch("node kind")
132
+ if expected in binding:
133
+ if binding[expected] is not actual:
134
+ raise _Mismatch("shared dataflow")
135
+ return
136
+ if actual in reverse:
137
+ raise _Mismatch("distinct dataflow")
138
+ actual_meta, expected_meta = _metadata(actual.meta.get("val")), _metadata(expected.meta.get("val"))
139
+ if actual_meta is None or actual_meta != expected_meta:
140
+ raise _Mismatch("tensor metadata")
141
+ if expected.op != "placeholder":
142
+ if actual.op != expected.op or actual.target != expected.target:
143
+ raise _Mismatch("operator")
144
+ walk(actual.args, expected.args)
145
+ walk(actual.kwargs, expected.kwargs)
146
+ binding[expected] = actual
147
+ reverse[actual] = expected
148
+ elif isinstance(expected, (tuple, list)):
149
+ if not isinstance(actual, (tuple, list)) or len(actual) != len(expected):
150
+ raise _Mismatch("argument list")
151
+ for av, ev in zip(actual, expected, strict=True):
152
+ walk(av, ev)
153
+ elif isinstance(expected, dict):
154
+ if not isinstance(actual, dict) or set(actual) != set(expected):
155
+ raise _Mismatch("keyword arguments")
156
+ for key in expected:
157
+ walk(actual[key], expected[key])
158
+ elif isinstance(expected, torch.device):
159
+ if not isinstance(actual, torch.device) or actual.type != expected.type:
160
+ raise _Mismatch("device")
161
+ elif type(actual) is not type(expected) or actual != expected:
162
+ raise _Mismatch("literal")
163
+
164
+ output = next(node for node in pattern.graph.nodes if node.op == "output").args[0]
165
+ walk(candidate, output)
166
+ inputs = tuple(binding[node] for node in pattern.graph.nodes if node.op == "placeholder")
167
+ interior = {actual for expected, actual in binding.items() if expected.op != "placeholder"}
168
+ retained = set()
169
+
170
+ def retain_view(node):
171
+ if node not in interior or node in retained:
172
+ return
173
+ # AOT CSE can share the exact zero-filled scatter base across cores.
174
+ # Its complete literal recipe was matched above; preserve it while
175
+ # another region still uses it, without retaining computed arithmetic.
176
+ if node.target == torch.ops.aten.full.default and not node.all_input_nodes:
177
+ retained.add(node)
178
+ return
179
+ if node.target not in (torch.ops.aten.reshape.default, torch.ops.aten.permute.default):
180
+ raise _Mismatch("external intermediate consumer")
181
+ retained.add(node)
182
+ for parent in node.all_input_nodes:
183
+ retain_view(parent)
184
+
185
+ for node in interior - {candidate}:
186
+ if any(user not in interior for user in node.users):
187
+ # The raw output-gradient's head view also feeds the independent
188
+ # attention/memory VJPs. Retain that metadata-only chain in place;
189
+ # the opaque op takes its original contiguous MM result directly.
190
+ retain_view(node)
191
+ if len({value.meta["val"].device for value in inputs}) != 1:
192
+ raise _Mismatch("mixed input devices")
193
+ _validate_producers(inputs)
194
+ return inputs, interior - retained
195
+
196
+
197
+ def _validate_producers(inputs):
198
+ """The local branches must belong to one causal dropout-zero Flash VJP."""
199
+ qm, ql, _sin, _cos, kw, _tk, kl, vl, dy, memory, local, _gate, uw, _tu = inputs
200
+ flash = None
201
+ for node, index in ((ql, 0), (kl, 1), (vl, 2)):
202
+ if (node.op != "call_function" or node.target is not operator.getitem
203
+ or len(node.args) != 2 or node.args[1] != index):
204
+ raise _Mismatch("Flash gradient ordering")
205
+ if flash is None:
206
+ flash = node.args[0]
207
+ elif node.args[0] is not flash:
208
+ raise _Mismatch("separate Flash VJPs")
209
+ if (not isinstance(flash, Node)
210
+ or flash.target != torch.ops.aten._scaled_dot_product_flash_attention_backward.default
211
+ or len(flash.args) != 14 or flash.args[8:12] != (128, 128, 0.0, True)
212
+ or dict(flash.kwargs) != {"scale": 0.125} or flash.args[4] is not local):
213
+ raise _Mismatch("Flash attention settings")
214
+ for node in (qm, kw, uw):
215
+ if node.op != "call_function" or node.target != torch.ops.aten.bmm.default:
216
+ raise _Mismatch("native backward GEMM producer")
217
+ if dy.op != "call_function" or dy.target != torch.ops.aten.mm.default:
218
+ raise _Mismatch("native output gradient producer")
219
+
220
+
221
+ @custom_op("warpstate::paired_backward_layout", mutates_args=())
222
+ def _paired_backward_layout(q_memory: torch.Tensor, q_local: torch.Tensor,
223
+ sin: torch.Tensor, cos: torch.Tensor, k_write: torch.Tensor,
224
+ tanh_k: torch.Tensor, k_local: torch.Tensor, v_local: torch.Tensor,
225
+ grad_y: torch.Tensor, memory: torch.Tensor, local: torch.Tensor,
226
+ gate: torch.Tensor, u_write: torch.Tensor,
227
+ tanh_u: torch.Tensor) -> torch.Tensor:
228
+ """Opaque pure launch using native input storage, with no layout copies.
229
+
230
+ Fake/compile guards already require native shapes and strides. The entrypoint
231
+ is internal to this strict rewrite; shape views and fresh output allocation
232
+ are the only tensor operations. Runtime evidence remains valid on cache hits.
233
+ """
234
+ from ops.backward_layout import _paired_kernel
235
+
236
+ args = (q_memory, q_local, sin, cos, k_write, tanh_k, k_local, v_local,
237
+ grad_y, memory, local, gate, u_write, tanh_u)
238
+ _validate_native_inputs(args)
239
+ output = torch.empty((4, 2048, 10240), dtype=torch.bfloat16, device=q_memory.device)
240
+ # All contiguous native shapes have exactly the raw layouts consumed by the
241
+ # prototype kernel; passing native pointers needs no view or contiguous call.
242
+ launch = (*args, output, 16777216, 32, 128, 64, 0.125, 256)
243
+ _paired_kernel[(32768,)](*launch, True, num_warps=8, enable_fp_fusion=True)
244
+ _paired_kernel[(65536,)](*launch, False, num_warps=8, enable_fp_fusion=True)
245
+ _COUNTS["runtime_calls"] += 1
246
+ return output
247
+
248
+
249
+ @_paired_backward_layout.register_fake
250
+ def _paired_backward_layout_fake(*inputs):
251
+ _validate_native_inputs(inputs)
252
+ return inputs[0].new_empty((4, 2048, 10240))
253
+
254
+
255
+ def _validate_native_inputs(inputs):
256
+ for name, value, shape in zip(_NAMES, inputs, _NATIVE_SHAPES, strict=True):
257
+ dtype = torch.float32 if name in ("sin", "cos") else torch.bfloat16
258
+ if (tuple(value.shape) != shape or value.dtype != dtype or not value.is_contiguous()
259
+ or value.device != inputs[0].device or value.device.type != "cuda"):
260
+ raise ValueError(f"Unsupported paired backward input {name}")
261
+
262
+
263
+ class BackwardLayoutPass(CustomGraphPass):
264
+ def __init__(self, mode, identity):
265
+ self.mode, self.identity = mode, identity
266
+
267
+ def uuid(self):
268
+ return self.identity
269
+
270
+ def __call__(self, graph):
271
+ _COUNTS["graphs"] += 1
272
+ candidates = [node for node in graph.nodes if node.op == "call_function"
273
+ and node.target == torch.ops.aten.cat.default
274
+ and isinstance(node.args[0], (tuple, list)) and len(node.args[0]) == 5]
275
+ for candidate in candidates:
276
+ _COUNTS["candidates"] += 1
277
+ try:
278
+ inputs, interior = _match(candidate, _pattern())
279
+ except _Mismatch as error:
280
+ _MISSES[str(error)] += 1
281
+ continue
282
+ _COUNTS["matches"] += 1
283
+ if self.mode == "inspect":
284
+ continue
285
+ with graph.inserting_before(candidate):
286
+ replacement = graph.call_function(torch.ops.warpstate.paired_backward_layout.default, inputs)
287
+ replacement.meta = dict(candidate.meta)
288
+ candidate.replace_all_uses_with(replacement)
289
+ for node in reversed(list(graph.nodes)):
290
+ if node in interior:
291
+ graph.erase_node(node)
292
+ _COUNTS["replacements"] += 1
293
+
294
+
295
+ def _identity(mode):
296
+ directory = Path(__file__).resolve().parent
297
+ sources = {"ops/" + name: hashlib.sha256((directory / name).read_bytes()).hexdigest()
298
+ for name in ("backward_layout_execution.py", "backward_layout.py")}
299
+ metadata = {"format": "warpstate-backward-layout-v1", "mode": mode,
300
+ "source_sha256": sources, "torch_version": torch.__version__,
301
+ "shape": list(_SHAPE), "algorithm": "paired", "block": 256, "warps": 8}
302
+ metadata["identity"] = hashlib.sha256(json.dumps(metadata, sort_keys=True).encode()).hexdigest()
303
+ return metadata
304
+
305
+
306
+ def validate_backward_layout_config(cfg):
307
+ """Reject unsupported model recipes before allocation or resume mutation."""
308
+ required = {"d_model": 2048, "n_heads": 32, "chunk_size": 128,
309
+ "parallel_memory": True, "dropout": 0.0}
310
+ for name, value in required.items():
311
+ if getattr(cfg, name, None) != value:
312
+ raise ValueError(f"Paired backward layout requires {name}={value}")
313
+
314
+
315
+ def configure_backward_layout(mode="baseline"):
316
+ """Configure once before compilation, returning checkpoint/benchmark identity.
317
+
318
+ A pre-existing custom pass is rejected, never replaced or silently composed;
319
+ composition could change the arithmetic boundary. Use a fresh process when
320
+ changing modes, matching Compiled Autograd's process-wide configuration.
321
+ """
322
+ global _ACTIVE_MODE, _ACTIVE_PASS
323
+ if mode not in BACKWARD_LAYOUT_MODES:
324
+ raise ValueError(f"Unknown backward layout mode: {mode}")
325
+ from torch._inductor import config
326
+ if _ACTIVE_PASS is not None:
327
+ raise RuntimeError("Backward layout is already configured; use a fresh process")
328
+ if mode == "baseline":
329
+ return None
330
+ if config.post_grad_custom_post_pass is not None:
331
+ raise RuntimeError("A post-gradient custom pass already exists; refusing to replace it")
332
+ if (config.emulate_precision_casts or not config.triton.codegen_upcast_to_fp32
333
+ or config.eager_numerics.disable_ftz):
334
+ raise ValueError("Paired backward layout requires the profiled FP32 pointwise fusion settings")
335
+ metadata = _identity(mode)
336
+ _ACTIVE_MODE = mode
337
+ _ACTIVE_PASS = BackwardLayoutPass(mode, metadata["identity"])
338
+ config.post_grad_custom_post_pass = _ACTIVE_PASS
339
+ return metadata
340
+
341
+
342
+ def backward_layout_stats():
343
+ """Process-local compilation and runtime evidence; no device synchronization."""
344
+ return {"mode": _ACTIVE_MODE, **{name: _COUNTS[name] for name in
345
+ ("graphs", "candidates", "matches", "replacements", "runtime_calls")},
346
+ "miss_reasons": dict(_MISSES)}
347
+
348
+
349
+ def require_backward_layout_dispatch(min_calls=1):
350
+ if isinstance(min_calls, bool) or not isinstance(min_calls, int) or min_calls < 1:
351
+ raise ValueError("min_calls must be a positive integer")
352
+ if _ACTIVE_MODE != "paired" or _COUNTS["runtime_calls"] < min_calls:
353
+ raise RuntimeError(f"Paired backward layout expected at least {min_calls} runtime calls; "
354
+ f"observed {backward_layout_stats()}")
355
+ return backward_layout_stats()
ops/benchmark-supervisord.conf ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [program:warpstate-benchmark]
2
+ command=/workspace/WarpState/rtx6000_40b/ops/benchmark.sh --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt --micro-batch 4 --no-activation-checkpointing --compile-cores --autocast-cache --loss-chunk-size 1024 --vram-limit-gib 88 --warmup-updates 2 --measured-updates 3
3
+ directory=/workspace/WarpState/rtx6000_40b
4
+ autostart=false
5
+ autorestart=false
6
+ startsecs=5
7
+ startretries=0
8
+ stopsignal=TERM
9
+ stopwaitsecs=30
10
+ stopasgroup=true
11
+ killasgroup=true
12
+ stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/benchmark.log
13
+ stdout_logfile_maxbytes=20MB
14
+ stdout_logfile_backups=3
15
+ redirect_stderr=true
16
+ environment=TORCHINDUCTOR_COMPILE_THREADS="8",WARPSTATE_BENCH_DEVICES="0,1",WARPSTATE_BENCH_PROCESSES="2"
ops/benchmark.sh ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ ulimit -c 0
4
+ source /workspace/WarpState/rtx6000_40b/ops/training-env.sh
5
+ cd /workspace/WarpState/rtx6000_40b
6
+ export CUDA_VISIBLE_DEVICES="${WARPSTATE_BENCH_DEVICES:-0}"
7
+ if [[ "${WARPSTATE_BENCH_PROCESSES:-1}" == 1 ]]; then
8
+ exec .server-venv/bin/python -u ops/benchmark_training.py "$@"
9
+ else
10
+ exec .server-venv/bin/torchrun --standalone \
11
+ --nproc-per-node="$WARPSTATE_BENCH_PROCESSES" \
12
+ --log-dir=work/logs/torchrun-benchmark ops/benchmark_training.py "$@"
13
+ fi
ops/benchmark_gemm.py ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Time isolated BF16 GEMMs with the dominant WarpState projection layouts.
3
+
4
+ Source ops/training-env.sh first. Only deterministic random fixtures are used;
5
+ this program never opens a checkpoint or token dataset. Compilation/autotuning
6
+ are excluded from CUDA-event timings. These timings exclude communication,
7
+ optimizer updates and most Python overhead, so are not training throughput.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import contextlib
13
+ import datetime as dt
14
+ import hashlib
15
+ import json
16
+ import math
17
+ import os
18
+ from pathlib import Path
19
+ import statistics
20
+ import subprocess
21
+ import time
22
+
23
+ PACKAGE = Path(__file__).resolve().parents[1]
24
+ WORKSPACE = PACKAGE.parent
25
+ GIB = 1024**3
26
+ FAMILIES = {"ffn_up_gate": (2048, 14336), "in_proj": (2048, 10240),
27
+ "ffn_down": (7168, 2048), "out_proj": (2048, 2048)}
28
+ BACKENDS = {"eager": None, "aten": "ATEN", "triton": "TRITON", "mixed": "ATEN,TRITON"}
29
+
30
+
31
+ def projection_cases(families, rows, operations):
32
+ """Return M,N,K and storage layouts for Y=X@W.T, dX=dY@W, dW=dY.T@X."""
33
+ cases = []
34
+ for family in families:
35
+ input_dim, output_dim = FAMILIES[family]
36
+ specs = {
37
+ "fwd": (rows, output_dim, input_dim, False, True),
38
+ "dx": (rows, input_dim, output_dim, False, False),
39
+ "dw": (output_dim, input_dim, rows, True, False),
40
+ }
41
+ for operation in operations:
42
+ m, n, k, at, bt = specs[operation]
43
+ cases.append({"name": family + "." + operation, "family": family,
44
+ "operation": operation, "m": m, "n": n, "k": k,
45
+ "a_transposed": at, "b_transposed": bt,
46
+ "a_shape": [m, k], "b_shape": [k, n],
47
+ "a_stride": [1, m] if at else [k, 1],
48
+ "b_stride": [1, k] if bt else [n, 1],
49
+ "flops": 2 * m * n * k})
50
+ return cases
51
+
52
+
53
+ def arguments(argv=None):
54
+ parser = argparse.ArgumentParser(description=__doc__)
55
+ parser.add_argument("--families", nargs="+", choices=FAMILIES,
56
+ default=["ffn_up_gate", "in_proj", "ffn_down"])
57
+ parser.add_argument("--operations", nargs="+", choices=["fwd", "dx", "dw"],
58
+ default=["fwd", "dx", "dw"])
59
+ parser.add_argument("--rows", type=int, default=8192,
60
+ help="Micro-batch times sequence length; production uses 8192")
61
+ parser.add_argument("--backends", nargs="+", choices=BACKENDS,
62
+ default=["eager", "aten", "triton"])
63
+ parser.add_argument("--warmup", type=int, default=10)
64
+ parser.add_argument("--repeats", type=int, default=30)
65
+ parser.add_argument("--trials", type=int, default=5)
66
+ parser.add_argument("--burnin-seconds", type=float, default=2.0,
67
+ help="Sustain GEMM load before timing to reduce cold boost effects")
68
+ parser.add_argument("--seed", type=int, default=1337)
69
+ parser.add_argument("--device", type=int, default=0,
70
+ help="CUDA device index after CUDA_VISIBLE_DEVICES filtering")
71
+ parser.add_argument("--blas-backend", choices=["default", "cublas", "cublaslt"],
72
+ default="default")
73
+ parser.add_argument("--allow-reduced-precision-reduction", action=argparse.BooleanOptionalAction,
74
+ default=True,
75
+ help="Allow BF16 intermediate reductions, matching the current production setting")
76
+ parser.add_argument("--list-shapes", action="store_true",
77
+ help="Print selected shapes/strides and exit without importing torch")
78
+ parser.add_argument("--output-dir", type=Path,
79
+ default=PACKAGE / "work" / ("benchmark-gemm-" +
80
+ dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%S")))
81
+ args = parser.parse_args(argv)
82
+ for key in ("rows", "warmup", "repeats", "trials"):
83
+ if getattr(args, key) < 1:
84
+ parser.error("--" + key + " must be positive")
85
+ if args.device < 0:
86
+ parser.error("--device must be nonnegative")
87
+ if not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0:
88
+ parser.error("--burnin-seconds must be finite and nonnegative")
89
+ for key in ("families", "operations", "backends"):
90
+ if len(getattr(args, key)) != len(set(getattr(args, key))):
91
+ parser.error("--" + key + " must not contain duplicates")
92
+ args.output_dir = args.output_dir.expanduser().resolve()
93
+ work = PACKAGE / "work"
94
+ if (not args.output_dir.is_relative_to(work) or
95
+ not args.output_dir.relative_to(work).parts or
96
+ not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")):
97
+ parser.error("--output-dir must be below rtx6000_40b/work/benchmark-*")
98
+ if not args.list_shapes:
99
+ for key in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH",
100
+ "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"):
101
+ if not os.environ.get(key) or not Path(os.environ[key]).resolve().is_relative_to(WORKSPACE):
102
+ parser.error("Source ops/training-env.sh first; " + key + " must stay in the workspace")
103
+ return args
104
+
105
+
106
+ def gpu_telemetry():
107
+ try:
108
+ result = subprocess.run(
109
+ ["nvidia-smi", "--query-gpu=index,uuid,name,power.draw,power.limit,clocks.sm,"
110
+ "clocks.mem,temperature.gpu,utilization.gpu,memory.used", "--format=csv,noheader"],
111
+ capture_output=True, text=True, timeout=10, check=False)
112
+ return {"csv": result.stdout.strip(), "returncode": result.returncode}
113
+ except (OSError, subprocess.TimeoutExpired) as error:
114
+ return {"error": str(error)}
115
+
116
+
117
+ def main():
118
+ args = arguments()
119
+ cases = projection_cases(args.families, args.rows, args.operations)
120
+ if args.list_shapes:
121
+ print(json.dumps(cases, indent=2))
122
+ return
123
+ # Reject an existing directory to preserve previous measurements.
124
+ args.output_dir.mkdir(parents=True, exist_ok=False)
125
+ import torch
126
+ from torch._inductor import config
127
+
128
+ if not torch.cuda.is_available():
129
+ raise SystemExit("CUDA is required for timings; --list-shapes is CPU-only")
130
+ torch.cuda.set_device(args.device)
131
+ device = torch.device("cuda", args.device)
132
+ if not torch.cuda.is_bf16_supported():
133
+ raise SystemExit("This GPU does not support BF16")
134
+ torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = args.allow_reduced_precision_reduction
135
+ torch.backends.cuda.matmul.allow_tf32 = False
136
+ torch.backends.cuda.preferred_blas_library(args.blas_backend)
137
+ properties = torch.cuda.get_device_properties(device)
138
+ report = {
139
+ "started_utc": dt.datetime.now(dt.UTC).isoformat(),
140
+ "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
141
+ "arguments": {key: str(value) if isinstance(value, Path) else value
142
+ for key, value in vars(args).items()},
143
+ "torch_version": torch.__version__, "cuda_version": torch.version.cuda,
144
+ "dtype": "bfloat16", "output_dtype": "bfloat16",
145
+ "allow_bf16_reduced_precision_reduction": args.allow_reduced_precision_reduction,
146
+ "compile_mode": "max-autotune-no-cudagraphs",
147
+ "autotune_fallback_to_aten": False,
148
+ "device": {"name": properties.name, "capability": list(torch.cuda.get_device_capability(device)),
149
+ "multiprocessors": properties.multi_processor_count,
150
+ "total_memory_gib": properties.total_memory / GIB},
151
+ "environment": {key: os.environ.get(key) for key in
152
+ ("CUDA_VISIBLE_DEVICES", "CUBLAS_WORKSPACE_CONFIG", "OMP_NUM_THREADS",
153
+ "TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR", "PYTORCH_ALLOC_CONF",
154
+ "TORCHINDUCTOR_COMPILE_THREADS", "ENABLE_PERSISTENT_TMA_MATMUL",
155
+ "TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_BACKENDS",
156
+ "TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_SEARCH_SPACE", "CUTLASS_EPILOGUE_FUSION")},
157
+ "telemetry_before": gpu_telemetry(), "results": [],
158
+ "notes": ["Shapes represent isolated linear algebra, not an end-to-end backward graph.",
159
+ "Each repeat allocates a BF16 output, as torch.mm normally does.",
160
+ "Timings reuse inputs and omit compilation/autotuning; compare to full training separately.",
161
+ "Error is against eager cuBLAS BF16 output, not an exact FP32 reference.",
162
+ "Telemetry is a snapshot, not a synchronized power integral."],
163
+ }
164
+
165
+ def save():
166
+ target = args.output_dir / "results.json"
167
+ temporary = args.output_dir / "results.json.partial"
168
+ temporary.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
169
+ temporary.replace(target)
170
+
171
+ save()
172
+ with torch.inference_mode():
173
+ for case_index, case in enumerate(cases):
174
+ generator = torch.Generator(device=device).manual_seed(args.seed + case_index)
175
+
176
+ def fixture(rows, columns, transposed):
177
+ shape = (columns, rows) if transposed else (rows, columns)
178
+ tensor = torch.randn(shape, device=device, dtype=torch.bfloat16, generator=generator)
179
+ return tensor.T if transposed else tensor
180
+
181
+ a = fixture(case["m"], case["k"], case["a_transposed"])
182
+ b = fixture(case["k"], case["n"], case["b_transposed"])
183
+ assert list(a.stride()) == case["a_stride"]
184
+ assert list(b.stride()) == case["b_stride"]
185
+ reference = torch.mm(a, b)
186
+ torch.cuda.synchronize(device)
187
+ for backend in args.backends:
188
+ row = {**case, "backend": backend, "status": "running",
189
+ "max_autotune_gemm_backends": BACKENDS[backend],
190
+ "telemetry_before": gpu_telemetry()}
191
+ print(json.dumps({"case": case["name"], "backend": backend, "status": "starting"}), flush=True)
192
+ try:
193
+ torch._dynamo.reset()
194
+ settings = ({"max_autotune_gemm_backends": BACKENDS[backend],
195
+ "autotune_fallback_to_aten": False}
196
+ if BACKENDS[backend] else {})
197
+ with config.patch(settings) if settings else contextlib.nullcontext():
198
+ operation = torch.mm if backend == "eager" else torch.compile(
199
+ lambda x, y: torch.mm(x, y), fullgraph=True, dynamic=False,
200
+ mode="max-autotune-no-cudagraphs")
201
+ compile_start = time.perf_counter()
202
+ output = operation(a, b)
203
+ torch.cuda.synchronize(device)
204
+ row["first_call_seconds"] = time.perf_counter() - compile_start
205
+ difference = output.float().sub_(reference.float())
206
+ reference_rms = reference.float().square_().mean().sqrt().item()
207
+ row["max_abs_error"] = difference.abs().max().item()
208
+ row["rms_error"] = difference.square_().mean().sqrt().item()
209
+ row["relative_rms_error"] = row["rms_error"] / max(reference_rms, 1e-30)
210
+ if not all(math.isfinite(row[key]) for key in
211
+ ("max_abs_error", "rms_error", "relative_rms_error")):
212
+ raise RuntimeError("Nonfinite output error")
213
+ del difference, output
214
+ for _ in range(args.warmup):
215
+ output = operation(a, b)
216
+ del output
217
+ torch.cuda.synchronize(device)
218
+ burnin_start = time.perf_counter()
219
+ burnin_iterations = 0
220
+ while time.perf_counter() - burnin_start < args.burnin_seconds:
221
+ for _ in range(10):
222
+ output = operation(a, b)
223
+ del output
224
+ torch.cuda.synchronize(device)
225
+ burnin_iterations += 10
226
+ row["burnin_elapsed_seconds"] = time.perf_counter() - burnin_start
227
+ row["burnin_iterations"] = burnin_iterations
228
+ torch.cuda.reset_peak_memory_stats(device)
229
+ row["baseline_allocated_gib"] = torch.cuda.memory_allocated(device) / GIB
230
+ trial_ms = []
231
+ for _ in range(args.trials):
232
+ start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
233
+ start.record()
234
+ for _ in range(args.repeats):
235
+ output = operation(a, b)
236
+ end.record()
237
+ end.synchronize()
238
+ trial_ms.append(start.elapsed_time(end) / args.repeats)
239
+ del output
240
+ row["trial_ms_per_gemm"] = trial_ms
241
+ row["median_ms"] = statistics.median(trial_ms)
242
+ row["min_ms"] = min(trial_ms)
243
+ row["tflops"] = case["flops"] / (row["median_ms"] * 1e9)
244
+ row["peak_allocated_gib"] = torch.cuda.max_memory_allocated(device) / GIB
245
+ row["peak_reserved_gib"] = torch.cuda.max_memory_reserved(device) / GIB
246
+ row["telemetry_after"] = gpu_telemetry()
247
+ row["status"] = "ok"
248
+ except Exception as error:
249
+ row["status"] = "error"
250
+ row["error"] = type(error).__name__ + ": " + str(error)[:4000]
251
+ report["results"].append(row)
252
+ save()
253
+ print(json.dumps({key: row[key] for key in
254
+ ("name", "backend", "status", "median_ms", "tflops",
255
+ "max_abs_error", "rms_error", "error") if key in row}), flush=True)
256
+ del a, b, reference
257
+ torch.cuda.empty_cache()
258
+ report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat()
259
+ report["telemetry_after"] = gpu_telemetry()
260
+ save()
261
+ print("Results: " + str(args.output_dir / "results.json"), flush=True)
262
+ if any(row["status"] != "ok" for row in report["results"]):
263
+ raise SystemExit(1)
264
+
265
+
266
+ if __name__ == "__main__":
267
+ main()