Upload 139 files
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +25 -0
- DATA_NOTICE.md +14 -0
- PACKAGE_MANIFEST.json +42 -0
- README.md +280 -1
- README_UK.md +170 -0
- SOURCE_PACKAGE_NOTES.md +29 -0
- VALIDATION.json +19 -0
- assets/warpstate-architecture.svg +48 -0
- assets/warpstate-banner.svg +29 -0
- chat.py +341 -0
- dpo.py +647 -0
- ops/B200_HANDOFF.md +214 -0
- ops/B200_OPTIMIZATION_PLAN.md +120 -0
- ops/B200_PERFORMANCE.md +231 -0
- ops/B200_SCHEDULING.md +83 -0
- ops/B200_SETUP.md +176 -0
- ops/BACKWARD_LAYOUT_EXPERIMENT.md +204 -0
- ops/BACKWARD_LAYOUT_INTEGRATION.md +307 -0
- ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md +147 -0
- ops/CHAT.md +134 -0
- ops/COMMUNICATION_EXPLORATION_20260912.md +254 -0
- ops/DATA_PREPARATION.md +78 -0
- ops/DPO_CHAT.md +57 -0
- ops/DPO_HANDOFF.md +144 -0
- ops/LOCAL_RESUME.md +207 -0
- ops/LOSS_FORECAST_20260913.md +208 -0
- ops/MATMUL_PROFILE_20260912.md +316 -0
- ops/MATMUL_TUNING.md +342 -0
- ops/OPTIMIZATION_20260912.md +349 -0
- ops/OPTIMIZED_TRAINING.md +228 -0
- ops/PERFORMANCE.md +236 -0
- ops/POINTWISE_HEADROOM_20260913.md +195 -0
- ops/POSTTRAIN.md +165 -0
- ops/RESTART_HANDOFF.md +97 -0
- ops/RTX_ASYNC_TRAINING.md +114 -0
- ops/RTX_PROFILE_20260910.md +453 -0
- ops/TRAINING.md +148 -0
- ops/activation_execution.py +253 -0
- ops/analyze_communication_profile.py +227 -0
- ops/archive_dpo.py +302 -0
- ops/b200-requirements.txt +2 -0
- ops/b200-supervisord.conf +35 -0
- ops/b200ctl.sh +5 -0
- ops/backward_layout.py +283 -0
- ops/backward_layout_baseline.py +201 -0
- ops/backward_layout_benchmark.py +164 -0
- ops/backward_layout_execution.py +355 -0
- ops/benchmark-supervisord.conf +16 -0
- ops/benchmark.sh +13 -0
- ops/benchmark_gemm.py +267 -0
.gitignore
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Python
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
.mypy_cache/
|
| 6 |
+
.ruff_cache/
|
| 7 |
+
.venv/
|
| 8 |
+
.server-venv/
|
| 9 |
+
venv/
|
| 10 |
+
|
| 11 |
+
# Model / training outputs
|
| 12 |
+
work/
|
| 13 |
+
checkpoints/
|
| 14 |
+
*.pt
|
| 15 |
+
*.pth
|
| 16 |
+
*.ckpt
|
| 17 |
+
*.safetensors
|
| 18 |
+
*.bin
|
| 19 |
+
*.log
|
| 20 |
+
*.pid
|
| 21 |
+
|
| 22 |
+
# Secrets and local environment
|
| 23 |
+
.env
|
| 24 |
+
.env.*
|
| 25 |
+
!.env.example
|
DATA_NOTICE.md
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dataset attribution
|
| 2 |
+
|
| 3 |
+
Default upstream dataset: **FineWeb-Edu**, Hugging Face / HuggingFaceFW.
|
| 4 |
+
Repository: https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu
|
| 5 |
+
Configuration: sample-100BT. Language: English. Dataset card lists ODC-BY.
|
| 6 |
+
Read the upstream dataset card and license for applicable attribution and
|
| 7 |
+
underlying web-content terms. This ZIP distributes no dataset texts.
|
| 8 |
+
The immutable source revision is recorded in work/data/manifest.json at preparation.
|
| 9 |
+
|
| 10 |
+
The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale:
|
| 11 |
+
https://arxiv.org/abs/2406.17557
|
| 12 |
+
|
| 13 |
+
The WarpState code and tokenizer are copied from the user's existing project.
|
| 14 |
+
No new ownership or license over those user-provided assets is asserted here.
|
PACKAGE_MANIFEST.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_40b.py": {
|
| 3 |
+
"sha256": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
|
| 4 |
+
"bytes": 50988
|
| 5 |
+
},
|
| 6 |
+
"README_UK.md": {
|
| 7 |
+
"sha256": "b401dc313e25aea48ab1a0bd24945a9923cd98adcb254c77cb1e130dd16c1335",
|
| 8 |
+
"bytes": 13562
|
| 9 |
+
},
|
| 10 |
+
"DATA_NOTICE.md": {
|
| 11 |
+
"sha256": "6b11284648bcb6fe9d1c74204d54e64d228b05d78749330c5caab119806b2a79",
|
| 12 |
+
"bytes": 735
|
| 13 |
+
},
|
| 14 |
+
"test_server.py": {
|
| 15 |
+
"sha256": "66ebfd733f8d71a462108c5d99dc469a99d1023bb7a0cd611b501a58b9269bff",
|
| 16 |
+
"bytes": 6351
|
| 17 |
+
},
|
| 18 |
+
"tokenizer.model": {
|
| 19 |
+
"sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 20 |
+
"bytes": 545080
|
| 21 |
+
},
|
| 22 |
+
"configs/warpstate_2b.json": {
|
| 23 |
+
"sha256": "884ba9e523af2a065ff0e11a4fd86eddfdc5ef656196dc318f94ec455be63009",
|
| 24 |
+
"bytes": 269
|
| 25 |
+
},
|
| 26 |
+
"warpstate/__init__.py": {
|
| 27 |
+
"sha256": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
|
| 28 |
+
"bytes": 113
|
| 29 |
+
},
|
| 30 |
+
"warpstate/config.py": {
|
| 31 |
+
"sha256": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
|
| 32 |
+
"bytes": 1187
|
| 33 |
+
},
|
| 34 |
+
"warpstate/model.py": {
|
| 35 |
+
"sha256": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161",
|
| 36 |
+
"bytes": 15500
|
| 37 |
+
},
|
| 38 |
+
"VALIDATION.json": {
|
| 39 |
+
"sha256": "168bb8233b04db144021f185b6a3890c06c137c3c5c29c1f18464ee8474ce1b8",
|
| 40 |
+
"bytes": 614
|
| 41 |
+
}
|
| 42 |
+
}
|
README.md
CHANGED
|
@@ -1,3 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<div align="center">
|
| 2 |
+
<img src="assets/warpstate-banner.svg" alt="WarpState — memory-native language modeling" width="100%" />
|
| 3 |
+
|
| 4 |
+
<h1>⚡ WarpState</h1>
|
| 5 |
+
<p><strong>A memory-native language model built around local attention, fast/slow temporal states, and depth-wise computation.</strong></p>
|
| 6 |
+
<p><em>Experimental architecture · Original 2.005B-parameter research run · Pretraining → SFT → DPO</em></p>
|
| 7 |
+
|
| 8 |
+
<p>
|
| 9 |
+
<img alt="Research" src="https://img.shields.io/badge/status-research%20prototype-8B5CF6?style=for-the-badge" />
|
| 10 |
+
<img alt="Documented model size" src="https://img.shields.io/badge/model-2.005B%20parameters-22D3EE?style=for-the-badge" />
|
| 11 |
+
<img alt="Tokens trained" src="https://img.shields.io/badge/pretraining-10B%20tokens-0EA5E9?style=for-the-badge" />
|
| 12 |
+
<img alt="PyTorch" src="https://img.shields.io/badge/framework-PyTorch-EE4C2C?style=for-the-badge&logo=pytorch&logoColor=white" />
|
| 13 |
+
</p>
|
| 14 |
+
<p><strong>Created by <a href="https://github.com/gtausa197-svg">Volodymyr Makarenko</a> · Co-authored & sponsored by <a href="https://github.com/ykim078">@ykim078</a></strong></p>
|
| 15 |
+
<p>
|
| 16 |
+
<a href="#-what-is-warpstate">Overview</a> ·
|
| 17 |
+
<a href="#-architecture">Architecture</a> ·
|
| 18 |
+
<a href="#-training--evaluation">Results</a> ·
|
| 19 |
+
<a href="#-getting-started">Quick start</a> ·
|
| 20 |
+
<a href="#-project-leadership--sponsorship">Team & sponsor</a>
|
| 21 |
+
</p>
|
| 22 |
+
</div>
|
| 23 |
+
|
| 24 |
---
|
| 25 |
+
|
| 26 |
+
## 🧠 What is WarpState?
|
| 27 |
+
|
| 28 |
+
**WarpState** is an experimental autoregressive language model exploring a different way to balance *local precision* and *long-range continuity*. Instead of building a full sequence-wide attention matrix at every layer, it combines:
|
| 29 |
+
|
| 30 |
+
- **Causal local attention** within fixed **128-token chunks**.
|
| 31 |
+
- **Two decaying temporal memory states** (fast and slow) that carry compressed information from completed chunks.
|
| 32 |
+
- **Learned gating** that blends locally attended features with historical memory.
|
| 33 |
+
- **Depth-wise computation** through reusable model cores with depth-specific modulation.
|
| 34 |
+
- A full language-model training stack, including **BF16 computation / FP32 optimizer state**, checkpoint-resumable pretraining, supervised fine-tuning (SFT), and direct preference optimization (DPO).
|
| 35 |
+
|
| 36 |
+
The architecture is implemented in [`warpstate/model.py`](warpstate/model.py) and [`warpstate/config.py`](warpstate/config.py). **WarpState is not Project Nord's SNN architecture**: the supplied Project Nord README was used as a *presentation reference*, not as a source of WarpState features or benchmark claims.
|
| 37 |
+
|
| 38 |
+
> [!IMPORTANT]
|
| 39 |
+
> **Release contents:** The supplied source archives include Python code, the SentencePiece tokenizer, tests, and operational documentation, **but not pretrained weights, training data, or the `configs/` directory** referenced by several launchers. This README documents the historical research run described in the included records; the uploaded source package alone is **not** a one-command pretrained-model download. A complete runnable release must include the correct configuration files and access to trusted checkpoints.
|
| 40 |
+
|
| 41 |
+
## ✨ At a glance
|
| 42 |
+
|
| 43 |
+
| Research dimension | Documented state |
|
| 44 |
+
| :--- | :--- |
|
| 45 |
+
| **Model** | 2,005,111,424 parameters in the recorded full-size training run |
|
| 46 |
+
| **Tokenizer** | SentencePiece, 32,768-token vocabulary |
|
| 47 |
+
| **Training sequence** | 2,048 tokens; not equivalent to an exact 2,048-token persistent memory |
|
| 48 |
+
| **Local attention** | Causal SDPA, 128 tokens per chunk |
|
| 49 |
+
| **Persistent state** | Fast + slow fixed-size matrix state per attention head and core execution |
|
| 50 |
+
| **Training corpus prepared** | 40,000,000,000 train tokens + 1,048,576 validation tokens |
|
| 51 |
+
| **Pretraining completed** | 10,000,007,168 tokens, at step 76,294, per the September 2026 handoff |
|
| 52 |
+
| **Post-training** | 10,312 SFT updates; 2,334 DPO updates, reported complete |
|
| 53 |
+
| **Compute format** | BF16 forward/backward; FP32 parameters, gradients, and AdamW states |
|
| 54 |
+
| **Project stage** | Research implementation with historical training logs and tests; no independently verified general-purpose LLM benchmark in this package |
|
| 55 |
+
|
| 56 |
+
<sub>Numbers are sourced from the included source files and operational handoff documents, primarily `ops/RESTART_HANDOFF.md`, `ops/TRAINING.md`, `ops/DPO_HANDOFF.md`, and `ops/DPO_CHAT.md`. Training outcomes are historical records, not fresh measurements performed for this README.</sub>
|
| 57 |
+
|
| 58 |
---
|
| 59 |
+
|
| 60 |
+
## 🧬 Architecture
|
| 61 |
+
|
| 62 |
+
<div align="center">
|
| 63 |
+
<img src="assets/warpstate-architecture.svg" alt="WarpState: token input, local causal attention, fast and slow memory, gated fusion, repeated cores and language modeling head" width="100%" />
|
| 64 |
+
</div>
|
| 65 |
+
|
| 66 |
+
### How one WarpState core works
|
| 67 |
+
|
| 68 |
+
1. **Normalize and project.** RMSNorm feeds one fused projection into queries, keys, values, a gate, and memory-write features.
|
| 69 |
+
2. **Attend locally.** Rotary position embeddings and causal scaled dot-product attention operate within each 128-token tile.
|
| 70 |
+
3. **Read compressed history.** Queries read from two memory matrices containing information written by *previously completed* tiles.
|
| 71 |
+
4. **Fuse the paths.** A learned sigmoid gate mixes local attention and fast/slow historical memory.
|
| 72 |
+
5. **Update memory.** Bounded, outer-product-like writes update the fast/slow states with separate learned decay coefficients.
|
| 73 |
+
6. **Refine the representation.** Residual projection, RMSNorm, and SwiGLU complete the block; the language model reuses cores across logical depth passes.
|
| 74 |
+
|
| 75 |
+
### Memory, not magic
|
| 76 |
+
|
| 77 |
+
The streaming path retains compact fast/slow state instead of an ever-growing full-attention key/value history. That does **not** imply infinite context, lossless recall, perfect retrieval, or constant runtime in every implementation mode. The optional parallel-memory path materializes tile-level interactions during training, and end-to-end cost depends on the kernel and execution configuration.
|
| 78 |
+
|
| 79 |
+
### Implementation highlights
|
| 80 |
+
|
| 81 |
+
| Module | What it does | Source |
|
| 82 |
+
| :--- | :--- | :--- |
|
| 83 |
+
| **WarpStateCore** | Local attention + two-timescale associative memory | [`warpstate/model.py`](warpstate/model.py) |
|
| 84 |
+
| **WarpStateLM** | Token embeddings, depth modulation, shared cores, tied output head | [`warpstate/model.py`](warpstate/model.py) |
|
| 85 |
+
| **Training runner** | Streaming data preparation, AdamW, validation, checkpoint/resume | [`run_40b.py`](run_40b.py) |
|
| 86 |
+
| **Distributed and kernel paths** | DDP, execution optimizations, matmul and backward experiments | [`ops/`](ops/) |
|
| 87 |
+
| **SFT** | Assistant-token-masked supervised fine-tuning | [`posttrain.py`](posttrain.py) |
|
| 88 |
+
| **DPO** | Preference optimization with a fixed reference model | [`dpo.py`](dpo.py) |
|
| 89 |
+
| **Inference probes** | CPU base-model chat and archived DPO inference tooling | [`chat.py`](chat.py), [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) |
|
| 90 |
+
|
| 91 |
+
---
|
| 92 |
+
|
| 93 |
+
## 📈 Training & evaluation
|
| 94 |
+
|
| 95 |
+
### Documented experimental milestones
|
| 96 |
+
|
| 97 |
+
| Phase | Recorded output | Interpretation |
|
| 98 |
+
| :--- | :--- | :--- |
|
| 99 |
+
| **Raw pretraining** | **10,000,007,168 tokens** · **76,294 updates** · validation loss **2.3497** | Pretraining token-level cross-entropy; **not** a claim that all 40B prepared tokens were trained |
|
| 100 |
+
| **SFT** | **10,312 updates** · **329,973 examples** · validation loss **1.1157** | Assistant-token-masked fine-tuning; its loss uses a **different objective** |
|
| 101 |
+
| **DPO** | **2,334 updates** · **74,661 preference pairs** · held-out DPO loss **0.5790** | Preference objective; **not** model perplexity or accuracy |
|
| 102 |
+
|
| 103 |
+
The included DPO record also reports positive relative reward margins for **245 / 339 held-out pairs (72.27%)**. This is a *margin statistic*, **not** an end-user generation win rate. Source: [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md).
|
| 104 |
+
|
| 105 |
+
### Selected historical throughput observations
|
| 106 |
+
|
| 107 |
+
| Setup | Recorded observation | Scope / caveat |
|
| 108 |
+
| :--- | :--- | :--- |
|
| 109 |
+
| 2 × RTX PRO 6000 Blackwell Max-Q | **31,199 tokens/s** | September 13 analysis of a repeated 20-update, non-profiled configuration; excludes validation/checkpoint pauses |
|
| 110 |
+
| 1 × NVIDIA B200 | **20,752 sustained tokens/s**; **72,499 median tokens/s** | Specific 20-update autotuned benchmark with provider scheduling waits; excludes initialization, validation and saves |
|
| 111 |
+
|
| 112 |
+
These measurements describe **training throughput on the named server GPUs**, not inference speed, consumer GPU performance, or a reproducible speed promise. See [`ops/POINTWISE_HEADROOM_20260913.md`](ops/POINTWISE_HEADROOM_20260913.md) and [`ops/B200_PERFORMANCE.md`](ops/B200_PERFORMANCE.md) for test conditions and the discrepancy between median and sustained rates.
|
| 113 |
+
|
| 114 |
+
**What remains to be demonstrated publicly:** standardized language tasks, instruction following, multilingual quality, reasoning, independent reproducibility, and memory-retention tests. A low training loss alone cannot establish any of these capabilities.
|
| 115 |
+
|
| 116 |
+
---
|
| 117 |
+
|
| 118 |
+
## 🚀 Getting started
|
| 119 |
+
|
| 120 |
+
### 1. Assemble the source tree
|
| 121 |
+
|
| 122 |
+
The distributed archives contain complementary portions of the same project. Combine their contents under **one project root**, preserving `warpstate/`, `ops/`, root Python files, and `tokenizer.model`:
|
| 123 |
+
|
| 124 |
+
```text
|
| 125 |
+
WarpState/
|
| 126 |
+
├── README.md # This document
|
| 127 |
+
├── assets/ # README visuals
|
| 128 |
+
├── warpstate/
|
| 129 |
+
│ ├── __init__.py
|
| 130 |
+
│ ├── config.py
|
| 131 |
+
│ └── model.py
|
| 132 |
+
├── ops/ # Training, optimization, SFT/DPO tooling
|
| 133 |
+
├── configs/ # REQUIRED for full training; not in supplied ZIPs
|
| 134 |
+
├── run_40b.py # Pretraining entry point
|
| 135 |
+
├── posttrain.py # SFT runner
|
| 136 |
+
├── dpo.py # DPO runner
|
| 137 |
+
├── chat.py # CPU base-model probe
|
| 138 |
+
├── tokenizer.model
|
| 139 |
+
├── README_UK.md # Existing Ukrainian server guide
|
| 140 |
+
└── test_*.py # Test suite
|
| 141 |
+
```
|
| 142 |
+
|
| 143 |
+
> [!WARNING]
|
| 144 |
+
> The 2.005B training configuration and the SFT/DPO plans under `configs/` were **not present** in the supplied archives. `WarpStateConfig()` has smaller development defaults and does **not** instantiate the documented 2.005B run. Restore the original configuration/plan files from your trusted project copy before attempting to reproduce the historical results.
|
| 145 |
+
|
| 146 |
+
### 2. Set up a Python environment
|
| 147 |
+
|
| 148 |
+
Use **Linux, Python 3.12, a supported NVIDIA GPU and a CUDA-compatible PyTorch build** for the documented full training workflow. The original server setup used high-memory RTX PRO 6000 or B200-class hardware; the 2B full-optimizer recipe is **not** an 8 GB VRAM training preset.
|
| 149 |
+
|
| 150 |
+
```bash
|
| 151 |
+
cd WarpState
|
| 152 |
+
python3.12 -m venv .venv
|
| 153 |
+
source .venv/bin/activate
|
| 154 |
+
|
| 155 |
+
# Select a PyTorch/CUDA build compatible with YOUR GPU and driver:
|
| 156 |
+
# https://pytorch.org/get-started/locally/
|
| 157 |
+
python -m pip install torch sentencepiece numpy
|
| 158 |
+
```
|
| 159 |
+
|
| 160 |
+
The historical automatic bootstrap is implemented in `run_40b.py`. Detailed, pinned deployment requirements are in `ops/training-requirements-resolved.txt`, `ops/B200_SETUP.md`, and `README_UK.md`; package resolution depends on your actual host.
|
| 161 |
+
|
| 162 |
+
### 3. Inspect the architecture without a checkpoint
|
| 163 |
+
|
| 164 |
+
```python
|
| 165 |
+
from warpstate import WarpStateConfig, WarpStateLM
|
| 166 |
+
|
| 167 |
+
config = WarpStateConfig() # Development-sized defaults; NOT the historical 2.005B config
|
| 168 |
+
model = WarpStateLM(config)
|
| 169 |
+
print(f"Parameters: {model.num_parameters():,}")
|
| 170 |
+
```
|
| 171 |
+
|
| 172 |
+
This inspects randomly initialized code only. It does **not** provide a trained model.
|
| 173 |
+
|
| 174 |
+
### 4. Run a trusted base-model checkpoint
|
| 175 |
+
|
| 176 |
+
With the original compatible tokenizer and full `warpstate-server-v1` checkpoint available:
|
| 177 |
+
|
| 178 |
+
```bash
|
| 179 |
+
python3.12 chat.py \
|
| 180 |
+
--checkpoint /path/to/trusted/latest.pt \
|
| 181 |
+
--tokenizer ./tokenizer.model \
|
| 182 |
+
--prompt "Explain how memory works." \
|
| 183 |
+
--temperature 0
|
| 184 |
+
```
|
| 185 |
+
|
| 186 |
+
`chat.py` is a **CPU FP32 base-pretraining probe**. The DPO-tuned assistant uses a **different interface and archived snapshot format**; follow [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) and [`ops/RESTART_HANDOFF.md`](ops/RESTART_HANDOFF.md) on a host containing the original archives. Some operational scripts embed the historical `/workspace/WarpState/rtx6000_40b` layout and require path adaptation elsewhere.
|
| 187 |
+
|
| 188 |
+
### 5. Start or resume training (advanced)
|
| 189 |
+
|
| 190 |
+
Only after restoring the **matching** 2.005B config, tokenizer, trusted weights or explicit from-scratch initialization, GPU environment, disk space and dataset permissions:
|
| 191 |
+
|
| 192 |
+
```bash
|
| 193 |
+
# Example for a NEW run initialized from an existing trusted WarpState checkpoint
|
| 194 |
+
python3.12 run_40b.py \
|
| 195 |
+
--weights /path/to/trusted/latest.pt \
|
| 196 |
+
--config ./configs/warpstate_2b.json \
|
| 197 |
+
--work-dir /path/to/warpstate-work
|
| 198 |
+
```
|
| 199 |
+
|
| 200 |
+
Alternatively, `--from-scratch` requests random initialization; it is **not** a way to load the trained 2B model. By default, the runner prepares **40B train tokens (~80 GB of `uint16` token data)** before full training. Even `--max-steps 10` does **not** skip full corpus preparation. Preserve checkpoint, tokenizer and data provenance when resuming. Never load untrusted `.pt`/pickle files.
|
| 201 |
+
|
| 202 |
+
Read [`README_UK.md`](README_UK.md), [`ops/TRAINING.md`](ops/TRAINING.md), and [`ops/LOCAL_RESUME.md`](ops/LOCAL_RESUME.md) before using production launchers.
|
| 203 |
+
|
| 204 |
+
---
|
| 205 |
+
|
| 206 |
+
## 🔍 Reproducibility & engineering
|
| 207 |
+
|
| 208 |
+
- **Deterministic corpus recipe:** FineWeb-Edu streaming, SentencePiece token counting, train/holdout split, provenance and SHA-256 manifests (see [`README_UK.md`](README_UK.md)).
|
| 209 |
+
- **Recoverable checkpoints:** complete FP32 model and AdamW state, RNG state, training cursor, run signature and guarded resume.
|
| 210 |
+
- **Precision discipline:** BF16 compute with FP32 master weights and optimizer; kernel changes require numerical comparisons.
|
| 211 |
+
- **Multiple execution modes:** compiled cores, gradient/communication overlap experiments, distributed training, asynchronous checkpoint writing and optional kernel tuning.
|
| 212 |
+
- **Testing:** unit and integration tests for training, checkpointing, transfer, post-training and distributed execution are included. Historical validation reports are not a substitute for rerunning them on a new host.
|
| 213 |
+
|
| 214 |
+
The repository also contains dated operational documents. **They describe specific machines and historical run states**, not universally safe commands to execute automatically on a new machine.
|
| 215 |
+
|
| 216 |
+
### Data and model safety
|
| 217 |
+
|
| 218 |
+
The original base data used [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu). Respect upstream dataset licenses and documentation, keep private checkpoints/logs out of Git, and treat PyTorch pickle checkpoints as trusted-code inputs. No datasets or trained weights are embedded in these README assets.
|
| 219 |
+
|
| 220 |
+
---
|
| 221 |
+
|
| 222 |
+
## 🗺️ Roadmap
|
| 223 |
+
|
| 224 |
+
- [x] Implement chunked causal attention with fast/slow temporal memory.
|
| 225 |
+
- [x] Implement depth-modulated reusable cores and tied language-model head.
|
| 226 |
+
- [x] Build resumable pretraining and GPU execution/optimization tooling.
|
| 227 |
+
- [x] Record a 10B-token pretraining milestone for the 2.005B experiment.
|
| 228 |
+
- [x] Record completed SFT and DPO stages in historical run documentation.
|
| 229 |
+
- [ ] Publish complete, reproducible configs and a verified release manifest.
|
| 230 |
+
- [ ] Publish licensed, integrity-checked downloadable checkpoints (if distribution is authorized).
|
| 231 |
+
- [ ] Add standardized benchmarks, memory-retention evaluations and representative generations.
|
| 232 |
+
- [ ] Consolidate deployment scripts and add CI for supported hardware paths.
|
| 233 |
+
- [ ] Document scaling, memory behavior and real inference performance across devices.
|
| 234 |
+
|
| 235 |
+
---
|
| 236 |
+
|
| 237 |
+
## 🤝 Project leadership & sponsorship
|
| 238 |
+
|
| 239 |
+
<table>
|
| 240 |
+
<tr>
|
| 241 |
+
<td align="center" width="50%">
|
| 242 |
+
<strong>🧠 Project creator & lead developer</strong><br/>
|
| 243 |
+
<a href="https://github.com/gtausa197-svg"><strong>Volodymyr Makarenko</strong></a><br/>
|
| 244 |
+
<sub>Core research direction, architecture and development</sub>
|
| 245 |
+
</td>
|
| 246 |
+
<td align="center" width="50%">
|
| 247 |
+
<strong>💜 Project sponsor & co-author</strong><br/>
|
| 248 |
+
<a href="https://github.com/ykim078"><strong>@ykim078</strong></a><br/>
|
| 249 |
+
<sub>Recognized as both sponsor and co-author of WarpState</sub>
|
| 250 |
+
</td>
|
| 251 |
+
</tr>
|
| 252 |
+
</table>
|
| 253 |
+
|
| 254 |
+
**Special thanks to [@ykim078](https://github.com/ykim078)** for supporting the project and participating as a **co-author**. Sponsorship and co-authorship are distinct roles; both are intentionally credited here at the project owner's request. Specific financial terms and individual code contributions are not claimed.
|
| 255 |
+
|
| 256 |
+
Contributions are welcome through issues and pull requests once the repository is published with a contribution policy and clear license.
|
| 257 |
+
|
| 258 |
+
## 📚 Documentation
|
| 259 |
+
|
| 260 |
+
| Guide | Purpose |
|
| 261 |
+
| :--- | :--- |
|
| 262 |
+
| [`README_UK.md`](README_UK.md) | Ukrainian source-package and GPU training guide |
|
| 263 |
+
| [`ops/RESTART_HANDOFF.md`](ops/RESTART_HANDOFF.md) | Recorded completed raw / SFT / DPO run and archive state |
|
| 264 |
+
| [`ops/B200_PERFORMANCE.md`](ops/B200_PERFORMANCE.md) | B200 benchmark conditions and results |
|
| 265 |
+
| [`ops/OPTIMIZED_TRAINING.md`](ops/OPTIMIZED_TRAINING.md) | Dual RTX PRO 6000 training measurements |
|
| 266 |
+
| [`ops/POSTTRAIN.md`](ops/POSTTRAIN.md) | SFT execution and data provenance |
|
| 267 |
+
| [`ops/DPO_HANDOFF.md`](ops/DPO_HANDOFF.md) | Preference optimization design |
|
| 268 |
+
| [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) | Post-DPO inference instructions and caveats |
|
| 269 |
+
|
| 270 |
+
## 📄 License & citation
|
| 271 |
+
|
| 272 |
+
**License:** No root `LICENSE` file was included in the supplied WarpState archives. **Do not assume an open-source license or reuse permission** until the repository owner publishes an explicit license. The Project Nord reference's Apache-2.0 label **does not automatically apply** to WarpState.
|
| 273 |
+
|
| 274 |
+
If citing the project, link to the **actual WarpState repository URL** once published; do not cite the unrelated Nord repository as its code source. Credit the creator and co-author as shown above.
|
| 275 |
+
|
| 276 |
+
---
|
| 277 |
+
|
| 278 |
+
<div align="center">
|
| 279 |
+
<strong>⚡ WarpState</strong><br/>
|
| 280 |
+
<em>Local attention for immediate detail. Temporal memory for continuity.</em><br/><br/>
|
| 281 |
+
<sub>Built as an experimental research system. Documented results, explicit limitations, reproducible engineering.</sub>
|
| 282 |
+
</div>
|
README_UK.md
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WarpState 2B: один запуск на RTX PRO 6000 Blackwell 96 ГБ
|
| 2 |
+
|
| 3 |
+
Архів містить код вашої WarpState, точний наявний токенізатор і єдиний
|
| 4 |
+
керівний скрипт `run_40b.py`. Ваг моделі та 80 ГБ даних в архіві немає.
|
| 5 |
+
Ваги беруться з вашого сервера, дані готуються на сервері автоматично.
|
| 6 |
+
|
| 7 |
+
## Запуск
|
| 8 |
+
|
| 9 |
+
Потрібні Linux x86_64, **Python 3.12–3.14 з модулем venv**, робочий NVIDIA
|
| 10 |
+
драйвер, сумісний із CUDA 13.0, одна RTX PRO 6000 Blackwell 96 ГБ,
|
| 11 |
+
інтернет і рекомендовано **250 ГБ вільного SSD/NVMe**. Бажано 128 ГБ RAM;
|
| 12 |
+
фактичні потреби залежать від завантаження checkpoint і підготовки даних.
|
| 13 |
+
Скрипт не встановлює системний драйвер і не змінює інші Python-середовища.
|
| 14 |
+
|
| 15 |
+
Розпакуйте ZIP, перейдіть у його папку та підставте справжній шлях до ваг:
|
| 16 |
+
|
| 17 |
+
```bash
|
| 18 |
+
cd rtx6000_40b
|
| 19 |
+
python3.12 run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
Ця **одна команда**:
|
| 23 |
+
|
| 24 |
+
1. Створює `.server-venv` та встановлює CUDA PyTorch і залежності.
|
| 25 |
+
2. Перевіряє GPU, формат/архітектуру ваг, розмір словника й доступне місце.
|
| 26 |
+
3. Потоково читає FineWeb-Edu та готує **рівно 40 000 000 000 train-токенів**
|
| 27 |
+
вашим токенізатором плюс 1 048 576 validation-токенів.
|
| 28 |
+
4. Запускає повне навчання всіх параметрів із GPU AdamW та BF16-обчисленнями.
|
| 29 |
+
5. Пише метрики, validation і checkpoint з можливістю відновлення.
|
| 30 |
+
|
| 31 |
+
Для запуску з від’єднанням від SSH використовуйте вже встановлений tmux
|
| 32 |
+
або стандартний nohup. Наприклад:
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
nohup python3.12 -u run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b > server.log 2>&1 &
|
| 36 |
+
echo $! > server.pid
|
| 37 |
+
tail -f server.log
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
Завершення через Ctrl+C у передньому плані або `kill -TERM <PID>` для процесу
|
| 41 |
+
скрипта зберігає прогрес. Дочекайтеся `Saved ...latest.pt` та виходу процесу.
|
| 42 |
+
У підготовці корпусу зберігається позиція читання й записані токени.
|
| 43 |
+
Не використовуйте `kill -9`, коли потрібні останні незбережені оновлення.
|
| 44 |
+
Повторна **та сама команда** відновлює запуск. Одночасний запуск у тому
|
| 45 |
+
самому `--work-dir` захищено блокуванням.
|
| 46 |
+
|
| 47 |
+
## Які ваги підходять
|
| 48 |
+
|
| 49 |
+
Це код **вашої WarpState 2B**, а не універсальний тренер для Llama/Qwen.
|
| 50 |
+
Очікується ваш PyTorch `latest.pt` з полями `model` і `config`.
|
| 51 |
+
Для старого CPU-offload checkpoint використовуються точні FP32 master-ваги,
|
| 52 |
+
якщо вони є. Токенізатор в архіві — той самий `tokenizer.model`, що у проєкті.
|
| 53 |
+
Не замінюйте його на інший зі схожим розміром словника: перевірка розміру
|
| 54 |
+
не доводить однакової відповідності ID словам.
|
| 55 |
+
|
| 56 |
+
Перше завантаження через `--weights` — **початок нового 40B-запуску з ваших
|
| 57 |
+
ваг**: старий 300M-розклад і старі моменти AdamW не переносяться.
|
| 58 |
+
Записується походження ваг і попередній лічильник токенів.
|
| 59 |
+
У наступних перезапусках `work/run/latest.pt` відновлюються ваги,
|
| 60 |
+
AdamW, RNG, позиція корпусу, лічильник та розклад нового запуску.
|
| 61 |
+
Оригінальний файл `--weights` не перезаписується.
|
| 62 |
+
|
| 63 |
+
## Дані та очікування якості
|
| 64 |
+
|
| 65 |
+
Джерело: [HuggingFaceFW/fineweb-edu, sample-100BT](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu).
|
| 66 |
+
Це відфільтрований **англомовний** корпус для попереднього навчання.
|
| 67 |
+
Число токенів у його назві пораховане іншим токенізатором; скрипт сам
|
| 68 |
+
відміряє 40 млрд саме вашим SentencePiece. Ці 40 млрд — додаткові токени
|
| 69 |
+
нового запуску, незалежно від кількості токенів у початкових вагах.
|
| 70 |
+
|
| 71 |
+
У кож��н документ додається EOS. Незмінний SHA256 тексту визначає train
|
| 72 |
+
або holdout; усі holdout-документи виключаються з train навіть після
|
| 73 |
+
заповнення validation. Повтор того самого тексту потрапляє в той самий
|
| 74 |
+
розділ; це не додаткова глобальна дедуплікація майже однакових документів.
|
| 75 |
+
Ревізія джерела фіксується як commit SHA. Підготовка відновлюється через
|
| 76 |
+
стан HF IterableDataset; незавершений хвіст вихідного файла відкочується
|
| 77 |
+
до останньої узгодженої точки. Якщо джерело закінчиться раніше, скрипт
|
| 78 |
+
зупиниться, а не доповнюватиме корпус нескінченними повтореннями.
|
| 79 |
+
|
| 80 |
+
Два uint16-файли займають приблизно **80,0 ГБ / 74,5 GiB**. Обсяг мережевого
|
| 81 |
+
трафіку більший; вихідні тексти не завантажуються до ZIP. Підготовка може
|
| 82 |
+
зайняти години або дні. Це не вже готовий 40B-корпус у маленькому архіві.
|
| 83 |
+
Наприкінці обчислюються SHA256 файлів. Повна повторна перевірка даних:
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
python3.12 run_40b.py --stage train --work-dir /srv/warpstate40b --weights /srv/models/latest.pt --verify-data
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
40 млрд токенів — суттєвий бюджет для експерименту з 2B, але кількість токенів
|
| 90 |
+
не гарантує якість. Тут експериментальна архітектура; оцінюйте validation,
|
| 91 |
+
мовні/фактологічні тести й реальні відповіді. Англомовний pretraining сам
|
| 92 |
+
по собі не робить модель якісним українським чат-асистентом. Для цього
|
| 93 |
+
потрібні відповідні українські дані, instruction tuning та окрема оцінка;
|
| 94 |
+
вони не підмінені випадковою сумішшю в цьому пакеті.
|
| 95 |
+
|
| 96 |
+
## Параметри й продуктивність
|
| 97 |
+
|
| 98 |
+
За замовчуванням: sequence **2048**, micro-batch **8**, накопичення **8**,
|
| 99 |
+
**131 072 токени/оновлення**, повний FP32 GPU AdamW, BF16 autocast,
|
| 100 |
+
activation checkpointing, TF32 для відповідних FP32 матричних операцій,
|
| 101 |
+
loss блоками 1024, clipping 1,0, betas 0,9/0,95, weight decay 0,1.
|
| 102 |
+
LR 0,00015 → 0,000015, warmup 1000 кроків, cosine decay. Приблизно
|
| 103 |
+
305 176 оновлень; останнє маскується, щоб не перевищити рівно 40 млрд токенів.
|
| 104 |
+
Циклічне читання проходить корпус послідовно, а не вибирає випадкові
|
| 105 |
+
послідовності з повторенням. На останній межі останній токен передбачає перший.
|
| 106 |
+
|
| 107 |
+
Локальний блок архітектури залишається 128 токенів; 2048 — довжина
|
| 108 |
+
послідовності навчання, а не обіцянка точного зберігання всього контексту.
|
| 109 |
+
Тривала пам’ять лишається стисненими fast/slow станами вашої архітектури.
|
| 110 |
+
|
| 111 |
+
Перед довгим навчанням можна обмежити перший запуск десятьма оновленнями:
|
| 112 |
+
|
| 113 |
+
```bash
|
| 114 |
+
python3.12 run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b --max-steps 10
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
Підготовка все одно створить повний 40B-корпус. Потім приберіть `--max-steps`
|
| 118 |
+
і повторіть команду: продовжиться той самий 40B-розклад.
|
| 119 |
+
Опційний `--compile` вмикає Linux Inductor; потрібні сумісні компілятори,
|
| 120 |
+
перший запуск довший. Вибирайте його **до першого навчального запуску**.
|
| 121 |
+
За замовчуванням він вимкнений, оскільки повний compile цього сценарію на
|
| 122 |
+
RTX PRO 6000 тут не перевірявся. Так само не виміряно оптимальний micro-batch.
|
| 123 |
+
За OOM зменшіть micro-batch і відповідно збільште накопичення **до початку
|
| 124 |
+
нового запуску**; відновлення навмисно не дозволяє мовчки змінювати налаштування.
|
| 125 |
+
|
| 126 |
+
Фактична швидкість виводиться після кожного кроку й включає forward,
|
| 127 |
+
backward, clipping, AdamW ��а отримання пакета. Вона не включає підготовку
|
| 128 |
+
корпусу, validation, компіляцію й запис checkpoint. **10 000 токенів/с не
|
| 129 |
+
гарантуються**: повна модель на цій серверній карті тут не запускалась.
|
| 130 |
+
Для масштабу, суто арифметично: 40B при 10 000 токенів/с — 46,3 доби
|
| 131 |
+
чистого навчання, при 20 000 — 23,1 доби, без додаткових витрат.
|
| 132 |
+
|
| 133 |
+
Validation виконується кожні 500 кроків на незмінних позиціях; checkpoint —
|
| 134 |
+
кожні 1000 та при штатній зупинці. Одне збереження моделі з GPU AdamW
|
| 135 |
+
займає приблизно 24 ГБ; атомарна заміна потребує місця для двох файлів.
|
| 136 |
+
Довге навчання означає великий сумарний обсяг записів на диск.
|
| 137 |
+
|
| 138 |
+
## Файли й перевірки
|
| 139 |
+
|
| 140 |
+
- `work/data/train.bin`, `val.bin`, `manifest.json`, `prepare-state.pkl` — дані та їхній прогрес.
|
| 141 |
+
- `work/run/latest.pt` — відновлюваний серверний checkpoint.
|
| 142 |
+
- `work/run/metrics.jsonl`, `validation.jsonl`, `settings.json` — журнал і параметри.
|
| 143 |
+
- `.server-venv/requirements-resolved.txt` — точний список встановлених залежностей.
|
| 144 |
+
- `PACKAGE_MANIFEST.json` — контрольні суми файлів архіву.
|
| 145 |
+
|
| 146 |
+
Код підготовки й навчання локально перевірений на маленькій моделі:
|
| 147 |
+
маска останнього пакета, відновлення підготовки з відкиданням незбереженого
|
| 148 |
+
хвоста, точний збіг ваг і станів AdamW між безперервним і відновленим
|
| 149 |
+
CUDA-навчанням, відхилення зміненого LR. Це не серверний бенчмарк 2B/96 ГБ.
|
| 150 |
+
Також перевірено справжній HF streaming на 10 000 train + 64 validation
|
| 151 |
+
токенах: відновлення дає побайтово ті самі файли, SHA256 збігаються.
|
| 152 |
+
Локальна перевірка виконувалася на Windows/RTX 5070; Linux bootstrap та
|
| 153 |
+
повне навчання 2B на 96 ГБ потрібно перевірити на вашому сервері.
|
| 154 |
+
|
| 155 |
+
Для відтворення тестів у встановленому середовищі:
|
| 156 |
+
|
| 157 |
+
```bash
|
| 158 |
+
.server-venv/bin/python -m pip install pytest==9.1.1
|
| 159 |
+
.server-venv/bin/python -m pytest test_server.py -q
|
| 160 |
+
```
|
| 161 |
+
|
| 162 |
+
Лише trusted локальні checkpoint/state-файли: PyTorch та pickle можуть
|
| 163 |
+
виконувати код під час читання. Не підставляйте невідомі файли з інтернету.
|
| 164 |
+
|
| 165 |
+
## Джерела
|
| 166 |
+
|
| 167 |
+
- [FineWeb-Edu: опис, англійська мова, sample-100BT, ODC-BY](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu).
|
| 168 |
+
- [Hugging Face: streaming та відновлення IterableDataset](https://huggingface.co/docs/datasets/stream).
|
| 169 |
+
- [NVIDIA: RTX PRO 6000 Blackwell 96 ГБ](https://www.nvidia.com/en-sg/data-center/rtx-pro-6000-blackwell-server-edition/).
|
| 170 |
+
- [CUDA-версія PyTorch](https://download.pytorch.org/whl/cu130/torch/).
|
SOURCE_PACKAGE_NOTES.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WarpState — assembled source bundle
|
| 2 |
+
|
| 3 |
+
This bundle combines the **three source archives you provided** with the newly designed README and SVG assets. Existing source files were preserved byte-for-byte. No Python implementation was rewritten. Python bytecode caches (`__pycache__`, `.pyc`) were omitted.
|
| 4 |
+
|
| 5 |
+
## Included
|
| 6 |
+
|
| 7 |
+
- `warpstate/model.py`, `warpstate/config.py`, `warpstate/__init__.py` — model implementation
|
| 8 |
+
- `run_40b.py` — pretraining
|
| 9 |
+
- `posttrain.py` — supervised fine-tuning
|
| 10 |
+
- `dpo.py` — preference optimization
|
| 11 |
+
- `chat.py` — chat inference/probe
|
| 12 |
+
- `ops/*.py` and `ops/*.sh` — execution and training operations
|
| 13 |
+
- `test_*.py` — tests
|
| 14 |
+
- `tokenizer.model` — supplied tokenizer
|
| 15 |
+
- `README.md` and `assets/` — updated GitHub presentation
|
| 16 |
+
|
| 17 |
+
## Not supplied in the uploaded archives
|
| 18 |
+
|
| 19 |
+
- Trained model checkpoints / weights
|
| 20 |
+
- Training datasets
|
| 21 |
+
- `configs/` directory, including `configs/warpstate_2b.json` referenced by supplied `PACKAGE_MANIFEST.json` and training documents
|
| 22 |
+
- Explicit license file
|
| 23 |
+
|
| 24 |
+
**Note:** This is a *complete assembly of the available source*, not a turnkey pretrained model. Before publishing, review all source and documentation, choose a license, restore valid configs, and independently validate environment-specific commands. Scripts with old server-specific paths may require adaptation. Do not upload private secrets or checkpoints.
|
| 25 |
+
|
| 26 |
+
Original archives:
|
| 27 |
+
- `drive-download-20261026T200043Z-1-001.zip`
|
| 28 |
+
- `warpstate-20261026T200055Z-1-001.zip`
|
| 29 |
+
- `ops-20261026T200120Z-1-001.zip`
|
VALIDATION.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"unit_tests": 3,
|
| 3 |
+
"unit_test_status": "passed",
|
| 4 |
+
"tests": [
|
| 5 |
+
"partial final token batch",
|
| 6 |
+
"preparation interruption and rollback",
|
| 7 |
+
"small GPU full-optimizer resume equals uninterrupted run; changed LR rejected"
|
| 8 |
+
],
|
| 9 |
+
"live_fineweb_test": {
|
| 10 |
+
"train_tokens": 10000,
|
| 11 |
+
"val_tokens": 64,
|
| 12 |
+
"resume_byte_identical": true,
|
| 13 |
+
"sha256_verified": true
|
| 14 |
+
},
|
| 15 |
+
"test_platform": "Windows / RTX 5070 Laptop / torch 2.14.0+cu130",
|
| 16 |
+
"target_platform": "Linux / one RTX PRO 6000 Blackwell 96 GB",
|
| 17 |
+
"full_target_hardware_tested": false,
|
| 18 |
+
"full_40b_corpus_prepared_locally": false
|
| 19 |
+
}
|
assets/warpstate-architecture.svg
ADDED
|
|
assets/warpstate-banner.svg
ADDED
|
|
chat.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""CPU-only chat/completion probe of an independent WarpState weight snapshot.
|
| 3 |
+
|
| 4 |
+
Only load trusted local torch checkpoints. This does not stop training, touch
|
| 5 |
+
CUDA, load Adam into a model, or write to the training checkpoint directory.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import gc
|
| 11 |
+
import hashlib
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
import stat
|
| 17 |
+
import tempfile
|
| 18 |
+
import time
|
| 19 |
+
|
| 20 |
+
import sentencepiece as spm
|
| 21 |
+
import torch
|
| 22 |
+
|
| 23 |
+
from warpstate import WarpStateConfig, WarpStateLM
|
| 24 |
+
|
| 25 |
+
ROOT = Path(__file__).resolve().parent
|
| 26 |
+
FORMAT = 'warpstate-chat-v1'
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def sha256(path):
|
| 30 |
+
digest = hashlib.sha256()
|
| 31 |
+
with Path(path).open('rb') as stream:
|
| 32 |
+
for block in iter(lambda: stream.read(8 * 1024**2), b''):
|
| 33 |
+
digest.update(block)
|
| 34 |
+
return digest.hexdigest()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def capture_latest(source, tokenizer, snapshot_root):
|
| 38 |
+
"""Pin one atomically published inode, export only its original FP32 model.
|
| 39 |
+
|
| 40 |
+
latest.pt is published by rename, never updated in place. An open descriptor
|
| 41 |
+
remains bound to that complete version even if the trainer publishes again.
|
| 42 |
+
No hardlink to the production checkpoint is created.
|
| 43 |
+
"""
|
| 44 |
+
source, tokenizer, snapshot_root = map(Path, (source, tokenizer, snapshot_root))
|
| 45 |
+
if source.suffix != '.pt' or source.is_symlink():
|
| 46 |
+
raise ValueError('Select a completed regular .pt checkpoint, not a temporary file')
|
| 47 |
+
tokenizer_sha = sha256(tokenizer)
|
| 48 |
+
fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW)
|
| 49 |
+
try:
|
| 50 |
+
before = os.fstat(fd)
|
| 51 |
+
if not stat.S_ISREG(before.st_mode):
|
| 52 |
+
raise ValueError('Checkpoint must be a regular file')
|
| 53 |
+
checkpoint = torch.load(f'/proc/self/fd/{fd}', map_location='cpu',
|
| 54 |
+
weights_only=False, mmap=True)
|
| 55 |
+
if checkpoint.get('format') == FORMAT:
|
| 56 |
+
raise ValueError(
|
| 57 |
+
f'Chat snapshot requires its original snapshot.json beside {source}. '
|
| 58 |
+
'Restore that metadata or export a fresh snapshot from a training checkpoint.')
|
| 59 |
+
if checkpoint.get('format') != 'warpstate-server-v1':
|
| 60 |
+
raise ValueError('Expected a full warpstate-server-v1 training checkpoint')
|
| 61 |
+
recorded_tokenizer = checkpoint['signature']['data']['signature']['tokenizer_sha256']
|
| 62 |
+
if recorded_tokenizer != tokenizer_sha:
|
| 63 |
+
raise ValueError('Tokenizer SHA256 differs from training checkpoint')
|
| 64 |
+
snapshot_root.mkdir(parents=True, exist_ok=True)
|
| 65 |
+
folder = Path(tempfile.mkdtemp(prefix=f'step-{checkpoint["step"]}-', dir=snapshot_root))
|
| 66 |
+
destination = folder / 'model.pt'
|
| 67 |
+
temporary = folder / 'model.pt.tmp'
|
| 68 |
+
metadata = dict(format=FORMAT, inference_only=True, config=checkpoint['config'],
|
| 69 |
+
step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'],
|
| 70 |
+
val_loss=checkpoint.get('val_loss'), tokenizer_sha256=tokenizer_sha,
|
| 71 |
+
captured_at_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
| 72 |
+
source=dict(path=str(source.resolve()), device=before.st_dev,
|
| 73 |
+
inode=before.st_ino, size=before.st_size,
|
| 74 |
+
mtime_ns=before.st_mtime_ns))
|
| 75 |
+
print(f'Capturing completed step {metadata["step"]:,} / '
|
| 76 |
+
f'{metadata["tokens_seen"]:,} tokens; model weights only...', flush=True)
|
| 77 |
+
with temporary.open('xb') as stream:
|
| 78 |
+
torch.save(dict(metadata, model=checkpoint['model']), stream)
|
| 79 |
+
stream.flush()
|
| 80 |
+
os.fsync(stream.fileno())
|
| 81 |
+
after = os.fstat(fd)
|
| 82 |
+
# Atomic unlink/rename may change ctime/nlink without changing contents.
|
| 83 |
+
if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
|
| 84 |
+
after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns):
|
| 85 |
+
raise RuntimeError('Pinned checkpoint contents changed; snapshot not published')
|
| 86 |
+
os.chmod(temporary, 0o444)
|
| 87 |
+
os.replace(temporary, destination)
|
| 88 |
+
metadata['snapshot_sha256'] = sha256(destination)
|
| 89 |
+
metadata['snapshot_bytes'] = destination.stat().st_size
|
| 90 |
+
with (folder / 'snapshot.json').open('x') as stream:
|
| 91 |
+
json.dump(metadata, stream, indent=2)
|
| 92 |
+
stream.write('\n')
|
| 93 |
+
stream.flush()
|
| 94 |
+
os.fsync(stream.fileno())
|
| 95 |
+
directory = os.open(folder, os.O_RDONLY | os.O_DIRECTORY)
|
| 96 |
+
try:
|
| 97 |
+
os.fsync(directory)
|
| 98 |
+
finally:
|
| 99 |
+
os.close(directory)
|
| 100 |
+
return destination
|
| 101 |
+
finally:
|
| 102 |
+
os.close(fd)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def resolve_snapshot(source, tokenizer, snapshot_root):
|
| 106 |
+
"""Reuse a sealed export, or safely capture a full training checkpoint.
|
| 107 |
+
|
| 108 |
+
Missing metadata is not permission to load unverified inference weights:
|
| 109 |
+
capture_latest accepts only the embedded training-checkpoint format and
|
| 110 |
+
pins its inode across atomic replacements by the trainer.
|
| 111 |
+
"""
|
| 112 |
+
source = Path(source)
|
| 113 |
+
if not source.is_file():
|
| 114 |
+
raise ValueError(f'Checkpoint file does not exist or is not a regular file: {source}')
|
| 115 |
+
if source.with_name('snapshot.json').exists():
|
| 116 |
+
return source
|
| 117 |
+
return capture_latest(source, tokenizer, snapshot_root)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def load_model(snapshot, tokenizer):
|
| 121 |
+
snapshot, tokenizer = Path(snapshot), Path(tokenizer)
|
| 122 |
+
proof_path = snapshot.with_name('snapshot.json')
|
| 123 |
+
try:
|
| 124 |
+
proof = json.loads(proof_path.read_text())
|
| 125 |
+
except FileNotFoundError as exc:
|
| 126 |
+
raise ValueError(
|
| 127 |
+
f'Missing snapshot metadata: {proof_path}. '
|
| 128 |
+
'Use --checkpoint to export training weights, or restore the original metadata.') from exc
|
| 129 |
+
if not isinstance(proof, dict) or not isinstance(proof.get('snapshot_sha256'), str):
|
| 130 |
+
raise ValueError(f'Invalid snapshot metadata: {proof_path}')
|
| 131 |
+
if proof['snapshot_sha256'] != sha256(snapshot):
|
| 132 |
+
raise ValueError('Inference snapshot SHA256 mismatch')
|
| 133 |
+
checkpoint = torch.load(snapshot, map_location='cpu', weights_only=False, mmap=True)
|
| 134 |
+
if checkpoint.get('format') != FORMAT or not checkpoint.get('inference_only'):
|
| 135 |
+
raise ValueError('Use the independent inference snapshot, not live latest.pt')
|
| 136 |
+
if checkpoint['tokenizer_sha256'] != sha256(tokenizer):
|
| 137 |
+
raise ValueError('Tokenizer SHA256 mismatch')
|
| 138 |
+
config = WarpStateConfig(**checkpoint['config'])
|
| 139 |
+
tokenizer_model = spm.SentencePieceProcessor(model_file=str(tokenizer))
|
| 140 |
+
if tokenizer_model.vocab_size() != config.vocab_size:
|
| 141 |
+
raise ValueError('Tokenizer vocabulary differs from model')
|
| 142 |
+
with torch.device('meta'):
|
| 143 |
+
model = WarpStateLM(config)
|
| 144 |
+
model.load_state_dict(checkpoint['model'], strict=True, assign=True)
|
| 145 |
+
model.eval().requires_grad_(False)
|
| 146 |
+
if any(p.device.type != 'cpu' or p.dtype != torch.float32 for p in model.parameters()):
|
| 147 |
+
raise ValueError('This CPU probe requires the original FP32 model parameters')
|
| 148 |
+
metadata = {key: value for key, value in checkpoint.items() if key != 'model'}
|
| 149 |
+
del checkpoint
|
| 150 |
+
gc.collect()
|
| 151 |
+
return model, tokenizer_model, metadata
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def banned_ngram_tokens(tokens, size):
|
| 155 |
+
"""Block completions of n-grams already present in this reply, not its prompt."""
|
| 156 |
+
if size == 0 or len(tokens) + 1 < size:
|
| 157 |
+
return set()
|
| 158 |
+
prefix = tuple(tokens[-(size - 1):]) if size > 1 else ()
|
| 159 |
+
return {tokens[start + size - 1] for start in range(len(tokens) - size + 1)
|
| 160 |
+
if tuple(tokens[start:start + size - 1]) == prefix}
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def sample_token(logits, temperature, top_k, top_p, generator, *,
|
| 164 |
+
previous_tokens=(), repetition_penalty=1.0, banned_tokens=()):
|
| 165 |
+
logits = logits.float().clone()
|
| 166 |
+
if not torch.isfinite(logits).all():
|
| 167 |
+
raise ValueError('Non-finite generation logits')
|
| 168 |
+
if not math.isfinite(temperature) or temperature < 0 or not 0 < top_p <= 1 or top_k < 0:
|
| 169 |
+
raise ValueError('Invalid sampling settings')
|
| 170 |
+
if not math.isfinite(repetition_penalty) or repetition_penalty < 1:
|
| 171 |
+
raise ValueError('Require a finite repetition-penalty >= 1')
|
| 172 |
+
if previous_tokens and repetition_penalty != 1:
|
| 173 |
+
seen = list(set(previous_tokens))
|
| 174 |
+
scores = logits[seen]
|
| 175 |
+
# Multiplying negative scores and dividing positive ones lowers both.
|
| 176 |
+
logits[seen] = torch.where(scores < 0, scores * repetition_penalty,
|
| 177 |
+
scores / repetition_penalty)
|
| 178 |
+
if banned_tokens:
|
| 179 |
+
logits[list(banned_tokens)] = -torch.inf
|
| 180 |
+
if not torch.isfinite(logits).any():
|
| 181 |
+
return None # No legal continuation; never sample a masked token.
|
| 182 |
+
if temperature == 0:
|
| 183 |
+
return int(logits.argmax())
|
| 184 |
+
values = logits / temperature
|
| 185 |
+
if top_k:
|
| 186 |
+
threshold = values.topk(min(top_k, values.numel())).values[-1]
|
| 187 |
+
values = values.masked_fill(values < threshold, -torch.inf)
|
| 188 |
+
ordered, indices = values.sort(descending=True)
|
| 189 |
+
probability = ordered.softmax(dim=-1)
|
| 190 |
+
# Keep the token crossing the cumulative threshold, including at least one.
|
| 191 |
+
ordered = ordered.masked_fill(probability.cumsum(-1) - probability > top_p, -torch.inf)
|
| 192 |
+
picked = torch.multinomial(ordered.softmax(dim=-1), 1, generator=generator)
|
| 193 |
+
return int(indices[picked].item())
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@torch.inference_mode()
|
| 197 |
+
def generate(model, sp, prompt, *, max_new_tokens=256, temperature=0.8,
|
| 198 |
+
top_k=40, top_p=0.9, seed=1337, max_context=2048,
|
| 199 |
+
repetition_penalty=1.2, no_repeat_ngram_size=4):
|
| 200 |
+
if max_new_tokens < 1 or max_context <= max_new_tokens:
|
| 201 |
+
raise ValueError('Require 0 < max-new-tokens < max-context')
|
| 202 |
+
if not math.isfinite(repetition_penalty) or repetition_penalty < 1:
|
| 203 |
+
raise ValueError('Require a finite repetition-penalty >= 1')
|
| 204 |
+
if not isinstance(no_repeat_ngram_size, int) or no_repeat_ngram_size < 0:
|
| 205 |
+
raise ValueError('Require an integer no-repeat-ngram-size >= 0')
|
| 206 |
+
ids = sp.encode(prompt, out_type=int)
|
| 207 |
+
if not ids:
|
| 208 |
+
raise ValueError('Prompt contains no tokens')
|
| 209 |
+
ids = ids[-(max_context - max_new_tokens):]
|
| 210 |
+
caches = model.init_generation_cache(1, torch.device('cpu'), torch.float32)
|
| 211 |
+
token = torch.empty((1, 1), dtype=torch.long)
|
| 212 |
+
for value in ids:
|
| 213 |
+
token.fill_(value)
|
| 214 |
+
logits = model.step(token, caches)[0, -1]
|
| 215 |
+
generator = torch.Generator(device='cpu').manual_seed(seed)
|
| 216 |
+
generated = []
|
| 217 |
+
for index in range(max_new_tokens):
|
| 218 |
+
value = sample_token(logits, temperature, top_k, top_p, generator,
|
| 219 |
+
previous_tokens=generated, repetition_penalty=repetition_penalty,
|
| 220 |
+
banned_tokens=banned_ngram_tokens(generated, no_repeat_ngram_size))
|
| 221 |
+
if value is None or value == sp.eos_id():
|
| 222 |
+
break
|
| 223 |
+
generated.append(value)
|
| 224 |
+
yield value
|
| 225 |
+
if index + 1 < max_new_tokens:
|
| 226 |
+
token.fill_(value)
|
| 227 |
+
logits = model.step(token, caches)[0, -1]
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def reply(model, sp, prompt, args):
|
| 231 |
+
ids, emitted = [], ''
|
| 232 |
+
begin, first = time.perf_counter(), None
|
| 233 |
+
for value in generate(model, sp, prompt, max_new_tokens=args.max_new_tokens,
|
| 234 |
+
temperature=args.temperature, top_k=args.top_k, top_p=args.top_p,
|
| 235 |
+
seed=args.seed, max_context=args.max_context,
|
| 236 |
+
repetition_penalty=args.repetition_penalty,
|
| 237 |
+
no_repeat_ngram_size=args.no_repeat_ngram_size):
|
| 238 |
+
if first is None:
|
| 239 |
+
first = time.perf_counter()
|
| 240 |
+
ids.append(value)
|
| 241 |
+
text = sp.decode(ids)
|
| 242 |
+
# Delay incomplete UTF-8 byte-fallback characters until they are complete.
|
| 243 |
+
safe = text.split('\ufffd', 1)[0]
|
| 244 |
+
if safe.startswith(emitted):
|
| 245 |
+
print(safe[len(emitted):], end='', flush=True)
|
| 246 |
+
emitted = safe
|
| 247 |
+
text = sp.decode(ids)
|
| 248 |
+
if text.startswith(emitted):
|
| 249 |
+
print(text[len(emitted):], end='', flush=True)
|
| 250 |
+
elapsed = time.perf_counter() - begin
|
| 251 |
+
ttft = '-' if first is None else f'{first - begin:.2f}s'
|
| 252 |
+
print(f'\n[{len(ids)} tokens; {elapsed:.1f}s total; TTFT {ttft}; CPU FP32]', flush=True)
|
| 253 |
+
return text
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def main():
|
| 257 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 258 |
+
parser.add_argument('--checkpoint', type=Path, default=ROOT / 'work/run/latest.pt')
|
| 259 |
+
parser.add_argument('--snapshot', type=Path,
|
| 260 |
+
help='Reuse a sealed model-only snapshot, or export a training checkpoint')
|
| 261 |
+
parser.add_argument('--snapshot-root', type=Path, default=ROOT / 'work/chat')
|
| 262 |
+
parser.add_argument('--tokenizer', type=Path, default=ROOT / 'tokenizer.model')
|
| 263 |
+
parser.add_argument('--capture-only', action='store_true')
|
| 264 |
+
parser.add_argument('--prompt', help='One-shot probe instead of interactive input')
|
| 265 |
+
parser.add_argument('--raw', action='store_true', help='Plain text completion instead of Q/A labels')
|
| 266 |
+
parser.add_argument('--threads', type=int, default=16)
|
| 267 |
+
parser.add_argument('--max-new-tokens', type=int, default=256)
|
| 268 |
+
parser.add_argument('--max-context', type=int, default=2048)
|
| 269 |
+
parser.add_argument('--temperature', type=float, default=0.8, help='0 = greedy')
|
| 270 |
+
parser.add_argument('--top-k', type=int, default=40)
|
| 271 |
+
parser.add_argument('--top-p', type=float, default=0.9)
|
| 272 |
+
parser.add_argument('--repetition-penalty', type=float, default=1.2,
|
| 273 |
+
help='Lower scores of tokens already generated in this reply; 1 disables')
|
| 274 |
+
parser.add_argument('--no-repeat-ngram-size', type=int, default=4,
|
| 275 |
+
help='Block repeated token sequences within a reply; 0 disables')
|
| 276 |
+
parser.add_argument('--seed', type=int, default=1337)
|
| 277 |
+
args = parser.parse_args()
|
| 278 |
+
if args.threads < 1 or not 0 < args.max_new_tokens < args.max_context:
|
| 279 |
+
parser.error('Require threads >= 1 and 0 < max-new-tokens < max-context')
|
| 280 |
+
if not math.isfinite(args.temperature) or args.temperature < 0 or args.top_k < 0 or not 0 < args.top_p <= 1:
|
| 281 |
+
parser.error('Invalid temperature / top-k / top-p')
|
| 282 |
+
if not math.isfinite(args.repetition_penalty) or args.repetition_penalty < 1 or args.no_repeat_ngram_size < 0:
|
| 283 |
+
parser.error('Require finite repetition-penalty >= 1 and no-repeat-ngram-size >= 0')
|
| 284 |
+
if args.prompt is not None and not args.prompt.strip():
|
| 285 |
+
parser.error('A one-shot prompt must not be empty')
|
| 286 |
+
torch.set_num_threads(args.threads)
|
| 287 |
+
torch.set_num_interop_threads(1)
|
| 288 |
+
try:
|
| 289 |
+
if args.snapshot is not None:
|
| 290 |
+
snapshot = resolve_snapshot(args.snapshot, args.tokenizer, args.snapshot_root)
|
| 291 |
+
else:
|
| 292 |
+
snapshot = capture_latest(args.checkpoint, args.tokenizer, args.snapshot_root)
|
| 293 |
+
print(f'Snapshot: {snapshot}', flush=True)
|
| 294 |
+
if args.capture_only:
|
| 295 |
+
return
|
| 296 |
+
model, sp, metadata = load_model(snapshot, args.tokenizer)
|
| 297 |
+
except (OSError, ValueError) as exc:
|
| 298 |
+
parser.error(str(exc))
|
| 299 |
+
print(f'WarpState {model.num_parameters() / 1e9:.3f}B | CPU FP32 | '
|
| 300 |
+
f'step {metadata["step"]:,} | {metadata["tokens_seen"]:,} training tokens', flush=True)
|
| 301 |
+
print('Base pretrained model, not instruction-tuned. Q/A labels are only a probe.\n'
|
| 302 |
+
'Commands: /new, /raw, /chat, /info, /help, /exit. Ctrl-C cancels a reply.\n'
|
| 303 |
+
'Each turn replays up to the latest context tokens; long prompts take time on CPU.', flush=True)
|
| 304 |
+
print(f'Repetition protection: penalty={args.repetition_penalty:g}, '
|
| 305 |
+
f'no-repeat-ngram-size={args.no_repeat_ngram_size} (per reply).', flush=True)
|
| 306 |
+
history = ''
|
| 307 |
+
while True:
|
| 308 |
+
try:
|
| 309 |
+
user = args.prompt if args.prompt is not None else input('\nYou: ').strip()
|
| 310 |
+
if user is None or (args.prompt is None and user in ('/exit', '/quit')):
|
| 311 |
+
break
|
| 312 |
+
if not user:
|
| 313 |
+
continue
|
| 314 |
+
if args.prompt is None and user in ('/new', '/raw', '/chat'):
|
| 315 |
+
history = ''
|
| 316 |
+
if user != '/new':
|
| 317 |
+
args.raw = user == '/raw'
|
| 318 |
+
print('Context cleared. Mode: ' + ('raw completion' if args.raw else 'Q/A probe'))
|
| 319 |
+
continue
|
| 320 |
+
if args.prompt is None and user == '/info':
|
| 321 |
+
print(json.dumps(metadata, indent=2))
|
| 322 |
+
continue
|
| 323 |
+
if args.prompt is None and user == '/help':
|
| 324 |
+
print('/new clears context; /raw plain completion; /chat Q/A; /info snapshot; /exit quit')
|
| 325 |
+
continue
|
| 326 |
+
prompt = history + (user if args.raw else f'Question: {user}\nAnswer:')
|
| 327 |
+
print('Model: ', end='', flush=True)
|
| 328 |
+
answer = reply(model, sp, prompt, args)
|
| 329 |
+
history = prompt + ('' if args.raw else ' ') + answer + '\n\n'
|
| 330 |
+
if args.prompt is not None:
|
| 331 |
+
break
|
| 332 |
+
except EOFError:
|
| 333 |
+
break
|
| 334 |
+
except KeyboardInterrupt:
|
| 335 |
+
print('\nReply cancelled; training was not affected.', flush=True)
|
| 336 |
+
if args.prompt is not None:
|
| 337 |
+
break
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
if __name__ == '__main__':
|
| 341 |
+
main()
|
dpo.py
ADDED
|
@@ -0,0 +1,647 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""BF16 DPO with an immutable SFT reference, exact resume and response-only loss.
|
| 3 |
+
|
| 4 |
+
The reference is scored once, before allocating the policy optimizer. Its cache
|
| 5 |
+
is bound to weights, data, masking, execution and source hashes. Policy training
|
| 6 |
+
uses fresh FP32 Adam state and saves only into its own output directory.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import contextlib
|
| 13 |
+
import gc
|
| 14 |
+
import hashlib
|
| 15 |
+
import json
|
| 16 |
+
import math
|
| 17 |
+
import os
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
import random
|
| 20 |
+
import signal
|
| 21 |
+
import time
|
| 22 |
+
import types
|
| 23 |
+
|
| 24 |
+
import numpy as np
|
| 25 |
+
import torch
|
| 26 |
+
import torch.distributed as dist
|
| 27 |
+
import torch.nn.functional as F
|
| 28 |
+
from torch.utils.checkpoint import checkpoint as activation_checkpoint
|
| 29 |
+
|
| 30 |
+
from posttrain import (WORKSPACE, atomic_json, base_pipeline_lock, digest,
|
| 31 |
+
restore_rng, rng_state, update_indices, validate_paths)
|
| 32 |
+
from ops.dpo_data import MASK_FORMAT, collate_pairs, load_data
|
| 33 |
+
from ops.posttrain_data import RENDER_FORMAT, renderer_sha256
|
| 34 |
+
|
| 35 |
+
FORMAT = 'warpstate-dpo-v1'
|
| 36 |
+
CACHE_FORMAT = 'warpstate-dpo-reference-v1'
|
| 37 |
+
STOP = False
|
| 38 |
+
DEFAULTS = dict(sequence_length=2048, micro_batch=2, grad_accum=8, lr=5e-7,
|
| 39 |
+
min_lr=0., warmup_ratio=.1, epochs=1, max_steps=None,
|
| 40 |
+
weight_decay=0., seed=42, eval_every=100, save_every=100,
|
| 41 |
+
eval_batches=32, compile_core=True, bf16_residuals=True,
|
| 42 |
+
gradient_checkpointing=False, workers=4, deterministic=True,
|
| 43 |
+
clip_grad_norm=1., beta=5., objective='length-normalized-dpo',
|
| 44 |
+
head_chunk_size=256)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def load_plan(path):
|
| 48 |
+
plan = json.loads(Path(path).read_text())
|
| 49 |
+
if plan.get('format') != 'warpstate-dpo-plan-v1' or plan.get('stage') != 'dpo':
|
| 50 |
+
raise ValueError('Expected warpstate-dpo-plan-v1 stage dpo')
|
| 51 |
+
supplied = plan.get('training', {})
|
| 52 |
+
unknown = set(supplied) - set(DEFAULTS)
|
| 53 |
+
if unknown:
|
| 54 |
+
raise ValueError(f'Unknown DPO training settings: {sorted(unknown)}')
|
| 55 |
+
settings = DEFAULTS | supplied
|
| 56 |
+
for key in ('sequence_length', 'micro_batch', 'grad_accum', 'epochs', 'eval_every',
|
| 57 |
+
'save_every', 'eval_batches', 'workers', 'head_chunk_size'):
|
| 58 |
+
if type(settings[key]) is not int or settings[key] < 1:
|
| 59 |
+
raise ValueError(f'{key} must be a positive integer')
|
| 60 |
+
if settings['max_steps'] is not None and (type(settings['max_steps']) is not int or settings['max_steps'] < 1):
|
| 61 |
+
raise ValueError('max_steps must be null or a positive integer')
|
| 62 |
+
for key in ('lr', 'min_lr', 'warmup_ratio', 'weight_decay', 'clip_grad_norm', 'beta'):
|
| 63 |
+
value = settings[key]
|
| 64 |
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
|
| 65 |
+
raise ValueError(f'{key} must be finite and nonnegative')
|
| 66 |
+
if not 0 <= settings['warmup_ratio'] < 1 or not 0 <= settings['min_lr'] <= settings['lr']:
|
| 67 |
+
raise ValueError('Invalid LR schedule')
|
| 68 |
+
if not settings['lr'] or not settings['clip_grad_norm'] or not settings['beta']:
|
| 69 |
+
raise ValueError('lr, clip_grad_norm and beta must be positive')
|
| 70 |
+
for key in ('compile_core', 'bf16_residuals', 'gradient_checkpointing', 'deterministic'):
|
| 71 |
+
if type(settings[key]) is not bool:
|
| 72 |
+
raise ValueError(f'{key} must be a boolean')
|
| 73 |
+
if type(settings['seed']) is not int or settings['seed'] < 0:
|
| 74 |
+
raise ValueError('seed must be a nonnegative integer')
|
| 75 |
+
if settings['objective'] not in ('length-normalized-dpo', 'summed-dpo'):
|
| 76 |
+
raise ValueError('Unsupported DPO objective')
|
| 77 |
+
return plan, settings
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def source_identity():
|
| 81 |
+
from posttrain import source_identity as shared_sources
|
| 82 |
+
root = Path(__file__).parent
|
| 83 |
+
return shared_sources() | {str(path.relative_to(root)): digest(path)
|
| 84 |
+
for path in (Path(__file__), root / 'ops/dpo_data.py')}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def identity_digest(identity):
|
| 88 |
+
return hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(',', ':'),
|
| 89 |
+
allow_nan=False).encode()).hexdigest()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def learning_rate(step, total_steps, settings):
|
| 93 |
+
"""Fixed linear warmup/decay schedule, independent of a pause boundary."""
|
| 94 |
+
warmup = min(math.ceil(settings['warmup_ratio'] * total_steps), max(0, total_steps - 1))
|
| 95 |
+
if step < warmup:
|
| 96 |
+
return settings['lr'] * (step + 1) / warmup
|
| 97 |
+
progress = min(1., (step - warmup) / max(1, total_steps - warmup - 1))
|
| 98 |
+
return settings['min_lr'] + (settings['lr'] - settings['min_lr']) * (1. - progress)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def reference_identity(weights_hash, manifest_hash, tokenizer_hash, settings, device, world):
|
| 102 |
+
return dict(format=CACHE_FORMAT, weights_sha256=weights_hash, data_sha256=manifest_hash,
|
| 103 |
+
tokenizer_sha256=tokenizer_hash, renderer_sha256=renderer_sha256(),
|
| 104 |
+
mask_format=MASK_FORMAT, sources=source_identity(), settings={key: settings[key]
|
| 105 |
+
for key in ('sequence_length', 'micro_batch', 'head_chunk_size', 'compile_core',
|
| 106 |
+
'bf16_residuals', 'deterministic', 'workers')},
|
| 107 |
+
dropout=0., logps='response-token-sum-float32', dtype='bfloat16',
|
| 108 |
+
device_type=device.type, world_size=world, torch_version=torch.__version__)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def validate_cache(directory, identity, sizes):
|
| 112 |
+
directory = Path(directory)
|
| 113 |
+
manifest = json.loads((directory / 'manifest.json').read_text())
|
| 114 |
+
if (manifest.get('format') != CACHE_FORMAT or manifest.get('complete') is not True or
|
| 115 |
+
manifest.get('identity') != identity or manifest.get('identity_sha256') != identity_digest(identity)):
|
| 116 |
+
raise ValueError('Reference cache identity/completion differs')
|
| 117 |
+
result = []
|
| 118 |
+
for name, size in zip(('train.npy', 'val.npy'), sizes):
|
| 119 |
+
path = directory / name
|
| 120 |
+
info = manifest.get('files', {}).get(name, {})
|
| 121 |
+
if not path.is_file() or path.is_symlink() or digest(path) != info.get('sha256') or info.get('rows') != size:
|
| 122 |
+
raise ValueError(f'Reference cache file changed: {name}')
|
| 123 |
+
values = np.load(path, allow_pickle=False, mmap_mode='r')
|
| 124 |
+
if values.shape != (size, 2) or values.dtype != np.float64 or not np.isfinite(values).all():
|
| 125 |
+
raise ValueError(f'Invalid reference log probabilities: {name}')
|
| 126 |
+
result.append(values)
|
| 127 |
+
return *result, digest(directory / 'manifest.json')
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def apply_sequence_logps(model, chunk_size):
|
| 131 |
+
"""Return per-sequence log-probability sums without retaining vocabulary logits.
|
| 132 |
+
|
| 133 |
+
Vocabulary matmuls use BF16 autocast; cross-entropy reductions use FP32. Each
|
| 134 |
+
head chunk is recomputed in backward, so DPO keeps a vector of branch scores.
|
| 135 |
+
The backbone modules and canonical parameter/state names are unchanged.
|
| 136 |
+
"""
|
| 137 |
+
def forward(self, input_ids, targets=None, *, return_logits=False):
|
| 138 |
+
if targets is None or return_logits:
|
| 139 |
+
raise ValueError('DPO forward requires response targets and no returned logits')
|
| 140 |
+
hidden = self.embed(input_ids)
|
| 141 |
+
for depth in range(self.cfg.logical_depth):
|
| 142 |
+
hidden = hidden * (1. + self.depth_scale[depth]) + self.depth_bias[depth]
|
| 143 |
+
core = self.cores[depth % self.cfg.n_cores]
|
| 144 |
+
if self.training and self.cfg.gradient_checkpointing:
|
| 145 |
+
hidden = activation_checkpoint(core, hidden, use_reentrant=False)
|
| 146 |
+
else:
|
| 147 |
+
hidden = core(hidden)
|
| 148 |
+
hidden = self.final_norm(hidden)
|
| 149 |
+
|
| 150 |
+
def score(h, weight, labels):
|
| 151 |
+
logits = F.linear(h, weight).float()
|
| 152 |
+
losses = F.cross_entropy(logits.reshape(-1, logits.size(-1)), labels.reshape(-1),
|
| 153 |
+
reduction='none', ignore_index=-100)
|
| 154 |
+
return -losses.reshape(labels.shape).sum(dim=1)
|
| 155 |
+
|
| 156 |
+
logps = hidden.new_zeros((hidden.shape[0],), dtype=torch.float32)
|
| 157 |
+
for start in range(0, hidden.shape[1], chunk_size):
|
| 158 |
+
args = (hidden[:, start:start + chunk_size], self.embed.weight,
|
| 159 |
+
targets[:, start:start + chunk_size])
|
| 160 |
+
value = activation_checkpoint(score, *args, use_reentrant=False) if torch.is_grad_enabled() else score(*args)
|
| 161 |
+
logps = logps + value
|
| 162 |
+
return None, logps
|
| 163 |
+
model.forward = types.MethodType(forward, model)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def preference_metrics(policy, reference, counts, beta, objective):
|
| 167 |
+
"""Pair vectors, retaining gradients only through policy log probabilities."""
|
| 168 |
+
reference = reference.detach().to(policy.dtype)
|
| 169 |
+
if objective == 'length-normalized-dpo':
|
| 170 |
+
policy = policy / counts.clamp_min(1)
|
| 171 |
+
reference = reference / counts.clamp_min(1)
|
| 172 |
+
elif objective != 'summed-dpo':
|
| 173 |
+
raise ValueError('Unsupported DPO objective')
|
| 174 |
+
rewards = beta * (policy - reference)
|
| 175 |
+
margin = rewards[:, 0] - rewards[:, 1]
|
| 176 |
+
return -F.logsigmoid(margin), margin, (margin > 0).to(policy.dtype)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def stop_requested(args, device, world):
|
| 180 |
+
value = STOP or bool(args.stop_file and Path(args.stop_file).exists())
|
| 181 |
+
if world > 1:
|
| 182 |
+
flag = torch.tensor(int(value), device=device)
|
| 183 |
+
dist.all_reduce(flag, op=dist.ReduceOp.MAX)
|
| 184 |
+
value = bool(flag.item())
|
| 185 |
+
return value
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def load_model(checkpoint, settings, device, tokenizer):
|
| 189 |
+
from warpstate import WarpStateConfig, WarpStateLM
|
| 190 |
+
from run_40b import compile_training_cores
|
| 191 |
+
from ops.residual_experiment import apply_bf16_residuals
|
| 192 |
+
cfg = WarpStateConfig(**checkpoint['config'])
|
| 193 |
+
cfg.gradient_checkpointing = settings['gradient_checkpointing']
|
| 194 |
+
# DPO compares a deterministic policy/reference scoring rule, with dropout disabled.
|
| 195 |
+
cfg.dropout = 0.
|
| 196 |
+
cfg.validate()
|
| 197 |
+
if cfg.vocab_size != tokenizer.vocab_size() or settings['sequence_length'] % cfg.chunk_size:
|
| 198 |
+
raise ValueError('Model/tokenizer vocabulary or sequence/chunk mismatch')
|
| 199 |
+
with torch.device('meta'):
|
| 200 |
+
raw = WarpStateLM(cfg)
|
| 201 |
+
raw.load_state_dict(checkpoint['model'], strict=True, assign=True)
|
| 202 |
+
# Clone on CPU as well: mmap initialization is immutable and must not alias training.
|
| 203 |
+
raw.to(device=device, dtype=torch.float32)
|
| 204 |
+
if device.type == 'cpu':
|
| 205 |
+
for parameter in raw.parameters():
|
| 206 |
+
parameter.data = parameter.data.clone()
|
| 207 |
+
if settings['compile_core']:
|
| 208 |
+
compile_training_cores(raw)
|
| 209 |
+
apply_sequence_logps(raw, settings['head_chunk_size'])
|
| 210 |
+
if settings['bf16_residuals']:
|
| 211 |
+
apply_bf16_residuals(raw)
|
| 212 |
+
return raw, cfg
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def atomic_array(path, array):
|
| 216 |
+
temporary = Path(str(path) + '.tmp')
|
| 217 |
+
with temporary.open('wb') as stream:
|
| 218 |
+
np.save(stream, array, allow_pickle=False)
|
| 219 |
+
stream.flush()
|
| 220 |
+
os.fsync(stream.fileno())
|
| 221 |
+
os.replace(temporary, path)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def prepare_reference(args, identity, records, settings, device, rank, world, tokenizer):
|
| 225 |
+
"""Create/reuse distributed resumable score shards, then seal a checked cache."""
|
| 226 |
+
directory = Path(args.reference_cache_dir)
|
| 227 |
+
if (directory / 'manifest.json').exists():
|
| 228 |
+
return validate_cache(directory, identity, [len(rows) for rows in records])
|
| 229 |
+
directory.mkdir(parents=True, exist_ok=True)
|
| 230 |
+
marker = directory / 'identity.json'
|
| 231 |
+
if marker.exists() and json.loads(marker.read_text()) != identity:
|
| 232 |
+
raise ValueError('Partial reference cache identity differs')
|
| 233 |
+
if rank == 0:
|
| 234 |
+
atomic_json(marker, identity)
|
| 235 |
+
if world > 1:
|
| 236 |
+
dist.barrier()
|
| 237 |
+
checkpoint = torch.load(args.weights, map_location='cpu', weights_only=False, mmap=True)
|
| 238 |
+
raw, _ = load_model(checkpoint, settings, device, tokenizer)
|
| 239 |
+
raw.requires_grad_(False).eval()
|
| 240 |
+
del checkpoint
|
| 241 |
+
pad_id = tokenizer.pad_id() if tokenizer.pad_id() >= 0 else tokenizer.eos_id()
|
| 242 |
+
started = time.monotonic()
|
| 243 |
+
try:
|
| 244 |
+
for split, rows in zip(('train', 'val'), records):
|
| 245 |
+
indices = list(range(rank * settings['micro_batch'], len(rows), world * settings['micro_batch']))
|
| 246 |
+
shard = directory / f'{split}-rank-{rank}.npz'
|
| 247 |
+
done, scored_indices, scores = 0, [], []
|
| 248 |
+
if shard.exists():
|
| 249 |
+
with np.load(shard, allow_pickle=False) as old:
|
| 250 |
+
if str(old['identity']) != identity_digest(identity):
|
| 251 |
+
raise ValueError('Partial reference shard identity differs')
|
| 252 |
+
done = int(old['batches'])
|
| 253 |
+
scored_indices = old['indices'].tolist()
|
| 254 |
+
scores = old['scores'].tolist()
|
| 255 |
+
expected_indices = [i for start in indices[:done]
|
| 256 |
+
for i in range(start, min(start + settings['micro_batch'], len(rows)))]
|
| 257 |
+
if scored_indices != expected_indices or len(scores) != len(scored_indices):
|
| 258 |
+
raise ValueError('Partial reference shard cursor differs')
|
| 259 |
+
|
| 260 |
+
def save_shard():
|
| 261 |
+
temporary = Path(str(shard) + '.tmp')
|
| 262 |
+
with temporary.open('wb') as stream:
|
| 263 |
+
np.savez(stream, identity=identity_digest(identity), batches=done,
|
| 264 |
+
indices=np.asarray(scored_indices, dtype=np.int64),
|
| 265 |
+
scores=np.asarray(scores, dtype=np.float64).reshape(-1, 2))
|
| 266 |
+
stream.flush()
|
| 267 |
+
os.fsync(stream.fileno())
|
| 268 |
+
os.replace(temporary, shard)
|
| 269 |
+
|
| 270 |
+
# Every rank executes equal stop/barrier counts, even on uneven final shards.
|
| 271 |
+
max_batches = math.ceil(len(rows) / (world * settings['micro_batch']))
|
| 272 |
+
for batch in range(max_batches):
|
| 273 |
+
if stop_requested(args, device, world):
|
| 274 |
+
save_shard()
|
| 275 |
+
return None
|
| 276 |
+
if batch < done or batch >= len(indices):
|
| 277 |
+
continue
|
| 278 |
+
start = indices[batch]
|
| 279 |
+
selected = list(range(start, min(start + settings['micro_batch'], len(rows))))
|
| 280 |
+
padded = [rows[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected))
|
| 281 |
+
x, y, _ = collate_pairs(padded, settings['sequence_length'], pad_id)
|
| 282 |
+
with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False):
|
| 283 |
+
_, logps = raw(x.to(device), y.to(device), return_logits=False)
|
| 284 |
+
values = logps.reshape(-1, 2).double().cpu().numpy()[:len(selected)]
|
| 285 |
+
if not np.isfinite(values).all():
|
| 286 |
+
raise RuntimeError('Non-finite frozen reference scores')
|
| 287 |
+
scored_indices.extend(selected)
|
| 288 |
+
scores.extend(values.tolist())
|
| 289 |
+
done = batch + 1
|
| 290 |
+
if done % 128 == 0 or done == len(indices):
|
| 291 |
+
save_shard()
|
| 292 |
+
print(json.dumps(dict(event='reference_progress', rank=rank, split=split,
|
| 293 |
+
batches=done, total_batches=len(indices), scored_pairs=len(scores),
|
| 294 |
+
elapsed_seconds=time.monotonic() - started)), flush=True)
|
| 295 |
+
save_shard()
|
| 296 |
+
if world > 1:
|
| 297 |
+
dist.barrier()
|
| 298 |
+
if rank == 0:
|
| 299 |
+
files = {}
|
| 300 |
+
for split, rows in zip(('train', 'val'), records):
|
| 301 |
+
combined = np.full((len(rows), 2), np.nan, dtype=np.float64)
|
| 302 |
+
seen = np.zeros(len(rows), dtype=np.bool_)
|
| 303 |
+
for owner in range(world):
|
| 304 |
+
with np.load(directory / f'{split}-rank-{owner}.npz', allow_pickle=False) as shard:
|
| 305 |
+
if str(shard['identity']) != identity_digest(identity):
|
| 306 |
+
raise ValueError('Reference shard identity differs')
|
| 307 |
+
index = shard['indices']
|
| 308 |
+
if seen[index].any():
|
| 309 |
+
raise ValueError('Duplicate reference indices')
|
| 310 |
+
combined[index] = shard['scores']
|
| 311 |
+
seen[index] = True
|
| 312 |
+
if not seen.all() or not np.isfinite(combined).all():
|
| 313 |
+
raise ValueError('Reference cache is incomplete')
|
| 314 |
+
path = directory / f'{split}.npy'
|
| 315 |
+
atomic_array(path, combined)
|
| 316 |
+
files[path.name] = dict(sha256=digest(path), rows=len(rows))
|
| 317 |
+
atomic_json(directory / 'manifest.json', dict(format=CACHE_FORMAT, complete=True,
|
| 318 |
+
identity=identity, identity_sha256=identity_digest(identity), files=files))
|
| 319 |
+
if world > 1:
|
| 320 |
+
dist.barrier()
|
| 321 |
+
return validate_cache(directory, identity, [len(rows) for rows in records])
|
| 322 |
+
finally:
|
| 323 |
+
del raw
|
| 324 |
+
# Patched bound forwards and functional BF16 wrappers form cycles;
|
| 325 |
+
# collect them before allocating the policy and FP32 Adam state.
|
| 326 |
+
gc.collect()
|
| 327 |
+
if device.type == 'cuda':
|
| 328 |
+
torch.cuda.empty_cache()
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def training_update(model, optimizer, microbatches, global_pairs, world, device, settings):
|
| 332 |
+
optimizer.zero_grad(set_to_none=True)
|
| 333 |
+
totals = torch.zeros(5, device=device, dtype=torch.float64)
|
| 334 |
+
for index, (x, y, active, reference) in enumerate(microbatches):
|
| 335 |
+
sync = model.no_sync() if world > 1 and index < len(microbatches) - 1 else contextlib.nullcontext()
|
| 336 |
+
x, y, active, reference = (value.to(device) for value in (x, y, active, reference))
|
| 337 |
+
counts = y.ne(-100).sum(dim=1).reshape(-1, 2)
|
| 338 |
+
with sync:
|
| 339 |
+
with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False):
|
| 340 |
+
_, logps = model(x, y, return_logits=False)
|
| 341 |
+
logps = logps.reshape(-1, 2)
|
| 342 |
+
losses, margins, accuracy = preference_metrics(logps, reference, counts,
|
| 343 |
+
settings['beta'], settings['objective'])
|
| 344 |
+
if not torch.isfinite(losses).all():
|
| 345 |
+
raise RuntimeError('Non-finite DPO training loss')
|
| 346 |
+
# DDP averages rank gradients; this is the true global pair mean,
|
| 347 |
+
# including a partial final update and entirely empty rank slots.
|
| 348 |
+
(losses * active).sum().mul(world / global_pairs).backward()
|
| 349 |
+
totals += torch.stack(((losses.detach() * active).double().sum(),
|
| 350 |
+
(margins.detach() * active).double().sum(),
|
| 351 |
+
(accuracy.detach() * active).double().sum(),
|
| 352 |
+
((-logps[:, 0].detach() / counts[:, 0].clamp_min(1)) * active).double().sum(),
|
| 353 |
+
active.double().sum()))
|
| 354 |
+
norm = torch.nn.utils.clip_grad_norm_(model.parameters(), settings['clip_grad_norm'], error_if_nonfinite=True)
|
| 355 |
+
optimizer.step()
|
| 356 |
+
if world > 1:
|
| 357 |
+
dist.all_reduce(totals)
|
| 358 |
+
if int(totals[-1]) != global_pairs:
|
| 359 |
+
raise RuntimeError('Distributed DPO pair count differs')
|
| 360 |
+
values = (totals[:4] / totals[-1]).tolist()
|
| 361 |
+
return dict(zip(('loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)), float(norm)
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def evaluate(model, records, reference, settings, device, pad_id, *, full=False):
|
| 365 |
+
state = rng_state(device)
|
| 366 |
+
was_training = model.training
|
| 367 |
+
model.eval()
|
| 368 |
+
totals = torch.zeros(4, dtype=torch.float64, device=device)
|
| 369 |
+
count = 0
|
| 370 |
+
limit = len(records) if full else min(len(records), settings['eval_batches'] * settings['micro_batch'])
|
| 371 |
+
try:
|
| 372 |
+
with torch.no_grad():
|
| 373 |
+
for start in range(0, limit, settings['micro_batch']):
|
| 374 |
+
selected = records[start:min(start + settings['micro_batch'], limit)]
|
| 375 |
+
n = len(selected)
|
| 376 |
+
selected += [None] * (settings['micro_batch'] - n)
|
| 377 |
+
x, y, active = collate_pairs(selected, settings['sequence_length'], pad_id)
|
| 378 |
+
x, y, active = x.to(device), y.to(device), active.to(device)
|
| 379 |
+
ref = torch.zeros((settings['micro_batch'], 2), device=device)
|
| 380 |
+
ref[:n] = torch.tensor(np.array(reference[start:start + n]), device=device)
|
| 381 |
+
with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False):
|
| 382 |
+
_, logps = model(x, y, return_logits=False)
|
| 383 |
+
logps = logps.reshape(-1, 2)
|
| 384 |
+
counts = y.ne(-100).sum(dim=1).reshape(-1, 2)
|
| 385 |
+
losses, margins, accuracy = preference_metrics(logps, ref, counts,
|
| 386 |
+
settings['beta'], settings['objective'])
|
| 387 |
+
totals += torch.stack(((losses * active).double().sum(), (margins * active).double().sum(),
|
| 388 |
+
(accuracy * active).double().sum(),
|
| 389 |
+
((-logps[:, 0] / counts[:, 0].clamp_min(1)) * active).double().sum()))
|
| 390 |
+
count += n
|
| 391 |
+
finally:
|
| 392 |
+
model.train(was_training)
|
| 393 |
+
restore_rng(state, device)
|
| 394 |
+
values = (totals / count).tolist()
|
| 395 |
+
if not all(math.isfinite(value) for value in values):
|
| 396 |
+
raise RuntimeError('Non-finite DPO validation')
|
| 397 |
+
return dict(zip(('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)) | dict(evaluated_samples=count)
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def validate_resume(checkpoint, signature, world):
|
| 401 |
+
if checkpoint.get('format') != FORMAT or checkpoint.get('signature') != signature:
|
| 402 |
+
raise ValueError('DPO resume signature differs; use a new output directory')
|
| 403 |
+
if len(checkpoint.get('rng_by_rank', [])) != world:
|
| 404 |
+
raise ValueError('DPO checkpoint lacks exact per-rank RNG state')
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def validate_reference_path(args):
|
| 408 |
+
cache = Path(args.reference_cache_dir).resolve()
|
| 409 |
+
paths = [Path(path).resolve() for path in (args.work_dir, args.data_dir, args.weights.parent)]
|
| 410 |
+
if not cache.is_relative_to(WORKSPACE) or cache == WORKSPACE:
|
| 411 |
+
raise ValueError('Reference cache must stay inside the workspace')
|
| 412 |
+
if any(cache == path or cache.is_relative_to(path) or path.is_relative_to(cache) for path in paths):
|
| 413 |
+
raise ValueError('Reference cache must be separate from training outputs and inputs')
|
| 414 |
+
return cache
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def train(args):
|
| 418 |
+
import sentencepiece as spm
|
| 419 |
+
from ops.checkpoint_writer import DistributedCheckpointWriter
|
| 420 |
+
work, data_dir, weights, tokenizer_path, config = validate_paths(
|
| 421 |
+
args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config)
|
| 422 |
+
cache = validate_reference_path(args)
|
| 423 |
+
plan, settings = load_plan(config)
|
| 424 |
+
rank = int(os.environ.get('RANK', '0'))
|
| 425 |
+
world = int(os.environ.get('WORLD_SIZE', '1'))
|
| 426 |
+
local_rank = int(os.environ.get('LOCAL_RANK', '0'))
|
| 427 |
+
device = torch.device('cuda', local_rank) if args.device == 'cuda' else torch.device('cpu')
|
| 428 |
+
if device.type == 'cuda':
|
| 429 |
+
torch.cuda.set_device(device)
|
| 430 |
+
if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported():
|
| 431 |
+
raise RuntimeError('BF16-capable CUDA GPU required')
|
| 432 |
+
torch.set_num_threads(settings['workers'])
|
| 433 |
+
torch.use_deterministic_algorithms(settings['deterministic'])
|
| 434 |
+
random.seed(settings['seed'] + rank)
|
| 435 |
+
np.random.seed(settings['seed'] + rank)
|
| 436 |
+
torch.manual_seed(settings['seed'] + rank)
|
| 437 |
+
tokenizer_hash, base_hash = digest(tokenizer_path), digest(weights)
|
| 438 |
+
sp = spm.SentencePieceProcessor(model_file=str(tokenizer_path))
|
| 439 |
+
train_data, val_data, manifest, manifest_hash = load_data(data_dir, sp, tokenizer_hash, settings['sequence_length'])
|
| 440 |
+
source = torch.load(weights, map_location='cpu', weights_only=False, mmap=True)
|
| 441 |
+
if (source.get('format') != 'warpstate-chat-v1' or source.get('inference_only') is not True or
|
| 442 |
+
source.get('stage') != 'sft' or source.get('tokenizer_sha256') != tokenizer_hash):
|
| 443 |
+
raise ValueError('Initialize DPO from an independent, tokenizer-matched SFT model export')
|
| 444 |
+
expected = plan.get('sft', plan.get('base', {}))
|
| 445 |
+
for key in ('step', 'tokens_seen', 'tokenizer_sha256'):
|
| 446 |
+
if key in expected and expected[key] != source.get(key):
|
| 447 |
+
raise ValueError(f'SFT source {key} differs from plan')
|
| 448 |
+
ancestor = source.get('origin', {})
|
| 449 |
+
for key in ('step', 'tokens_seen'):
|
| 450 |
+
expected_key = 'base_' + key
|
| 451 |
+
value = ancestor.get(key, ancestor.get(expected_key))
|
| 452 |
+
if expected_key in expected and expected[expected_key] != value:
|
| 453 |
+
raise ValueError(f'SFT source {expected_key} differs from plan')
|
| 454 |
+
origin = {key: value for key, value in source.items() if key != 'model'}
|
| 455 |
+
origin.update(weights_sha256=base_hash, weights_path=str(weights))
|
| 456 |
+
del source
|
| 457 |
+
identity = reference_identity(base_hash, manifest_hash, tokenizer_hash, settings, device, world)
|
| 458 |
+
if world > 1:
|
| 459 |
+
dist.init_process_group(backend='nccl' if device.type == 'cuda' else 'gloo')
|
| 460 |
+
try:
|
| 461 |
+
reference = prepare_reference(args, identity, (train_data, val_data), settings, device, rank, world, sp)
|
| 462 |
+
if reference is None:
|
| 463 |
+
print(json.dumps(dict(event='reference_paused', rank=rank)), flush=True)
|
| 464 |
+
return
|
| 465 |
+
reference_train, reference_val, reference_hash = reference
|
| 466 |
+
if getattr(args, 'reference_only', False):
|
| 467 |
+
print(json.dumps(dict(event='reference_complete', reference_sha256=reference_hash)), flush=True)
|
| 468 |
+
return
|
| 469 |
+
signature = dict(plan=plan, settings=settings, data_sha256=manifest_hash,
|
| 470 |
+
tokenizer_sha256=tokenizer_hash, renderer=RENDER_FORMAT, mask=MASK_FORMAT,
|
| 471 |
+
base_sha256=base_hash, reference_sha256=reference_hash,
|
| 472 |
+
world_size=world, device_type=device.type, torch_version=torch.__version__, sources=source_identity())
|
| 473 |
+
latest = work / 'latest.pt'
|
| 474 |
+
resuming = latest.exists()
|
| 475 |
+
saved = torch.load(latest if resuming else weights, map_location='cpu', weights_only=False, mmap=True)
|
| 476 |
+
if resuming:
|
| 477 |
+
validate_resume(saved, signature, world)
|
| 478 |
+
origin = saved['origin']
|
| 479 |
+
raw, cfg = load_model(saved, settings, device, sp)
|
| 480 |
+
raw.requires_grad_(True)
|
| 481 |
+
optimizer = torch.optim.AdamW(raw.parameters(), lr=settings['lr'], betas=(.9, .95),
|
| 482 |
+
eps=1e-8, weight_decay=settings['weight_decay'], fused=device.type == 'cuda')
|
| 483 |
+
step = cursor = tokens_seen = response_tokens_seen = 0
|
| 484 |
+
if resuming:
|
| 485 |
+
optimizer.load_state_dict(saved['optimizer'])
|
| 486 |
+
step, cursor = saved['step'], saved['sample_cursor']
|
| 487 |
+
tokens_seen, response_tokens_seen = saved['tokens_seen'], saved['response_tokens_seen']
|
| 488 |
+
resume_rng = saved['rng_by_rank'][rank] if resuming else None
|
| 489 |
+
del saved
|
| 490 |
+
if world > 1:
|
| 491 |
+
model = torch.nn.parallel.DistributedDataParallel(raw,
|
| 492 |
+
device_ids=[local_rank] if device.type == 'cuda' else None,
|
| 493 |
+
broadcast_buffers=False, gradient_as_bucket_view=True, bucket_cap_mb=128)
|
| 494 |
+
else:
|
| 495 |
+
model = raw
|
| 496 |
+
if resume_rng is not None:
|
| 497 |
+
restore_rng(resume_rng, device)
|
| 498 |
+
global_batch = settings['micro_batch'] * settings['grad_accum'] * world
|
| 499 |
+
total_steps = math.ceil(len(train_data) / global_batch) * settings['epochs']
|
| 500 |
+
if settings['max_steps'] is not None:
|
| 501 |
+
total_steps = min(total_steps, settings['max_steps'])
|
| 502 |
+
end_step = min(total_steps, step + args.max_steps) if args.max_steps is not None else total_steps
|
| 503 |
+
work.mkdir(parents=True, exist_ok=True)
|
| 504 |
+
if rank == 0:
|
| 505 |
+
atomic_json(work / 'settings.json', dict(signature=signature, origin=origin,
|
| 506 |
+
total_steps=total_steps, train_rows=len(train_data), val_rows=len(val_data)))
|
| 507 |
+
if step < total_steps:
|
| 508 |
+
(work / 'completion.json').unlink(missing_ok=True)
|
| 509 |
+
pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id()
|
| 510 |
+
last_validation = None
|
| 511 |
+
last_validation_step = None
|
| 512 |
+
|
| 513 |
+
def validation(full=False):
|
| 514 |
+
values = [evaluate(raw, val_data, reference_val, settings, device, pad_id, full=full) if rank == 0 else None]
|
| 515 |
+
if world > 1:
|
| 516 |
+
dist.broadcast_object_list(values, src=0, device=device)
|
| 517 |
+
return values[0]
|
| 518 |
+
|
| 519 |
+
def metric(values):
|
| 520 |
+
if rank == 0:
|
| 521 |
+
values.update(timestamp=time.time(), stage='dpo', step=step, tokens_seen=tokens_seen,
|
| 522 |
+
response_tokens_seen=response_tokens_seen, sample_cursor=cursor,
|
| 523 |
+
objective=settings['objective'])
|
| 524 |
+
with (work / 'metrics.jsonl').open('a') as stream:
|
| 525 |
+
stream.write(json.dumps(values, allow_nan=False) + '\n')
|
| 526 |
+
if values.get('event') == 'validation':
|
| 527 |
+
atomic_json(work / 'evaluation.json', dict(values, data_sha256=manifest_hash,
|
| 528 |
+
base_sha256=base_hash, reference_sha256=reference_hash,
|
| 529 |
+
tokenizer_sha256=tokenizer_hash, mask_format=MASK_FORMAT,
|
| 530 |
+
available_validation_samples=len(val_data)))
|
| 531 |
+
print(json.dumps(values, allow_nan=False), flush=True)
|
| 532 |
+
|
| 533 |
+
def save(writer):
|
| 534 |
+
state = rng_state(device)
|
| 535 |
+
states = [None] * world
|
| 536 |
+
if world > 1:
|
| 537 |
+
dist.all_gather_object(states, state)
|
| 538 |
+
else:
|
| 539 |
+
states = [state]
|
| 540 |
+
writer.submit(state_factory=lambda: dict(format=FORMAT, signature=signature, origin=origin,
|
| 541 |
+
config=vars(cfg), model=raw.state_dict(), optimizer=optimizer.state_dict(), step=step,
|
| 542 |
+
sample_cursor=cursor, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen,
|
| 543 |
+
val_loss=last_validation['val_loss'], validation=last_validation,
|
| 544 |
+
val_step=last_validation_step, rng_by_rank=states, completed=step >= total_steps,
|
| 545 |
+
total_steps=total_steps))
|
| 546 |
+
|
| 547 |
+
with DistributedCheckpointWriter(latest, rank=rank, device=device, distributed=world > 1) as writer:
|
| 548 |
+
last_validation = validation(full=True)
|
| 549 |
+
last_validation_step = step
|
| 550 |
+
metric(dict(event='validation', full_holdout=True, **last_validation))
|
| 551 |
+
last_saved_step = step if resuming else -1
|
| 552 |
+
model.train()
|
| 553 |
+
while step < end_step and not stop_requested(args, device, world):
|
| 554 |
+
writer.check()
|
| 555 |
+
indices, next_cursor = update_indices(len(train_data), settings['seed'], cursor,
|
| 556 |
+
settings['epochs'], global_batch)
|
| 557 |
+
if not indices:
|
| 558 |
+
break
|
| 559 |
+
microbatches = []
|
| 560 |
+
for micro in range(settings['grad_accum']):
|
| 561 |
+
offset = (micro * world + rank) * settings['micro_batch']
|
| 562 |
+
selected = indices[offset:offset + settings['micro_batch']]
|
| 563 |
+
padded = [train_data[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected))
|
| 564 |
+
x, y, active = collate_pairs(padded, settings['sequence_length'], pad_id)
|
| 565 |
+
ref = torch.zeros((settings['micro_batch'], 2), dtype=torch.float32)
|
| 566 |
+
if selected:
|
| 567 |
+
ref[:len(selected)] = torch.tensor(np.array(reference_train[selected]), dtype=torch.float32)
|
| 568 |
+
microbatches.append((x, y, active, ref))
|
| 569 |
+
lr = learning_rate(step, total_steps, settings)
|
| 570 |
+
for group in optimizer.param_groups:
|
| 571 |
+
group['lr'] = lr
|
| 572 |
+
started = time.perf_counter()
|
| 573 |
+
values, norm = training_update(model, optimizer, microbatches, len(indices), world, device, settings)
|
| 574 |
+
if device.type == 'cuda':
|
| 575 |
+
torch.cuda.synchronize(device)
|
| 576 |
+
seconds = time.perf_counter() - started
|
| 577 |
+
step += 1
|
| 578 |
+
cursor = next_cursor
|
| 579 |
+
actual_tokens = sum(len(train_data[i][branch]['input_ids']) - 1
|
| 580 |
+
for i in indices for branch in ('chosen', 'rejected'))
|
| 581 |
+
tokens_seen += actual_tokens
|
| 582 |
+
response_tokens_seen += sum(sum(value != -100 for value in train_data[i][branch]['labels'][1:])
|
| 583 |
+
for i in indices for branch in ('chosen', 'rejected'))
|
| 584 |
+
metric(dict(event='train', **values, lr=lr, grad_norm=norm, update_seconds=seconds,
|
| 585 |
+
tokens_per_second=actual_tokens / seconds, update_samples=len(indices)))
|
| 586 |
+
if step % settings['eval_every'] == 0 or step == end_step:
|
| 587 |
+
full = step >= total_steps
|
| 588 |
+
last_validation = validation(full=full)
|
| 589 |
+
last_validation_step = step
|
| 590 |
+
metric(dict(event='validation', full_holdout=full, **last_validation))
|
| 591 |
+
if step % settings['save_every'] == 0 or step == end_step:
|
| 592 |
+
save(writer)
|
| 593 |
+
last_saved_step = step
|
| 594 |
+
if last_saved_step != step:
|
| 595 |
+
save(writer)
|
| 596 |
+
writer.flush()
|
| 597 |
+
if rank == 0 and step >= total_steps:
|
| 598 |
+
atomic_json(work / 'completion.json', dict(format='warpstate-dpo-completion-v1',
|
| 599 |
+
stage='dpo', completed=True, paused_for_evaluation=True, step=step,
|
| 600 |
+
total_steps=total_steps, sample_cursor=cursor, tokens_seen=tokens_seen,
|
| 601 |
+
response_tokens_seen=response_tokens_seen, val_loss=last_validation['val_loss'],
|
| 602 |
+
evaluation_file=str(work / 'evaluation.json'), checkpoint=str(latest),
|
| 603 |
+
reference_sha256=reference_hash, timestamp=time.time()))
|
| 604 |
+
metric(dict(event='complete' if step >= total_steps else 'paused', total_steps=total_steps,
|
| 605 |
+
checkpoint=str(latest)))
|
| 606 |
+
validate_cache(cache, identity, (len(train_data), len(val_data)))
|
| 607 |
+
if digest(weights) != base_hash:
|
| 608 |
+
raise RuntimeError('Frozen SFT initialization changed during DPO')
|
| 609 |
+
finally:
|
| 610 |
+
if world > 1 and dist.is_initialized():
|
| 611 |
+
dist.destroy_process_group()
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def main():
|
| 615 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 616 |
+
for name in ('config', 'work-dir', 'data-dir', 'weights', 'tokenizer', 'reference-cache-dir'):
|
| 617 |
+
parser.add_argument('--' + name, required=True, type=Path)
|
| 618 |
+
parser.add_argument('--max-steps', type=int)
|
| 619 |
+
parser.add_argument('--stop-file', type=Path)
|
| 620 |
+
parser.add_argument('--device', choices=('cuda', 'cpu'), default='cuda')
|
| 621 |
+
parser.add_argument('--reference-only', action='store_true')
|
| 622 |
+
args = parser.parse_args()
|
| 623 |
+
if args.max_steps is not None and args.max_steps < 1:
|
| 624 |
+
parser.error('--max-steps must be positive')
|
| 625 |
+
if args.stop_file is None:
|
| 626 |
+
args.stop_file = args.work_dir / 'control/training-stop-request'
|
| 627 |
+
if not args.stop_file.resolve().is_relative_to(WORKSPACE):
|
| 628 |
+
parser.error('--stop-file must stay inside the workspace')
|
| 629 |
+
def stop(signum, frame):
|
| 630 |
+
global STOP
|
| 631 |
+
STOP = True
|
| 632 |
+
for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1):
|
| 633 |
+
signal.signal(signum, stop)
|
| 634 |
+
from run_40b import process_lock
|
| 635 |
+
validate_paths(args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config)
|
| 636 |
+
validate_reference_path(args)
|
| 637 |
+
args.work_dir.mkdir(parents=True, exist_ok=True)
|
| 638 |
+
args.reference_cache_dir.mkdir(parents=True, exist_ok=True)
|
| 639 |
+
plan, _ = load_plan(args.config)
|
| 640 |
+
rank = int(os.environ.get('RANK', '0'))
|
| 641 |
+
with process_lock(args.work_dir / f'rank-{rank}.lock'), \
|
| 642 |
+
process_lock(args.reference_cache_dir / f'rank-{rank}.lock'), base_pipeline_lock(plan):
|
| 643 |
+
train(args)
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
if __name__ == '__main__':
|
| 647 |
+
main()
|
ops/B200_HANDOFF.md
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Continue WarpState on the B200
|
| 2 |
+
|
| 3 |
+
This is the starting point for sessions running directly in
|
| 4 |
+
`/workspace/WarpState` on the **single visible B200** instance. Use the current
|
| 5 |
+
session state below; the original transfer details remain for provenance.
|
| 6 |
+
|
| 7 |
+
**Latest session state, September 10, 2026:** training is **STOPPED** at the
|
| 8 |
+
user's request. The graceful stop finished step **680 / 89,128,960 tokens**,
|
| 9 |
+
evaluated validation loss **4.39469**, and durably published the full checkpoint
|
| 10 |
+
at `work/run/latest.pt` before exit. This is the checkpoint for the next resume.
|
| 11 |
+
|
| 12 |
+
Checkpoint SHA256:
|
| 13 |
+
`bf689a10a030842699bfa3a25ff62e488cfa2d5b022b2b6e3bfc860359eb8b4f`.
|
| 14 |
+
Inspection verified all 2,005,111,424 finite FP32 parameters, 256 full AdamW
|
| 15 |
+
entries at step 680, FP32 moments, rank-zero RNG structure, unchanged data and
|
| 16 |
+
resume signature, and scheduled optimizer LR **0.000102**. The full inspection
|
| 17 |
+
is `work/validation/b200/paused-checkpoint-step-680.json`. The final CPU staging
|
| 18 |
+
took 4.93 seconds and background publication took 17.74 seconds; shutdown
|
| 19 |
+
waited for publication. No training GPU process remains.
|
| 20 |
+
|
| 21 |
+
Earlier snapshots `work/run/paused-step-488.pt` and
|
| 22 |
+
`work/run/validated-step-500.pt` remain preserved. The step-500 verification is
|
| 23 |
+
historical; use the newer step-680 report for the current resume state.
|
| 24 |
+
Resume only after the user requests it, using the controls below.
|
| 25 |
+
|
| 26 |
+
The B200 launcher now uses microbatch 16 / accumulation 4, compiled full loss,
|
| 27 |
+
BF16 kernel autotuning, and asynchronous full checkpoints. See
|
| 28 |
+
[B200_PERFORMANCE.md](B200_PERFORMANCE.md) for all measurements and validation,
|
| 29 |
+
and [B200_SCHEDULING.md](B200_SCHEDULING.md) for evidence linking recurring waits
|
| 30 |
+
to provider GPU scheduling. Check Supervisor status before starting another
|
| 31 |
+
job. The deployment record and handover tables below describe the original
|
| 32 |
+
step-448 transfer, not the current training cursor.
|
| 33 |
+
|
| 34 |
+
## First commands
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
cd /workspace/WarpState
|
| 38 |
+
source rtx6000_40b/ops/training-env.sh
|
| 39 |
+
source rtx6000_40b/ops/workspace-tools.sh
|
| 40 |
+
git status --short --branch
|
| 41 |
+
git log -3 --oneline
|
| 42 |
+
rtx6000_40b/ops/b200ctl.sh status
|
| 43 |
+
nvidia-smi
|
| 44 |
+
df -h /workspace/WarpState
|
| 45 |
+
rtx6000_40b/.server-venv/bin/python -m json.tool \
|
| 46 |
+
rtx6000_40b/work/validation/b200/deployment.json
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
The generated `work/validation/b200/deployment.json` is the final deployment
|
| 50 |
+
record: use it to check artifact hashes, environment validation, and the
|
| 51 |
+
handover state. If it is missing or reports pending work, finish those checks
|
| 52 |
+
before training. Reading this document alone does not establish that a large
|
| 53 |
+
file transfer completed. Published `train.bin` and `latest.pt` must have
|
| 54 |
+
verified full SHA256 hashes; `.uploading` files are incomplete transfers.
|
| 55 |
+
|
| 56 |
+
The shell setup keeps caches and temporary files inside the workspace and
|
| 57 |
+
adds the project-local Git installation to `PATH`. Source both scripts in
|
| 58 |
+
each new shell. All source and Git history have been transferred; imported
|
| 59 |
+
history includes `a7b25a9`, followed by the handoff documentation commit. Use
|
| 60 |
+
`git log` for the actual current revision. Keep the repository on `main`,
|
| 61 |
+
follow the root [AGENTS.md](../../AGENTS.md), and commit completed source
|
| 62 |
+
changes using `WarpState Automation <warpstate@localhost>`. Generated data,
|
| 63 |
+
environments, checkpoints, and validation reports remain ignored.
|
| 64 |
+
This is an independent Git copy with no push remote configured; commits on
|
| 65 |
+
the B200 are local to that instance until explicitly synchronized elsewhere.
|
| 66 |
+
|
| 67 |
+
## Environment and preserved training state
|
| 68 |
+
|
| 69 |
+
| Item | Handover state |
|
| 70 |
+
| --- | --- |
|
| 71 |
+
| Hardware | One NVIDIA B200; Ubuntu 24.04; Python 3.12 |
|
| 72 |
+
| Python environment | `/workspace/WarpState/rtx6000_40b/.server-venv` |
|
| 73 |
+
| Dependencies | PyTorch `2.14.0+cu130`; pinned [training requirements](training-requirements-resolved.txt) and [B200 requirements](b200-requirements.txt) |
|
| 74 |
+
| Model | 2,005,111,424 parameters; original tracked 32,768-token tokenizer |
|
| 75 |
+
| Training corpus | `work/data/train.bin`: 40,000,000,000 `uint16` token IDs; 80,000,000,000 bytes |
|
| 76 |
+
| Validation corpus | `work/data/val.bin`: 1,048,576 `uint16` token IDs; 2,097,152 bytes |
|
| 77 |
+
| Checkpoint | `work/run/latest.pt`: full checkpoint at step **448**, with **58,720,256** tokens consumed |
|
| 78 |
+
| Restored state | Model, full FP32 AdamW optimizer, token cursor, learning-rate schedule, and rank-zero random state |
|
| 79 |
+
| Precision | **BF16 computation only**, as explicitly required by the user; parameters, gradients, and AdamW state stay FP32 |
|
| 80 |
+
|
| 81 |
+
Paths beginning with `work/` above are relative to `rtx6000_40b/`. Preserve
|
| 82 |
+
the copied data manifest and preparation state. The original request was
|
| 83 |
+
revision `main`; the data recipe pins the immutable upstream revision
|
| 84 |
+
separately. Changing the requested revision for this existing work directory
|
| 85 |
+
breaks its resume signature. The existing data requires no new download or
|
| 86 |
+
tokenization.
|
| 87 |
+
|
| 88 |
+
The host's GCC/G++ 13.2 and Python development headers passed actual Inductor
|
| 89 |
+
compilation. No additional compiler setup is needed. The launcher also
|
| 90 |
+
supports an optional `tmp/remote-setup/toolchain-env.sh` if the toolchain is
|
| 91 |
+
changed later. Provisioning details are in [B200_SETUP.md](B200_SETUP.md).
|
| 92 |
+
|
| 93 |
+
## Resume and pause controls
|
| 94 |
+
|
| 95 |
+
Use [b200ctl.sh](b200ctl.sh) with [b200-supervisord.conf](b200-supervisord.conf).
|
| 96 |
+
Both `autostart` and `autorestart` are disabled. Starting the Supervisor alone
|
| 97 |
+
leaves training stopped.
|
| 98 |
+
|
| 99 |
+
If `b200ctl.sh status` reports that the control socket is absent, first check
|
| 100 |
+
whether the daemon is already starting. Initialize it only when absent:
|
| 101 |
+
|
| 102 |
+
```bash
|
| 103 |
+
mkdir -p rtx6000_40b/work/control rtx6000_40b/work/logs
|
| 104 |
+
rtx6000_40b/.server-venv/bin/supervisord \
|
| 105 |
+
-c rtx6000_40b/ops/b200-supervisord.conf
|
| 106 |
+
rtx6000_40b/ops/b200ctl.sh status
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
After deployment verification and a user request to resume training:
|
| 110 |
+
|
| 111 |
+
```bash
|
| 112 |
+
rtx6000_40b/ops/b200ctl.sh start warpstate-train
|
| 113 |
+
rtx6000_40b/ops/b200ctl.sh status
|
| 114 |
+
tail -n 30 rtx6000_40b/work/logs/b200-train.log
|
| 115 |
+
tail -n 5 rtx6000_40b/work/run/metrics.jsonl
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
Initial core compilation can take several minutes. Confirm the log restores
|
| 119 |
+
step 448, or a later checkpoint after subsequent training, and then reports
|
| 120 |
+
finite loss and completed optimizer updates. Measure throughput after warmup.
|
| 121 |
+
To follow the live log, use `tail -f` on
|
| 122 |
+
`rtx6000_40b/work/logs/b200-train.log`.
|
| 123 |
+
|
| 124 |
+
For interactive monitoring, the separate `warpstate-b200` tmux session has a
|
| 125 |
+
`training` window with the live training log and a second pane showing GPU
|
| 126 |
+
usage. Attach from the repository root:
|
| 127 |
+
|
| 128 |
+
```bash
|
| 129 |
+
rtx6000_40b/ops/tmux-workspace.sh attach -t warpstate-b200
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
Detach with `Ctrl-b`, then `d`. Training remains managed by Supervisor, so
|
| 133 |
+
detaching or closing a monitoring pane does not stop training. Continue to
|
| 134 |
+
use `b200ctl.sh` for start/stop controls. The wrapper keeps the tmux socket
|
| 135 |
+
under `work/control`; workspace-local installation details are recorded in
|
| 136 |
+
[tmux-workspace-install.md](tmux-workspace-install.md).
|
| 137 |
+
|
| 138 |
+
To pause and preserve the full continuation state:
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
rtx6000_40b/ops/b200ctl.sh stop warpstate-train
|
| 142 |
+
rtx6000_40b/ops/b200ctl.sh status
|
| 143 |
+
tail -n 30 rtx6000_40b/work/logs/b200-train.log
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
Stopping requests a graceful save after the current optimizer update and
|
| 147 |
+
allows up to 15 minutes. Verify the saved-checkpoint message and `STOPPED`
|
| 148 |
+
status. The next start resumes `latest.pt`. Preserve this checkpoint;
|
| 149 |
+
**never replace it with a fresh run or use `--from-scratch` to resume**.
|
| 150 |
+
|
| 151 |
+
The original `train-fast.sh` hardcodes the workstation's two GPU UUIDs.
|
| 152 |
+
The original `supervisord.conf`/`datactl.sh` belongs to that workstation,
|
| 153 |
+
specifies user `root`, and can automatically start preparation. Use the B200
|
| 154 |
+
controls on this instance. Run only one training launcher against `work`.
|
| 155 |
+
|
| 156 |
+
## Current single-GPU configuration
|
| 157 |
+
|
| 158 |
+
[train-b200.sh](train-b200.sh) selects one GPU and uses `--reconfigure` to
|
| 159 |
+
continue the existing full checkpoint with its data, objective, and global batch
|
| 160 |
+
unchanged:
|
| 161 |
+
|
| 162 |
+
| Setting | Value |
|
| 163 |
+
| --- | --- |
|
| 164 |
+
| Sequence length | 2,048 |
|
| 165 |
+
| Microbatch / accumulation | 16 / 4 |
|
| 166 |
+
| Tokens per optimizer update | **131,072** |
|
| 167 |
+
| Core compilation | Enabled; `max-autotune-no-cudagraphs` |
|
| 168 |
+
| Core activation checkpointing | Disabled |
|
| 169 |
+
| Loss | Compiled full vocabulary loss |
|
| 170 |
+
| BF16 autocast cache | Enabled |
|
| 171 |
+
| Allocator cap | 150 GiB |
|
| 172 |
+
| Additional required free VRAM at startup | 3 GiB beyond the cap |
|
| 173 |
+
| Required free SSD space at startup | At least 53 GiB beyond existing files |
|
| 174 |
+
| Save / evaluation interval | Every 50 / 100 updates |
|
| 175 |
+
| Checkpoint writing | Immutable CPU staging, then asynchronous serialization and atomic publication |
|
| 176 |
+
|
| 177 |
+
The allocator cap excludes CUDA context and library memory. The selected
|
| 178 |
+
preset measured about 72.5K median tokens/s and 139.3 GiB peak reserved memory.
|
| 179 |
+
Sustained throughput is much lower and varies with provider scheduling waits;
|
| 180 |
+
see [B200_PERFORMANCE.md](B200_PERFORMANCE.md) for complete measurements and
|
| 181 |
+
limitations. The dataset, 40B token budget, global batch and LR schedule remain
|
| 182 |
+
unchanged. Autotuning can extend initial compilation; eight compiler workers
|
| 183 |
+
are used by default.
|
| 184 |
+
|
| 185 |
+
Asynchronous saves need approximately 24 GB of additional host memory. CPU
|
| 186 |
+
staging finishes before the next update; only serialization and disk I/O overlap
|
| 187 |
+
training. One outstanding save bounds memory, failures stop the job, and graceful
|
| 188 |
+
exit waits for publication. Wait for the **Saved** message, not just **Staged**.
|
| 189 |
+
`--no-async-checkpoint` restores synchronous saves. CUDA graphs are validated
|
| 190 |
+
as an optional `--compile-mode reduce-overhead` override with persistent FP32
|
| 191 |
+
gradient buffers; they were not selected as the default. cuBLASLt preference
|
| 192 |
+
remains an optional experiment rather than a default override.
|
| 193 |
+
|
| 194 |
+
## Original deployment validation
|
| 195 |
+
|
| 196 |
+
The remote CPU suite passed **92 tests**, with **5 GPU cases deselected**.
|
| 197 |
+
This provider hung during collection with `CUDA_VISIBLE_DEVICES=''`; keep
|
| 198 |
+
normal GPU visibility and use the explicit test selection documented in
|
| 199 |
+
[B200_SETUP.md](B200_SETUP.md). Do not copy the workstation's empty-mask CPU
|
| 200 |
+
test command into this environment.
|
| 201 |
+
|
| 202 |
+
Both single-GPU full-state resume cases also passed (**2 passed, 3
|
| 203 |
+
deselected**, 80.37 seconds). Actual BF16 Inductor compilation passed with
|
| 204 |
+
one graph and 38 generated kernels; two fused AdamW updates retained finite
|
| 205 |
+
FP32 parameters, gradients and moments. These checks used small fixtures;
|
| 206 |
+
the production checkpoint was preserved. The large transfers' final hashes,
|
| 207 |
+
checkpoint inspection and final readiness state are recorded in
|
| 208 |
+
`work/validation/b200/deployment.json`; consult that record for completion.
|
| 209 |
+
Multi-GPU tests require more than this instance's one GPU.
|
| 210 |
+
|
| 211 |
+
The deployment checks above describe the original transfer. Current source
|
| 212 |
+
checks, full-state continuation tests, and production checkpoint validation are
|
| 213 |
+
recorded in [B200_PERFORMANCE.md](B200_PERFORMANCE.md). Read the current
|
| 214 |
+
checkpoint inspection and confirm job status before any later resume.
|
ops/B200_OPTIMIZATION_PLAN.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# B200 optimization research, September 10, 2026
|
| 2 |
+
|
| 3 |
+
This records the initial research before implementation. The subsequent
|
| 4 |
+
measurements, selected launcher settings, CUDA graph validation, backend
|
| 5 |
+
comparison and asynchronous checkpoint implementation are in
|
| 6 |
+
[B200_PERFORMANCE.md](B200_PERFORMANCE.md). See
|
| 7 |
+
[B200_SCHEDULING.md](B200_SCHEDULING.md) for the completed stall investigation.
|
| 8 |
+
|
| 9 |
+
At the time of this initial research, training was paused at step **488**, with
|
| 10 |
+
**63,963,136** tokens consumed and validation loss **4.7938699126**.
|
| 11 |
+
`work/run/latest.pt` and `work/run/paused-step-488.pt` then referenced
|
| 12 |
+
the same verified 24,061,671,905 bytes. Subsequent atomic replacements of
|
| 13 |
+
`latest.pt` preserved the step-488 hardlink. Consult [B200_HANDOFF.md](B200_HANDOFF.md)
|
| 14 |
+
for the current paused checkpoint.
|
| 15 |
+
|
| 16 |
+
Checkpoint SHA256:
|
| 17 |
+
`69d37e902c41f541d55a07b3d10dcb7643721240cc6aae032455cd8904c64608`.
|
| 18 |
+
Inspection verified 2,005,111,424 FP32 parameters, all 256 AdamW state entries
|
| 19 |
+
and their step-488 counters, FP32 moments, rank-zero RNG, token cursor, and
|
| 20 |
+
learning rate 0.0000732. Details are in the ignored generated report
|
| 21 |
+
`work/validation/b200/paused-checkpoint-step-488.json`.
|
| 22 |
+
|
| 23 |
+
The dataset and production execution settings are preserved. The constraints
|
| 24 |
+
for all candidates remain BF16 computation, FP32 master parameters and AdamW,
|
| 25 |
+
sequence length 2,048, and **131,072 global tokens per optimizer update**.
|
| 26 |
+
The original learning-rate schedule, objective, tokenizer, and data order
|
| 27 |
+
remain unchanged. No optimization benchmark or production restart was run
|
| 28 |
+
for this research request.
|
| 29 |
+
|
| 30 |
+
## Measured baseline and first priority
|
| 31 |
+
|
| 32 |
+
Production updates 450–488 processed 5,111,808 tokens in 351.161 seconds
|
| 33 |
+
inside the update timer: **14,557 sustained tokens/s**. The median individual
|
| 34 |
+
update ran at **61,888 tokens/s**. Nine updates took approximately 30–32
|
| 35 |
+
seconds; ordinary updates took approximately 2.12 seconds. The timer excludes
|
| 36 |
+
checkpoint writes and evaluation. Source: `work/run/metrics.jsonl`.
|
| 37 |
+
|
| 38 |
+
Peak allocation was **59.40 GiB**, with **60.48 GiB reserved**. The B200 reports
|
| 39 |
+
182,631 MiB total memory; the current allocator cap is 150 GiB. Raising that
|
| 40 |
+
cap alone does not increase useful work.
|
| 41 |
+
|
| 42 |
+
During sampled stalls, GPU utilization fell to zero and the trainer's main
|
| 43 |
+
thread waited on a futex without accumulating CPU execution time. CPU quota
|
| 44 |
+
throttling did not increase; cgroup CPU/I/O pressure averages were zero.
|
| 45 |
+
There is no matching periodic branch in the training loop. This does not
|
| 46 |
+
identify the cause or eliminate data/pinning, compiler, autograd, driver,
|
| 47 |
+
or provider waits. Removing the waits is the largest observed opportunity;
|
| 48 |
+
the median rate is evidence of active compute speed, not a promised sustained
|
| 49 |
+
rate after a fix.
|
| 50 |
+
|
| 51 |
+
Use the benchmark's optional `--diagnose-stalls` to attribute host time to
|
| 52 |
+
batch transfer, forward, backward, clipping, AdamW, and final synchronization.
|
| 53 |
+
It adds no CUDA synchronization and emits Python stacks every ten seconds
|
| 54 |
+
during measured updates. Its timings include implicit waits and are not GPU
|
| 55 |
+
kernel timings. Then correlate the waiting call with CUDA and OS-runtime
|
| 56 |
+
traces across several updates. The existing `--profile` captures only one
|
| 57 |
+
measured update and can miss these stalls. Installed Nsight Systems is
|
| 58 |
+
2024.3.2.0, so verify B200/tool compatibility before using options documented
|
| 59 |
+
for newer releases. Sources: [NVIDIA Nsight Systems](https://docs.nvidia.com/nsight-systems/UserGuide/index.html#os-runtime-libraries-trace),
|
| 60 |
+
[PyTorch profiler](https://docs.pytorch.org/docs/2.14/profiler.html).
|
| 61 |
+
|
| 62 |
+
## Candidates, in order
|
| 63 |
+
|
| 64 |
+
| Candidate | Concrete experiment | Why it may help and what remains unverified |
|
| 65 |
+
| --- | --- | --- |
|
| 66 |
+
| Larger microbatch | Compare microbatch/accumulation **4/16**, **8/8**, **16/4** at the same sequence length | Larger GEMMs and fewer microbatch dispatches can improve compute efficiency. Prior workstation memory was about 86 GiB at microbatch 8; roughly 137 GiB at 16 is an extrapolation, not a B200 measurement. Measure startup and steady peaks below the existing cap. [NVIDIA GEMM guidance](https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html) |
|
| 67 |
+
| Keep loss activations instead of recomputing | Existing `--loss-mode compiled-full` at the best fitting microbatch | Uses more memory to avoid repeating the vocabulary projection during backward. At microbatch 16, BF16 logits alone are 2 GiB, before FP32 loss intermediates. Earlier workstation gains were about 3%; B200 gains are unmeasured. [PyTorch recomputation tradeoff](https://pytorch.org/blog/activation-checkpointing-techniques/), [local prior results](PERFORMANCE.md) |
|
| 68 |
+
| Tune matrix kernels for the chosen shape | Existing `--compile-mode max-autotune-no-cudagraphs` | Profiles candidate implementations while preserving BF16. Compile time increases; exclude it from timing and compare numerical results. [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html) |
|
| 69 |
+
| Compare cuBLASLt and workspace choices | Compare `TORCH_BLAS_PREFER_CUBLASLT=1` against the current backend, recording effective workspace | NVIDIA recommends 32 MiB workspace for Blackwell sm10x. Current `CUBLAS_WORKSPACE_CONFIG=:4096:8` already provides 32 MiB. PyTorch's unified workspace can cap cuBLASLt at the cuBLAS allocation; changing only `CUBLASLT_WORKSPACE_SIZE` may have no effect. Library preference may fall back. [NVIDIA cuBLAS](https://docs.nvidia.com/cuda/archive/13.0.0/cublas/index.html#cublassetworkspace), [PyTorch BLAS controls](https://docs.pytorch.org/docs/2.14/backends.html#torch.backends.cuda.preferred_blas_library) |
|
| 70 |
+
| CUDA graphs when traces show dispatch overhead | Existing `--compile-mode reduce-overhead` | Can trade additional persistent GPU memory for lower host launch overhead. Static training shapes help, but accumulation, evaluation, and partial final updates require validation. Graphs are not an established remedy for the observed waits. [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html), [graph constraints](https://docs.pytorch.org/docs/2.14/notes/cuda.html#constraints) |
|
| 71 |
+
|
| 72 |
+
The model already uses compiled cores, BF16 autocast caching, fused AdamW,
|
| 73 |
+
and disabled core activation checkpointing. Recommending those as new changes
|
| 74 |
+
would not explain further gains. The prior workstation profile attributes
|
| 75 |
+
most matrix time to FFN and input projections; attention and the output head
|
| 76 |
+
are smaller fractions. Those proportions need confirmation on B200 before
|
| 77 |
+
investing in custom attention or loss kernels.
|
| 78 |
+
|
| 79 |
+
As a later wall-clock improvement, investigate checkpoint staging and
|
| 80 |
+
asynchronous serialization. PyTorch documents copying a stable model/optimizer
|
| 81 |
+
snapshot into CPU buffers before background writing, with one outstanding
|
| 82 |
+
save at a time. This needs approximately another checkpoint's worth of host
|
| 83 |
+
memory and careful completion/error handling. It requires integration work
|
| 84 |
+
to retain this runner's full-state format, atomic publication and graceful
|
| 85 |
+
stop guarantee. It would address save pauses, not the recurring waits inside
|
| 86 |
+
the measured updates. Keep the existing save cadence during initial compute
|
| 87 |
+
comparisons. [PyTorch asynchronous checkpoint recipe](https://docs.pytorch.org/tutorials/recipes/distributed_async_checkpoint_recipe.html).
|
| 88 |
+
|
| 89 |
+
## Comparison and promotion requirements
|
| 90 |
+
|
| 91 |
+
Use the preserved step-488 checkpoint for every disposable benchmark. The
|
| 92 |
+
existing benchmark copies model and optimizer tensors to separate CUDA
|
| 93 |
+
storage, reads prepared data in place, and writes no training checkpoints.
|
| 94 |
+
Its learning rate stays fixed at the loaded value and RNG is reseeded; its
|
| 95 |
+
updates are timing experiments rather than production continuation.
|
| 96 |
+
|
| 97 |
+
Start with two warmup updates and at least twenty measured updates, extending
|
| 98 |
+
the window if needed to capture several stalls. Report total tokens divided
|
| 99 |
+
by total update time, median update speed, slow-update count, VRAM peaks,
|
| 100 |
+
loss and gradient norm. Keep profiling runs separate from throughput runs.
|
| 101 |
+
Use fresh `work/benchmark-*` output directories and identical checkpoint/data
|
| 102 |
+
inputs for each candidate. Benchmark the baseline again beside the winner
|
| 103 |
+
to distinguish configuration effects from changing host conditions.
|
| 104 |
+
|
| 105 |
+
Before promoting a candidate, compare loss and gradients with the baseline
|
| 106 |
+
on identical tokens and validate full-state continuation. The existing CPU
|
| 107 |
+
loss tests preserve objective and parameter identity, but their compiled
|
| 108 |
+
case uses an eager backend. The compiled-full resume test currently requires
|
| 109 |
+
two GPUs. Add and run a single-GPU CUDA regression for compiled-full before
|
| 110 |
+
promoting it: continuous versus interrupted/resumed updates, canonical model
|
| 111 |
+
keys, full optimizer and RNG state, token cursor, and the final partial batch.
|
| 112 |
+
Microbatch-only changes still require the existing resume/signature checks
|
| 113 |
+
and a short production continuation with finite loss and a verified saved
|
| 114 |
+
checkpoint. Preserve schedule and data invariants through `--reconfigure`.
|
| 115 |
+
|
| 116 |
+
The currently committed diagnostics have only CPU validation: syntax and CLI
|
| 117 |
+
parsing, accumulated stage timing including exception cleanup, and watchdog
|
| 118 |
+
cancellation. GPU profiling and the candidate grid remain pending. Resume
|
| 119 |
+
optimization work when the user asks to continue; keep the B200 job stopped
|
| 120 |
+
until then.
|
ops/B200_PERFORMANCE.md
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# B200 BF16 execution and checkpoint validation
|
| 2 |
+
|
| 3 |
+
Measurements from September 10, 2026 use disposable copies of the full
|
| 4 |
+
step-488 checkpoint (63,963,136 tokens), including FP32 AdamW moments.
|
| 5 |
+
The prepared dataset, tokenizer, model architecture, sequence length 2,048,
|
| 6 |
+
and global batch of 131,072 tokens remain unchanged. No FP8 or FP16 training
|
| 7 |
+
is used. Production resumed from step 488 with validation loss 4.79387 after
|
| 8 |
+
the user requested continuation; see the production validation section below.
|
| 9 |
+
The preserved input is `work/run/paused-step-488.pt`, SHA256
|
| 10 |
+
`69d37e902c41f541d55a07b3d10dcb7643721240cc6aae032455cd8904c64608`.
|
| 11 |
+
|
| 12 |
+
## Latest pause
|
| 13 |
+
|
| 14 |
+
Training stopped gracefully at the user's request after step **680 /
|
| 15 |
+
89,128,960 tokens**, with training loss **4.60232** and validation loss
|
| 16 |
+
**4.39469**. The full `work/run/latest.pt` checkpoint was published before the
|
| 17 |
+
process exited. Final staging took **4.93 seconds**, followed by **17.74
|
| 18 |
+
seconds** of background writing. All 256 optimizer entries are at step 680;
|
| 19 |
+
model/AdamW values are finite FP32, the schedule and data signature are
|
| 20 |
+
unchanged, and the saved optimizer LR is **0.000102**. SHA256 and the complete
|
| 21 |
+
inspection are recorded in `work/validation/b200/paused-checkpoint-step-680.json`
|
| 22 |
+
and the current [handoff](B200_HANDOFF.md).
|
| 23 |
+
|
| 24 |
+
The final 100 completed updates (581-680) averaged **16,501 wall-clock
|
| 25 |
+
tokens/s**, including between-update overhead, with a median **72,418 tokens/s**
|
| 26 |
+
and 18 slow updates. This window ends before the final pause evaluation/save.
|
| 27 |
+
Peak reserved memory stayed **139.55 GiB**. The earlier measurements below
|
| 28 |
+
remain historical comparisons. Production is now stopped; source settings,
|
| 29 |
+
prepared data and earlier preserved checkpoint snapshots remain unchanged.
|
| 30 |
+
|
| 31 |
+
## Selected launcher and measured tradeoffs
|
| 32 |
+
|
| 33 |
+
[train-b200.sh](train-b200.sh) now selects microbatch 16 / accumulation 4,
|
| 34 |
+
compiled full loss, `max-autotune-no-cudagraphs`, and asynchronous full
|
| 35 |
+
checkpoints. BF16 autocast, FP32 master weights/gradients/AdamW, the 40B token
|
| 36 |
+
budget, original learning-rate schedule, seed, and save/evaluation cadence
|
| 37 |
+
remain unchanged. Core activation checkpointing stays disabled and the
|
| 38 |
+
allocator cap remains 150 GiB. Eight compiler workers are the default;
|
| 39 |
+
initial autotuning can take several minutes.
|
| 40 |
+
|
| 41 |
+
Every benchmark below uses two warmup and twenty measured updates. Sustained
|
| 42 |
+
throughput divides measured tokens by measured update seconds, including
|
| 43 |
+
provider scheduling waits. Median speed describes a typical update; it is
|
| 44 |
+
not continuous throughput or a guarantee of exclusive GPU capacity. Loading,
|
| 45 |
+
compilation, validation and checkpoint writes are outside these measurements.
|
| 46 |
+
|
| 47 |
+
| Microbatch / accumulation | Loss | Core mode / BLAS preference | Sustained tokens/s | Median tokens/s | Peak reserved GiB |
|
| 48 |
+
| --- | --- | --- | ---: | ---: | ---: |
|
| 49 |
+
| 4 / 16 | Checkpointed | Default; stall diagnostics | 13,808 | 61,853 | 60.44 |
|
| 50 |
+
| 4 / 16 | Checkpointed | Final baseline control | 15,344 | 61,839 | 60.44 |
|
| 51 |
+
| 8 / 8 | Checkpointed | Default | 16,362 | 65,636 | 86.09 |
|
| 52 |
+
| 16 / 4 | Checkpointed | Default | 16,467 | 67,421 | 137.72 |
|
| 53 |
+
| 16 / 4 | Compiled full | Default | 16,698 | 71,370 | 141.25 |
|
| 54 |
+
| 16 / 4 | Compiled full | Autotune, no graphs | 20,752 | 72,499 | 139.31 |
|
| 55 |
+
| 16 / 4 | Compiled full | Default / cuBLASLt preference | 18,718 | 71,309 | 141.25 |
|
| 56 |
+
| 16 / 4 | Compiled full | CUDA graphs, corrected timer | 17,432 | 70,660 | 141.31 |
|
| 57 |
+
|
| 58 |
+
Autotuning gave the fastest typical update, about 17.2% above the initial
|
| 59 |
+
baseline. Peak allocated memory was 138.18 GiB. Its first warmup took 206.16
|
| 60 |
+
seconds including compilation/tuning and scheduling waits, excluded above.
|
| 61 |
+
The baseline diagnostic run encountered five slow measured updates, versus
|
| 62 |
+
three in the autotune run. Scheduling phase contributes substantially to
|
| 63 |
+
aggregate differences, so sustained gains cannot be attributed solely to the
|
| 64 |
+
chosen execution settings. The final baseline control repeated a median of
|
| 65 |
+
61,839 tokens/s, within 0.03% of the original diagnostic baseline despite
|
| 66 |
+
different scheduling waits; the typical-speed gain is repeatable.
|
| 67 |
+
|
| 68 |
+
cuBLASLt preference produced exactly the same losses and gradient norms as
|
| 69 |
+
the default compiled-full case and no meaningful median speed improvement.
|
| 70 |
+
It is available for comparison but is not forced in the B200 launcher. The
|
| 71 |
+
recorded preferences were `_BlasBackend.Cublas` and `_BlasBackend.Cublaslt`;
|
| 72 |
+
a preference does not guarantee every operation uses that backend. Current `CUBLAS_WORKSPACE_CONFIG=:4096:8`
|
| 73 |
+
provides 32 MiB workspace. See [PyTorch backend controls](https://docs.pytorch.org/docs/2.14/backends.html#torch.backends.cuda.preferred_blas_library)
|
| 74 |
+
and [NVIDIA workspace guidance](https://docs.nvidia.com/cuda/archive/13.0.0/cublas/index.html#cublassetworkspace).
|
| 75 |
+
|
| 76 |
+
## CUDA graph implementation and evidence
|
| 77 |
+
|
| 78 |
+
`--compile-mode reduce-overhead` now preallocates FP32 gradient buffers outside
|
| 79 |
+
capture and zeros them in place between updates. Without this, the installed
|
| 80 |
+
PyTorch rejected accumulation into graph-owned gradient outputs. Other modes
|
| 81 |
+
retain `zero_grad(set_to_none=True)`. No per-core step markers were inserted;
|
| 82 |
+
completed forward/backward microbatches use PyTorch's iteration tracking.
|
| 83 |
+
|
| 84 |
+
The full-model graph run recorded **56 captures**, **zero compiler skips**,
|
| 85 |
+
and **4,480 successful nonempty replays during measured updates**. Losses and
|
| 86 |
+
gradient norms match the default compiled-full trajectory exactly. The
|
| 87 |
+
optional [diagnostic helper](cudagraph_diagnostics.py) counts successful tree
|
| 88 |
+
replay calls using private PyTorch 2.14 APIs, restores its wrapper on exit,
|
| 89 |
+
and adds no CUDA operations or synchronization. Counts are not GPU timings.
|
| 90 |
+
|
| 91 |
+
The first exploratory graph benchmark excluded gradient zeroing from the
|
| 92 |
+
timer and logged capture details with `TORCH_LOGS=cudagraphs`. It is retained
|
| 93 |
+
as evidence but not used in the table. The corrected run includes gradient
|
| 94 |
+
zeroing, matching the production update timer. Future benchmark configurations
|
| 95 |
+
all use this timing. First-update peak allocation was 138.15 GiB; steady
|
| 96 |
+
allocator peaks of 34.69 GiB during replay do not represent the complete graph
|
| 97 |
+
memory footprint. Reserved memory, including graph pools, remained 141.31 GiB.
|
| 98 |
+
|
| 99 |
+
Graph mode is validated and available as an override, but was not selected as
|
| 100 |
+
the default because it did not improve typical update speed. It captures cores;
|
| 101 |
+
embedding, depth modulation, output projection, optimizer and the separately
|
| 102 |
+
compiled full-loss reduction are outside those core graphs. See
|
| 103 |
+
[PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html)
|
| 104 |
+
and [CUDA graph constraints](https://docs.pytorch.org/docs/2.14/notes/cuda.html#constraints).
|
| 105 |
+
|
| 106 |
+
## Asynchronous full checkpoints
|
| 107 |
+
|
| 108 |
+
`--async-checkpoint` is optional in the general runner and enabled in the B200
|
| 109 |
+
launcher. It retains the existing `warpstate-server-v1` single-file checkpoint,
|
| 110 |
+
including model, complete AdamW state, RNG, token cursor, schedule/signature,
|
| 111 |
+
configuration and provenance. `--no-async-checkpoint` restores synchronous
|
| 112 |
+
saving. The asynchronous option currently requires a single training rank.
|
| 113 |
+
|
| 114 |
+
The main thread first creates a fully independent CPU snapshot. GPU-to-CPU
|
| 115 |
+
staging completes synchronously before the next optimizer update can mutate
|
| 116 |
+
live state. One background writer then serializes, flushes and fsyncs the file,
|
| 117 |
+
atomically replaces `latest.pt`, and fsyncs the parent directory. Only
|
| 118 |
+
serialization and disk I/O overlap training. Staging needs roughly 24 GB of
|
| 119 |
+
additional host memory; it does not allocate a full extra GPU checkpoint.
|
| 120 |
+
|
| 121 |
+
A new save waits for the previous writer before allocating another snapshot,
|
| 122 |
+
so only one snapshot/write can be outstanding. Background errors surface
|
| 123 |
+
between updates and again on flush/close. Normal exit, graceful stop and
|
| 124 |
+
training exceptions join any already-staged valid save before the pipeline
|
| 125 |
+
lock is released. Exceptions do not snapshot partially updated live weights.
|
| 126 |
+
Logs distinguish staging from completed publication and include snapshot
|
| 127 |
+
step/tokens. A **Staged** message alone is not a durable checkpoint.
|
| 128 |
+
This follows the stable-snapshot and bounded-outstanding-save principles in
|
| 129 |
+
[PyTorch's asynchronous checkpoint recipe](https://docs.pytorch.org/tutorials/recipes/distributed_async_checkpoint_recipe.html)
|
| 130 |
+
while preserving this repository's existing checkpoint format.
|
| 131 |
+
|
| 132 |
+
## Numerical and continuation checks
|
| 133 |
+
|
| 134 |
+
All 22 updates in each completed benchmark had finite loss and gradient norm.
|
| 135 |
+
Against the microbatch-4 trajectory, maximum absolute loss differences were
|
| 136 |
+
0.00006819 for microbatch 8, 0.00006199 for microbatch 16, 0.00031376 for compiled
|
| 137 |
+
full loss at microbatch 16, and 0.00006914 for its autotuned version. Maximum
|
| 138 |
+
relative gradient-norm differences were 0.2680%, 0.1688%, 1.1087%, and 0.1760%.
|
| 139 |
+
Executions preserve the objective but need not be bitwise equal across modes.
|
| 140 |
+
Checkpoint and dataset identities remained unchanged throughout the benchmarks.
|
| 141 |
+
|
| 142 |
+
The final CPU runner/loss/checkpoint/launcher selection passed **91 tests**
|
| 143 |
+
with nine GPU cases deselected. Default and autotuned compiled-full CUDA
|
| 144 |
+
continuation cases passed earlier; the graph and asynchronous cases then
|
| 145 |
+
passed **2/2** in 84.41 seconds. These regressions check exact model, full
|
| 146 |
+
optimizer and RNG state, canonical keys, FP32 state, learning-rate schedule,
|
| 147 |
+
finite validation, and the final partial update at token counts 32, 64 and 73.
|
| 148 |
+
The asynchronous case compares asynchronous pause/resume against synchronous
|
| 149 |
+
continuous training and verifies that the asynchronous path actually ran.
|
| 150 |
+
CPU writer tests cover immutable snapshots during AdamW mutation, atomic
|
| 151 |
+
publication and failure cleanup, bounded writes, persistent errors and both
|
| 152 |
+
normal/exceptional exit flushing. Only one GPU workload ran at a time.
|
| 153 |
+
|
| 154 |
+
## Production continuation and first asynchronous save
|
| 155 |
+
|
| 156 |
+
After the user requested continuation, production resumed under B200
|
| 157 |
+
Supervisor in tmux session `warpstate-b200`, window `training`, new pane `0.1`.
|
| 158 |
+
Initial validation loss matched the pause snapshot at **4.79387**. The first
|
| 159 |
+
update, step 489, took 8.11 seconds including cached compilation startup.
|
| 160 |
+
Steps **490-500** processed 1,441,792 tokens at **19,223 sustained tokens/s**,
|
| 161 |
+
with a median **72,509 tokens/s**, including provider waits in update timing.
|
| 162 |
+
Peak allocated/reserved memory was **138.18 / 139.55 GiB**. This window excludes
|
| 163 |
+
validation, CPU checkpoint staging and background disk-write overhead.
|
| 164 |
+
|
| 165 |
+
At step **500 / 65,536,000 tokens**, training loss was **5.0160** and validation
|
| 166 |
+
loss was **4.74471**. The different training batches explain why individual
|
| 167 |
+
training losses can be above five while the fixed validation loss is lower.
|
| 168 |
+
The complete snapshot took **35.15 seconds** to stage synchronously, followed
|
| 169 |
+
by **44.44 seconds** of background serialization/fsync/publication. Steps
|
| 170 |
+
**501-507** completed while that writer was active. These staging and write
|
| 171 |
+
times are observed wall times on this host; the staging wait was not separately
|
| 172 |
+
profiled and should not be attributed entirely to transfer bandwidth.
|
| 173 |
+
|
| 174 |
+
The published checkpoint retains all **2,005,111,424 FP32 parameters**, **256
|
| 175 |
+
full AdamW entries at step 500**, finite FP32 moments, one rank's RNG state,
|
| 176 |
+
the same data/origin, **131,072 tokens per update**, and scheduled optimizer
|
| 177 |
+
LR **0.000075**. The full SHA256 and inspection are recorded in
|
| 178 |
+
`work/validation/b200/optimized-checkpoint-step-500.json`. The checkpoint is
|
| 179 |
+
preserved as `work/run/validated-step-500.pt`; later saves advanced `latest.pt`.
|
| 180 |
+
The original `paused-step-488.pt` remained unchanged. Training continued after
|
| 181 |
+
this first-save validation, until the later step-680 pause recorded above.
|
| 182 |
+
Preparation, tokenizer, dataset, schedule and precision were preserved.
|
| 183 |
+
|
| 184 |
+
## Remaining bottleneck
|
| 185 |
+
|
| 186 |
+
The recurring extra 30-second waits are strongly associated with the provider's
|
| 187 |
+
HAIShare GPU scheduler, which waits for ownership beneath CUDA launch calls.
|
| 188 |
+
Data transfer took about 5-6 ms/update; AdamW and CPU quota throttling did not
|
| 189 |
+
account for sampled delays. See [B200_SCHEDULING.md](B200_SCHEDULING.md) for
|
| 190 |
+
evidence and uncertainty. No provider scheduling or interposition settings
|
| 191 |
+
were changed. Confirm exclusive compute with the provider; full visible VRAM
|
| 192 |
+
alone does not establish exclusive allocation.
|
| 193 |
+
|
| 194 |
+
## Reproduction
|
| 195 |
+
|
| 196 |
+
Use [b200ctl.sh](b200ctl.sh) to stop production gracefully and confirm the GPU
|
| 197 |
+
is free before benchmarking. Preserve the pause snapshot. Benchmarks copy
|
| 198 |
+
model and optimizer state into disposable CUDA storage and never save a
|
| 199 |
+
training checkpoint. Their learning rate stays fixed at the loaded value and
|
| 200 |
+
RNG is reseeded, so benchmark updates are timing experiments.
|
| 201 |
+
|
| 202 |
+
From `/workspace/WarpState`, choose a new output directory for each run:
|
| 203 |
+
|
| 204 |
+
```bash
|
| 205 |
+
source rtx6000_40b/ops/training-env.sh
|
| 206 |
+
source rtx6000_40b/ops/workspace-tools.sh
|
| 207 |
+
export TORCHINDUCTOR_COMPILE_THREADS=8
|
| 208 |
+
rtx6000_40b/ops/benchmark.sh \
|
| 209 |
+
--checkpoint /workspace/WarpState/rtx6000_40b/work/run/paused-step-488.pt \
|
| 210 |
+
--micro-batch 16 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 211 |
+
--compile-cores --compile-mode max-autotune-no-cudagraphs \
|
| 212 |
+
--no-activation-checkpointing --autocast-cache --loss-mode compiled-full \
|
| 213 |
+
--loss-chunk-size 1024 --vram-limit-gib 150 \
|
| 214 |
+
--warmup-updates 2 --measured-updates 20 \
|
| 215 |
+
--output-dir /workspace/WarpState/rtx6000_40b/work/benchmark-b200-reproduction
|
| 216 |
+
```
|
| 217 |
+
|
| 218 |
+
For backend comparisons use `--compile-mode default` and select
|
| 219 |
+
`--blas-backend default`, `--blas-backend cublas`, or `--blas-backend cublaslt`
|
| 220 |
+
in otherwise identical cases. The completed cuBLASLt case used the equivalent
|
| 221 |
+
`TORCH_BLAS_PREFER_CUBLASLT=1` environment preference, recorded in its metadata.
|
| 222 |
+
For graph comparison use `--compile-mode reduce-overhead --cudagraph-diagnostics`.
|
| 223 |
+
The same compile-mode override is accepted by `train-b200.sh` on a later
|
| 224 |
+
production launch; the general runner's async flag is independent.
|
| 225 |
+
|
| 226 |
+
Raw `summary.json` and rank records are under `work/benchmark-b200-*`;
|
| 227 |
+
`work/validation/b200/optimization-grid.json` aggregates comparisons.
|
| 228 |
+
Metadata records source hashes, input identities, versions, backend options,
|
| 229 |
+
precision and each update's measurement. Shared logs are shown in tmux session
|
| 230 |
+
`warpstate-b200`, window `optimization`; attach with
|
| 231 |
+
`rtx6000_40b/ops/tmux-workspace.sh attach -t warpstate-b200`.
|
ops/B200_SCHEDULING.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# B200 scheduling investigation
|
| 2 |
+
|
| 3 |
+
On September 10, 2026, a BF16 benchmark reproduced the recurring training
|
| 4 |
+
stalls while production remained paused at step 488 / 63,963,136 tokens.
|
| 5 |
+
The evidence strongly implicates the provider's HAIShare GPU scheduler.
|
| 6 |
+
The current compute entitlement and pool configuration have not been verified.
|
| 7 |
+
|
| 8 |
+
## Measured behavior
|
| 9 |
+
|
| 10 |
+
The diagnostic benchmark restored the full paused checkpoint, including AdamW
|
| 11 |
+
state, and used microbatch 4, accumulation 16, sequence length 2,048, compiled
|
| 12 |
+
cores, and checkpointed 1,024-token loss chunks. It performed two warmup and
|
| 13 |
+
20 measured updates of 131,072 tokens each, without checkpoint writes.
|
| 14 |
+
Generated evidence remains in
|
| 15 |
+
`work/benchmark-b200-baseline-diagnostic/rank-0.jsonl` and
|
| 16 |
+
`work/logs/b200-optimization.log`, relative to `rtx6000_40b/`.
|
| 17 |
+
|
| 18 |
+
Sustained measured throughput was **13,807.94 tokens/s**, versus a median
|
| 19 |
+
per-update rate of **61,853.22 tokens/s**. Most updates took about 2.12 seconds.
|
| 20 |
+
Five slow updates account for the large gap:
|
| 21 |
+
|
| 22 |
+
| Benchmark update, including warmup | Total seconds | Forward wall seconds |
|
| 23 |
+
| --- | ---: | ---: |
|
| 24 |
+
| 3 | 29.1877 | 27.8748 |
|
| 25 |
+
| 8 | 32.1939 | 30.7726 |
|
| 26 |
+
| 12 | 32.2163 | 30.8157 |
|
| 27 |
+
| 17 | 32.1830 | 30.8250 |
|
| 28 |
+
| 22 | 32.1977 | 30.8020 |
|
| 29 |
+
|
| 30 |
+
These are host wall times, including implicit synchronization waits, not GPU
|
| 31 |
+
kernel durations. Batch preparation and transfer took only 4.6-6.5 milliseconds
|
| 32 |
+
on those slow updates. During observed stalls, the main process slept in
|
| 33 |
+
`futex_wait_queue` without accumulating CPU work and GPU utilization dropped
|
| 34 |
+
to zero. Diagnostic Python stacks stopped inside the existing Triton launcher:
|
| 35 |
+
compiled model forward -> Inductor generated call -> Triton runtime `run`
|
| 36 |
+
-> generated `launcher`, line 5. This was the steady kernel launch path;
|
| 37 |
+
the captured stack did not show active compilation.
|
| 38 |
+
The final slow update also produced a stack in ordinary eager depth-modulation
|
| 39 |
+
tensor arithmetic, without Triton or Inductor frames. The wait therefore
|
| 40 |
+
occurs across CUDA entrypoints rather than only one compiled kernel path.
|
| 41 |
+
|
| 42 |
+
## Runtime evidence and interpretation
|
| 43 |
+
|
| 44 |
+
Read-only inspection found `/usr/lib/haishare/libhaishare.so` in
|
| 45 |
+
`/etc/ld.so.preload` and in both benchmark process maps. `nvidia-smi -q` also
|
| 46 |
+
identified a HAIShare wrapper before returning its report. The library exports
|
| 47 |
+
`cuLaunchKernel`, `continue_with_lock`, `own_lock`, `own_lock_cv`, and scheduler
|
| 48 |
+
state symbols. Disassembly of `continue_with_lock` showed a GPU ownership
|
| 49 |
+
check, a lock request, and `pthread_cond_timedwait` on `own_lock_cv`, retried
|
| 50 |
+
at five-second intervals until ownership is granted.
|
| 51 |
+
|
| 52 |
+
Together, CUDA launch interception and the scheduler's condition-variable
|
| 53 |
+
wait provide a concrete mechanism for the observed futex sleep beneath the
|
| 54 |
+
Python launcher. A native stack tracing every intermediate frame was not
|
| 55 |
+
captured, so the association remains an inference rather than a complete
|
| 56 |
+
dynamic call trace. Static symbols alone do not establish the active quota.
|
| 57 |
+
|
| 58 |
+
The GPU report showed zero ECC errors, no active thermal or power slowdown,
|
| 59 |
+
healthy fabric, and no recovery action. MIG and driver virtualization were
|
| 60 |
+
disabled; those settings do not exclude user-space GPU scheduling.
|
| 61 |
+
|
| 62 |
+
The provider's [GPUaaS FAQ](https://docs.hosted.ai/user-panel/faqs/gpuaas-faqs)
|
| 63 |
+
documents time sharing with a default **30-second time quantum**. Guaranteed
|
| 64 |
+
compute depends on the pool's sharing ratio, while idle tenants can allow
|
| 65 |
+
others to use more of the GPU. This closely matches the recurring extra
|
| 66 |
+
30-second waits. The provider's
|
| 67 |
+
[pool configuration documentation](https://docs.hosted.ai/admin-panel/gpuaas/creating-a-gpuaas-pool)
|
| 68 |
+
describes sharing ratio, scheduling mode, and time-quantum controls.
|
| 69 |
+
|
| 70 |
+
## Follow-up
|
| 71 |
+
|
| 72 |
+
Ask the provider to confirm this instance's guaranteed compute percentage,
|
| 73 |
+
pool sharing ratio, time quantum, scheduling mode, and exclusive-allocation
|
| 74 |
+
status. Correlating scheduler grants and releases with the benchmark timestamps
|
| 75 |
+
would confirm the cause and distinguish expected allocation from a scheduler
|
| 76 |
+
fault. A full-memory B200 view alone does not establish exclusive compute.
|
| 77 |
+
|
| 78 |
+
No provider configuration or interposition settings were changed. Local BF16
|
| 79 |
+
optimization continues within the assigned allocation; benchmark comparisons
|
| 80 |
+
must report sustained throughput including waits as well as typical update
|
| 81 |
+
speed. See [B200_OPTIMIZATION_PLAN.md](B200_OPTIMIZATION_PLAN.md) for the
|
| 82 |
+
performance experiments and [B200_HANDOFF.md](B200_HANDOFF.md) for checkpoint
|
| 83 |
+
and production controls.
|
ops/B200_SETUP.md
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Single-B200 training environment
|
| 2 |
+
|
| 3 |
+
A new deployment starts **paused**. See [B200_HANDOFF.md](B200_HANDOFF.md)
|
| 4 |
+
for the latest checkpoint and current job status. Starting its Supervisor
|
| 5 |
+
does not start training. Use this configuration on the B200 host after
|
| 6 |
+
verifying the transferred dataset and checkpoint.
|
| 7 |
+
|
| 8 |
+
The repository and all environments, caches, data, logs, and temporary files
|
| 9 |
+
must remain under `/workspace/WarpState`. The Python 3.12 environment at
|
| 10 |
+
`rtx6000_40b/.server-venv` needs
|
| 11 |
+
[`training-requirements-resolved.txt`](training-requirements-resolved.txt)
|
| 12 |
+
and `supervisor==4.3.0`, combined in
|
| 13 |
+
[`b200-requirements.txt`](b200-requirements.txt). Source
|
| 14 |
+
[`training-env.sh`](training-env.sh) before provisioning. If present, the launcher also sources the generated
|
| 15 |
+
`/workspace/WarpState/tmp/remote-setup/toolchain-env.sh` for the host's
|
| 16 |
+
workspace-local compiler toolchain.
|
| 17 |
+
|
| 18 |
+
For a fresh environment on Ubuntu 24.04 x86-64 with Python 3.12 and a
|
| 19 |
+
CUDA-13-compatible driver:
|
| 20 |
+
|
| 21 |
+
```bash
|
| 22 |
+
cd /workspace/WarpState
|
| 23 |
+
source rtx6000_40b/ops/training-env.sh
|
| 24 |
+
python3 -m venv rtx6000_40b/.server-venv
|
| 25 |
+
rtx6000_40b/.server-venv/bin/python -m pip install \
|
| 26 |
+
--extra-index-url https://download.pytorch.org/whl/cu130 \
|
| 27 |
+
-r rtx6000_40b/ops/b200-requirements.txt
|
| 28 |
+
rtx6000_40b/.server-venv/bin/python -m pip check
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
A copy of the existing environment is also usable when the OS architecture,
|
| 32 |
+
Python version and absolute environment path match. Verify its dependency
|
| 33 |
+
versions and CUDA execution on the destination; omit compiler caches from
|
| 34 |
+
the transfer. The system needs GCC/G++, libc and Python development headers
|
| 35 |
+
for compilation. Do not install an additional CUDA driver inside the project.
|
| 36 |
+
|
| 37 |
+
If the host has no Git, download and extract its distribution package under
|
| 38 |
+
the workspace instead of modifying system directories:
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
mkdir -p /workspace/WarpState/tmp/remote-setup /workspace/WarpState/.tools
|
| 42 |
+
cd /workspace/WarpState/tmp/remote-setup
|
| 43 |
+
apt-get download git
|
| 44 |
+
dpkg-deb -x git_*.deb /workspace/WarpState/.tools
|
| 45 |
+
cd /workspace/WarpState
|
| 46 |
+
source rtx6000_40b/ops/workspace-tools.sh
|
| 47 |
+
git --version
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
This uses the host's existing shared libraries. Source
|
| 51 |
+
[`workspace-tools.sh`](workspace-tools.sh) in subsequent shells to use this
|
| 52 |
+
Git installation. Transfer Git history with a bundle of all refs; exclude
|
| 53 |
+
private keys, local credentials and generated artifacts from that bundle.
|
| 54 |
+
|
| 55 |
+
[`train-b200.sh`](train-b200.sh) uses one visible GPU, defaulting to index 0;
|
| 56 |
+
an existing `CUDA_VISIBLE_DEVICES` value takes precedence. It selects
|
| 57 |
+
sequence length 2,048, microbatch 16, and accumulation 4, preserving
|
| 58 |
+
**131,072 tokens per optimizer update**. Matrix computation uses BF16
|
| 59 |
+
autocast; parameters, gradients, and full AdamW state remain FP32. Core
|
| 60 |
+
compilation uses `max-autotune-no-cudagraphs`, core activation checkpointing
|
| 61 |
+
is disabled, and full vocabulary loss is compiled. Full checkpoints use
|
| 62 |
+
immutable CPU staging followed by asynchronous writing. The 150 GiB allocator
|
| 63 |
+
cap requires another 3 GiB free at startup and excludes CUDA context and
|
| 64 |
+
library memory. Current measurements and validation are recorded in
|
| 65 |
+
[B200_PERFORMANCE.md](B200_PERFORMANCE.md).
|
| 66 |
+
|
| 67 |
+
Copy the complete prepared `work/data/` directory, including its original
|
| 68 |
+
manifest, and `work/run/latest.pt`. The original transferred checkpoint was
|
| 69 |
+
step 448 with 58,720,256 tokens consumed; consult the handoff for the latest
|
| 70 |
+
saved step. Keep the manifest's requested revision `main`; its immutable source revision is recorded separately. The runner
|
| 71 |
+
restores model, full optimizer, token cursor, learning-rate schedule, and
|
| 72 |
+
rank-zero random state. `--reconfigure` permits the transition from two
|
| 73 |
+
GPUs to one while checking the unchanged data, objective, and global batch.
|
| 74 |
+
Different grouping can produce floating-point rounding differences.
|
| 75 |
+
|
| 76 |
+
## Environment checks
|
| 77 |
+
|
| 78 |
+
On this host, setting `CUDA_VISIBLE_DEVICES=''` caused pytest collection
|
| 79 |
+
to hang. Keep normal device visibility and explicitly deselect the CUDA
|
| 80 |
+
training cases when running the CPU suite:
|
| 81 |
+
|
| 82 |
+
```bash
|
| 83 |
+
cd /workspace/WarpState
|
| 84 |
+
source rtx6000_40b/ops/training-env.sh
|
| 85 |
+
rtx6000_40b/.server-venv/bin/python -m pytest \
|
| 86 |
+
rtx6000_40b/test_server.py rtx6000_40b/test_training_execution.py \
|
| 87 |
+
rtx6000_40b/test_training_launcher.py rtx6000_40b/test_loss_execution.py \
|
| 88 |
+
rtx6000_40b/test_checkpoint_writer.py \
|
| 89 |
+
rtx6000_40b/test_residual_execution.py rtx6000_40b/test_artifact_transfer.py \
|
| 90 |
+
-k 'not gpu_full_state_resume_and_exact_partial_update and not single_gpu_compiled_full_state_resume and not two_gpu_full_state_resume_and_empty_final_rank and not one_to_two_gpu_reconfiguration_preserves_training_state' \
|
| 91 |
+
--basetemp=/workspace/WarpState/tmp/b200-cpu-tests -q
|
| 92 |
+
```
|
| 93 |
+
|
| 94 |
+
This runs the CPU tests and performs device-availability queries during
|
| 95 |
+
collection. Small CUDA resume tests can be run separately with the B200
|
| 96 |
+
available, using their own temporary checkpoints:
|
| 97 |
+
|
| 98 |
+
```bash
|
| 99 |
+
rtx6000_40b/.server-venv/bin/python -m pytest rtx6000_40b/test_server.py \
|
| 100 |
+
-k gpu_full_state_resume_and_exact_partial_update \
|
| 101 |
+
--basetemp=/workspace/WarpState/tmp/b200-cuda-tests -q
|
| 102 |
+
```
|
| 103 |
+
|
| 104 |
+
The prepared corpus occupies 80,002,097,152 bytes, and the full checkpoint
|
| 105 |
+
about 24.06 GB. Keep at least 53 GiB additionally free for checkpoint writing
|
| 106 |
+
and atomic replacement. Generated files remain ignored by Git.
|
| 107 |
+
|
| 108 |
+
## Large artifact transfers
|
| 109 |
+
|
| 110 |
+
[`transfer_artifacts.py`](transfer_artifacts.py) sends large files over
|
| 111 |
+
independent SSH connections when one connection limits throughput. Run it
|
| 112 |
+
on the source host with the destination's already verified host-key file:
|
| 113 |
+
|
| 114 |
+
```bash
|
| 115 |
+
cd /workspace/WarpState
|
| 116 |
+
source rtx6000_40b/ops/training-env.sh
|
| 117 |
+
python3 rtx6000_40b/ops/transfer_artifacts.py \
|
| 118 |
+
/workspace/WarpState/rtx6000_40b/work/data/train.bin \
|
| 119 |
+
/workspace/WarpState/rtx6000_40b/work/data/train.bin \
|
| 120 |
+
--host ubuntu@HOST --port PORT \
|
| 121 |
+
--key /workspace/WarpState/id_ed25519 \
|
| 122 |
+
--known-hosts /workspace/WarpState/tmp/remote-setup/known_hosts \
|
| 123 |
+
--connections 12 --chunk-mib 512 --compress \
|
| 124 |
+
--expected-sha256 237a9ea6a90831769ec22fb8089c2a26c02fb991492629bb28f2b5b10d6b9114
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
Use the same command with the checkpoint's source/destination paths and
|
| 128 |
+
its actual SHA256. Copy the small data manifest, preparation state,
|
| 129 |
+
validation tokens and run metadata separately. The uploader retains
|
| 130 |
+
`.uploading`, `.upload-state.json`, `.upload-chunks/` and `.upload.lock`
|
| 131 |
+
sidecars next to each destination. These are inside ignored `work/`.
|
| 132 |
+
Interrupted uploads resume by rerunning the same command; completed chunks
|
| 133 |
+
are checked again before reuse. Full source and destination hashes must
|
| 134 |
+
match before the final filename appears. An existing different destination
|
| 135 |
+
is preserved. Keep training stopped until all artifacts have been verified.
|
| 136 |
+
|
| 137 |
+
## Controls on the remote host
|
| 138 |
+
|
| 139 |
+
Initialize the dedicated Supervisor once after provisioning:
|
| 140 |
+
|
| 141 |
+
```bash
|
| 142 |
+
cd /workspace/WarpState
|
| 143 |
+
source rtx6000_40b/ops/training-env.sh
|
| 144 |
+
mkdir -p rtx6000_40b/work/control rtx6000_40b/work/logs
|
| 145 |
+
rtx6000_40b/.server-venv/bin/supervisord \
|
| 146 |
+
-c rtx6000_40b/ops/b200-supervisord.conf
|
| 147 |
+
rtx6000_40b/ops/b200ctl.sh status
|
| 148 |
+
```
|
| 149 |
+
|
| 150 |
+
It runs as the invoking user and uses its own `b200-*` control files and
|
| 151 |
+
logs. It does not manage preparation or benchmarks. Training remains
|
| 152 |
+
stopped until explicitly started and never restarts automatically after
|
| 153 |
+
failure or a daemon restart.
|
| 154 |
+
|
| 155 |
+
When ready to resume on the B200:
|
| 156 |
+
|
| 157 |
+
```bash
|
| 158 |
+
cd /workspace/WarpState
|
| 159 |
+
nvidia-smi
|
| 160 |
+
rtx6000_40b/ops/b200ctl.sh start warpstate-train
|
| 161 |
+
rtx6000_40b/ops/b200ctl.sh status
|
| 162 |
+
tail -n 10 rtx6000_40b/work/logs/b200-train.log
|
| 163 |
+
tail -n 3 rtx6000_40b/work/run/metrics.jsonl
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
To pause and save a full checkpoint:
|
| 167 |
+
|
| 168 |
+
```bash
|
| 169 |
+
rtx6000_40b/ops/b200ctl.sh stop warpstate-train
|
| 170 |
+
```
|
| 171 |
+
|
| 172 |
+
The stop request waits for the current optimizer update and checkpoint,
|
| 173 |
+
with a 15-minute Supervisor timeout. The next start resumes `latest.pt`.
|
| 174 |
+
Initial compilation can take several minutes. Use `b200ctl.sh` for this
|
| 175 |
+
daemon; the original `datactl.sh` addresses the separate workstation
|
| 176 |
+
Supervisor. Do not run two launchers against the same `work` directory.
|
ops/BACKWARD_LAYOUT_EXPERIMENT.md
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Attention backward layout experiment — September 12, 2026
|
| 2 |
+
|
| 3 |
+
The isolated paired-lane implementation reduces this region from **0.519935 ms
|
| 4 |
+
to 0.357944 ms** on one RTX PRO 6000 Blackwell Max-Q GPU: **31.16% less time**.
|
| 5 |
+
It matches the original profiled kernels byte for byte on the tested production
|
| 6 |
+
shape. This isolated experiment did not include a training adapter or establish
|
| 7 |
+
a full-model speedup. The subsequent opt-in graph integration and its training
|
| 8 |
+
measurements are documented in
|
| 9 |
+
[BACKWARD_LAYOUT_INTEGRATION.md](BACKWARD_LAYOUT_INTEGRATION.md).
|
| 10 |
+
|
| 11 |
+
## Boundary and provenance
|
| 12 |
+
|
| 13 |
+
The existing compiled-autograd overlap profile contains two adjacent kernels:
|
| 14 |
+
|
| 15 |
+
| Original kernel | Rank-0 time per update | Calls per update |
|
| 16 |
+
| --- | ---: | ---: |
|
| 17 |
+
| `triton_poi_fused_17`: rotary backward and Q/K scratch | 31.907 ms | 224 |
|
| 18 |
+
| `triton_poi_fused_18`: packed Q/K/V/gate/U gradient | 77.576 ms | 224 |
|
| 19 |
+
|
| 20 |
+
The evidence is
|
| 21 |
+
`work/benchmark-opt-20260912/profile-ca-overlap/profile-rank-0-kernels.tsv`.
|
| 22 |
+
Their combined **109.483 ms** is an existing kernel-time budget, not an amount
|
| 23 |
+
already removed from training. The isolated timing uses a different execution
|
| 24 |
+
context; do not scale it into a claimed training throughput improvement.
|
| 25 |
+
|
| 26 |
+
`ops/backward_layout_baseline.py` retains the exact two profiled arithmetic and
|
| 27 |
+
indexing bodies as source, with their original launch dimensions. It records
|
| 28 |
+
the original generated source identifiers
|
| 29 |
+
`cq665jkyfy6uac7wrzvwa5aigwids6rzt4n7vzip6xvufai4ajjw` and
|
| 30 |
+
`cbr6gfx3vjhnzjbykfch3pg5peunfivhgdrfvl5hnzolhvfjfgdy` for provenance.
|
| 31 |
+
Running the experiment recompiles these tracked bodies; it does not load
|
| 32 |
+
ignored compiler-cache files or depend on preserving them.
|
| 33 |
+
|
| 34 |
+
The shape is `[B,N,H,C,D] = [4,16,32,128,64]`: microbatch four, 2,048-token
|
| 35 |
+
sequences, 32 heads, and a 64-element head dimension. Saved activations and
|
| 36 |
+
gradients are BF16. The original saved rotary sine/cosine tables are FP32.
|
| 37 |
+
K-write gradients have `[B,N,H,D,C]` storage; the output gradient is token-major.
|
| 38 |
+
The final packed tensor has shape `[4,2048,10240]`, ordered Q/K/V/gate/U.
|
| 39 |
+
|
| 40 |
+
## Arithmetic and implementation
|
| 41 |
+
|
| 42 |
+
`ops/backward_layout.py` provides an independent tensor reference, deterministic
|
| 43 |
+
synthetic saved inputs, the validated `launch_layout` interface, and
|
| 44 |
+
`prepare_layout`, which returns an allocation-free callable for timing.
|
| 45 |
+
|
| 46 |
+
The new rotary kernel processes even/odd pairs together. It reads the separate
|
| 47 |
+
local and memory Q gradients and adds them in FP32, applies the rotary backward
|
| 48 |
+
transform, and rounds the final Q result to BF16. It preserves the original
|
| 49 |
+
interleave's inactive-lane `+0.0`, including signed-zero behavior.
|
| 50 |
+
|
| 51 |
+
K first combines the transposed write gradient through the tanh derivative with
|
| 52 |
+
the local gradient in FP32. Its sine/rotary contribution is **rounded to BF16
|
| 53 |
+
before** adding the cosine contribution, as in the original two-kernel path.
|
| 54 |
+
That rounded intermediate stays in registers. K then rounds to BF16 on output.
|
| 55 |
+
|
| 56 |
+
A second kernel copies V, computes the sigmoid gate derivative in the original
|
| 57 |
+
operation order, and applies the U tanh derivative. Both kernels write directly
|
| 58 |
+
to their disjoint slices of the packed output. The `single` variant launches
|
| 59 |
+
both sets of programs in one grid using a uniform per-program branch. Keeping
|
| 60 |
+
two launches was slightly faster in the final comparison.
|
| 61 |
+
|
| 62 |
+
There are no reductions, GEMMs, parameter-gradient updates, collectives, or
|
| 63 |
+
optimizer operations in this boundary. Model parameters, FP32 gradient
|
| 64 |
+
accumulation, FP32 communication, and FP32 optimizer state are untouched.
|
| 65 |
+
Inputs, output, and profile scratch have explicit dtype/shape/device/stride
|
| 66 |
+
checks; output and scratch storage may not overwrite saved inputs or each other.
|
| 67 |
+
|
| 68 |
+
## Measurement and correctness
|
| 69 |
+
|
| 70 |
+
The initial screen covered two launch algorithms, blocks 256/512/1024, and
|
| 71 |
+
4/8 warps. All twelve configurations matched the original outputs numerically.
|
| 72 |
+
The largest single-grid configuration was slightly slower than the baseline,
|
| 73 |
+
so fewer launches alone did not establish a win. The initial report's equality
|
| 74 |
+
field compared values; a later stricter byte comparison detected a few signed
|
| 75 |
+
zeros, prompting the explicit inactive-lane zero addition in the final source.
|
| 76 |
+
|
| 77 |
+
The final confirmation used a different seed, block 256, eight warps, nine
|
| 78 |
+
trials, and 100 calls per timing interval. Each candidate interval was bracketed
|
| 79 |
+
by original-kernel intervals. Inputs, output, and original scratch were
|
| 80 |
+
preallocated; both paths were warmed before measurements. CUDA events measure
|
| 81 |
+
the queued execution, including possible launch scheduling gaps.
|
| 82 |
+
|
| 83 |
+
| Final algorithm | Median original | Median candidate | Time reduction |
|
| 84 |
+
| --- | ---: | ---: | ---: |
|
| 85 |
+
| Paired, two launches | 0.519935 ms | 0.357944 ms | 31.16% |
|
| 86 |
+
| Single grid | 0.520078 ms | 0.358521 ms | 31.06% |
|
| 87 |
+
|
| 88 |
+
Both final candidates had **zero byte differences** across every Q/K/V/gate/U
|
| 89 |
+
projection: 83,886,080 BF16 output elements per comparison. Production-shape
|
| 90 |
+
tests use another seed and strict integer-view byte equality. Ragged-shape
|
| 91 |
+
tests compare each projection independently against tensor expressions, use
|
| 92 |
+
unequal even/odd rotary coefficients, and verify saved inputs remain unchanged.
|
| 93 |
+
CPU checks include the model's actual rotary VJP, gate autograd, packing order,
|
| 94 |
+
observable intermediate BF16 rounding, and malformed input rejection.
|
| 95 |
+
|
| 96 |
+
The independent eager FP32 tensor expression and original compiled kernels can
|
| 97 |
+
have different FMA contraction. In the final run their maximum relative RMS
|
| 98 |
+
error was `8.72e-6`; V/gate/U were exactly equal. Therefore the independent
|
| 99 |
+
reference establishes mathematical indexing/arithmetic, while the original
|
| 100 |
+
compiled bodies supply the exact GPU-rounding oracle. These finite synthetic
|
| 101 |
+
fixtures do not prove equivalence for every possible activation distribution.
|
| 102 |
+
|
| 103 |
+
The final GPU suite passed **18 tests**. The focused suite also supports CPU-only
|
| 104 |
+
execution. Reports remain ignored:
|
| 105 |
+
|
| 106 |
+
- `work/benchmark-backward-layout-20260912/results.json`: initial twelve-way screen.
|
| 107 |
+
- `work/benchmark-backward-layout-confirm-20260912/results.json`: signed-zero diagnostic.
|
| 108 |
+
- `work/benchmark-backward-layout-final-20260912/results.json`: final source, strict byte comparison, and confirmation timings.
|
| 109 |
+
- `work/validation/rtx-opt-20260912/backward-layout-tests.log`: final GPU tests.
|
| 110 |
+
|
| 111 |
+
The final timing report records source hashes, arguments, Torch `2.14.0+cu130`,
|
| 112 |
+
CUDA 13.0, device identity, and per-projection errors. Later test/documentation
|
| 113 |
+
additions do not change kernel arithmetic. After measurement, one extra EOF
|
| 114 |
+
blank line was removed from `backward_layout_baseline.py` for the Git whitespace
|
| 115 |
+
check. Its parsed AST is identical; the original and final source hashes are
|
| 116 |
+
recorded in `work/validation/communication-exploration/backward-source-normalization.json`.
|
| 117 |
+
|
| 118 |
+
## Traffic and integration limits
|
| 119 |
+
|
| 120 |
+
The rewrite avoids the logical stores and subsequent reads of two
|
| 121 |
+
16,777,216-element BF16 scratch tensors: **134,217,728 bytes (128 MiB) per
|
| 122 |
+
region invocation**. This is a source-level traffic calculation, not a measured
|
| 123 |
+
DRAM counter. Cache behavior and other reads also affect the measured timing.
|
| 124 |
+
The production compiler reuses a dead buffer for one scratch tensor, so this
|
| 125 |
+
does **not** establish two fewer allocations or a corresponding peak-VRAM gain.
|
| 126 |
+
|
| 127 |
+
The prototype accepts separate local and memory gradients at the same backward
|
| 128 |
+
boundary as the profiled graph. A naive custom forward that returns one shared
|
| 129 |
+
BF16 Q tensor can cause autograd to round the local-plus-memory gradient sum
|
| 130 |
+
before rotary backward; the current graph adds those contributions in FP32.
|
| 131 |
+
Integrating the kernel requires preserving that join and the K rounding point,
|
| 132 |
+
then validating real model gradients, optimizer updates, checkpoint resumption,
|
| 133 |
+
and full-model timing. No such integration is selected by this experiment.
|
| 134 |
+
|
| 135 |
+
## Recommended next integration experiment
|
| 136 |
+
|
| 137 |
+
The narrowest route is a **post-gradient graph replacement**, leaving the
|
| 138 |
+
forward and autograd ownership intact. The installed PyTorch exposes
|
| 139 |
+
`torch._inductor.config.post_grad_custom_post_pass`; its
|
| 140 |
+
`fx_passes/post_grad.py` applies this hook to normalized, functionalized graphs
|
| 141 |
+
before final topological sorting and fake-tensor metadata updates. Implement a
|
| 142 |
+
`CustomGraphPass` that replaces only the existing packed-gradient region with a
|
| 143 |
+
pure opaque `torch.library.custom_op` and a matching fake implementation.
|
| 144 |
+
`ops/matmul_execution.py` already uses that opaque backward-kernel boundary to
|
| 145 |
+
avoid this PyTorch version's Compiled Autograd Triton higher-order tracing issue.
|
| 146 |
+
These are version-dependent compiler interfaces; their availability is a
|
| 147 |
+
proposed route, not evidence that this replacement has been integrated.
|
| 148 |
+
|
| 149 |
+
The pass should match the actual backward dataflow and arithmetic, not generated
|
| 150 |
+
kernel names or an inference flag. Require the exact supported shapes, dtypes,
|
| 151 |
+
strides, projection ordering, separate Q gradient branches, K rounding boundary,
|
| 152 |
+
and gate scale/expression. Verify that removed intermediates have no consumers
|
| 153 |
+
outside the replacement. On any mismatch, leave the original graph intact and
|
| 154 |
+
report the missed replacement; an opt-in validation run must fail if expected
|
| 155 |
+
dispatches are absent. Initially support only the measured parallel-memory,
|
| 156 |
+
dropout-zero production shape. Pass existing native-layout tensors into the
|
| 157 |
+
opaque operator and reshape views inside it, avoiding new contiguous copies
|
| 158 |
+
merely to satisfy a custom-operator input layout.
|
| 159 |
+
|
| 160 |
+
Use `CustomGraphPass.uuid()` with hashes of the matcher, kernel, and wrapper
|
| 161 |
+
sources plus all specialization settings, so a changed replacement cannot reuse
|
| 162 |
+
an incompatible compiled artifact. The installed `custom_graph_pass.py`
|
| 163 |
+
documents `get_hash_for_files` for this purpose. Record the same execution
|
| 164 |
+
identity in benchmark and checkpoint metadata, with explicit reconfiguration
|
| 165 |
+
for resumption changes. A new forward custom-autograd splitter is less suitable:
|
| 166 |
+
separate Q outputs address its gradient join, but the complete gate/packing
|
| 167 |
+
fusion also needs the later blend's raw output gradient and saved local/memory
|
| 168 |
+
values. Capturing that region through a forward boundary would require broader
|
| 169 |
+
ownership of the attention/memory computation or extra materialized gradients.
|
| 170 |
+
|
| 171 |
+
Begin with a match-only graph inspection, then replace one supported core.
|
| 172 |
+
Require generated-kernel/profile evidence that the expected regions actually
|
| 173 |
+
dispatch the paired implementation, with no added layout copies. Compare
|
| 174 |
+
forward outputs, all parameter gradients, accumulated FP32 gradients and Adam
|
| 175 |
+
updates against the same compiled baseline; cover first and later accumulation
|
| 176 |
+
microbatches, the final DDP synchronization, and checkpoint continuation. Then
|
| 177 |
+
measure repeated full-model runs from the same checkpoint/token cursor,
|
| 178 |
+
including peak VRAM and communication overlap. Select the replacement only
|
| 179 |
+
after those correctness and full-model performance checks pass.
|
| 180 |
+
|
| 181 |
+
## Reproduction
|
| 182 |
+
|
| 183 |
+
Reserve one idle GPU and keep training paused before running CUDA commands.
|
| 184 |
+
Use a new output directory each time. All source, caches, and artifacts stay
|
| 185 |
+
inside `/workspace/WarpState`.
|
| 186 |
+
|
| 187 |
+
```bash
|
| 188 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 189 |
+
source ops/training-env.sh
|
| 190 |
+
CUDA_VISIBLE_DEVICES='' .server-venv/bin/python -B -m pytest -q test_backward_layout.py
|
| 191 |
+
CUDA_VISIBLE_DEVICES=1 WARPSTATE_TEST_CUDA=1 \
|
| 192 |
+
.server-venv/bin/python -B -m pytest -q test_backward_layout.py
|
| 193 |
+
CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B \
|
| 194 |
+
ops/backward_layout_benchmark.py \
|
| 195 |
+
--output-dir work/benchmark-backward-layout-screen-new
|
| 196 |
+
CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B \
|
| 197 |
+
ops/backward_layout_benchmark.py --algorithms paired single \
|
| 198 |
+
--blocks 256 --warps 8 --seed 9929 --trials 9 --repeats 100 \
|
| 199 |
+
--output-dir work/benchmark-backward-layout-confirm-new
|
| 200 |
+
```
|
| 201 |
+
|
| 202 |
+
The benchmark needs no dataset or model checkpoint. It reports candidates
|
| 203 |
+
even when their bytes differ, so inspect correctness fields before interpreting
|
| 204 |
+
timings. It does not generate a training recipe or modify the production run.
|
ops/BACKWARD_LAYOUT_INTEGRATION.md
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BF16 backward-layout integration experiment
|
| 2 |
+
|
| 3 |
+
The paired attention-backward layout prototype is integrated into the compiled
|
| 4 |
+
training graph. Repeated complete-model measurements improve update throughput
|
| 5 |
+
by 0.637%, so `train-local-resume.sh` selects `--backward-layout paired` for the
|
| 6 |
+
next resumption. The CLI default remains `baseline`. The conservative 128 MiB
|
| 7 |
+
communication preset remains selected; the separate producer-scheduling trial
|
| 8 |
+
is available but disabled. Production training remains paused at step 40,081.
|
| 9 |
+
|
| 10 |
+
## Execution contract
|
| 11 |
+
|
| 12 |
+
Both `run_40b.py` and `ops/benchmark_training.py` accept
|
| 13 |
+
`--backward-layout baseline|inspect|paired`. `baseline` preserves existing
|
| 14 |
+
execution. `inspect` checks the graph without replacing it. `paired` replaces
|
| 15 |
+
only the recognized backward region and requires actual kernel dispatches.
|
| 16 |
+
|
| 17 |
+
The initial supported envelope is microbatch four, sequence length 2,048,
|
| 18 |
+
32 heads of width 64, chunk length 128, parallel memory, and zero dropout.
|
| 19 |
+
Use BF16 residuals, compiled cores in default mode, baseline activation and
|
| 20 |
+
normalization implementations, no activation checkpointing, and
|
| 21 |
+
`compiled-autograd-overlap`. Separate matmul recipes are excluded from this
|
| 22 |
+
initial experiment. Master parameters, gradient accumulation, collectives, and
|
| 23 |
+
AdamW state retain their existing FP32 representation; model compute uses BF16.
|
| 24 |
+
|
| 25 |
+
The graph pass matches normalized ATen operations, constants, tensor metadata,
|
| 26 |
+
branch relationships, and external consumers. Generated kernel names are not
|
| 27 |
+
match keys. Separate local and memory Q gradients and the K intermediate
|
| 28 |
+
rounding boundary are preserved. Unsupported regions retain the original graph;
|
| 29 |
+
an opted-in training or benchmark update fails if expected dispatches are absent.
|
| 30 |
+
|
| 31 |
+
Checkpoint execution signatures and benchmark metadata contain the implementation
|
| 32 |
+
identity and source hashes. A changed identity requires `--reconfigure` on
|
| 33 |
+
resumption. Compiler cache identity also includes the pass implementation.
|
| 34 |
+
Runtime dispatch counters establish execution even when a compiler cache hit
|
| 35 |
+
bypasses the graph matcher. Benchmarks require exactly one replacement per
|
| 36 |
+
logical layer per accumulation microbatch in each measured update.
|
| 37 |
+
|
| 38 |
+
The opaque operator follows PyTorch's [functional custom-operator contract](https://docs.pytorch.org/tutorials/advanced/python_custom_ops_functional.html):
|
| 39 |
+
it mutates no input, returns a fresh output, and registers fake output metadata.
|
| 40 |
+
The hook and arithmetic specialization are tied to the installed PyTorch
|
| 41 |
+
version. This is a backward graph transformation; it does not replace the
|
| 42 |
+
model's forward autograd ownership.
|
| 43 |
+
|
| 44 |
+
## Validation and timing protocol
|
| 45 |
+
|
| 46 |
+
The isolated prototype's 31% reduction applies only to its two-kernel region.
|
| 47 |
+
Roughly 34 ms saved per training update, or 0.8% overall, was the pre-integration
|
| 48 |
+
estimate. The measured complete-model result appears below.
|
| 49 |
+
|
| 50 |
+
Validation compares the compiled baseline and candidate at production attention
|
| 51 |
+
geometry using a smaller number of layers. It covers forward outputs, every
|
| 52 |
+
parameter gradient after initial and accumulated backwards, optimizer updates,
|
| 53 |
+
and two-rank continuation through a final partly masked update. Continuation
|
| 54 |
+
within an execution mode is a separate check from numerical agreement between
|
| 55 |
+
different execution modes.
|
| 56 |
+
|
| 57 |
+
Complete-model timings use the retained checkpoint and data cursor, microbatch
|
| 58 |
+
four per rank, eight accumulation microbatches, and 131,072 global tokens per
|
| 59 |
+
update. Both GPUs retain their 300 W power limits. Full resumed AdamW state and
|
| 60 |
+
an 88 GiB allocator cap remain enabled. No production checkpoint is written.
|
| 61 |
+
|
| 62 |
+
Repeated runs use fresh processes and interleave baseline and candidate order.
|
| 63 |
+
Warmup and measured counts are matched. GPU telemetry is sampled outside the
|
| 64 |
+
timed Python region. Compare total tokens divided by total update time, timing
|
| 65 |
+
drift, matched losses/gradient norms, and allocated/reserved VRAM. Profiles are
|
| 66 |
+
separate runs; their instrumented throughput is excluded from timing results.
|
| 67 |
+
|
| 68 |
+
## Reproduction
|
| 69 |
+
|
| 70 |
+
Keep production training paused and reserve both GPUs before running:
|
| 71 |
+
|
| 72 |
+
```bash
|
| 73 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 74 |
+
source ops/training-env.sh
|
| 75 |
+
CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO \
|
| 76 |
+
NCCL_DEBUG_SUBSYS=INIT,GRAPH,TUNING,ENV \
|
| 77 |
+
.server-venv/bin/torchrun --standalone --nproc-per-node=2 \
|
| 78 |
+
ops/benchmark_training.py \
|
| 79 |
+
--checkpoint work/run/latest.pt --data work/data/train.bin \
|
| 80 |
+
--micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 81 |
+
--no-activation-checkpointing --autocast-cache \
|
| 82 |
+
--compile-cores --compile-mode default --loss-mode compiled-full \
|
| 83 |
+
--bf16-residuals --gradient-mode compiled-autograd-overlap \
|
| 84 |
+
--ddp-bucket-cap-mb 128 --optimizer-state resume --vram-limit-gib 88 \
|
| 85 |
+
--warmup-updates 8 --measured-updates 40 --backward-layout paired \
|
| 86 |
+
--output-dir work/benchmark-backward-layout-reproduction/paired-1
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
Use `--backward-layout baseline` for the control and a fresh output directory for
|
| 90 |
+
every process. For dispatch and overlap profiles, use `--profile` with one
|
| 91 |
+
measured update after the same warmup. Existing comparison and communication
|
| 92 |
+
profile tools consume the saved JSON reports without opening model weights.
|
| 93 |
+
|
| 94 |
+
The underlying kernel arithmetic and original isolated measurements are in
|
| 95 |
+
[BACKWARD_LAYOUT_EXPERIMENT.md](BACKWARD_LAYOUT_EXPERIMENT.md).
|
| 96 |
+
|
| 97 |
+
## Training validation results
|
| 98 |
+
|
| 99 |
+
One production-shaped core matched the compiled baseline exactly for forward
|
| 100 |
+
output, input gradients, and all nine core parameter gradients. The opaque
|
| 101 |
+
operator's schema-purity and fake-metadata checks passed on CUDA.
|
| 102 |
+
|
| 103 |
+
The two-rank training fixture uses production attention/FFN dimensions, one
|
| 104 |
+
logical layer, three accumulation microbatches, two full updates, and a final
|
| 105 |
+
nine-token update with an entirely masked rank. It compares every parameter
|
| 106 |
+
gradient after the first, middle, and final synchronization backward, then
|
| 107 |
+
checks each parameter and Adam-state tensor after every optimizer update.
|
| 108 |
+
|
| 109 |
+
An initial 512-vocabulary fixture with repeated input tokens passed numerical
|
| 110 |
+
comparison but failed byte-exact continuation: 33 embedding-weight elements
|
| 111 |
+
differed by at most `3.7253e-9`. Differences in optimizer state were also confined
|
| 112 |
+
to the embedding. Other model weights, RNG states, token counters, and execution
|
| 113 |
+
identity matched. Its report is retained as a diagnostic of the existing
|
| 114 |
+
embedding accumulation's nondeterminism, not counted as an exact-resume pass.
|
| 115 |
+
|
| 116 |
+
The follow-up `--unique-inputs` fixture uses vocabulary 8,192 and a deterministic
|
| 117 |
+
permutation so each embedding row appears once per backward. It retains the
|
| 118 |
+
same attention geometry and precision without relaxing comparison thresholds.
|
| 119 |
+
Both ranks passed exact continuation for model, full optimizer, RNG, and cursor
|
| 120 |
+
state. All Adam parameters and moments also matched the baseline byte for byte
|
| 121 |
+
at every update. The largest gradient relative L2 difference was `1.22e-7`,
|
| 122 |
+
with maximum absolute difference `2.73e-12`, confined to the small
|
| 123 |
+
`memory_mix_logit` reduction. Both ranks agreed exactly on final model and Adam
|
| 124 |
+
state within each run. Every expected layout dispatch occurred.
|
| 125 |
+
|
| 126 |
+
Reproduce the strict continuation fixture with a new output directory:
|
| 127 |
+
|
| 128 |
+
```bash
|
| 129 |
+
CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/python -B \
|
| 130 |
+
ops/validate_backward_layout_training.py --world-size 2 --unique-inputs \
|
| 131 |
+
--output-dir work/validation/backward-layout-training-reproduction
|
| 132 |
+
```
|
| 133 |
+
|
| 134 |
+
The fixture initializes deterministic synthetic weights, tokens, and nonzero
|
| 135 |
+
FP32 Adam moments. This checks integration and continuation at production
|
| 136 |
+
attention geometry; it is not a byte-exact continuation claim for the complete
|
| 137 |
+
28-layer model on repeated-token corpus data. The reports are
|
| 138 |
+
`work/validation/backward-layout-training-20260912/` and
|
| 139 |
+
`work/validation/backward-layout-training-unique-20260912/`.
|
| 140 |
+
|
| 141 |
+
A two-core follow-up covers the compiler's sharing of an identical zero constant
|
| 142 |
+
across layers. The matcher retains that strictly matched constant while another
|
| 143 |
+
region still uses it, then removes it after its final replacement. Shared
|
| 144 |
+
computed arithmetic still causes rejection. The installed compiler's actual
|
| 145 |
+
common-subexpression pass is exercised by a focused structural test.
|
| 146 |
+
|
| 147 |
+
The final two-core, two-GPU fixture also passed exact model/optimizer/RNG/cursor
|
| 148 |
+
continuation. It required two dispatches per backward and observed every one.
|
| 149 |
+
All updated parameters and Adam states matched baseline byte for byte; maximum
|
| 150 |
+
gradient errors were `3.64e-12` absolute and `1.27e-7` relative L2. Add `--cores 2`
|
| 151 |
+
to the command above to reproduce it. Results are in
|
| 152 |
+
`work/validation/backward-layout-training-two-core-20260912/results.json`.
|
| 153 |
+
See [BACKWARD_LAYOUT_TRAINING_VALIDATION.md](BACKWARD_LAYOUT_TRAINING_VALIDATION.md)
|
| 154 |
+
for the full validation scope and source identities.
|
| 155 |
+
|
| 156 |
+
The separate producer-scheduling fixture also passed the existing two-GPU
|
| 157 |
+
asynchronous-save/stop/resume test, including full model/optimizer/RNG equality
|
| 158 |
+
and a final masked rank. Its log is
|
| 159 |
+
`work/validation/backward-layout-training-producer-resume-20260912.log`.
|
| 160 |
+
|
| 161 |
+
## Complete-model timing results
|
| 162 |
+
|
| 163 |
+
Four fresh-process runs used the order control, paired, paired, control, each
|
| 164 |
+
with eight warmup and 40 measured updates. All started from step 40,081 and
|
| 165 |
+
token cursor 5,253,496,832 with the same retained checkpoint, data, optimizer,
|
| 166 |
+
and source identities. Profiling and evaluation/checkpoint pauses are
|
| 167 |
+
excluded; these numbers describe training-update throughput.
|
| 168 |
+
|
| 169 |
+
| Run | Layout | Tokens/second |
|
| 170 |
+
| --- | --- | ---: |
|
| 171 |
+
| `control-1` | Baseline | 31,010.48 |
|
| 172 |
+
| `paired-2` | Paired | 31,197.36 |
|
| 173 |
+
| `paired-3` | Paired | 31,200.70 |
|
| 174 |
+
| `control-2` | Baseline | 30,992.67 |
|
| 175 |
+
|
| 176 |
+
Aggregating total tokens over total measured time gives **31,001.57 to
|
| 177 |
+
31,199.03 tokens/second**, a **0.637%** gain. Mean update time falls from
|
| 178 |
+
4.22791 to 4.20116 seconds, saving **26.76 ms per update**. The two controls
|
| 179 |
+
differ by 0.0575%; the two candidates differ by 0.011%. Both candidates beat
|
| 180 |
+
both controls. This is a small, repeatable improvement on this hardware and
|
| 181 |
+
preset; the isolated region's 31% gain does not apply to whole-model training.
|
| 182 |
+
|
| 183 |
+
Peak live allocator usage is unchanged at **65.7119 GiB per GPU**. Peak reserved
|
| 184 |
+
allocator memory falls from **66.9785 to 66.4785 GiB per GPU**. The 0.5 GiB
|
| 185 |
+
reduction is reservation, not a reduction in peak live tensors. Power limits
|
| 186 |
+
remain 300 W and measured-window temperatures and clock rates are comparable.
|
| 187 |
+
|
| 188 |
+
Every measured candidate update dispatches exactly 224 paired operations per
|
| 189 |
+
rank: 28 logical layers times eight accumulation microbatches. The second
|
| 190 |
+
candidate run loads compiler caches and still records every runtime dispatch,
|
| 191 |
+
despite the graph matcher itself not being invoked on those cache hits.
|
| 192 |
+
|
| 193 |
+
The first update's loss and gradient norm are exactly equal across all four
|
| 194 |
+
runs. Later trajectories are not byte-identical, including the repeated control.
|
| 195 |
+
Against the first control, the second control's maximum loss difference is
|
| 196 |
+
`1.28e-4` and relative gradient-norm difference is 0.219%. Candidate maxima are
|
| 197 |
+
`1.48e-4` / 0.179% and `1.35e-4` / 0.307%. These small scalar differences are
|
| 198 |
+
consistent with nondeterministic training reductions; they do not establish
|
| 199 |
+
byte equality of the complete model's gradients. The fixture checks above
|
| 200 |
+
provide the separate tensor-level evidence.
|
| 201 |
+
|
| 202 |
+
Run artifacts are under `work/benchmark-backward-layout-integration-20260912/`.
|
| 203 |
+
The comparison and telemetry summary is
|
| 204 |
+
`work/validation/backward-layout-integration/abba-summary.json`.
|
| 205 |
+
The manually stopped `paired-1` compilation probe has no completed update and
|
| 206 |
+
is excluded from all timing results.
|
| 207 |
+
|
| 208 |
+
## Profile verification
|
| 209 |
+
|
| 210 |
+
Separate baseline and paired runs each use eight warmup updates and one profiled
|
| 211 |
+
update. Each candidate rank records 224 custom-operator calls and 448 actual
|
| 212 |
+
`_paired_kernel` launches: 224 rotary and 224 value/gate/U kernels. The original
|
| 213 |
+
rotary and projection-packing launch signatures are absent. Their combined
|
| 214 |
+
region time falls from **114.95 to 78.96 ms** on rank zero, and from **114.65 to
|
| 215 |
+
78.91 ms** on rank one, about 31% less CUDA time for this region. This saves
|
| 216 |
+
roughly 36 ms of kernel time; the repeated unprofiled whole-update improvement
|
| 217 |
+
is the smaller 26.76 ms reported above.
|
| 218 |
+
|
| 219 |
+
The total CUDA kernel count stays at 17,477 per rank, and every copy/fill count
|
| 220 |
+
and byte total is unchanged. There are no added layout-materialization kernels.
|
| 221 |
+
The cached synchronized backward graph retains 45 identical generated Triton
|
| 222 |
+
bodies and the same 225 MM, 224 BMM, and one ADDMM calls; only the two recognized
|
| 223 |
+
layout bodies are removed. The replacement consumes native buffers directly.
|
| 224 |
+
These checks are recorded in
|
| 225 |
+
`work/validation/backward-layout-integration/profile-kernel-copy-comparison.json`
|
| 226 |
+
and `generated-sync-graph-comparison.json` in the same directory.
|
| 227 |
+
|
| 228 |
+
The complete captured update retains all 59 collectives and
|
| 229 |
+
8,020,445,708 logical input bytes per rank, with identical collective signatures.
|
| 230 |
+
The following values are local-device timeline unions, not sums across GPUs:
|
| 231 |
+
|
| 232 |
+
| Profile metric (ms/update) | Baseline rank 0 | Paired rank 0 | Baseline rank 1 | Paired rank 1 |
|
| 233 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 234 |
+
| Noncommunication CUDA activity | 3,957.68 | 3,935.16 | 3,834.21 | 3,796.91 |
|
| 235 |
+
| Compute/communication overlap | 192.63 | 192.56 | 191.92 | 191.57 |
|
| 236 |
+
| Communication with no overlapping compute | 149.87 | 154.23 | 281.09 | 290.27 |
|
| 237 |
+
|
| 238 |
+
Overlap is preserved. The exposed communication duration varies slightly upward
|
| 239 |
+
in this single profile pair; the measured layout gain is not a communication
|
| 240 |
+
speedup. Removing computation can also reduce work available to hide communication.
|
| 241 |
+
Instrumented profile durations are excluded from the repeated timing result.
|
| 242 |
+
The timeline accounting is in
|
| 243 |
+
`work/validation/backward-layout-integration/profile-communication.json`.
|
| 244 |
+
|
| 245 |
+
## Separate producer scheduling experiment
|
| 246 |
+
|
| 247 |
+
`--overlap-producer-schedule` is an opt-in communication experiment. Its
|
| 248 |
+
two separate runs use `--backward-layout baseline` and 128 MiB buckets, with
|
| 249 |
+
the same checkpoint, cursor, geometry, warmup, timing, and telemetry protocol
|
| 250 |
+
above. `producer-1` reached **31,133.27 tokens/second** (+0.425% against the
|
| 251 |
+
aggregate control); `producer-2` reached **31,090.54** (+0.287%). The repeat is
|
| 252 |
+
0.137% slower than the first screen, and its gain falls just below the approximate
|
| 253 |
+
0.3% threshold used to justify testing a combined preset. Peak allocated and
|
| 254 |
+
reserved VRAM match the control's 65.7119 and 66.9785 GiB per GPU.
|
| 255 |
+
|
| 256 |
+
Both runs show a small positive average difference, but the evidence is weaker
|
| 257 |
+
than the layout's repeated gain. The producer option remains available for
|
| 258 |
+
experimentation and is **not selected** in the production launcher. No combined
|
| 259 |
+
layout/producer run was started; additive gains have not been established.
|
| 260 |
+
This is a conservative selection decision, not proof that the scheduler cannot
|
| 261 |
+
help. The run summaries remain under the same benchmark directory as the ABBA
|
| 262 |
+
measurements.
|
| 263 |
+
|
| 264 |
+
The installed scheduler already moves ready collective launches earlier through
|
| 265 |
+
`simple_overlap`. This experiment additionally lets
|
| 266 |
+
`reorder_communication_preserving_peak_memory` move a collective with its
|
| 267 |
+
packing/casting producers, followed by the existing `sink_waits_iterative` pass.
|
| 268 |
+
Without runtime estimates, producer grouping stops at GEMM-producing nodes.
|
| 269 |
+
It therefore targets earlier bucket readiness rather than promising to reorder
|
| 270 |
+
all independent backward GEMMs.
|
| 271 |
+
|
| 272 |
+
The installed `comms.py` docstring names a `sink_comms_and_waits` prerequisite
|
| 273 |
+
that has no callable implementation in this version. The installed `config.py`
|
| 274 |
+
instead recommends producer reordering and iterative wait sinking; ordinary
|
| 275 |
+
peak-memory scheduling runs before these passes. The initial experiment uses one
|
| 276 |
+
producer pass followed by one wait sink, retaining the existing final wait
|
| 277 |
+
scheduling behavior.
|
| 278 |
+
|
| 279 |
+
Both passes explicitly prohibit changing collective launch order and allow
|
| 280 |
+
zero increase in their estimated peak memory. Producer groups cannot include
|
| 281 |
+
another collective. These settings override the producer pass's installed
|
| 282 |
+
unsafe-order and 20% memory defaults. The flag requires
|
| 283 |
+
`compiled-autograd-overlap` and initially rejects wait swapping, runtime
|
| 284 |
+
estimation, and a nonzero memory allowance. Its immutable execution setting is
|
| 285 |
+
recorded in benchmark metadata and checkpoint signatures; changing it on resume
|
| 286 |
+
requires `--reconfigure`. The timing and continuation checks above measure this
|
| 287 |
+
option separately from the paired layout.
|
| 288 |
+
|
| 289 |
+
## Final checks and resume state
|
| 290 |
+
|
| 291 |
+
The final CPU suite passed **356 tests**, with 24 CUDA tests skipped in that
|
| 292 |
+
CPU-only invocation and three existing Dynamo cache-wrapper warnings. Separate
|
| 293 |
+
CUDA operator checks, production-shaped one-/two-core gradient/optimizer and
|
| 294 |
+
exact-continuation fixtures, and the producer scheduling async-resume test are
|
| 295 |
+
documented above. The final CPU log is
|
| 296 |
+
`work/validation/backward-layout-integration/pytest-final-cpu.log`.
|
| 297 |
+
|
| 298 |
+
All ten package-manifest entries validate. The launcher passes `bash -n`.
|
| 299 |
+
The retained full checkpoint at `work/run/latest.pt` passes the before/after
|
| 300 |
+
SHA256 check, retaining
|
| 301 |
+
`9745d93345dff7c389ac6fa65948dc78ec021c90f56b40e9caed8ddcd84e8c54`.
|
| 302 |
+
The verification is saved in `checkpoint-after.log` beside the CPU log.
|
| 303 |
+
It remains at step **40,081**, **5,253,496,832 tokens**, with recorded validation
|
| 304 |
+
loss **2.4545078724622726**. Production and preparation jobs remain stopped,
|
| 305 |
+
both GPUs are idle, and their power limits remain 300 W. Archived weights are
|
| 306 |
+
untouched. The selected launcher will use the paired layout when training is
|
| 307 |
+
next resumed; this experiment does not resume production.
|
ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Backward-layout training validation — September 12, 2026
|
| 2 |
+
|
| 3 |
+
The paired graph replacement was validated with two-GPU training and checkpoint
|
| 4 |
+
continuation fixtures with production attention and FFN dimensions. The fixtures
|
| 5 |
+
use one or two physical cores/logical layers, not the complete 28-layer model.
|
| 6 |
+
They verify complete parameter-gradient and optimizer behavior for those fixtures;
|
| 7 |
+
they are not throughput measurements or proof about every trained activation.
|
| 8 |
+
|
| 9 |
+
`ops/validate_backward_layout_training.py` retains microbatch four, sequence
|
| 10 |
+
length 2,048, 32 heads, head dimension 64, chunk size 128, and FFN width 7,168.
|
| 11 |
+
It uses BF16 autocast/residuals, compiled cores and full loss, conservative
|
| 12 |
+
compiled-autograd communication overlap, 128 MiB DDP buckets, and FP32 master
|
| 13 |
+
parameters, gradients, all-reduces, and fused AdamW state. Weights and tokens are
|
| 14 |
+
deterministic synthetic fixtures. Adam starts with deterministic nonzero FP32
|
| 15 |
+
moments, rather than fresh first-update moments or imported production state.
|
| 16 |
+
|
| 17 |
+
Three accumulation microbatches exercise separate backward variants: first
|
| 18 |
+
`no_sync()` with absent gradients, later `no_sync()` accumulation, and final
|
| 19 |
+
accumulation with DDP synchronization. There are two full updates and a final
|
| 20 |
+
nine-token update, during which rank one has entirely masked targets. Each
|
| 21 |
+
candidate backward must execute the paired operation; an earlier dispatch cannot
|
| 22 |
+
hide fallback in a later graph. Expected totals per rank are seven dispatches
|
| 23 |
+
for continuous execution, three for the partial run, and four after resumption.
|
| 24 |
+
|
| 25 |
+
The harness runs baseline, continuous candidate, partial candidate, and resumed
|
| 26 |
+
candidate in separate processes. It compares initial forward logits, every
|
| 27 |
+
parameter gradient after every backward, and every parameter/Adam state after
|
| 28 |
+
every update. It also requires byte-identical final model, optimizer, RNG, token
|
| 29 |
+
cursor, configuration, and execution/source identity across candidate stop/resume.
|
| 30 |
+
Both ranks' final model and optimizer states must agree exactly in each run.
|
| 31 |
+
Source hashes are checked between phases, and outputs require a fresh directory
|
| 32 |
+
under `work/validation`. No production checkpoint or dataset is opened or changed.
|
| 33 |
+
|
| 34 |
+
The later `--cores` option selects a positive number of distinct physical cores
|
| 35 |
+
and the same number of logical layers. It scales both the per-backward dispatch
|
| 36 |
+
requirement and exact final dispatch count with logical depth. This covers
|
| 37 |
+
cross-core compilation that a single-core fixture cannot test.
|
| 38 |
+
|
| 39 |
+
The numerical limits were declared before CUDA measurements. Gradient and Adam
|
| 40 |
+
moment errors must satisfy both L2 error at most 0.5% of reference L2 plus
|
| 41 |
+
`1e-8`, and maximum error at most 1% of reference maximum plus `1e-8`. Updated
|
| 42 |
+
parameters have a `2e-5` maximum absolute limit; losses have a `2e-4` absolute
|
| 43 |
+
limit. Adam step counters, initial forward logits, and complete candidate resume
|
| 44 |
+
state require byte equality. Every comparison separately records exact bytes,
|
| 45 |
+
finite values, and numerical error; a tolerance pass is not labeled byte equality.
|
| 46 |
+
|
| 47 |
+
The initial fixture used a 512-token vocabulary with repeated input indices.
|
| 48 |
+
Its candidate gradient/optimizer comparisons passed, but strict continuation
|
| 49 |
+
failed. Differences were confined to the tied embedding: 33 parameter elements
|
| 50 |
+
with maximum difference `3.725290298461914e-9`, 50 first-moment elements with
|
| 51 |
+
maximum `9.516952559351921e-8`, and 28 second-moment elements with maximum
|
| 52 |
+
`4.3655745685100555e-11`. All other parameter/state tensors, RNG states, and
|
| 53 |
+
execution/cursor metadata matched exactly. Every run's two ranks agreed. This
|
| 54 |
+
is consistent with the existing relaxed embedding atomic accumulation for
|
| 55 |
+
repeated tokens. The original failure and numerical diagnostics remain recorded;
|
| 56 |
+
no comparison limit was relaxed to turn it into a pass.
|
| 57 |
+
|
| 58 |
+
The successful `--unique-inputs` fixture uses an 8,192-token vocabulary and a
|
| 59 |
+
deterministically shuffled permutation of all input indices in every microbatch.
|
| 60 |
+
Each embedding row is therefore visited once per backward. Targets can still
|
| 61 |
+
repeat. This removes that repeated-index accumulation from the exact-continuation
|
| 62 |
+
fixture without changing the model's training implementation. The guarded pass
|
| 63 |
+
also gained a rejection for unsupported `disable_ftz=True` before this fresh run;
|
| 64 |
+
the measured default setting and kernel arithmetic were unchanged.
|
| 65 |
+
|
| 66 |
+
The single-core run passed on both RTX PRO 6000 Blackwell Max-Q GPUs:
|
| 67 |
+
|
| 68 |
+
| Check | Result |
|
| 69 |
+
| --- | --- |
|
| 70 |
+
| Initial forward logits | Byte-identical |
|
| 71 |
+
| Every required backward dispatch | Present; one per microbatch |
|
| 72 |
+
| Gradient comparisons across candidate/partial/resumed ranks | 28 complete parameter sets |
|
| 73 |
+
| Maximum gradient absolute / relative L2 error | `2.7284841053187847e-12` / `1.2185588970205807e-7` |
|
| 74 |
+
| Remaining gradient differences | Only the 32-element memory-mix parameter |
|
| 75 |
+
| Parameters and Adam moments after all 12 compared rank-updates | Byte-identical to baseline |
|
| 76 |
+
| Full candidate checkpoint continuation on both ranks | Byte-identical for every saved field |
|
| 77 |
+
| Model/optimizer agreement between ranks in every run | Byte-identical |
|
| 78 |
+
| Maximum allocated / reserved CUDA memory | 3.064 / 3.445 GiB |
|
| 79 |
+
|
| 80 |
+
For that single-core run, the harness source hash was
|
| 81 |
+
`a2d24d27b81db2f24daac76fb23096a67116e86feed2b43621a866ec5b7d1656`;
|
| 82 |
+
and the guarded graph-pass hash was
|
| 83 |
+
`4029da453f54b470f8c091f3faf825913583d357d0285450fccf82082c4557c3`.
|
| 84 |
+
The JSON reports contain all other source hashes, per-tensor results, and runtime
|
| 85 |
+
dispatch/match counts. That run found four matching compiled backward
|
| 86 |
+
variants per rank with no missed candidates. The initial and final vocabulary
|
| 87 |
+
fixtures are different correctness experiments and must not be compared for speed.
|
| 88 |
+
|
| 89 |
+
While the full-model probe was compiling, a CPU reproduction exposed a limitation
|
| 90 |
+
that a single-core fixture cannot detect: installed AOT common-subexpression
|
| 91 |
+
elimination shares the same zero constant between cores. The original matcher's
|
| 92 |
+
external-user guard rejected both regions in that reproduction. The disposable
|
| 93 |
+
full-model probe was stopped before its first update completed; it supplies no
|
| 94 |
+
timing result. The matcher was corrected to retain only that strictly
|
| 95 |
+
matched, immutable zero constant while another region still uses it; external
|
| 96 |
+
users of all other matched arithmetic remain rejected. The arithmetic kernels
|
| 97 |
+
were unchanged. A separate CPU test applies the installed CSE pass to two regions
|
| 98 |
+
and checks that both replacements succeed.
|
| 99 |
+
|
| 100 |
+
The fresh two-core run used `--cores 2 --unique-inputs` and passed all the same
|
| 101 |
+
checks with unchanged numerical limits. It found eight matched regions per rank
|
| 102 |
+
across four backward variants, with no misses. Each backward dispatched twice;
|
| 103 |
+
the continuous/partial/resumed totals were exactly 14/6/8 per rank. Across 28
|
| 104 |
+
complete gradient comparisons, the largest difference was
|
| 105 |
+
`3.637978807091713e-12` absolute and `1.2654258902165433e-7` relative L2. Every
|
| 106 |
+
parameter and Adam tensor after all 12 compared rank-updates remained
|
| 107 |
+
byte-identical to baseline. Complete candidate stop/resume state and model/Adam
|
| 108 |
+
agreement between ranks were byte-identical for every saved field. Maximum
|
| 109 |
+
allocated/reserved CUDA memory was 5.366/5.736 GiB.
|
| 110 |
+
|
| 111 |
+
For the two-core run, the final harness source hash is
|
| 112 |
+
`5f2d161aa624b670b07205345c63177b3b0f46baa67eaef7e9d4d15482f42f6d`
|
| 113 |
+
and the graph-pass hash is
|
| 114 |
+
`174e97d06602471c258c70b7c0cef9efba7d14e7b78d71251ea012136cbc5ff9`.
|
| 115 |
+
These supersede the single-core hashes above; all older reports are preserved.
|
| 116 |
+
The CUDA kernel source stayed unchanged across both successful fixtures.
|
| 117 |
+
|
| 118 |
+
The expanded CPU oracle suite passed 18 tests. Independently, the producer-scheduling
|
| 119 |
+
option passed the existing two-GPU asynchronous checkpoint/resumption test:
|
| 120 |
+
one passed, five deselected, in 71.06 seconds. That smaller fixture validates the
|
| 121 |
+
producer scheduler's continuation behavior; it does not exercise the paired
|
| 122 |
+
kernel or establish a combined full-model performance gain.
|
| 123 |
+
|
| 124 |
+
Ignored evidence is retained at:
|
| 125 |
+
|
| 126 |
+
- `work/validation/backward-layout-training-20260912/results.json`: repeated-index fixture, including strict-resume failure.
|
| 127 |
+
- `work/validation/backward-layout-training-20260912/resume-differences.json`: embedding-only continuation differences.
|
| 128 |
+
- `work/validation/backward-layout-training-unique-20260912/results.json`: successful single-core fixture and all source hashes.
|
| 129 |
+
- `work/validation/backward-layout-training-two-core-20260912/results.json`: successful two-core fixture after the shared-constant fix.
|
| 130 |
+
- `work/validation/backward-layout-training-producer-resume-20260912.log`: producer-scheduling continuation test.
|
| 131 |
+
|
| 132 |
+
To reproduce, reserve both GPUs, keep production training paused, and use a new
|
| 133 |
+
output directory. Use `--unique-inputs` for the strict continuation fixture;
|
| 134 |
+
omitting it retains the repeated-index diagnostic, which can fail exact resume
|
| 135 |
+
because of the existing embedding accumulation nondeterminism.
|
| 136 |
+
|
| 137 |
+
```bash
|
| 138 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 139 |
+
source ops/training-env.sh
|
| 140 |
+
CUDA_VISIBLE_DEVICES='' .server-venv/bin/python -B -m pytest -q test_backward_layout_training.py
|
| 141 |
+
CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/python -B \
|
| 142 |
+
ops/validate_backward_layout_training.py --world-size 2 --unique-inputs --cores 2 \
|
| 143 |
+
--output-dir work/validation/backward-layout-training-two-core-new
|
| 144 |
+
CUDA_VISIBLE_DEVICES=0,1 WARPSTATE_TEST_CUDA=1 \
|
| 145 |
+
.server-venv/bin/python -B -m pytest -q test_training_optimization.py \
|
| 146 |
+
-k producer-scheduled-overlap
|
| 147 |
+
```
|
ops/CHAT.md
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CPU chat with the latest completed checkpoint
|
| 2 |
+
|
| 3 |
+
This is a text-generation probe of the pretrained **2.005B WarpState** model,
|
| 4 |
+
not a chat-finetuned assistant. The training corpus is English text plus EOS,
|
| 5 |
+
without role tokens. The default `Question: ... / Answer:` wrapper is only a
|
| 6 |
+
prompting convention. Use `/raw` to test ordinary text completion instead.
|
| 7 |
+
Fluency or one good answer does not by itself establish model quality.
|
| 8 |
+
|
| 9 |
+
Run from `/workspace/WarpState`:
|
| 10 |
+
|
| 11 |
+
```bash
|
| 12 |
+
bash rtx6000_40b/ops/chat-cpu.sh
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
The launcher uses CPU FP32 inference and 16 intra-op threads; it never
|
| 16 |
+
initializes CUDA or signals the training job. This allows both GPUs to keep
|
| 17 |
+
training. CPU responses are slower, and CPU/disk contention is still possible.
|
| 18 |
+
No training/model implementation, checkpoint writer, optimizer or data setting
|
| 19 |
+
is modified. No quantization, compilation or external service is introduced.
|
| 20 |
+
|
| 21 |
+
## Snapshot safety
|
| 22 |
+
|
| 23 |
+
At startup the tool opens the completed `work/run/latest.pt` once and pins that
|
| 24 |
+
inode using a read-only descriptor. The existing training writer publishes by
|
| 25 |
+
atomic rename, so a later save does not change the opened version. The tool
|
| 26 |
+
memory-maps that descriptor on CPU and exports only original model weights,
|
| 27 |
+
configuration, step/token boundary and provenance into a fresh directory under
|
| 28 |
+
`work/chat/`. Adam/RNG/dataset state are not exported: this is **inference-only**,
|
| 29 |
+
not a training-resume checkpoint. The training checkpoint is not overwritten,
|
| 30 |
+
hardlinked, chmodded, or otherwise mutated.
|
| 31 |
+
|
| 32 |
+
An export uses about 7.47 GiB. Its new weight file is read-only, and
|
| 33 |
+
`snapshot.json` records its SHA256, tokenizer hash and source inode/boundary.
|
| 34 |
+
The hash is verified before strict model loading. Only load trusted local torch
|
| 35 |
+
checkpoints. A snapshot is frozen at its printed step, not refreshed silently
|
| 36 |
+
while chatting. To test a later checkpoint, quit and launch again without
|
| 37 |
+
`--snapshot`; this creates another independent export. Reuse an existing export
|
| 38 |
+
without consuming another export's storage:
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/work/chat/step-N-suffix/model.pt
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
`--snapshot` also accepts a full training checkpoint such as `work/run/latest.pt`.
|
| 45 |
+
When no `snapshot.json` is present, the tool validates the embedded training
|
| 46 |
+
format and creates an independent model-only export under `work/chat/` before
|
| 47 |
+
loading it. This creates a fresh export on each launch and prints its path:
|
| 48 |
+
|
| 49 |
+
```bash
|
| 50 |
+
bash rtx6000_40b/ops/chat-cpu.sh --snapshot /workspace/WarpState/rtx6000_40b/work/run/latest.pt \
|
| 51 |
+
--max-new-tokens 128 --repetition-penalty 1.2 --no-repeat-ngram-size 4
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
`--checkpoint /path/to/latest.pt` remains the explicit way to capture training
|
| 55 |
+
weights. A chat export whose original `snapshot.json` is missing or invalid is
|
| 56 |
+
rejected; its integrity check is never bypassed or replaced with new metadata.
|
| 57 |
+
Restore the original metadata or create a fresh export from a training checkpoint.
|
| 58 |
+
|
| 59 |
+
No generated snapshots, metadata or transcripts are committed to Git.
|
| 60 |
+
|
| 61 |
+
## Controls and one-shot probes
|
| 62 |
+
|
| 63 |
+
- `/new`: clear conversation context.
|
| 64 |
+
- `/raw`: clear context and switch to plain text completion.
|
| 65 |
+
- `/chat`: clear context and restore the Q/A wrapper.
|
| 66 |
+
- `/info`: show the exact snapshot boundary and metadata.
|
| 67 |
+
- `/help`, `/exit` (or Ctrl-D): help / quit.
|
| 68 |
+
- Ctrl-C during generation cancels that reply, not training.
|
| 69 |
+
|
| 70 |
+
Each turn replays the retained prompt with one independent generation cache per
|
| 71 |
+
logical layer. A completed tile updates fast/slow memory only after producing
|
| 72 |
+
its own outputs. The total context budget defaults to 2048 tokens, including
|
| 73 |
+
the requested new tokens; an oversized prompt drops its oldest tokens. Long
|
| 74 |
+
histories therefore have substantial CPU prefill latency.
|
| 75 |
+
|
| 76 |
+
```bash
|
| 77 |
+
# A short reproducible greedy probe; no interactive session.
|
| 78 |
+
bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/model.pt \
|
| 79 |
+
--prompt 'What is Facebook?' --temperature 0 --max-new-tokens 32
|
| 80 |
+
|
| 81 |
+
# Sampling settings are explicit and do not affect training RNG.
|
| 82 |
+
bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/model.pt \
|
| 83 |
+
--threads 16 --temperature 0.8 --top-k 40 --top-p 0.9 --seed 1337 \
|
| 84 |
+
--max-new-tokens 128
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
For a separate terminal, attach to the prepared session using
|
| 88 |
+
`tmux attach -t warpstate-chat`. Detach with Ctrl-b, then d. Chatting in that
|
| 89 |
+
session does not attach to or control the training console.
|
| 90 |
+
|
| 91 |
+
## Repetition protection
|
| 92 |
+
|
| 93 |
+
Chat and raw completion now default to `--repetition-penalty 1.2` and
|
| 94 |
+
`--no-repeat-ngram-size 4`. Already generated tokens receive a lower score,
|
| 95 |
+
and a token cannot complete a four-token sequence already emitted in the
|
| 96 |
+
same reply. These controls run before greedy selection, top-k and top-p;
|
| 97 |
+
EOS can still end a reply normally. The prompt and earlier turns are excluded
|
| 98 |
+
from the restrictions, so the model can reuse names and phrases from a question.
|
| 99 |
+
Protection resets for each reply. The active settings are printed at startup.
|
| 100 |
+
|
| 101 |
+
This prevents exact token loops of that size and discourages shorter repeats;
|
| 102 |
+
it does not guarantee distinct meanings or turn a pretrained model into an
|
| 103 |
+
instruction-following assistant. Strict n-gram blocking can also restrict
|
| 104 |
+
intentional repetition in code, quotations or lists. For an unfiltered baseline,
|
| 105 |
+
pass `--repetition-penalty 1 --no-repeat-ngram-size 0`. Both options affect only
|
| 106 |
+
inference; no weights or training settings change.
|
| 107 |
+
|
| 108 |
+
After updating a running chat, use `/exit` and relaunch it with the same
|
| 109 |
+
`--snapshot` path. This loads the new defaults and clears old repetitive
|
| 110 |
+
conversation context. Use `/new` to clear context later without reloading weights.
|
| 111 |
+
|
| 112 |
+
## Verification
|
| 113 |
+
|
| 114 |
+
The CPU test suite covers strict state loading, tokenizer identity, independent
|
| 115 |
+
depth caches, greedy/seeded sampling, repetition penalties for both logit signs,
|
| 116 |
+
phrase-loop blocking, per-reply reset, EOS, nonfinite logits, and snapshot
|
| 117 |
+
independence when `latest.pt` is atomically replaced during loading. It also
|
| 118 |
+
checks the CLI with `--snapshot latest.pt`, sealed snapshot reuse and rejection
|
| 119 |
+
of missing or invalid export metadata. It compares
|
| 120 |
+
incremental logits against full forward across three tile boundaries, with
|
| 121 |
+
both sequential and parallel memory. Agreement is tolerance-based: their
|
| 122 |
+
floating-point reduction orders are different, and CPU FP32 inference is not
|
| 123 |
+
claimed bitwise-equivalent to BF16 training.
|
| 124 |
+
|
| 125 |
+
```bash
|
| 126 |
+
cd /workspace/WarpState
|
| 127 |
+
source rtx6000_40b/ops/training-env.sh
|
| 128 |
+
CUDA_VISIBLE_DEVICES='' rtx6000_40b/.server-venv/bin/python -B -m pytest \
|
| 129 |
+
rtx6000_40b/test_chat.py -q --basetemp=/workspace/WarpState/tmp/pytest-chat
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
The initial real-model smoke probe loaded an independent step-3250 snapshot
|
| 133 |
+
(425,984,000 training tokens) and produced finite greedy output on CPU. This is
|
| 134 |
+
a functionality check, not a new validation benchmark or a quality guarantee.
|
ops/COMMUNICATION_EXPLORATION_20260912.md
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Communication scheduling exploration — September 12, 2026
|
| 2 |
+
|
| 3 |
+
These experiments use disposable copies of the step-40,081 full checkpoint.
|
| 4 |
+
Production training remains paused. The model, corpus cursor, 2,048-token
|
| 5 |
+
context, 131,072-token update, BF16 computation, FP32 gradient communication,
|
| 6 |
+
and full FP32 AdamW state are held fixed. The production resume preset retains
|
| 7 |
+
the previously selected conservative communication scheduler.
|
| 8 |
+
|
| 9 |
+
## Explicit scheduler controls
|
| 10 |
+
|
| 11 |
+
`run_40b.py` and `ops/benchmark_training.py` expose three opt-in controls for
|
| 12 |
+
`--gradient-mode compiled-autograd-overlap`:
|
| 13 |
+
|
| 14 |
+
| Flag | Default | Meaning |
|
| 15 |
+
| --- | --- | --- |
|
| 16 |
+
| `--overlap-swap-collectives` | Off | Move a wait and its dependent copies past later collective launches while preserving collective launch order. |
|
| 17 |
+
| `--overlap-runtime-estimates` | Off | Use measured GEMM estimates, aligned across distributed ranks, to guide wait placement. |
|
| 18 |
+
| `--overlap-memory-budget FRACTION` | `0.0` | Permit a fractional increase in the scheduler's estimated peak memory, bounded to 0–0.25. This does not increase the CUDA allocator cap. |
|
| 19 |
+
|
| 20 |
+
The helper always disables unsafe collective reordering. Settings are immutable,
|
| 21 |
+
validated before compiler configuration, and recorded in execution metadata.
|
| 22 |
+
The runner records all three controls in checkpoint signatures and requires
|
| 23 |
+
explicit `--reconfigure` for changes. Legacy signatures normalize to the same
|
| 24 |
+
False/False/0 defaults, preserving existing resume behavior.
|
| 25 |
+
|
| 26 |
+
Runtime estimation uses installed experimental PyTorch compiler APIs. It adds
|
| 27 |
+
GEMM benchmarking during compilation and distributed estimate alignment; this
|
| 28 |
+
is separate from the timed training updates. Configurations use fresh processes
|
| 29 |
+
because compiler/reducer settings are process-wide. Both ranks must compile the
|
| 30 |
+
same collective-bearing graphs.
|
| 31 |
+
|
| 32 |
+
The installed FX cache explicitly includes `config_comms` in its key, and the
|
| 33 |
+
AOT cache uses the same payload. Actual cache-key checks distinguish these
|
| 34 |
+
options; they cannot silently reuse a differently configured scheduler merely
|
| 35 |
+
because the forward graph is unchanged. The runtime-estimate screen also
|
| 36 |
+
records an additional pre-update AllGather consistent with rank alignment.
|
| 37 |
+
`TORCH_LOGS=overlap` alone does not print the detailed swap decisions: the
|
| 38 |
+
installed pass additionally requires `reorder_sink_verbose_logging=True`.
|
| 39 |
+
|
| 40 |
+
## Complete-model screens
|
| 41 |
+
|
| 42 |
+
All rows use the same source versions and checkpoint. Changes are relative to
|
| 43 |
+
the fresh 128 MiB conservative control, not the earlier eager-gradient baseline.
|
| 44 |
+
|
| 45 |
+
| Configuration / artifact directory | Tokens/s | Change | Reserved GiB/rank |
|
| 46 |
+
| --- | ---: | ---: | ---: |
|
| 47 |
+
| Conservative, 128 MiB (`control-128`) | 31,174.13 | — | 66.98 |
|
| 48 |
+
| Waits past launches, 128 MiB (`swap-128`) | 31,201.81 | +0.09% | 66.98 |
|
| 49 |
+
| Waits past launches, 64 MiB (`swap-64`) | 30,824.54 | -1.12% | 66.98 |
|
| 50 |
+
| Waits past launches, 256 MiB (`swap-256`) | 30,385.00 | -2.53% | 66.98 |
|
| 51 |
+
| Waits past launches + runtime estimates, 128 MiB (`swap-runtime-128`) | 31,117.39 | -0.18% | 66.46 / 66.96 |
|
| 52 |
+
| Same conservative control repeated (`control-repeat`) | 31,070.03 | -0.33% | 66.98 |
|
| 53 |
+
|
| 54 |
+
The wait-only and runtime-estimate results are effectively tied with control;
|
| 55 |
+
neither establishes a throughput gain. The runtime-estimate choice reduces
|
| 56 |
+
allocator reservations by approximately 0.5 GiB on rank zero; rank one is
|
| 57 |
+
essentially unchanged. Peak allocated memory remains 65.71188 GiB on both
|
| 58 |
+
ranks, so this does not demonstrate a smaller model working set.
|
| 59 |
+
Smaller and larger buckets both lose throughput here. None of these results
|
| 60 |
+
justifies changing the production preset.
|
| 61 |
+
|
| 62 |
+
Repeating the unchanged control moves throughput by -0.33%, larger than either
|
| 63 |
+
apparent wait-only or runtime-estimate effect. Its maximum loss difference is
|
| 64 |
+
0.00007796 and maximum relative gradient-norm difference is 0.0836%. Thus even
|
| 65 |
+
the unchanged full model does not give a bitwise scalar trajectory in these
|
| 66 |
+
screens. The other configurations' norm differences can be larger; a pair of
|
| 67 |
+
scalar traces is not proof of per-parameter equivalence or long-run quality.
|
| 68 |
+
|
| 69 |
+
Control and wait-only runs both slow by about 1.14% from their first five to
|
| 70 |
+
last five measured updates. Telemetry shows similar temperatures and clocks
|
| 71 |
+
across the control, wait-only, and 64 MiB runs; their mean GPU temperatures are
|
| 72 |
+
approximately 52 C and 55 C by rank. One-second clock samples mix GEMM and
|
| 73 |
+
non-GEMM phases and are not measurements of sustained GEMM clock alone.
|
| 74 |
+
|
| 75 |
+
`ops/compare_communication_benchmarks.py` validates checkpoint/data identities,
|
| 76 |
+
batch geometry, update cursors, recorded precision, and rank consistency before
|
| 77 |
+
comparing complete unprofiled windows. Its report preserves paired loss/norm
|
| 78 |
+
differences, timing variation, per-rank memory peaks, and source/input hashes.
|
| 79 |
+
These screens' largest loss difference is 0.00018311 and largest relative
|
| 80 |
+
gradient-norm difference is 0.366%; those are observed differences, not general
|
| 81 |
+
acceptance thresholds or evidence of improved model quality.
|
| 82 |
+
|
| 83 |
+
Generated conservative and wait-only synchronization graphs have identical
|
| 84 |
+
Triton arithmetic bodies, GEMM/attention order, collective launch order, and
|
| 85 |
+
statement multisets; only wait/copy placement changes. Existing embedding
|
| 86 |
+
backward uses relaxed FP32 atomic additions to repeated-token rows. Small
|
| 87 |
+
cross-run differences are therefore consistent with existing nondeterministic
|
| 88 |
+
reductions amplified by subsequent BF16 rounding, although this observation
|
| 89 |
+
alone does not prove their cause. Both ranks agree on gradient norms in each
|
| 90 |
+
run. Same-mode repetition and explicit continuation tests are separate checks.
|
| 91 |
+
|
| 92 |
+
The paired report is
|
| 93 |
+
`work/validation/communication-exploration/comparison-final.json`. It can be
|
| 94 |
+
rebuilt from the saved benchmark summaries and rank logs with:
|
| 95 |
+
|
| 96 |
+
```bash
|
| 97 |
+
.server-venv/bin/python ops/compare_communication_benchmarks.py \
|
| 98 |
+
--control work/benchmark-communication-20260912/control-128 \
|
| 99 |
+
--candidate work/benchmark-communication-20260912/swap-128 \
|
| 100 |
+
work/benchmark-communication-20260912/swap-64 \
|
| 101 |
+
work/benchmark-communication-20260912/swap-256 \
|
| 102 |
+
work/benchmark-communication-20260912/swap-runtime-128 \
|
| 103 |
+
work/benchmark-communication-20260912/control-repeat \
|
| 104 |
+
--output work/validation/communication-exploration/comparison-rebuilt.json
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
## Method
|
| 108 |
+
|
| 109 |
+
### Fresh trace comparison
|
| 110 |
+
|
| 111 |
+
Separate profiles use eight warmup updates and one instrumented update, with
|
| 112 |
+
identical checkpoint/corpus offsets. Their throughput is not included in the
|
| 113 |
+
unprofiled comparison above.
|
| 114 |
+
|
| 115 |
+
| CUDA interval budget | Conservative rank 0 / rank 1 | Wait swap rank 0 / rank 1 |
|
| 116 |
+
| --- | ---: | ---: |
|
| 117 |
+
| NCCL overlapping other CUDA work | 191.805 / 191.069 ms | 193.185 / 191.983 ms |
|
| 118 |
+
| NCCL with no other CUDA work | 153.136 / 266.595 ms | 152.460 / 274.337 ms |
|
| 119 |
+
|
| 120 |
+
Profiled update wall time is 4.17505 seconds for control and 4.18611 for swap.
|
| 121 |
+
Both contain 59 collective calls including scalar metrics, with identical CPU
|
| 122 |
+
operator/shape/type submission fingerprints. The structural wait movement does
|
| 123 |
+
not create a material increase in observed overlap on this model.
|
| 124 |
+
|
| 125 |
+
Rank one's larger exposed interval includes waiting for its peer: its
|
| 126 |
+
noncommunication CUDA union is approximately 3.833 seconds, versus 3.946 on
|
| 127 |
+
rank zero in the control. Exposed intervals must not be added across ranks or
|
| 128 |
+
treated as wholly recoverable communication overhead.
|
| 129 |
+
|
| 130 |
+
The report is `work/validation/communication-exploration/profile-comparison.json`.
|
| 131 |
+
Rebuild it with `ops/analyze_communication_profile.py`, the four
|
| 132 |
+
`profile-{control,swap}/profile-rank-{0,1}-trace.json` files under the benchmark
|
| 133 |
+
directory, and a fresh `--output` JSON path under `work/`.
|
| 134 |
+
|
| 135 |
+
### Next scheduling work
|
| 136 |
+
|
| 137 |
+
The installed compiler already enables `simple_overlap` after the iterative
|
| 138 |
+
wait pass. Enabling it again is not a new optimization. Its adjacent launch
|
| 139 |
+
moves stop at a gradient producer, an earlier collective, or estimated memory
|
| 140 |
+
growth. The next useful experiment would move the producer work with the
|
| 141 |
+
collective, making a bucket ready sooner rather than only delaying its wait.
|
| 142 |
+
|
| 143 |
+
`reorder_communication_preserving_peak_memory` can group packing/casting
|
| 144 |
+
producers with collectives. A future screen must explicitly set
|
| 145 |
+
`reorder_iterative_unsafe_collectives_reorder=False` and
|
| 146 |
+
`reorder_iterative_peak_memory_budget=0.0`; the installed defaults are True and
|
| 147 |
+
0.2. Its non-estimated mode stops at GEMM producers; aligned runtime estimates
|
| 148 |
+
allow considering larger producer groups. Another option, `raise_comms`,
|
| 149 |
+
prioritizes the next collective's producer ancestors but has no peak-memory
|
| 150 |
+
guard. Neither pass was implemented or benchmarked by this exploration.
|
| 151 |
+
|
| 152 |
+
The nonzero memory-allowance option is exercised by the continuation fixture;
|
| 153 |
+
nonzero allowances were not included in the complete-model throughput sweep.
|
| 154 |
+
Bucket readiness still depends on every member gradient, and the final
|
| 155 |
+
reductions must complete before clipping and AdamW. These are real limits on
|
| 156 |
+
how much overlap any scheduler can obtain.
|
| 157 |
+
|
| 158 |
+
### Timing procedure
|
| 159 |
+
|
| 160 |
+
The unprofiled comparisons use eight complete warmup updates followed by twenty
|
| 161 |
+
timed updates, with microbatch four per GPU and eight accumulation microbatches.
|
| 162 |
+
Each starts at the same checkpoint and token offset. The benchmark neither
|
| 163 |
+
modifies nor writes production checkpoints. Both GPUs retain their 300 W limits
|
| 164 |
+
and an 88 GiB PyTorch allocator cap. Matrix recipes, external weight caching,
|
| 165 |
+
and standalone activation replacements are off.
|
| 166 |
+
|
| 167 |
+
NVIDIA-SMI samples clocks, power, temperature, and memory once per second outside
|
| 168 |
+
the timed Python regions. Initial diagnostic comparisons use identical NCCL
|
| 169 |
+
logging settings. A single sub-percent timing difference is insufficient to
|
| 170 |
+
select a default, particularly when sustained clock or thermal drift is similar
|
| 171 |
+
in size. Loss and gradient-norm sequences must be compared at identical update
|
| 172 |
+
indices, including warmups.
|
| 173 |
+
|
| 174 |
+
Profiles are separate diagnostics. CUDA intervals are unioned per rank/device;
|
| 175 |
+
overlapping streams and annotation ranges are not summed as elapsed time.
|
| 176 |
+
Communication-only time is the union of NCCL intervals outside other CUDA
|
| 177 |
+
activity. It is an exposed-work budget, not a guarantee that every millisecond
|
| 178 |
+
can be hidden. Ranks' times must not be added together.
|
| 179 |
+
|
| 180 |
+
`ops/analyze_communication_profile.py` produces reproducible overlap reports
|
| 181 |
+
from the benchmark's Chrome traces. Collective payload metadata describes
|
| 182 |
+
logical input bytes, not measured physical link traffic. Its CPU collective
|
| 183 |
+
sequence fingerprint covers operation names and shapes; it is supporting
|
| 184 |
+
evidence and does not establish tensor identity or numerical correctness.
|
| 185 |
+
|
| 186 |
+
## Transport evidence
|
| 187 |
+
|
| 188 |
+
The fresh control's NCCL initialization confirms direct `P2P/CUMEM` transport
|
| 189 |
+
in both directions and two collective channels. Its tuning messages identify
|
| 190 |
+
large all-reduces as RING/SIMPLE. Network/socket initialization messages are not
|
| 191 |
+
evidence that gradient traffic uses sockets; kernel symbol suffixes alone also
|
| 192 |
+
do not establish the selected protocol. The GPUs have a SYS path across CPU
|
| 193 |
+
NUMA nodes, so direct peer access still traverses the system interconnect.
|
| 194 |
+
|
| 195 |
+
The benchmark records NCCL version, relevant environment overrides, and CPU
|
| 196 |
+
affinity. Raw logs and telemetry are under
|
| 197 |
+
`work/benchmark-communication-20260912/` and remain ignored.
|
| 198 |
+
|
| 199 |
+
## Reproduction
|
| 200 |
+
|
| 201 |
+
Reserve both GPUs with production training paused, then use a fresh output
|
| 202 |
+
directory for each process:
|
| 203 |
+
|
| 204 |
+
```bash
|
| 205 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 206 |
+
source ops/training-env.sh
|
| 207 |
+
CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO \
|
| 208 |
+
NCCL_DEBUG_SUBSYS=INIT,GRAPH,TUNING,ENV \
|
| 209 |
+
.server-venv/bin/torchrun --standalone --nproc-per-node=2 \
|
| 210 |
+
ops/benchmark_training.py \
|
| 211 |
+
--checkpoint work/run/latest.pt --data work/data/train.bin \
|
| 212 |
+
--micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 213 |
+
--no-activation-checkpointing --autocast-cache \
|
| 214 |
+
--compile-cores --compile-mode default --loss-mode compiled-full \
|
| 215 |
+
--bf16-residuals --gradient-mode compiled-autograd-overlap \
|
| 216 |
+
--optimizer-state resume --vram-limit-gib 88 \
|
| 217 |
+
--warmup-updates 8 --measured-updates 20 \
|
| 218 |
+
--output-dir work/benchmark-communication-reproduction/control-128
|
| 219 |
+
```
|
| 220 |
+
|
| 221 |
+
Append `--overlap-swap-collectives` for wait movement. Vary
|
| 222 |
+
`--ddp-bucket-cap-mb 64|128|256` in separate runs. The Python reducer groups
|
| 223 |
+
whole parameter gradients; a nominal bucket capacity does not split a larger
|
| 224 |
+
individual parameter. Runtime estimates and an estimated memory allowance
|
| 225 |
+
must be identified separately in results.
|
| 226 |
+
|
| 227 |
+
The backward memory-traffic prototype is a separate experiment; see
|
| 228 |
+
[BACKWARD_LAYOUT_EXPERIMENT.md](BACKWARD_LAYOUT_EXPERIMENT.md). Isolated kernel
|
| 229 |
+
speedups cannot be added to complete-model throughput results.
|
| 230 |
+
|
| 231 |
+
## Validation and retained state
|
| 232 |
+
|
| 233 |
+
The final CPU suite passed **214 tests** with 17 CUDA checks skipped in that
|
| 234 |
+
CPU-only invocation. It covers option validation, legacy signatures, explicit
|
| 235 |
+
execution reconfiguration, gradient accumulation, rank-aligned estimates,
|
| 236 |
+
trace interval accounting, paired-run consistency, and backward-layout math.
|
| 237 |
+
The separate backward-layout CUDA comparisons are documented in its report.
|
| 238 |
+
|
| 239 |
+
Both selected two-GPU continuation fixtures passed in 136.90 seconds:
|
| 240 |
+
wait movement with a 2% estimated memory allowance and 64 MiB buckets, and
|
| 241 |
+
wait movement with aligned runtime estimates and 128 MiB buckets. Each compares
|
| 242 |
+
continuous training with an asynchronous save/stop/resume, including a final
|
| 243 |
+
update with an entirely masked rank. Model tensors, complete FP32 optimizer
|
| 244 |
+
state, and both ranks' RNG state match bitwise within the same execution mode.
|
| 245 |
+
These small-model continuation checks are distinct from the full-model
|
| 246 |
+
cross-configuration numerical differences above.
|
| 247 |
+
|
| 248 |
+
Validation logs and reports are under
|
| 249 |
+
`work/validation/communication-exploration/`, including `pytest-final-cpu.log`,
|
| 250 |
+
`pytest-resume.log`, `comparison-final.json`, and `profile-comparison.json`.
|
| 251 |
+
The retained step-40,081 checkpoint passes its before/after SHA256 check
|
| 252 |
+
(`checkpoint-before.sha256`, `checkpoint-after.log`). All package manifest
|
| 253 |
+
entries also verify. Production remains paused, and no new scheduler setting
|
| 254 |
+
or backward-layout adapter is selected in the production preset.
|
ops/DATA_PREPARATION.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WarpState training data preparation
|
| 2 |
+
|
| 3 |
+
The managed job prepares 40,000,000,000 training tokens and 1,048,576
|
| 4 |
+
validation tokens from `HuggingFaceFW/fineweb-edu`, configuration
|
| 5 |
+
`sample-100BT`, using the package's unchanged SentencePiece tokenizer.
|
| 6 |
+
The upstream commit is pinned in the preparation state and final manifest.
|
| 7 |
+
|
| 8 |
+
All job files, dependencies, caches, logs, and temporary files are below
|
| 9 |
+
`/workspace/WarpState`. This job uses CPU and network resources, with 16
|
| 10 |
+
tokenizer threads and a lower CPU scheduling priority. It does not train a
|
| 11 |
+
model. The package's original Python files are unchanged.
|
| 12 |
+
|
| 13 |
+
The launcher uses `ops/prepare_data.py` to avoid a reproduced PyArrow/Python
|
| 14 |
+
shutdown crash on this host. It waits for the original pipeline to return
|
| 15 |
+
(after token/state files are flushed and committed), flushes console output,
|
| 16 |
+
then exits without interpreter finalization. Exceptions are logged and keep
|
| 17 |
+
a nonzero exit code, so Supervisor can restart from committed progress.
|
| 18 |
+
Garbage collection alone did not eliminate the crash. This follows the
|
| 19 |
+
same process-exit approach documented in NVIDIA's
|
| 20 |
+
[Cosmos launcher](https://github.com/NVIDIA/cosmos-framework/blob/main/cosmos_framework/scripts/train.py).
|
| 21 |
+
|
| 22 |
+
The launcher was checked on a real FineWeb-Edu stream with 1,000,000 train
|
| 23 |
+
tokens and 64 validation tokens. A SIGTERM stop committed its partial batch
|
| 24 |
+
and exited successfully. Resuming produced byte-identical files to an
|
| 25 |
+
uninterrupted run, and both completion manifests matched the computed
|
| 26 |
+
SHA256 checksums. Changed data settings were rejected with a nonzero exit
|
| 27 |
+
code. The results are in
|
| 28 |
+
`work/validation/live-resume-exitguard/validation.json`.
|
| 29 |
+
|
| 30 |
+
## Paths
|
| 31 |
+
|
| 32 |
+
- Python environment: `rtx6000_40b/.data-venv`
|
| 33 |
+
- Resolved Python dependencies: `rtx6000_40b/ops/data-requirements-resolved.txt`
|
| 34 |
+
- Training tokens: `rtx6000_40b/work/data/train.bin`
|
| 35 |
+
- Validation tokens: `rtx6000_40b/work/data/val.bin`
|
| 36 |
+
- Resumable state: `rtx6000_40b/work/data/prepare-state.pkl`
|
| 37 |
+
- Completion manifest and SHA256 checksums: `rtx6000_40b/work/data/manifest.json`
|
| 38 |
+
- Preparation log: `rtx6000_40b/work/logs/prepare.log`
|
| 39 |
+
- Hugging Face cache: `rtx6000_40b/work/hf-cache`
|
| 40 |
+
- Package download cache: `.cache/uv`
|
| 41 |
+
- Temporary files: `tmp`
|
| 42 |
+
|
| 43 |
+
The final token files contain little-endian uint16 IDs and occupy
|
| 44 |
+
80,002,097,152 bytes together. An incomplete `train.bin` is not a completed
|
| 45 |
+
dataset. Completion requires `manifest.json` with `complete: true`; the job
|
| 46 |
+
computes both file checksums before writing that manifest.
|
| 47 |
+
|
| 48 |
+
## Monitor and control
|
| 49 |
+
|
| 50 |
+
Run from `/workspace/WarpState/rtx6000_40b`:
|
| 51 |
+
|
| 52 |
+
```bash
|
| 53 |
+
ops/datactl.sh status
|
| 54 |
+
tail -f work/logs/prepare.log
|
| 55 |
+
|
| 56 |
+
# Gracefully stop, committing the current preparation batch:
|
| 57 |
+
ops/datactl.sh stop warpstate-prepare
|
| 58 |
+
|
| 59 |
+
# Resume from the saved iterator and committed token counts:
|
| 60 |
+
ops/datactl.sh start warpstate-prepare
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
Supervisor runs independently of the terminal, restarts the job after an
|
| 64 |
+
unexpected exit, and leaves it stopped after successful completion. Its
|
| 65 |
+
configuration, Unix socket, PID, and logs are local to this project. It does
|
| 66 |
+
not change the instance's system Supervisor configuration or expose a port.
|
| 67 |
+
|
| 68 |
+
If this project's Supervisor itself is no longer running, start it with:
|
| 69 |
+
|
| 70 |
+
```bash
|
| 71 |
+
TMPDIR=/workspace/WarpState/tmp PYTHONDONTWRITEBYTECODE=1 \
|
| 72 |
+
/usr/local/bin/supervisord -c ops/supervisord.conf
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
The prepared data can later be used with the existing trainer's
|
| 76 |
+
`--stage train --work-dir /workspace/WarpState/rtx6000_40b/work` and its
|
| 77 |
+
default dataset, revision, tokenizer, and token-count arguments. Training
|
| 78 |
+
also requires a CUDA PyTorch environment and compatible initial weights.
|
ops/DPO_CHAT.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Final DPO archive and chat
|
| 2 |
+
|
| 3 |
+
DPO finished on 2026-09-15 at 01:05:06.882 UTC, with the guardian's final
|
| 4 |
+
verification complete at 01:05:42 UTC. It completed 2,334 updates over 74,661
|
| 5 |
+
preference pairs. Training remains paused.
|
| 6 |
+
|
| 7 |
+
The historical archive is `work/archives/step-00076294-dpo-final/` relative to
|
| 8 |
+
`rtx6000_40b`. It preserves the full FP32 model/Adam/RNG checkpoint and a
|
| 9 |
+
tensor-exact inference export, plus the tokenizer, evaluation, recipes,
|
| 10 |
+
reference scores, and provenance. Files are sealed read-only and checksummed.
|
| 11 |
+
Existing raw and SFT archives are preserved.
|
| 12 |
+
|
| 13 |
+
From `/workspace/WarpState`, create or verify the archive and prepare the
|
| 14 |
+
independent chat copy:
|
| 15 |
+
|
| 16 |
+
```bash
|
| 17 |
+
source rtx6000_40b/ops/training-env.sh
|
| 18 |
+
rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/archive_dpo.py
|
| 19 |
+
rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/prepare_dpo_chat.py
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
The chat copy is `rtx6000_40b/work/chat/dpo-final/`: separate physical files,
|
| 23 |
+
with hashes equal to the archive. The chat launcher reads this copy by default.
|
| 24 |
+
Neither chat nor preparation modifies archived weights.
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
rtx6000_40b/ops/chat-dpo.sh
|
| 28 |
+
# Or attach to the already-open interface:
|
| 29 |
+
tmux attach -t warpstate-chat
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
Type a message at `You:`. Ctrl-C discards unfinished input or cancels the current
|
| 33 |
+
reply and returns to `You:`, preserving completed conversation turns.
|
| 34 |
+
`/reset` clears conversation history; `/quit` exits.
|
| 35 |
+
Detach tmux with Ctrl-B followed by D. The interface uses GPU 0, BF16 compute,
|
| 36 |
+
original FP32 master weights, a 2,048-token context, and greedy decoding.
|
| 37 |
+
Responses print when complete. Alternate CUDA devices can be selected with
|
| 38 |
+
`--device cuda:1`; `--temperature 0.7` enables sampling.
|
| 39 |
+
|
| 40 |
+
Prompt serialization matches the segmented role headers, BOS and assistant EOS
|
| 41 |
+
used during posttraining. Cached inference follows the model's parallel chunk
|
| 42 |
+
memory; shape-dependent BF16 rounding can differ from full-sequence execution.
|
| 43 |
+
|
| 44 |
+
Run reproducible sample prompts, choosing a new output filename each time:
|
| 45 |
+
|
| 46 |
+
```bash
|
| 47 |
+
rtx6000_40b/ops/chat-dpo.sh --prompts-json rtx6000_40b/configs/posttrain_smoke_prompts.json --max-new-tokens 128 --output-json rtx6000_40b/work/validation/dpo-smoke.json
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
These few prompts are sanity checks, not a capability benchmark. DPO's final
|
| 51 |
+
held-out preference loss was 0.5790; 245 of 339 pairs had positive relative
|
| 52 |
+
reward margins. This 72.27% is not a generation win rate or answer accuracy.
|
| 53 |
+
Exponentiating DPO loss does not produce language-model perplexity.
|
| 54 |
+
|
| 55 |
+
For a matched full SFT holdout evaluation, `ops/evaluate_sft_snapshot.py`
|
| 56 |
+
accepts `--snapshot`, `--device`, and a new `--output` path. Reports and generated
|
| 57 |
+
responses stay under ignored `work/`; source scripts and prompts are tracked.
|
ops/DPO_HANDOFF.md
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Archive completed SFT and continue into DPO
|
| 2 |
+
|
| 3 |
+
The user authorized this sequence on 2026-09-14:
|
| 4 |
+
|
| 5 |
+
**Finish SFT → archive its full state → independent initialization → DPO → pause.**
|
| 6 |
+
|
| 7 |
+
The separate `warpstate-dpo` Supervisor job waits while the existing SFT job
|
| 8 |
+
continues unchanged. The original SFT plan still says pause because it is part
|
| 9 |
+
of that run's immutable resume signature; its guardian must complete and exit
|
| 10 |
+
before this newly authorized continuation can start. The active DPO plan is
|
| 11 |
+
[`../configs/dpo_10b.json`](../configs/dpo_10b.json).
|
| 12 |
+
|
| 13 |
+
## Completion and archival gate
|
| 14 |
+
|
| 15 |
+
The DPO guardian requires all of the following before any GPU work:
|
| 16 |
+
|
| 17 |
+
- SFT Supervisor state `EXITED`, exit status 0, and its guardian's durable state
|
| 18 |
+
`complete`.
|
| 19 |
+
- Exclusive SFT guardian, base launcher and pipeline locks, then recheck.
|
| 20 |
+
- Exactly 10,312 SFT updates, 329,973 samples, 212,321,372 shifted input tokens,
|
| 21 |
+
and 143,695,039 supervised assistant tokens.
|
| 22 |
+
- A completed full SFT checkpoint with Adam and both ranks' RNG state, and its
|
| 23 |
+
matching final evaluation of all 1,603 held-out conversations.
|
| 24 |
+
- Unchanged parent SFT plan, tokenizer, prepared-data manifest and source pins.
|
| 25 |
+
|
| 26 |
+
The archive is `rtx6000_40b/work/archives/step-00076294-sft-final/`. It contains:
|
| 27 |
+
|
| 28 |
+
| File | Purpose |
|
| 29 |
+
| --- | --- |
|
| 30 |
+
| `checkpoint.pt` | Physical byte-for-byte copy of the full SFT checkpoint, including optimizer/RNG/cursor/signatures |
|
| 31 |
+
| `model.pt`, `snapshot.json` | Original FP32 model-only export, with every tensor compared exactly to the full checkpoint |
|
| 32 |
+
| `tokenizer.model` | Exact original SentencePiece tokenizer |
|
| 33 |
+
| `evaluation.json`, `completion.json`, `settings.json`, `metrics.jsonl` | Original final evaluation, completion, execution settings and complete SFT history |
|
| 34 |
+
| `sft-execution.json`, `sft-handoff.json`, `PROVENANCE.json` | Source/data identities and both raw-pretraining and SFT stage counters |
|
| 35 |
+
| `ARCHIVE.md`, `SHA256SUMS` | Retention instructions and integrity checks |
|
| 36 |
+
|
| 37 |
+
All files are sealed read-only before the complete directory is atomically
|
| 38 |
+
published without replacement. Existing archives are validated and reused, or
|
| 39 |
+
rejected without alteration. Interrupted hidden staging directories remain for
|
| 40 |
+
diagnosis. The original SFT resume checkpoint and all raw historical archives
|
| 41 |
+
remain in place.
|
| 42 |
+
|
| 43 |
+
DPO reads a physically independent copy of `model.pt`, `snapshot.json` and the
|
| 44 |
+
tokenizer under `work/posttrain/step-00076294-dpo/initialization/`. It does not
|
| 45 |
+
inherit SFT Adam state or write into either SFT archive or SFT run directory.
|
| 46 |
+
The full SFT archive is roughly 32 GB decimal; its counters describe SFT after
|
| 47 |
+
the raw step-76,294 model, not an additional 10B-token pretraining run.
|
| 48 |
+
|
| 49 |
+
## Preference objective and execution
|
| 50 |
+
|
| 51 |
+
This first DPO experiment uses:
|
| 52 |
+
|
| 53 |
+
| Setting | Value |
|
| 54 |
+
| --- | --- |
|
| 55 |
+
| Data | 74,661 training pairs and 339 held-out pairs already prepared |
|
| 56 |
+
| Objective | Length-normalized DPO, beta 5 |
|
| 57 |
+
| Epochs | 1; 2,334 optimizer updates |
|
| 58 |
+
| Global batch | 32 preference pairs; 2 pairs per rank × 2 ranks × 8 accumulation microbatches |
|
| 59 |
+
| Context | 2,048 tokens per chosen/rejected branch; no truncation or cross-conversation packing |
|
| 60 |
+
| Learning rate | Peak `5e-7`, 10% warmup, linear decay to zero |
|
| 61 |
+
| Optimizer | Fresh FP32 AdamW, betas 0.9/0.95, zero weight decay, clipping at 1 |
|
| 62 |
+
| Precision | BF16 compute, original FP32 parameters/gradients/Adam; no FP8 |
|
| 63 |
+
| Evaluation/save | Every 100 DPO updates; full holdout at start and completion |
|
| 64 |
+
|
| 65 |
+
For each branch, sum the log probabilities of **only the final assistant
|
| 66 |
+
response and its actual EOS token**. System/user/tool messages, earlier
|
| 67 |
+
assistant turns, role headers and padding are conditioning or masked. Divide
|
| 68 |
+
each policy-minus-reference log-probability sum by that branch's response-token
|
| 69 |
+
count, subtract rejected from chosen, multiply by beta, and apply negative
|
| 70 |
+
log-sigmoid. Average over actual preference pairs across ranks; short final
|
| 71 |
+
batches and empty rank slots do not change the normalization.
|
| 72 |
+
|
| 73 |
+
This uses the length-normalized objective and conservative hyperparameters
|
| 74 |
+
described in the [Tülu 3 paper, equation 6 and table 20](https://arxiv.org/html/2411.15124v3).
|
| 75 |
+
It is a pilot for this 2B/10B base, not a reproduced or tuned Tülu result.
|
| 76 |
+
Ordinary summed-log-probability DPO is a separate supported objective and its
|
| 77 |
+
usual beta cannot be substituted for beta 5 here without changing the recipe.
|
| 78 |
+
|
| 79 |
+
Before policy updates, score the immutable SFT model in BF16 on all selected
|
| 80 |
+
chosen/rejected branches. Each GPU saves resumable reference-score shards; the
|
| 81 |
+
final cache is bound to SFT weights, data, tokenizer, masks, execution settings,
|
| 82 |
+
library version and source hashes. Reusing or rebuilding it must preserve that
|
| 83 |
+
identity. The reference remains the completed SFT model after every policy
|
| 84 |
+
checkpoint/resume. Once scores are cached, release the reference model before
|
| 85 |
+
allocating policy optimizer state, avoiding a second resident full model.
|
| 86 |
+
|
| 87 |
+
The output head uses checkpointed token chunks to avoid retaining full-sequence
|
| 88 |
+
vocabulary logits for both responses. A tiny two-GPU BF16 preflight checks the
|
| 89 |
+
actual DPO/reference/continuation path after SFT releases the GPUs. CPU setup
|
| 90 |
+
tests do not claim that this deferred GPU gate or production-size startup has
|
| 91 |
+
already run. Any failed gate stops automatic progression with the archive kept.
|
| 92 |
+
|
| 93 |
+
## State, monitoring and controls
|
| 94 |
+
|
| 95 |
+
Paths below are relative to `rtx6000_40b/work/posttrain/step-00076294-dpo/`.
|
| 96 |
+
|
| 97 |
+
| Path | Purpose |
|
| 98 |
+
| --- | --- |
|
| 99 |
+
| `control/handoff.json`, `control/execution.json` | Guardian phase and frozen execution provenance |
|
| 100 |
+
| `initialization/` | Independent immutable SFT model copy |
|
| 101 |
+
| `preflight/` | Tiny GPU validation and its acceptance report |
|
| 102 |
+
| `reference-cache/` | Fixed SFT reference log probabilities and resumable scoring shards |
|
| 103 |
+
| `run/latest.pt` | Independent full DPO resume checkpoint, format `warpstate-dpo-v1` |
|
| 104 |
+
| `run/metrics.jsonl`, `run/evaluation.json`, `run/completion.json` | Training history, evaluation and durable completion |
|
| 105 |
+
|
| 106 |
+
From `/workspace/WarpState`:
|
| 107 |
+
|
| 108 |
+
```bash
|
| 109 |
+
rtx6000_40b/ops/datactl.sh status warpstate-posttrain warpstate-dpo
|
| 110 |
+
rtx6000_40b/ops/dpo-handoff.sh --check
|
| 111 |
+
tmux attach -t warpstate-dpo
|
| 112 |
+
tail -f rtx6000_40b/work/logs/dpo.log
|
| 113 |
+
```
|
| 114 |
+
|
| 115 |
+
To cancel automatic DPO while SFT continues, or gracefully stop active DPO:
|
| 116 |
+
|
| 117 |
+
```bash
|
| 118 |
+
rtx6000_40b/ops/datactl.sh stop warpstate-dpo
|
| 119 |
+
```
|
| 120 |
+
|
| 121 |
+
To explicitly resume the unchanged guardian/reference preparation/DPO state:
|
| 122 |
+
|
| 123 |
+
```bash
|
| 124 |
+
rtx6000_40b/ops/datactl.sh start warpstate-dpo
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
No restart of SFT is needed. The new guardian preserves the SFT guardian and
|
| 128 |
+
base-launch exclusion locks throughout DPO. Every worker holds a shared base
|
| 129 |
+
pipeline lock, so surviving workers block accidental base/SFT/DPO relaunches
|
| 130 |
+
even if their launcher dies. Stop files and lock files are never shared with
|
| 131 |
+
the live SFT stop request. The program has `autostart=false` and
|
| 132 |
+
`autorestart=false`; a whole-machine/Supervisor restart requires explicitly
|
| 133 |
+
restarting the appropriate jobs. A completed DPO run stays paused when its
|
| 134 |
+
guardian is started again.
|
| 135 |
+
|
| 136 |
+
On normal completion, validate final counters, all 339 held-out pairs, fixed
|
| 137 |
+
reference-cache identity and full archive/init hashes before recording success.
|
| 138 |
+
No additional training stage starts automatically. Next compare the archived
|
| 139 |
+
SFT model with DPO on instruction following and tool use. DPO loss, reward
|
| 140 |
+
margin and preference accuracy describe this objective; exponentiating DPO
|
| 141 |
+
loss does not produce language-model perplexity.
|
| 142 |
+
|
| 143 |
+
All generated data, scores, archives, checkpoints, logs and environments remain
|
| 144 |
+
ignored under `work/`. Only source/configuration/tests/documentation are tracked.
|
ops/LOCAL_RESUME.md
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Resume the B200 checkpoint on the RTX workstation
|
| 2 |
+
|
| 3 |
+
This configuration continues the B200 training run on the local RTX PRO 6000
|
| 4 |
+
workstation.
|
| 5 |
+
The source checkpoint was saved on the B200 at **step 680 / 89,128,960
|
| 6 |
+
tokens**, with validation loss **4.39469** and SHA256:
|
| 7 |
+
|
| 8 |
+
```text
|
| 9 |
+
bf689a10a030842699bfa3a25ff62e488cfa2d5b022b2b6e3bfc860359eb8b4f
|
| 10 |
+
```
|
| 11 |
+
|
| 12 |
+
After the verified import, `work/run/latest.pt` is the local resume checkpoint.
|
| 13 |
+
Later local saves replace it atomically; the checksum above identifies the
|
| 14 |
+
imported step-680 checkpoint. [B200_HANDOFF.md](B200_HANDOFF.md) and the B200
|
| 15 |
+
performance documents record the remote run and its historical controls.
|
| 16 |
+
Use the workstation controls below on this host.
|
| 17 |
+
The B200 source history was imported through commit `752563e`. The generated
|
| 18 |
+
local migration record is `work/validation/local-resume/import.json`, relative
|
| 19 |
+
to `rtx6000_40b`; it records the verified checkpoint and continuation state.
|
| 20 |
+
|
| 21 |
+
[`transfer_artifacts.py`](transfer_artifacts.py) accepts `--pull` to retrieve
|
| 22 |
+
a remote source into a local destination with parallel SSH connections.
|
| 23 |
+
Use `--expected-sha256` with the checkpoint hash above, keep the private key
|
| 24 |
+
on the local host, and use the existing verified host-key file. The download
|
| 25 |
+
uses resumable chunks and verifies the entire file before publishing its
|
| 26 |
+
final name. No second download of the unchanged 40B-token corpus is needed.
|
| 27 |
+
|
| 28 |
+
The prepared 40,000,000,000 training tokens and 1,048,576 validation tokens
|
| 29 |
+
are unchanged. Their immutable upstream revision, tokenizer hash, split rule,
|
| 30 |
+
and preparation dependencies remain in
|
| 31 |
+
[`../configs/data_40b.json`](../configs/data_40b.json). The existing data
|
| 32 |
+
signature retains the original requested revision `main`, which was resolved
|
| 33 |
+
to the pinned upstream commit. Preserve that signature when resuming.
|
| 34 |
+
|
| 35 |
+
## Local execution
|
| 36 |
+
|
| 37 |
+
[`train-local-resume.sh`](train-local-resume.sh) requires an existing full
|
| 38 |
+
checkpoint and selects both physical workstation GPUs by UUID. Both GPUs
|
| 39 |
+
became available before the local resume. This launcher uses the measured
|
| 40 |
+
BF16 configuration and continues the existing run. Check both GPUs'
|
| 41 |
+
availability before every start:
|
| 42 |
+
|
| 43 |
+
| Setting | Value |
|
| 44 |
+
| --- | --- |
|
| 45 |
+
| Sequence length | 2,048 |
|
| 46 |
+
| Microbatch per GPU / gradient accumulation | 4 / 8 |
|
| 47 |
+
| GPUs / global tokens per optimizer update | 2 / 131,072 |
|
| 48 |
+
| Parameters, gradients, full AdamW moments | FP32 on the GPU |
|
| 49 |
+
| Forward/backward computation / residual activations | BF16 autocast / BF16 |
|
| 50 |
+
| Core activation checkpointing | Disabled |
|
| 51 |
+
| Core compilation / autocast weight cache | Enabled, default mode / enabled |
|
| 52 |
+
| Vocabulary loss | Compiled full loss |
|
| 53 |
+
| Gradient execution / communication | Compiled autograd with delayed waits/copies; FP32 reduction, 128 MiB buckets |
|
| 54 |
+
| Allocator cap / extra startup headroom per GPU | 88 GiB / 3 GiB |
|
| 55 |
+
| Checkpoint writing | Synchronous immutable CPU snapshot; background serialization and atomic replacement |
|
| 56 |
+
| Save / evaluation interval | Every 100 / 100 updates |
|
| 57 |
+
| Validation display | Fresh results only, labeled with their evaluation step |
|
| 58 |
+
|
| 59 |
+
The current preset includes gradient conversion/accumulation fusion and moves
|
| 60 |
+
collective waits with their dependent copies past independent backward GEMMs.
|
| 61 |
+
The repeated 20-update comparison measured **31,045 tokens/s** versus
|
| 62 |
+
**30,578 tokens/s** without those changes (**1.53% faster**), with **66.98 GiB
|
| 63 |
+
reserved per GPU**. Both GPU ranks remain data-parallel; communication overlaps
|
| 64 |
+
their local computation. See [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md)
|
| 65 |
+
for the profile, numerical comparisons and exact two-GPU checkpoint/resume tests.
|
| 66 |
+
Shape-specific BF16 matrix plans are available through `--matmul-plan`. The
|
| 67 |
+
experimental fused-epilogue/TMA recipe measured 31,156 versus 31,022 tokens/s
|
| 68 |
+
with the same overlap mode in one comparison (+0.43%); that additional gain is
|
| 69 |
+
not yet confirmed across repeated windows, so the launcher keeps its current
|
| 70 |
+
matrix implementation. See [MATMUL_TUNING.md](MATMUL_TUNING.md) for the recipe,
|
| 71 |
+
tile tuning, persistent kernels, precision checks, and full-model measurements.
|
| 72 |
+
The earlier loss/residual changes are documented in
|
| 73 |
+
[RTX_PROFILE_20260910.md](RTX_PROFILE_20260910.md), and asynchronous checkpoints
|
| 74 |
+
in [RTX_ASYNC_TRAINING.md](RTX_ASYNC_TRAINING.md).
|
| 75 |
+
|
| 76 |
+
These rates exclude evaluation, checkpoint pauses and compilation. Compiled
|
| 77 |
+
autograd can spend approximately three minutes compiling its first full update;
|
| 78 |
+
even with cached kernels, each new process has graph-capture/setup costs.
|
| 79 |
+
The allocator cap excludes CUDA context/library allocations. Startup requires
|
| 80 |
+
91 GiB free on each GPU and does not reserve memory for other jobs.
|
| 81 |
+
Keep at least **53 GiB of free SSD space** beyond existing files for full
|
| 82 |
+
checkpoint writing and replacement.
|
| 83 |
+
|
| 84 |
+
Asynchronous saves use about **22.4 GiB of additional host memory** for one
|
| 85 |
+
full model/optimizer snapshot. GPU-to-CPU staging still pauses both workers;
|
| 86 |
+
serialization, file synchronization and publication overlap later updates.
|
| 87 |
+
There is at most one outstanding snapshot, and a new save waits for the
|
| 88 |
+
previous write if necessary. A logged `Staged step` is not yet durable: use
|
| 89 |
+
the corresponding `Saved ... (step ..., ... background write ...)` message.
|
| 90 |
+
Normal completion and graceful stop wait for the final write. A save failure
|
| 91 |
+
is reported to both workers at the next coordinated check, save or close and
|
| 92 |
+
stops training; a failure during an update can allow that update to finish.
|
| 93 |
+
|
| 94 |
+
Training step lines display `val ... (step ...)` only when that step has a
|
| 95 |
+
fresh validation result. Every step still records `val_loss`, `val_step` and
|
| 96 |
+
`val_tokens` in `metrics.jsonl`; these fields retain the latest measured
|
| 97 |
+
result between evaluations. Checkpoints retain the same validation metadata.
|
| 98 |
+
This adds no per-step evaluation work.
|
| 99 |
+
Startup still evaluates the resumed weights, and graceful stop/final completion
|
| 100 |
+
still evaluates and saves even when the step is not a multiple of 100.
|
| 101 |
+
|
| 102 |
+
`--reconfigure` preserves the model, full optimizer, token cursor, objective,
|
| 103 |
+
and learning-rate schedule while recording execution changes in the checkpoint.
|
| 104 |
+
The original B200 import restored rank-zero random states and seeded the new
|
| 105 |
+
rank reproducibly. Subsequent two-GPU resumes restore both ranks' states.
|
| 106 |
+
BF16 residual activations retain FP32 master parameters, gradients and Adam
|
| 107 |
+
moments, but change activation rounding. A small validation change at the
|
| 108 |
+
same saved weights is not training progress. Disabling them requires
|
| 109 |
+
`--no-bf16-residuals` with an explicit reconfiguration.
|
| 110 |
+
|
| 111 |
+
## Controls and current progress
|
| 112 |
+
|
| 113 |
+
Run from `/workspace/WarpState`:
|
| 114 |
+
|
| 115 |
+
```bash
|
| 116 |
+
rtx6000_40b/ops/datactl.sh status
|
| 117 |
+
nvidia-smi --query-gpu=index,memory.used,memory.free,utilization.gpu --format=csv
|
| 118 |
+
rtx6000_40b/ops/datactl.sh start warpstate-train
|
| 119 |
+
tail -n 10 rtx6000_40b/work/logs/train.log
|
| 120 |
+
tail -n 3 rtx6000_40b/work/run/metrics.jsonl
|
| 121 |
+
```
|
| 122 |
+
|
| 123 |
+
The status command and latest metrics identify the actual running state and
|
| 124 |
+
current cursor. This document records the import and launcher configuration;
|
| 125 |
+
it does not imply that a job is running. Preparation and training both have
|
| 126 |
+
automatic startup and restart disabled. After changing Supervisor
|
| 127 |
+
configuration, use `datactl.sh reread` and `datactl.sh update` before an
|
| 128 |
+
explicit start.
|
| 129 |
+
|
| 130 |
+
Stop with `rtx6000_40b/ops/datactl.sh stop warpstate-train`. The launcher
|
| 131 |
+
requests a stop through its stop file, waits for the current update and full
|
| 132 |
+
checkpoint, and has a 900-second Supervisor stop allowance. Check the saved
|
| 133 |
+
step in the log before treating an in-memory update as durable.
|
| 134 |
+
|
| 135 |
+
## Resuming after a deliberate pause
|
| 136 |
+
|
| 137 |
+
Keep a deliberately paused run stopped until the user requests resumption.
|
| 138 |
+
As requested on 2026-09-14, the workstation Supervisor command includes
|
| 139 |
+
`--stop-at-step 76294`: the trainer finishes optimizer update 76,294, evaluates,
|
| 140 |
+
and publishes the full checkpoint before exiting. This boundary corresponds to
|
| 141 |
+
10,000,007,168 consumed tokens. No update 76,295 runs. Async checkpoint writing
|
| 142 |
+
is flushed before exit, and Supervisor has automatic restart disabled.
|
| 143 |
+
|
| 144 |
+
`--stop-at-step` is an absolute limit, so an intervening restart cannot move
|
| 145 |
+
the boundary. `--max-steps` still limits additional updates in one invocation;
|
| 146 |
+
if both are supplied, the earlier boundary wins. Neither option changes the
|
| 147 |
+
40B-token learning-rate schedule or the checkpoint's training signature.
|
| 148 |
+
The active boundary is also recorded in `work/run/settings.json` under
|
| 149 |
+
`runtime.stop_at_step` and `runtime.end_step`.
|
| 150 |
+
|
| 151 |
+
When the user requests training beyond this boundary, first remove or increase
|
| 152 |
+
`--stop-at-step` in the `warpstate-train` command in `ops/supervisord.conf`, then
|
| 153 |
+
run `ops/datactl.sh reread` and `ops/datactl.sh update warpstate-train` from the
|
| 154 |
+
package directory before starting the job. A restart retaining a boundary at
|
| 155 |
+
or below the loaded checkpoint step performs no further optimizer updates.
|
| 156 |
+
|
| 157 |
+
The generated `work/validation/paused-run.json` records the last verified
|
| 158 |
+
pause: checkpoint path, SHA256, step, tokens consumed, validation loss,
|
| 159 |
+
optimizer/RNG checks, execution settings, source commit and next update.
|
| 160 |
+
`work/validation/paused-run.sha256` provides a directly checkable checksum.
|
| 161 |
+
Both records are excluded from Git. Subsequent training replaces `latest.pt`,
|
| 162 |
+
so these records identify the last deliberate pause rather than live progress.
|
| 163 |
+
|
| 164 |
+
Before resuming a stopped run, inspect its saved state and available hardware:
|
| 165 |
+
|
| 166 |
+
```bash
|
| 167 |
+
cd /workspace/WarpState
|
| 168 |
+
rtx6000_40b/ops/datactl.sh status
|
| 169 |
+
cat rtx6000_40b/work/validation/paused-run.json
|
| 170 |
+
sha256sum -c rtx6000_40b/work/validation/paused-run.sha256
|
| 171 |
+
nvidia-smi --query-gpu=index,memory.used,memory.free,utilization.gpu --format=csv
|
| 172 |
+
```
|
| 173 |
+
|
| 174 |
+
Use the existing full checkpoint at `work/run/latest.pt`, which contains
|
| 175 |
+
model weights, full AdamW state and both ranks' RNG states. Preserve
|
| 176 |
+
`work/data/`, `work/run/`, the `.server-venv` environment, tokenizer and
|
| 177 |
+
existing launchers. No data preparation or checkpoint conversion is needed.
|
| 178 |
+
Both workstation GPUs must have at least 91 GiB free before launch. The
|
| 179 |
+
launcher retains BF16 compute, the 131,072-token global batch, the 40B-token
|
| 180 |
+
schedule, asynchronous saves, and 100-step save/evaluation intervals.
|
| 181 |
+
|
| 182 |
+
When resumption is requested:
|
| 183 |
+
|
| 184 |
+
```bash
|
| 185 |
+
cd /workspace/WarpState
|
| 186 |
+
rtx6000_40b/ops/datactl.sh start warpstate-train
|
| 187 |
+
tmux attach -t warpstate-train
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
The launcher clears the previous stop request under its process lock. If the
|
| 191 |
+
host has rebooted and Supervisor is absent, initialize it once with
|
| 192 |
+
`/usr/local/bin/supervisord -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf`
|
| 193 |
+
before using the controls. Its training and preparation jobs start paused.
|
| 194 |
+
If the tmux monitor was lost during reboot, recreate it with:
|
| 195 |
+
|
| 196 |
+
```bash
|
| 197 |
+
tmux new-session -d -s warpstate-train -n training -c /workspace/WarpState \
|
| 198 |
+
'tail -n 30 -F /workspace/WarpState/rtx6000_40b/work/logs/train.log'
|
| 199 |
+
```
|
| 200 |
+
|
| 201 |
+
Keep the active token files, their manifest/preparation state, the latest full
|
| 202 |
+
checkpoint, active environment, and controls required to continue training.
|
| 203 |
+
Git retains source, pinned dependency specifications, tokenizer, and input
|
| 204 |
+
provenance for rebuilding. Obsolete checkpoints, experimental outputs,
|
| 205 |
+
download staging, caches, and disposable test artifacts can be removed once
|
| 206 |
+
their replacement or source recipe is verified; generated runtime files stay
|
| 207 |
+
out of Git.
|
ops/LOSS_FORECAST_20260913.md
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WarpState validation-loss forecasting, 2026-09-13
|
| 2 |
+
|
| 3 |
+
**Neither proposed token budget currently guarantees validation loss below 2.0.**
|
| 4 |
+
The observed curve supports continued improvement, but the crossing estimate is
|
| 5 |
+
extremely sensitive to the assumed eventual loss floor. A power-law fit to the
|
| 6 |
+
1–7B-token history crosses near 1.07T tokens; excluding the first billion tokens
|
| 7 |
+
of that fit moves its extrapolated floor above 2.0. This is uncertainty about the
|
| 8 |
+
forecast, not evidence of a demonstrated architecture limit.
|
| 9 |
+
|
| 10 |
+
This research did not change training, checkpoints, the dataset, or the schedule.
|
| 11 |
+
Generated snapshots, numerical results, and figures belong under
|
| 12 |
+
`work/validation/loss-forecast-20260913/` and remain excluded from Git.
|
| 13 |
+
|
| 14 |
+
## Current measurements and interpretation of the budgets
|
| 15 |
+
|
| 16 |
+
The frozen validation history ends at step 54,300: **7,117,209,600 tokens,
|
| 17 |
+
validation loss 2.42521**, with 555 evaluation records. Recent values fluctuate
|
| 18 |
+
around 2.41–2.43. The complete 6.75–7.00B bin averages 2.41219. Loss 2.0 corresponds
|
| 19 |
+
to perplexity `exp(2) = 7.389` on this tokenizer and evaluation distribution.
|
| 20 |
+
|
| 21 |
+
Interpret `8000*256` and `8000*1024` as **total optimizer steps**, not additional
|
| 22 |
+
steps or individual tokens. The active configuration processes
|
| 23 |
+
`2 GPUs * 4 sequences * 2048 tokens * 8 accumulation microsteps = 131072`
|
| 24 |
+
tokens per optimizer update.
|
| 25 |
+
|
| 26 |
+
| Budget | Optimizer steps | Total training tokens | Tokens per model parameter | Passes over the existing 40B-token corpus | Remaining wall time |
|
| 27 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 28 |
+
| `8000*256` | 2,048,000 | 268,435,456,000 | 133.88 | 6.711 | 100.5 days |
|
| 29 |
+
| `8000*1024` | 8,192,000 | 1,073,741,824,000 | 535.50 | 26.844 | 410.3 days |
|
| 30 |
+
|
| 31 |
+
The model has 2,005,111,424 parameters. Wall-time arithmetic uses the live metrics
|
| 32 |
+
at step 54,360 on 2026-09-13 11:41:14 UTC: the preceding 3,000 optimizer steps
|
| 33 |
+
averaged **4.35589249 seconds/update**, or **30,090.7 effective tokens/second**,
|
| 34 |
+
including periodic evaluation and checkpoint overhead. Durations assume the
|
| 35 |
+
current two-GPU workstation, unchanged batch size and throughput, and uninterrupted
|
| 36 |
+
operation. They estimate the time to consume a budget, not the time to attain a
|
| 37 |
+
particular loss. From zero, the budgets would take 103.3 and 413.0 days.
|
| 38 |
+
|
| 39 |
+
The configured objective and prepared token file are still **40B tokens**.
|
| 40 |
+
The larger budgets require an explicit continuation schedule and data plan.
|
| 41 |
+
If the question meant additional steps, add the current consumed tokens and
|
| 42 |
+
about 2.74 days to the corresponding remaining-duration arithmetic.
|
| 43 |
+
|
| 44 |
+
## What the primary research establishes
|
| 45 |
+
|
| 46 |
+
Language-model learning curves often admit power-law approximations after an
|
| 47 |
+
initial transient. These describe diminishing returns: each additional billion
|
| 48 |
+
tokens generally produces a smaller gain. Published coefficients depend on the
|
| 49 |
+
model, tokenization, data, and experimental setup.
|
| 50 |
+
[Kaplan et al., Sections 4–5](https://arxiv.org/pdf/2001.08361).
|
| 51 |
+
|
| 52 |
+
Chinchilla fits endpoint loss as
|
| 53 |
+
`L(N,D) = E + A/N^alpha + B/D^beta`. At fixed model size this has the form
|
| 54 |
+
`L(D) = F + B/D^beta`, where `F` includes the model-size contribution as well as
|
| 55 |
+
irreducible uncertainty. Its compute-optimal parameter/data allocation does not
|
| 56 |
+
define a stopping point for a fixed model. Appendix B also shows that the cosine
|
| 57 |
+
schedule duration must be calibrated to the intended training duration.
|
| 58 |
+
[Hoffmann et al., Section 3.3 and Appendix B](https://arxiv.org/pdf/2203.15556).
|
| 59 |
+
|
| 60 |
+
Schedule-aware work separates continuing optimization from the extra improvement
|
| 61 |
+
caused by lowering the learning rate. Tissue et al. model validation loss using
|
| 62 |
+
both accumulated learning rate and an annealing-history term; their Appendix C.3
|
| 63 |
+
uses shorter constant and cosine runs to forecast a 1.7B model trained on 1.4T
|
| 64 |
+
tokens. Such calibrated experiments support forecasting, but do not establish
|
| 65 |
+
that one early training trace can identify a 150-fold extension.
|
| 66 |
+
[Scaling Law with Learning Rate Annealing](https://arxiv.org/html/2408.11029).
|
| 67 |
+
|
| 68 |
+
Constant-rate training followed by a terminal cooldown provides a way to compare
|
| 69 |
+
different training budgets while reusing checkpoints. A loss reached after
|
| 70 |
+
cooldown need not have the same subsequent slope as a loss reached at a high
|
| 71 |
+
learning rate.
|
| 72 |
+
[Hägele et al., Sections 2–3](https://arxiv.org/html/2405.18392v3).
|
| 73 |
+
|
| 74 |
+
Long training of small models can remain useful. Sardana et al. study a 2.46B
|
| 75 |
+
model up to 500 tokens/parameter, close to the larger proposed budget here. Their
|
| 76 |
+
study also warns that fits from shorter training can overstate the benefits of
|
| 77 |
+
much longer training. Its reported final training loss and unique-data setup
|
| 78 |
+
cannot be converted into WarpState validation loss 2.0.
|
| 79 |
+
[Sardana et al., Sections 3–4 and Appendix C](https://arxiv.org/html/2401.00448).
|
| 80 |
+
Gadre et al. successfully forecast a 1.4B/900B-token model using calibrated
|
| 81 |
+
experiments, further supporting the feasibility of forecasting with the right
|
| 82 |
+
measurements.
|
| 83 |
+
[ICLR 2025 paper](https://proceedings.iclr.cc/paper_files/paper/2025/hash/a91869936a63d814971b6423990ecf6e-Abstract-Conference.html).
|
| 84 |
+
|
| 85 |
+
Repeated tokens have diminishing value compared with fresh tokens. Muennighoff
|
| 86 |
+
et al. find small penalties through roughly four epochs in their experiments,
|
| 87 |
+
with progressively smaller returns from further repetition. That finding is not
|
| 88 |
+
a universal epoch cutoff. Repeating our corpus 6.7 or 26.8 times is a different
|
| 89 |
+
experiment from acquiring 268B or 1.07T comparable fresh tokens.
|
| 90 |
+
[Scaling Data-Constrained Language Models](https://arxiv.org/html/2305.16264v3).
|
| 91 |
+
|
| 92 |
+
## Fits to WarpState: scenarios, not confidence intervals
|
| 93 |
+
|
| 94 |
+
Use actual evaluations from `work/run/validation.jsonl`, rather than the latest
|
| 95 |
+
validation value repeated in every training-metric row. Verify strictly increasing,
|
| 96 |
+
unique token counts, start at 1B tokens to exclude early execution changes, and
|
| 97 |
+
average complete 0.25B-token bins. Each bin receives equal weight, at its actual
|
| 98 |
+
mean token position. The frozen snapshot contains 24 complete bins from 1–7B;
|
| 99 |
+
the partial final bin is excluded from fitting.
|
| 100 |
+
|
| 101 |
+
Fit in loss space, using `t = tokens / 1e9`:
|
| 102 |
+
|
| 103 |
+
- Logarithmic: `L(t) = a + b*ln(t)`.
|
| 104 |
+
- Power law with a floor: `L(t) = F + A*t^(-beta)`, with `A >= 0`, `F >= 0`,
|
| 105 |
+
and positive `beta`.
|
| 106 |
+
- Fixed-floor sensitivity: choose an illustrative `F` and fit `A` and `beta`.
|
| 107 |
+
These chosen floors are assumptions, not measured properties of WarpState.
|
| 108 |
+
|
| 109 |
+
For the power-law scenario, a finite crossing exists only if `F < 2`, and its
|
| 110 |
+
location is `t = (A/(2-F))^(1/beta)`. A small uncertainty in `F` close to 2 thus
|
| 111 |
+
creates a very large uncertainty in the crossing time.
|
| 112 |
+
|
| 113 |
+
| Fit | Loss at 268.435B | Loss at 1.073742T | Crossing below 2.0 |
|
| 114 |
+
| --- | ---: | ---: | --- |
|
| 115 |
+
| Logarithmic, starts at 3/4/5B and ends at 7B | 1.68–1.85 | 1.40–1.64 | 53–103B |
|
| 116 |
+
| Power law, free floor, 1–7B history | 2.052 | approximately 2.000 | approximately 1.07T |
|
| 117 |
+
| Power law, free floor, 2–7B history | 2.096 | 2.058 | No finite crossing in this fitted curve |
|
| 118 |
+
| Power law, **assumed floor 1.5**, 1–7B history | 1.905 | 1.799 | 103B |
|
| 119 |
+
| Power law, **assumed floor 1.8**, 1–7B history | 2.001 | 1.932 | 272B |
|
| 120 |
+
| Power law, **assumed floor 1.9**, 1–7B history | 2.043 | 1.989 | 762B |
|
| 121 |
+
|
| 122 |
+
The 1–7B free-floor fit is approximately
|
| 123 |
+
`L(t) = 1.918124 + 0.975783*t^(-0.355255)`.
|
| 124 |
+
The 2–7B fit has floor 2.009366 and exponent 0.419261. Neither fitted floor is a
|
| 125 |
+
measured asymptotic limit. Shifting the start to 3, 4, or 5B yields floors around
|
| 126 |
+
2.19–2.33, showing how poorly the eventual floor is identified by this history.
|
| 127 |
+
The numerical 1.07T crossing should not be interpreted as a precise prediction
|
| 128 |
+
or as evidence that the larger budget was chosen correctly.
|
| 129 |
+
|
| 130 |
+
The logarithmic fit eventually predicts unboundedly negative loss, so it is a
|
| 131 |
+
local trend approximation. It is included because it illustrates an optimistic
|
| 132 |
+
extrapolation, not because it is a suitable global asymptotic law. Small residuals
|
| 133 |
+
on fitted data also do not establish long-range predictive accuracy.
|
| 134 |
+
|
| 135 |
+
Historical forward checks fit an earlier interval and predict the next complete
|
| 136 |
+
billion tokens. The table shows mean predicted minus observed loss; negative
|
| 137 |
+
values mean the forecast was too optimistic.
|
| 138 |
+
|
| 139 |
+
| Fit interval → held-out interval | Logarithmic bias | Free-floor power-law bias |
|
| 140 |
+
| --- | ---: | ---: |
|
| 141 |
+
| 2–4B → 4–5B | -0.00197 | +0.00695 |
|
| 142 |
+
| 2–5B → 5–6B | -0.00604 | -0.00194 |
|
| 143 |
+
| 2–6B → 6–7B | -0.01665 | -0.01023 |
|
| 144 |
+
|
| 145 |
+
Neither family wins every historical comparison. Both overpredicted the latest
|
| 146 |
+
interval's improvement, particularly the logarithmic fit. These checks use
|
| 147 |
+
0.25B-bin means, so their residuals should not be confused with errors on raw
|
| 148 |
+
individual validation points.
|
| 149 |
+
|
| 150 |
+
## Why there is no dependable crossing date yet
|
| 151 |
+
|
| 152 |
+
The observations extend to about 7.1B tokens, whereas the requested budgets are
|
| 153 |
+
roughly **38 and 151 times** that amount. Historical one-billion-token holdouts
|
| 154 |
+
can reject some overly optimistic local trends; they cannot validate such a
|
| 155 |
+
large extrapolation. Successive measurements are correlated observations of one
|
| 156 |
+
training run, not independent experimental replications.
|
| 157 |
+
|
| 158 |
+
The run still uses a learning rate near `1.40e-4`, about 93% of its `1.50e-4`
|
| 159 |
+
peak, with a planned 40B-token cosine decay toward `1.50e-5`. The simple fits
|
| 160 |
+
above do not model that future annealing. Cooldown may improve loss beyond their
|
| 161 |
+
predictions; a changed continuation schedule or repeated data may produce a
|
| 162 |
+
different curve. There is no fitted magnitude for those effects in WarpState.
|
| 163 |
+
|
| 164 |
+
Current evaluation samples eight fixed sequences of length 2,048: **16,384
|
| 165 |
+
target tokens**, drawn at fixed spaced positions from the 1,048,576-token
|
| 166 |
+
held-out file. A single reported value below 2 would be weaker evidence than a
|
| 167 |
+
sustained crossing and confirmation on more held-out tokens. Absolute loss 2
|
| 168 |
+
also cannot be compared directly with another paper's tokenizer or validation
|
| 169 |
+
distribution, and does not by itself determine downstream task performance.
|
| 170 |
+
|
| 171 |
+
The useful next forecast experiment is a matched control and cooldown branch
|
| 172 |
+
from the same full checkpoint, retaining optimizer state, data order, batch
|
| 173 |
+
geometry, BF16 computation, and the same evaluation metric. Evaluate additional
|
| 174 |
+
held-out tokens separately, preserving the historical metric for comparison.
|
| 175 |
+
Combining those measurements with later 10–15B-token checkpoints would help
|
| 176 |
+
separate continued-learning gains from annealing gains and allow an actual
|
| 177 |
+
out-of-sample test of a schedule-aware model. This document recommends that
|
| 178 |
+
experiment; it does not launch it.
|
| 179 |
+
|
| 180 |
+
For planning, treat sub-2.0 loss as an unresolved target. **268B tokens might be
|
| 181 |
+
enough, and 1.07T provides much more training, but the present evidence does not
|
| 182 |
+
justify promising either budget will cross the threshold.**
|
| 183 |
+
|
| 184 |
+
## Reproduction
|
| 185 |
+
|
| 186 |
+
The input snapshot SHA256 is
|
| 187 |
+
`13f7fa766a59b884d376515132e5f41fe07b771438e7e66ed294dce030e9f38e`.
|
| 188 |
+
The analysis program is `ops/loss_curve_forecast.py`. It requires NumPy;
|
| 189 |
+
Matplotlib is optional for the figure. The generated JSON records dependency
|
| 190 |
+
versions and hashes of both the snapshot and analysis source. The isolated
|
| 191 |
+
`work/analysis-venv` keeps analysis dependencies separate from training.
|
| 192 |
+
This analysis used Python 3.12.3, NumPy 2.5.3, and Matplotlib 3.11.2.
|
| 193 |
+
|
| 194 |
+
From `/workspace/WarpState/rtx6000_40b`:
|
| 195 |
+
|
| 196 |
+
```bash
|
| 197 |
+
source ops/training-env.sh
|
| 198 |
+
python3.12 -m venv work/analysis-venv # Only when rebuilding an absent environment.
|
| 199 |
+
work/analysis-venv/bin/python -m pip install 'numpy==2.5.3' 'matplotlib==3.11.2'
|
| 200 |
+
work/analysis-venv/bin/python -B ops/loss_curve_forecast.py \
|
| 201 |
+
--input work/validation/loss-forecast-20260913/validation-snapshot.jsonl \
|
| 202 |
+
--output work/validation/loss-forecast-20260913/forecast.json \
|
| 203 |
+
--plot work/validation/loss-forecast-20260913/loss-scenarios.png
|
| 204 |
+
```
|
| 205 |
+
|
| 206 |
+
The same command with an `.svg` plot extension generates a vector figure.
|
| 207 |
+
Use a new output directory when analyzing a later frozen snapshot so that the
|
| 208 |
+
original forecast remains available for comparison with subsequent observations.
|
ops/MATMUL_PROFILE_20260912.md
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WarpState matrix multiplication profile, 2026-09-12
|
| 2 |
+
|
| 3 |
+
The largest dense BF16 products already sustain approximately **243–263 TFLOPS
|
| 4 |
+
per GPU** in the refreshed isolated test. Their observed clocks and 300 W board
|
| 5 |
+
power explain much of the gap from the boost-clock specification. The best next
|
| 6 |
+
experiments are selective algorithm/tile selection and small, measured GEMM
|
| 7 |
+
fusions. A global backend replacement has not demonstrated a useful improvement.
|
| 8 |
+
|
| 9 |
+
This analysis uses the refreshed production-configuration profile and new
|
| 10 |
+
isolated GEMM timings. Production remains paused at step 40,081 with its full
|
| 11 |
+
resume state preserved. All step-8,000, step-16,000, and step-40,000 archives are
|
| 12 |
+
unchanged. No matrix experiment in this report writes production checkpoints.
|
| 13 |
+
|
| 14 |
+
## Profile and attribution
|
| 15 |
+
|
| 16 |
+
The configuration is the two RTX PRO 6000 Blackwell Max-Q GPUs, microbatch 4 per
|
| 17 |
+
rank, sequence length 2,048, eight accumulation microbatches, 131,072 global
|
| 18 |
+
tokens/update, compiled cores in default mode, compiled full loss, BF16 residual
|
| 19 |
+
activations, and the existing FP32 fused AdamW state. The unprofiled 20-update
|
| 20 |
+
baseline is **30,799 tokens/s during optimizer updates**; the separate profiled
|
| 21 |
+
update took 4.1781 seconds after two warmups.
|
| 22 |
+
|
| 23 |
+
Each GPU kernel's `External id` is joined to its `aten::mm`, `aten::bmm`, or
|
| 24 |
+
`aten::addmm` CPU event. Dimensions, strides, and dtypes come from that event.
|
| 25 |
+
The square output projection's forward and input-gradient shapes are identical;
|
| 26 |
+
the RHS transpose stride distinguishes them. This avoids assigning the same
|
| 27 |
+
square GEMM to both directions.
|
| 28 |
+
|
| 29 |
+
For `A[M,K] @ B[K,N]`, arithmetic is counted as `2*M*N*K` FLOPs, multiplied by
|
| 30 |
+
batch and call count. Achieved TFLOPS divide these FLOPs by matched GPU kernel
|
| 31 |
+
time, not CPU launch time or the entire training update. CPU operator totals can
|
| 32 |
+
also include associated fills/copies; the tables below use kernel durations.
|
| 33 |
+
|
| 34 |
+
All **5,400 matrix kernels per rank** are accounted for:
|
| 35 |
+
|
| 36 |
+
| Profile | Rank 0 matrix time | Rank 1 matrix time | FLOPs per rank/update |
|
| 37 |
+
| --- | ---: | ---: | ---: |
|
| 38 |
+
| Current baseline | 3,166.449 ms | 3,049.219 ms | 791.597 trillion |
|
| 39 |
+
| Compiled-autograd candidate | 3,122.129 ms | 3,005.404 ms | 791.597 trillion |
|
| 40 |
+
|
| 41 |
+
Baseline matrix kernels represent 71.19% and 68.62% of summed GPU activity on
|
| 42 |
+
ranks 0 and 1. Streams overlap with communication and other work, so these
|
| 43 |
+
percentages are not partitions of elapsed update time. The matrix operations
|
| 44 |
+
themselves execute on the main compute stream in this trace. The compiled
|
| 45 |
+
autograd comparison changes surrounding execution and communication overlap;
|
| 46 |
+
its slightly shorter matrix times do not establish a faster full training run.
|
| 47 |
+
|
| 48 |
+
## Which products dominate
|
| 49 |
+
|
| 50 |
+
For a linear layer, forward computes `Y = X @ W.T`, the input derivative
|
| 51 |
+
computes `dX = dY @ W`, and the weight derivative computes `dW = dY.T @ X`.
|
| 52 |
+
Every core projection executes 224 times per direction: 28 cores times eight
|
| 53 |
+
microbatches. Backward therefore includes two dense products for each forward.
|
| 54 |
+
|
| 55 |
+
| Projection family | Rank 0 forward | Rank 0 dX | Rank 0 dW | Total | Share of matrix time |
|
| 56 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 57 |
+
| FFN up and gate, 2,048 → 14,336 | 405.939 ms | 433.643 ms | 427.218 ms | 1,266.800 ms | 40.01% |
|
| 58 |
+
| Q/K/V/gate/memory input, 2,048 → 10,240 | 284.852 ms | 312.126 ms | 305.255 ms | 902.232 ms | 28.49% |
|
| 59 |
+
| FFN down, 7,168 → 2,048 | 218.817 ms | 209.116 ms | 198.243 ms | 626.176 ms | 19.78% |
|
| 60 |
+
| Core output, 2,048 → 2,048 | 57.486 ms | 56.861 ms | 57.828 ms | 172.176 ms | 5.44% |
|
| 61 |
+
| Vocabulary head, 2,048 → 32,768 | 33.869 ms | 34.131 ms | 35.123 ms | 103.124 ms | 3.26% |
|
| 62 |
+
| State-memory batched products | — | — | — | 95.942 ms | 3.03% |
|
| 63 |
+
|
| 64 |
+
FFN up/gate, input projection, and FFN down together consume **88.28% of matrix
|
| 65 |
+
time**. Dense linear products perform 788.350 trillion FLOPs per rank/update;
|
| 66 |
+
state-memory batched products account for the remaining 3.247 trillion.
|
| 67 |
+
|
| 68 |
+
The vocabulary head now uses eight complete 8,192-token products per direction.
|
| 69 |
+
The September 10 profile used chunked head loss and repeated head forward
|
| 70 |
+
products during backward. Its larger forward-head count should not be reused
|
| 71 |
+
to estimate the current configuration.
|
| 72 |
+
|
| 73 |
+
The dominant shapes and their rank-0 kernel performance are:
|
| 74 |
+
|
| 75 |
+
| Product | M | N | K | Mean kernel time | Achieved TFLOPS |
|
| 76 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 77 |
+
| FFN up/gate forward | 8,192 | 14,336 | 2,048 | 1.8122 ms | 265.4 |
|
| 78 |
+
| FFN up/gate dX | 8,192 | 2,048 | 14,336 | 1.9359 ms | 248.5 |
|
| 79 |
+
| FFN up/gate dW | 14,336 | 2,048 | 8,192 | 1.9072 ms | 252.2 |
|
| 80 |
+
| Input projection forward | 8,192 | 10,240 | 2,048 | 1.2717 ms | 270.2 |
|
| 81 |
+
| Input projection dX | 8,192 | 2,048 | 10,240 | 1.3934 ms | 246.6 |
|
| 82 |
+
| Input projection dW | 10,240 | 2,048 | 8,192 | 1.3627 ms | 252.1 |
|
| 83 |
+
| FFN down forward | 8,192 | 2,048 | 7,168 | 0.9769 ms | 246.2 |
|
| 84 |
+
| FFN down dX | 8,192 | 7,168 | 2,048 | 0.9336 ms | 257.6 |
|
| 85 |
+
| FFN down dW | 2,048 | 7,168 | 8,192 | 0.8850 ms | 271.8 |
|
| 86 |
+
|
| 87 |
+
These are timings within the full-model profile. They can differ from isolated
|
| 88 |
+
tests because clocks, cache state, memory pressure, and communication differ.
|
| 89 |
+
The isolated tests below use longer warmup and repeated timing trials.
|
| 90 |
+
|
| 91 |
+
## What executes inside the dense kernels
|
| 92 |
+
|
| 93 |
+
Most dense kernels use symbols beginning
|
| 94 |
+
`cutlass_80_tensorop_bf16_s16816gemm_relu_bf16`. Forward commonly selects
|
| 95 |
+
`128x256_32x3_tn_align8`; dX uses `256x128_32x3_nn_align8`; dW uses
|
| 96 |
+
`128x256_32x3_nt_align8`. The head forward uses a `256x128` TN variant.
|
| 97 |
+
|
| 98 |
+
These are tiled tensor-core kernels: tiles load operands, reuse them while
|
| 99 |
+
advancing through K in pipelined stages, accumulate partial products, then store
|
| 100 |
+
the output. The symbols encode tile/pipeline/operand variants. The embedded
|
| 101 |
+
`80` and `relu` text does not establish obsolete hardware execution or an actual
|
| 102 |
+
ReLU in this model; semantic attribution comes from the originating operators.
|
| 103 |
+
|
| 104 |
+
Observed launches use 256 threads, approximately 218–232 registers per thread,
|
| 105 |
+
and 73,728 bytes of dynamic shared memory. Several have a grid Z dimension of
|
| 106 |
+
2 or 4. That is consistent with additional partitioned work, but launch metadata
|
| 107 |
+
alone does not prove the precise split-K/reduction implementation. A proposed
|
| 108 |
+
split-K experiment must first identify what the selected library algorithm
|
| 109 |
+
already does.
|
| 110 |
+
|
| 111 |
+
Profiler occupancy estimates report zero for some kernels that demonstrably
|
| 112 |
+
execute. They are not valid measured occupancy. The previous hardware-counter
|
| 113 |
+
attempt was denied by the driver's `ERR_NVGPUCTRPERM`; this analysis makes no
|
| 114 |
+
new occupancy, tensor-pipe-utilization, or bandwidth-counter claim.
|
| 115 |
+
|
| 116 |
+
The large dense shapes are multiples of tensor-core alignment and ordinary
|
| 117 |
+
tile sizes. Their transpose operands are views with unit stride in one
|
| 118 |
+
dimension, not evidence of a full physical transpose before every GEMM.
|
| 119 |
+
Blindly adding `.contiguous()` would create extra traffic; any alternative
|
| 120 |
+
layout must include its preparation cost in the benchmark.
|
| 121 |
+
|
| 122 |
+
## Where parallel execution already happens
|
| 123 |
+
|
| 124 |
+
Both GPUs already train concurrently using data parallelism. Each holds the
|
| 125 |
+
model and processes **65,536 of the 131,072 global tokens per update**: eight
|
| 126 |
+
8,192-token microbatches per rank. There is no intentional schedule that runs
|
| 127 |
+
GPU 0's training first and GPU 1's afterward. Gradient synchronization combines
|
| 128 |
+
their contributions to the same optimizer update.
|
| 129 |
+
|
| 130 |
+
The original compiled-autograd candidate lost overlap between computation and
|
| 131 |
+
communication **within each GPU's schedule**. That is different from serializing
|
| 132 |
+
the two GPUs: both ranks still compute concurrently, but local waits block
|
| 133 |
+
independent backward GEMMs while collectives finish. Recovering that overlap
|
| 134 |
+
is the immediate parallelization target identified by the trace.
|
| 135 |
+
|
| 136 |
+
The refreshed `nvidia-smi topo -m` reports the GPU-to-GPU path as `SYS`, meaning
|
| 137 |
+
PCIe plus an inter-NUMA CPU interconnect, with no NVLink entry. Reported CPU
|
| 138 |
+
affinities are `0-95,192-287` for GPU 0 and `96-191,288-383` for GPU 1. Topology
|
| 139 |
+
identifies the route; it does not measure achieved communication bandwidth.
|
| 140 |
+
No CPU/GPU affinity or power settings were changed for this analysis.
|
| 141 |
+
|
| 142 |
+
Some dX and dW products are mathematically independent and could use separate
|
| 143 |
+
CUDA streams. However, the large GEMMs already consume much of each GPU's
|
| 144 |
+
compute and power capacity; concurrent launches share those resources rather
|
| 145 |
+
than doubling them. A focused stream experiment could test thin or final-tail
|
| 146 |
+
products, including their synchronization cost, before changing the full backward
|
| 147 |
+
schedule. Model or pipeline parallelism would add activation transfers and
|
| 148 |
+
potential pipeline bubbles over this topology, with additional implementation
|
| 149 |
+
complexity. It is not an automatic improvement over the existing two-GPU data
|
| 150 |
+
parallel execution.
|
| 151 |
+
|
| 152 |
+
## Refreshed isolated performance and compute headroom
|
| 153 |
+
|
| 154 |
+
The existing `benchmark_gemm.py` tested all 12 core shape/direction pairs on
|
| 155 |
+
GPU 0 using deterministic BF16 operands with production transpose strides.
|
| 156 |
+
Each case had at least two seconds of burn-in and five CUDA-event timing trials
|
| 157 |
+
of 30 products. The existing BF16 intermediate-reduction allowance was preserved;
|
| 158 |
+
no lower-precision model or gradient representation was introduced.
|
| 159 |
+
|
| 160 |
+
NVIDIA specifies 438.9 dense BF16 TFLOPS with FP32 accumulation at a 2,280 MHz
|
| 161 |
+
boost clock, 300 W board power, and 1,792 GB/s memory bandwidth for this Max-Q
|
| 162 |
+
GPU. Source: Appendix A of the
|
| 163 |
+
[RTX Blackwell architecture whitepaper](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46).
|
| 164 |
+
|
| 165 |
+
The final column below divides achieved performance by
|
| 166 |
+
`438.9 * observed_SM_clock_MHz / 2280`. It is an illustrative clock-scaled
|
| 167 |
+
comparison, not a hardware-counter utilization measurement. Clocks are telemetry
|
| 168 |
+
snapshots after each timed case, not time-integrated clock measurements.
|
| 169 |
+
|
| 170 |
+
| Product | Median time | TFLOPS | Observed SM clock | Clock-scaled comparison |
|
| 171 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 172 |
+
| FFN up/gate forward | 1.8473 ms | 260.4 | 1,492 MHz | 90.7% |
|
| 173 |
+
| FFN up/gate dX | 1.9148 ms | 251.2 | 1,402 MHz | 93.1% |
|
| 174 |
+
| FFN up/gate dW | 1.8925 ms | 254.2 | 1,425 MHz | 92.7% |
|
| 175 |
+
| Input projection forward | 1.3337 ms | 257.6 | 1,485 MHz | 90.1% |
|
| 176 |
+
| Input projection dX | 1.3958 ms | 246.2 | 1,417 MHz | 90.2% |
|
| 177 |
+
| Input projection dW | 1.3531 ms | 253.9 | 1,447 MHz | 91.2% |
|
| 178 |
+
| FFN down forward | 0.9879 ms | 243.5 | 1,440 MHz | 87.8% |
|
| 179 |
+
| FFN down dX | 0.9162 ms | 262.5 | 1,530 MHz | 89.1% |
|
| 180 |
+
| FFN down dW | 0.9399 ms | 255.9 | 1,485 MHz | 89.5% |
|
| 181 |
+
| Core output forward | 0.2656 ms | 258.8 | 1,612 MHz | 83.4% |
|
| 182 |
+
| Core output dX | 0.2625 ms | 261.8 | 1,620 MHz | 83.9% |
|
| 183 |
+
| Core output dW | 0.2578 ms | 266.5 | 1,725 MHz | 80.3% |
|
| 184 |
+
|
| 185 |
+
Every timed case's final power sample was approximately 300 W. The dominant
|
| 186 |
+
families remain close to the earlier measured range and approximately 88–93%
|
| 187 |
+
of this clock-scaled estimate. The smaller output projection has more apparent
|
| 188 |
+
relative headroom, but represents only 5.44% of matrix time. Neither comparison
|
| 189 |
+
implies all remaining theoretical capacity can be recovered by software.
|
| 190 |
+
|
| 191 |
+
## Compute versus operand traffic
|
| 192 |
+
|
| 193 |
+
For one idealized BF16 read of A/B and one write of C, arithmetic intensity is
|
| 194 |
+
`2*M*N*K / (2*(M*K + K*N + M*N))`. It is about 1,470 FLOPs/byte for the FFN
|
| 195 |
+
up/gate products, 1,412 for the input projection, 1,334 for FFN down, and 910
|
| 196 |
+
for the square output projection. Actual traffic includes reloads, cache effects,
|
| 197 |
+
and possible intermediate reductions; these are operand-footprint estimates.
|
| 198 |
+
|
| 199 |
+
This supports a compute-efficiency focus for the large dense GEMMs. It does not
|
| 200 |
+
prove that every tile load is perfectly hidden. A persistent/TMA implementation
|
| 201 |
+
can improve operand delivery or scheduling even when the overall dense product
|
| 202 |
+
is predominantly compute-limited.
|
| 203 |
+
|
| 204 |
+
State-memory products are different. Shapes with batch 128 and `M=16, N=4096,
|
| 205 |
+
K=16` have only about 8 FLOPs/byte under that estimate; the `128x64x64` and
|
| 206 |
+
`64x64x128` batched products have about 25.6. Observed rates are roughly
|
| 207 |
+
10–61 TFLOPS, with 16x16/32x32 WMMA kernel tiles. They merit smaller-shape or
|
| 208 |
+
producer/consumer fusion experiments, but their entire current kernel budget
|
| 209 |
+
is only 95.94 ms, approximately 2.3% of the profiled update.
|
| 210 |
+
|
| 211 |
+
## Prior experiments and specific next tests
|
| 212 |
+
|
| 213 |
+
The September 10 experiments already covered eager cuBLAS, cuBLASLt preference,
|
| 214 |
+
ATEN-only compilation, default Triton autotuning, and persistent TMA. See
|
| 215 |
+
[`RTX_PROFILE_20260910.md`](RTX_PROFILE_20260910.md).
|
| 216 |
+
|
| 217 |
+
| FFN up/gate product | Prior cuBLAS | Prior default Triton | Prior persistent TMA | Current cuBLAS |
|
| 218 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 219 |
+
| Forward | 1.828 ms | 1.981 ms | 1.798 ms | 1.847 ms |
|
| 220 |
+
| dX | 1.885 ms | 2.057 ms | 1.890 ms | 1.915 ms |
|
| 221 |
+
| dW | 1.871 ms | 1.869 ms | 1.810 ms | 1.892 ms |
|
| 222 |
+
|
| 223 |
+
These are separate runs with different observed clocks, not a controlled
|
| 224 |
+
same-clock speedup claim. Default Triton usually lost to cuBLAS; persistent TMA
|
| 225 |
+
closed much of that gap and won some shapes modestly. cuBLASLt preference was
|
| 226 |
+
mixed, and its earlier up/gate dW result coincided with a higher clock. Repeating
|
| 227 |
+
a global backend toggle is therefore a weak next experiment.
|
| 228 |
+
|
| 229 |
+
The following targeted experiments have a better basis:
|
| 230 |
+
|
| 231 |
+
1. **Select algorithms per dominant shape/direction.** Retest persistent TMA
|
| 232 |
+
against ATEN/cuBLAS for up/gate forward and dW first, with alternating warmed
|
| 233 |
+
trials and clock samples. Keep cuBLAS for shapes where it wins. Measure an
|
| 234 |
+
entire layer forward/backward and then the full two-GPU update so template
|
| 235 |
+
fusion and DDP overlap are included. Preserve the BF16 dW rounding before
|
| 236 |
+
FP32 accumulation.
|
| 237 |
+
2. **Test a GEMM epilogue that performs gradient conversion and accumulation.**
|
| 238 |
+
The current conversion/addition budget is 129.076 ms/update. A fused kernel
|
| 239 |
+
must retain the BF16 GEMM result's rounding before its FP32 addition and keep
|
| 240 |
+
autograd/DDP ownership correct. Direct FP32-output `addmm` was already tested:
|
| 241 |
+
it changes rounding and was not consistently better. The present compiled
|
| 242 |
+
autograd candidate also demonstrates why local fusion needs full-update
|
| 243 |
+
communication validation.
|
| 244 |
+
3. **Explore a paired up/gate epilogue only if its GEMM remains competitive.**
|
| 245 |
+
The combined projection is already one GEMM, with up and gate columns 7,168
|
| 246 |
+
apart. A tile/layout that places matching pairs together could compute SwiGLU
|
| 247 |
+
before writing its activated output. Backward still needs the projected up
|
| 248 |
+
and gate values, so their storage cannot simply disappear. The removable
|
| 249 |
+
standalone forward budget is about 41.16 ms/update; recomputing the projection
|
| 250 |
+
would cost about 405.94 ms and would defeat that purpose. Repacking must use
|
| 251 |
+
temporary views/copies and retain canonical archive/checkpoint layout.
|
| 252 |
+
4. **Explore the small state-memory GEMMs separately.** Their low intensity and
|
| 253 |
+
small dimensions make tiled producer/consumer fusion more plausible than
|
| 254 |
+
expecting dense-projection TFLOPS. Account for both gradient products and
|
| 255 |
+
layout preparation; a low TFLOPS number on its own does not establish waste.
|
| 256 |
+
|
| 257 |
+
The installed compiler already has epilogue/prologue fusion enabled for suitable
|
| 258 |
+
templates. Its `kernel/mm.py` adds persistent TMA choices only through the
|
| 259 |
+
max-autotune template route, gated by `ENABLE_PERSISTENT_TMA_MATMUL`. Enabling
|
| 260 |
+
that variable alone does not replace every existing library GEMM.
|
| 261 |
+
|
| 262 |
+
The distinction between GPU architectures matters: this workstation is SM120.
|
| 263 |
+
The installed `use_triton_blackwell_tma_template()` additionally requires
|
| 264 |
+
`is_datacenter_blackwell_arch()`, whose predicate is `100 <= arch < 110`.
|
| 265 |
+
The named datacenter-Blackwell warp-specialized template and the CUTEDSL route
|
| 266 |
+
therefore are not automatic options for this RTX GPU; generic persistent TMA
|
| 267 |
+
is the relevant already-tested path. The environment also lacks the `cutlass`
|
| 268 |
+
and `cutlass_library` packages and the configured CUTLASS source directory, so
|
| 269 |
+
CUTLASS/NVGEMM are not available simply by changing the backend flag.
|
| 270 |
+
|
| 271 |
+
As a conditional scale check, retaining the measured 3.1664 seconds of matrix
|
| 272 |
+
time and deleting every other cost would give about **41.4K tokens/s**. This is
|
| 273 |
+
not a hardware limit or an achievable configuration. Reaching 50K with all
|
| 274 |
+
other costs zero would still require about 17.2% less matrix time. The isolated
|
| 275 |
+
results give no evidence for a software-only reduction of that scale across all
|
| 276 |
+
products at the present power limit.
|
| 277 |
+
|
| 278 |
+
## Evidence and reproduction
|
| 279 |
+
|
| 280 |
+
Generated evidence remains ignored in Git:
|
| 281 |
+
|
| 282 |
+
- `work/validation/rtx-opt-20260912/matmul/matmul-attribution.json`: both ranks,
|
| 283 |
+
both profiles, shapes/strides/dtypes, call counts, exact kernel names and launch
|
| 284 |
+
metadata, FLOPs, operand intensities, and current clock-scaled comparisons.
|
| 285 |
+
- `work/benchmark-opt-20260912/profile-baseline/`: source traces, kernel TSVs,
|
| 286 |
+
operator tables, and category summaries.
|
| 287 |
+
- `work/benchmark-opt-20260912/profile-compiled-autograd/`: corresponding
|
| 288 |
+
candidate traces. Include its eight BF16 `aten::addmm` head-dW calls when
|
| 289 |
+
counting arithmetic; otherwise the head's weight-gradient FLOPs are omitted.
|
| 290 |
+
- `work/benchmark-gemm-20260912-current/results.json`: refreshed isolated trials
|
| 291 |
+
with telemetry for each product.
|
| 292 |
+
- `work/benchmark-gemm-20260910/`, `work/benchmark-gemm-cublaslt-20260910/`, and
|
| 293 |
+
`work/benchmark-gemm-tma-20260910/`: earlier backend comparisons.
|
| 294 |
+
|
| 295 |
+
Installed compiler source evidence is under
|
| 296 |
+
`.server-venv/lib/python3.12/site-packages/torch/_inductor/`: `kernel/mm.py`,
|
| 297 |
+
`config.py`, `utils.py`, and `codegen/cuda/cuda_env.py`. These observations apply
|
| 298 |
+
to the installed `2.14.0+cu130` build and should be rechecked after an upgrade.
|
| 299 |
+
|
| 300 |
+
To repeat the isolated sweep in an exclusive GPU window, with a new output
|
| 301 |
+
directory:
|
| 302 |
+
|
| 303 |
+
```bash
|
| 304 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 305 |
+
source ops/training-env.sh
|
| 306 |
+
CUDA_VISIBLE_DEVICES=0 .server-venv/bin/python -B ops/benchmark_gemm.py \
|
| 307 |
+
--families ffn_up_gate in_proj ffn_down out_proj \
|
| 308 |
+
--operations fwd dx dw --rows 8192 --backends eager \
|
| 309 |
+
--burnin-seconds 2 --repeats 30 --trials 5 \
|
| 310 |
+
--output-dir work/benchmark-gemm-20260912-recheck
|
| 311 |
+
```
|
| 312 |
+
|
| 313 |
+
The full-model profiling command and checkpoint protections are documented in
|
| 314 |
+
[`OPTIMIZATION_20260912.md`](OPTIMIZATION_20260912.md). Compilation and benchmark
|
| 315 |
+
outputs must stay under the workspace, and a new experiment must not overwrite
|
| 316 |
+
the retained reports or archived weights.
|
ops/MATMUL_TUNING.md
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
WarpState's matrix experiments select BF16 kernels by operation, exact matrix
|
| 2 |
+
dimensions, and operand strides. They are opt-in through `--matmul-plan` in
|
| 3 |
+
`run_40b.py` and `ops/benchmark_training.py`. The workstation preset selects
|
| 4 |
+
compiled autograd with communication overlap. Matrix kernels remain explicit
|
| 5 |
+
experiments; isolated GEMM improvements alone do not select them for production.
|
| 6 |
+
The complete two-GPU comparisons and validation are recorded below.
|
| 7 |
+
|
| 8 |
+
The implementation is split across these source files:
|
| 9 |
+
|
| 10 |
+
| File | Responsibility |
|
| 11 |
+
| --- | --- |
|
| 12 |
+
| [matmul_kernels.py](matmul_kernels.py) | BF16 ATen, tiled, persistent, and persistent TMA matrix launches; tile validation and shape/stride signatures. |
|
| 13 |
+
| [matmul_epilogue.py](matmul_epilogue.py) | Paired up/gate matrix multiplication with a SwiGLU epilogue; returns both packed projection and activation. |
|
| 14 |
+
| [matmul_execution.py](matmul_execution.py) | Strict plan loading, registered autograd, and selected core-linear/SwiGLU training adapters. |
|
| 15 |
+
| [tune_matmul.py](tune_matmul.py) | Disposable ordinary/fused FFN matrix tuning with warmed controls and generated candidate recipes. |
|
| 16 |
+
| [activation_execution.py](activation_execution.py) | Existing BF16 SwiGLU derivative used by the fused epilogue's backward. |
|
| 17 |
+
|
| 18 |
+
The adapter is applied before core compilation and DDP construction. Original
|
| 19 |
+
FP32 parameters remain the autograd leaves. Their differentiable BF16 casts stay
|
| 20 |
+
inside the visible forward graph; the adapter does not retain external casts
|
| 21 |
+
across updates or write leaf gradients through hooks. Matrix operands, returned
|
| 22 |
+
matrix gradients, projections, and activations are BF16. Custom dot products
|
| 23 |
+
accumulate in FP32. Parameter-gradient accumulation, clipping, DDP reduction,
|
| 24 |
+
FP32 master parameters, and AdamW state remain under the existing training
|
| 25 |
+
framework. The token cursor, objective, optimizer schedule, and checkpoint keys
|
| 26 |
+
retain their existing meanings. CPU and evaluation model execution use the
|
| 27 |
+
original module paths.
|
| 28 |
+
|
| 29 |
+
Selected kernel launches use opaque registered custom operations with fake
|
| 30 |
+
output metadata, surrounded by the adapter's registered autograd. This is
|
| 31 |
+
necessary because installed PyTorch 2.14 compiled autograd cannot retrace AOT's
|
| 32 |
+
functionalized Triton launch operator. Unselected ATen products remain visible
|
| 33 |
+
to the compiler. The CUDA suite exercises this second backward capture as well
|
| 34 |
+
as ordinary eager/AOT execution; a plain forward compilation test would miss it.
|
| 35 |
+
|
| 36 |
+
Each JSON plan has format `warpstate-matmul-plan-v1`, an optional description,
|
| 37 |
+
and an `entries` array. For `A[M,K] @ B[K,N]`, `m`, `n`, and `k` describe that
|
| 38 |
+
operation; strides are in elements. They are essential: a transposed weight and
|
| 39 |
+
a contiguous matrix with the same dimensions can require different kernels.
|
| 40 |
+
The complete dispatch key is `(kind, m, n, k, a_stride, b_stride)`. Missing entries
|
| 41 |
+
use ATen. An empty plan selects no custom kernels. Duplicate JSON keys, duplicate
|
| 42 |
+
dispatch entries, unknown fields, invalid tiles, and non-BF16 entries are rejected.
|
| 43 |
+
|
| 44 |
+
This is a **candidate example**, not a selected production recipe:
|
| 45 |
+
|
| 46 |
+
```json
|
| 47 |
+
{
|
| 48 |
+
"format": "warpstate-matmul-plan-v1",
|
| 49 |
+
"description": "Candidate paired FFN epilogue; requires complete-model validation.",
|
| 50 |
+
"entries": [
|
| 51 |
+
{
|
| 52 |
+
"kind": "swiglu",
|
| 53 |
+
"name": "ffn_up_gate.fwd",
|
| 54 |
+
"m": 8192,
|
| 55 |
+
"n": 14336,
|
| 56 |
+
"k": 2048,
|
| 57 |
+
"a_stride": [2048, 1],
|
| 58 |
+
"b_stride": [1, 2048],
|
| 59 |
+
"dtype": "bfloat16",
|
| 60 |
+
"algorithm": "tiled",
|
| 61 |
+
"tile": {
|
| 62 |
+
"block_m": 128,
|
| 63 |
+
"block_n": 128,
|
| 64 |
+
"block_k": 32,
|
| 65 |
+
"num_warps": 8,
|
| 66 |
+
"num_stages": 3
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
]
|
| 70 |
+
}
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
`kind: "mm"` selects an ordinary matrix product. Forward and backward products
|
| 74 |
+
have independent entries. The FFN up/gate forward uses `(M,N,K) =
|
| 75 |
+
(8192,14336,2048)` with strides `[2048,1]` and `[1,2048]`. Its weight-gradient
|
| 76 |
+
product uses `(14336,2048,8192)` with strides `[1,14336]` and `[2048,1]`, matching
|
| 77 |
+
the original `dY.T @ X` orientation. A forward winner therefore does not select
|
| 78 |
+
that weight-gradient kernel automatically.
|
| 79 |
+
|
| 80 |
+
`kind: "swiglu"` selects the fused forward. Here `N=2H` is the complete packed
|
| 81 |
+
projection width, while `block_n` counts columns in **each** half. Backward uses
|
| 82 |
+
the saved BF16 packed projection, the existing SwiGLU derivative, and independently
|
| 83 |
+
selected `mm` entries for `dX` and `dW`.
|
| 84 |
+
|
| 85 |
+
| Algorithm | Execution | Supported plan kinds |
|
| 86 |
+
| --- | --- | --- |
|
| 87 |
+
| `aten` | Calls the installed PyTorch matrix implementation with its existing process settings. | `mm`, `swiglu` |
|
| 88 |
+
| `tiled` | Launches one program for each output tile, with grouped row tiles for weight reuse. | `mm`, `swiglu` |
|
| 89 |
+
| `persistent` | Launches at most one program per SM; each program strides over several output tiles using ordinary loads. | `mm`, `swiglu` |
|
| 90 |
+
| `persistent_tma` | Persistent scheduling with Triton tensor descriptors and hardware TMA operand loads. | `mm` |
|
| 91 |
+
|
| 92 |
+
Persistent scheduling alone does not imply TMA or warp specialization. The TMA
|
| 93 |
+
implementation uses an opaque registered custom-op boundary because the installed
|
| 94 |
+
PyTorch 2.14 AOT path does not preserve the host descriptor metadata through a
|
| 95 |
+
plain `triton_op` launch. It does not secretly substitute another algorithm.
|
| 96 |
+
TMA operands need an inner stride of one and 16-byte-aligned rows and storage
|
| 97 |
+
offsets. The training selector falls back to ATen for an incompatible layout;
|
| 98 |
+
the explicit kernel launcher reports unsupported requests. The paired SwiGLU
|
| 99 |
+
kernel does not currently support TMA.
|
| 100 |
+
|
| 101 |
+
Tiles specify `(block_m, block_n, block_k, num_warps, num_stages)`. Block sizes
|
| 102 |
+
must be powers of two from 16 through 256, with 4 or 8 warps and 1 through 4
|
| 103 |
+
pipeline stages. A conservative SM120 operand-pipeline estimate rejects ordinary
|
| 104 |
+
tiles above `(BM+BN)*BK*2*stages = 96 KiB` and paired tiles above
|
| 105 |
+
`(BM+2*BN)*BK*2*stages = 96 KiB`. This is an estimate, not a guarantee about
|
| 106 |
+
compiler register/shared-memory allocation. The tuner records compilation failures
|
| 107 |
+
without promoting those candidates.
|
| 108 |
+
|
| 109 |
+
The paired epilogue arranges adjacent logical columns as up/gate pairs, using
|
| 110 |
+
their physical packed offsets `j` and `j+H`. One wider tensor-core dot reuses the
|
| 111 |
+
input tile, then splits the accumulator in registers. Both FP32 projection halves
|
| 112 |
+
are rounded to BF16 **before** the activation. The kernel stores that rounded
|
| 113 |
+
packed projection for backward and also writes the BF16 activation. It saves a
|
| 114 |
+
subsequent packed-projection read and activation launch; it does not remove the
|
| 115 |
+
projection storage required by backward.
|
| 116 |
+
|
| 117 |
+
The `eager_rounding` argument additionally rounds the SiLU result to BF16 before
|
| 118 |
+
multiplication when following eager execution. Compiled execution follows the
|
| 119 |
+
existing compiler's fused SiLU/product arithmetic, while still rounding the GEMM
|
| 120 |
+
projection first. No FP8, FP16, quantization, split-K atomics, or reduced-precision
|
| 121 |
+
gradient communication is introduced. Exact equality on measured aligned shapes
|
| 122 |
+
does not promise bitwise equality for every GEMM layout or compiler version.
|
| 123 |
+
For example, the tested ragged `K=193` cuBLAS reference used a different reduction
|
| 124 |
+
result: its packed relative-L2 difference was 0.269%, while the custom result
|
| 125 |
+
differed by 0.000394% from an FP32-dot reference. Tests separately check projection
|
| 126 |
+
accuracy and that activation consumes the saved rounded BF16 projection.
|
| 127 |
+
|
| 128 |
+
On GPU 1 of the RTX PRO 6000 Blackwell Max-Q workstation, the final paired kernel
|
| 129 |
+
was compared with `torch.compile(fullgraph=True)` around the complete BF16
|
| 130 |
+
`mm -> split -> SiLU -> multiply` forward, returning both packed and activation
|
| 131 |
+
outputs. Inputs had shape `[8192,2048]` and transposed weights `[2048,14336]`.
|
| 132 |
+
With seed 563, 400 warmup calls per variant, and 15 alternating-order batches of
|
| 133 |
+
40 CUDA-event-timed calls, the selected microbenchmark tile measured:
|
| 134 |
+
|
| 135 |
+
| Forward implementation | Median milliseconds |
|
| 136 |
+
| --- | ---: |
|
| 137 |
+
| Compiled ATen matrix multiplication plus SwiGLU | 1.950789 |
|
| 138 |
+
| Paired single-dot epilogue, tiled `(128,128,32,8,3)` | 1.905561 |
|
| 139 |
+
|
| 140 |
+
This is **2.37% greater primitive throughput**, or approximately 45 microseconds
|
| 141 |
+
less per call; paired batch gains ranged from 2.08% to 2.75%. Packed projections
|
| 142 |
+
and activations matched the compiled reference bitwise for two production-size
|
| 143 |
+
random input seeds. The final epilogue suite passed 33 tests, covering both
|
| 144 |
+
schedules, eager/compiled rounding, ragged dimensions, and the default/best tiles.
|
| 145 |
+
A subsequent CPU-only invocation passed 9 and skipped 24 CUDA fixtures.
|
| 146 |
+
|
| 147 |
+
Applying the measured 45-microsecond difference to 28 cores and eight microbatches
|
| 148 |
+
gives about 10 milliseconds per update before interaction with backward,
|
| 149 |
+
communication, compiler scheduling, and allocator behavior. That arithmetic is
|
| 150 |
+
not a measured complete-training gain. The earlier two-dot implementation lost
|
| 151 |
+
to the control; it is superseded by the single-dot source.
|
| 152 |
+
|
| 153 |
+
Generated evidence remains under
|
| 154 |
+
`work/validation/rtx-opt-20260912/epilogue/`:
|
| 155 |
+
|
| 156 |
+
- `paired-swiglu-confirm.json`: final steady-state comparison and second input seed.
|
| 157 |
+
- `paired-swiglu-single-dot.json`: final single-dot implementation's tile/schedule screen.
|
| 158 |
+
- `paired-swiglu.json` and `paired-swiglu-more-tiles.json`: superseded two-dot screens.
|
| 159 |
+
|
| 160 |
+
The ordinary GEMM screen tested three algorithms and five tiles for each FFN
|
| 161 |
+
up/gate forward and weight-gradient shape: 30 candidates total. Every candidate
|
| 162 |
+
matched the ATen BF16 output exactly. All forward candidates lost. The selected
|
| 163 |
+
weight-gradient candidate used persistent TMA with tile `(256,128,32,8,3)`:
|
| 164 |
+
median candidate time 1.811495 ms and median paired throughput gain 3.39%.
|
| 165 |
+
Its weakest of three paired trials gained 1.30%. The first trial's two controls
|
| 166 |
+
drifted from 1.589741 to 1.765839 ms, so this short screen alone is insufficient
|
| 167 |
+
to select a production configuration. The complete generated sweep and candidate
|
| 168 |
+
recipe are in `work/benchmark-matmul-tiles-20260912/`.
|
| 169 |
+
|
| 170 |
+
To reproduce the ordinary FFN forward/weight-gradient tuning, reserve an idle
|
| 171 |
+
GPU first. From the repository root, the following command uses physical GPU 0
|
| 172 |
+
and keeps outputs and compiler caches inside the workspace. The output directory
|
| 173 |
+
must be a new `work/benchmark-*` directory; existing directories are rejected.
|
| 174 |
+
|
| 175 |
+
```bash
|
| 176 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 177 |
+
source ops/training-env.sh
|
| 178 |
+
CUDA_VISIBLE_DEVICES=0 .server-venv/bin/python -B ops/tune_matmul.py \
|
| 179 |
+
--kind mm --operations fwd dw --algorithms tiled persistent persistent_tma \
|
| 180 |
+
--rows 8192 --seed 1337 --warmup 10 --burnin-seconds 1 \
|
| 181 |
+
--repeats 30 --trials 3 --minimum-gain 0.01 \
|
| 182 |
+
--maximum-relative-rms 0.003 \
|
| 183 |
+
--output-dir work/benchmark-matmul-tuning-reproduction
|
| 184 |
+
```
|
| 185 |
+
|
| 186 |
+
To recheck the measured fused tile with the same dimensions, seed, and operand
|
| 187 |
+
scale, use the fused mode on an idle GPU:
|
| 188 |
+
|
| 189 |
+
```bash
|
| 190 |
+
CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B ops/tune_matmul.py \
|
| 191 |
+
--kind swiglu --operations fwd --algorithms tiled \
|
| 192 |
+
--tile 128,128,32,8,3 --rows 8192 --seed 563 --input-scale 0.1 \
|
| 193 |
+
--warmup 400 --burnin-seconds 1 --repeats 40 --trials 15 \
|
| 194 |
+
--minimum-gain 0.01 --maximum-relative-rms 0.003 \
|
| 195 |
+
--output-dir work/benchmark-matmul-swiglu-reproduction
|
| 196 |
+
```
|
| 197 |
+
|
| 198 |
+
This uses the tuner's bracketed control/candidate/control protocol. The historical
|
| 199 |
+
15-batch report used alternating pairs, so compare the resulting measurements
|
| 200 |
+
as independent checks rather than expecting identical timing samples. Fused
|
| 201 |
+
mode compiles the complete matrix-plus-SwiGLU reference and validates both packed
|
| 202 |
+
projection and activation separately. It supports only `--operations fwd` with
|
| 203 |
+
`tiled` or `persistent`. Omit `--tile` to use that mode's bounded tile list, or
|
| 204 |
+
repeat it to specify several candidates. `--input-scale` defaults to 1.0 for both
|
| 205 |
+
operands; the recorded epilogue comparisons used 0.1.
|
| 206 |
+
|
| 207 |
+
`--list-cases` prints the requested dimensions and strides without using CUDA.
|
| 208 |
+
The tuner generates deterministic BF16 random operands from the supplied seed;
|
| 209 |
+
it does not open a model, checkpoint, or token database. In this disposable process
|
| 210 |
+
it disables BF16 reduced-precision reduction and TF32 for the ATen reference.
|
| 211 |
+
Every timed candidate has preceding and following warmed controls. Eligibility
|
| 212 |
+
requires finite results, the configured relative RMS tolerance, and at least
|
| 213 |
+
the configured gain in **every** trial. Compilation failures remain in
|
| 214 |
+
`results.json`. The best eligible entry for each tested operation is written to
|
| 215 |
+
`candidate-policy.json`; an empty recipe means no candidate passed selection.
|
| 216 |
+
Results include source hashes, software/device metadata, input signatures,
|
| 217 |
+
timing triples, per-output numerical error, and GPU telemetry. Generated reports and
|
| 218 |
+
candidate recipes remain ignored under `work/`; a recipe selected after complete
|
| 219 |
+
validation should be copied into tracked `configs/` with its provenance.
|
| 220 |
+
|
| 221 |
+
Use the same checkpoint, input offsets, microbatch, loss/gradient modes, and
|
| 222 |
+
warmup/measurement counts to compare a candidate with a control. For example,
|
| 223 |
+
after reserving both workstation GPUs, this disposable command tests the generated
|
| 224 |
+
recipe against the retained full checkpoint:
|
| 225 |
+
|
| 226 |
+
```bash
|
| 227 |
+
CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/torchrun --standalone --nproc-per-node=2 \
|
| 228 |
+
ops/benchmark_training.py --checkpoint work/run/latest.pt \
|
| 229 |
+
--data work/data/train.bin --micro-batch 4 --seq-len 2048 \
|
| 230 |
+
--global-tokens-per-update 131072 --no-activation-checkpointing \
|
| 231 |
+
--compile-cores --compile-mode default --loss-mode compiled-full \
|
| 232 |
+
--bf16-residuals --gradient-mode compiled-autograd-overlap \
|
| 233 |
+
--optimizer-state resume --warmup-updates 8 --measured-updates 20 \
|
| 234 |
+
--matmul-plan work/benchmark-matmul-tuning-reproduction/candidate-policy.json \
|
| 235 |
+
--output-dir work/benchmark-matmul-training-candidate
|
| 236 |
+
```
|
| 237 |
+
|
| 238 |
+
Run a matching control with `--matmul-plan` omitted and a different new output
|
| 239 |
+
directory. Neither benchmark advances the production checkpoint. Run numerical
|
| 240 |
+
backward/optimizer and checkpoint/resume tests before selecting a training recipe.
|
| 241 |
+
The production runner accepts the same `--matmul-plan PATH` option alongside its
|
| 242 |
+
existing resume arguments; it is not an instruction to start a new run or change
|
| 243 |
+
the established global token batch. External core-weight cast caching and the
|
| 244 |
+
`reduce-overhead` CUDA-graph mode are explicitly incompatible with this adapter.
|
| 245 |
+
Standalone `--activation-mode swiglu|rmsnorm|all` kernels require eager gradient
|
| 246 |
+
execution; their transparent backward launches have the same compiled-autograd
|
| 247 |
+
limitation described above. Both CLIs reject that unsupported combination.
|
| 248 |
+
Use `--activation-mode baseline` with compiled autograd; matrix-plan fused
|
| 249 |
+
epilogues still work through their separate opaque boundaries.
|
| 250 |
+
Combining a matrix plan and compiled autograd also requires activation
|
| 251 |
+
checkpointing off. The installed compiler cannot trace the saved-tensor unpack
|
| 252 |
+
hook when the runner wraps an already compiled core in an eager checkpoint.
|
| 253 |
+
Recomputation captured inside AOT passes the small kernel tests, but does not
|
| 254 |
+
establish support for that different runner nesting.
|
| 255 |
+
|
| 256 |
+
Plan loading happens once, before execution. It computes a canonical recipe
|
| 257 |
+
SHA256 and implementation SHA256 values for `matmul_execution.py` and
|
| 258 |
+
`matmul_kernels.py`; a fused plan also includes `matmul_epilogue.py` and
|
| 259 |
+
`activation_execution.py`. The runner records this metadata in settings and the
|
| 260 |
+
checkpoint training signature, and the benchmark records it with its source
|
| 261 |
+
provenance. A changed recipe or implementation therefore changes the execution
|
| 262 |
+
signature. Legacy checkpoints normalize to no matrix plan. Resuming with an
|
| 263 |
+
execution change requires the existing explicit `--reconfigure` mechanism,
|
| 264 |
+
which still protects the data, context, global token batch, and optimizer schedule.
|
| 265 |
+
Reformatting JSON without changing its parsed canonical content leaves the recipe
|
| 266 |
+
identity unchanged; changing a description or an entry changes it.
|
| 267 |
+
|
| 268 |
+
The complete-model comparisons resume the same step-40,081 checkpoint into
|
| 269 |
+
disposable GPU model/AdamW copies. Each uses eight warmup updates and 20 timed
|
| 270 |
+
updates, BF16 residuals, compiled full loss, the same 131,072-token global batch,
|
| 271 |
+
and compiled autograd with delayed communication waits. They exclude compilation,
|
| 272 |
+
evaluation and checkpoint pauses. The generated rank logs and summaries are
|
| 273 |
+
under `work/benchmark-opt-20260912/`.
|
| 274 |
+
|
| 275 |
+
| Configuration | Timed tokens/s | Change from control | Reserved GiB/rank |
|
| 276 |
+
| --- | ---: | ---: | ---: |
|
| 277 |
+
| Communication-overlap control (`matmul-control`) | 31,022.34 | — | 66.98 |
|
| 278 |
+
| Persistent TMA dW only (`matmul-dw`) | 31,063.57 | +0.13% | 66.99 |
|
| 279 |
+
| Fused forward + TMA dW, initial boundary (`matmul-combined`) | 31,002.83 | -0.06% | 66.54 |
|
| 280 |
+
| Fused forward + TMA dW, shared weight buffer (`matmul-combined-native`) | 31,156.20 | +0.43% | 66.98 |
|
| 281 |
+
|
| 282 |
+
These first comparisons are effectively tied. Over all 28 updates, the initial
|
| 283 |
+
combined candidate's maximum loss difference from the fresh control was
|
| 284 |
+
0.00008535, and the maximum relative gradient-norm difference was 0.124%.
|
| 285 |
+
Small BF16 reduction/fusion differences do not imply training progress.
|
| 286 |
+
|
| 287 |
+
After the weight-layout correction, the candidate averaged 4.20693 seconds per
|
| 288 |
+
update versus 4.22508 for the control. Its maximum loss difference was 0.00010061
|
| 289 |
+
and its maximum relative gradient-norm difference was below 0.114%; all observed losses
|
| 290 |
+
and norms were finite. This single comparison shows a small +0.43% gain, which
|
| 291 |
+
has not been confirmed across repeated timing windows. The matrix recipe remains
|
| 292 |
+
opt-in. The resume launcher retains the repeatedly verified communication-overlap
|
| 293 |
+
configuration from [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md), which
|
| 294 |
+
measured a 1.53% improvement over eager gradients in the repeated comparison.
|
| 295 |
+
These percentages come from separate experiments and must not be added into a
|
| 296 |
+
claimed combined speedup. The generated numerical/timing comparison is
|
| 297 |
+
`work/validation/rtx-opt-20260912/matmul/full-model-comparison.json`.
|
| 298 |
+
|
| 299 |
+
Generated-code inspection verified the selected forward and dW routes, with
|
| 300 |
+
224 uses each per rank/update inferred from the compiled graphs and loop counts.
|
| 301 |
+
It also found an avoidable cost in the initial fused boundary: passing `W.T`
|
| 302 |
+
into an opaque operation made Inductor store two identical BF16 weight buffers,
|
| 303 |
+
while the control shared one between forward and backward. The final adapter
|
| 304 |
+
passes native-layout BF16 `W` and forms its transpose inside the launch. The
|
| 305 |
+
actual-size compiled core now shares one allocation and one cast with backward,
|
| 306 |
+
removing 58,720,256 extra bytes of stores per core call, or 13,153,337,344 bytes
|
| 307 |
+
per rank/update. This is generated-code evidence, not a measured bandwidth
|
| 308 |
+
counter or a guaranteed wall-time saving. The TMA dW route already reuses its
|
| 309 |
+
gradient buffer and required no corresponding change.
|
| 310 |
+
|
| 311 |
+
The tracked
|
| 312 |
+
[`matmul_rtx6000_bf16_experimental.json`](../configs/matmul_rtx6000_bf16_experimental.json)
|
| 313 |
+
contains both exact candidate shapes and tiles. The final native-weight boundary
|
| 314 |
+
comparison uses this recipe directly. Static dispatch and buffer evidence is
|
| 315 |
+
recorded in `work/validation/rtx-opt-20260912/matmul/compiled-dispatch-evidence.json`
|
| 316 |
+
and `dw-dispatch-evidence.json`. Production remains paused; these experiments
|
| 317 |
+
leave the step-40,081 checkpoint and archived weights unchanged.
|
| 318 |
+
|
| 319 |
+
The integration validation passed 208 CPU tests, with 83 CUDA fixtures skipped
|
| 320 |
+
and three existing compiler warnings. Separate reserved-GPU runs passed the
|
| 321 |
+
33-test epilogue suite and 35 adapter tests, including tiled, persistent, and
|
| 322 |
+
TMA forward/dX/dW, FP32 gradient accumulation and AdamW, ordinary AOT, and compiled
|
| 323 |
+
autograd. The adapter suite includes two alignment regressions: shifted inputs
|
| 324 |
+
use exact ATen fallback in eager mode and match a compiled ATen control when
|
| 325 |
+
compiled. Inductor can realign an input before the opaque boundary, making
|
| 326 |
+
TMA valid; the original caller's offset need not survive compilation.
|
| 327 |
+
|
| 328 |
+
A two-GPU runner test combined the fused persistent epilogue, persistent TMA
|
| 329 |
+
weight gradients, and delayed communication waits. Continuous execution matched
|
| 330 |
+
asynchronous save/stop/resume exactly for model weights, full FP32 AdamW state,
|
| 331 |
+
both ranks' RNG state, losses, and token cursors. Its final partial update also
|
| 332 |
+
covered a rank with entirely masked targets. The final implementation passed in
|
| 333 |
+
73.63 seconds; its log is
|
| 334 |
+
`work/validation/rtx-opt-20260912/pytest-matmul-resume-final.log`.
|
| 335 |
+
The CPU and alignment regression logs are `pytest-matmul-final-cpu.log` and
|
| 336 |
+
`pytest-offset-guard.log` in the same validation directory. These checks prove
|
| 337 |
+
continuation consistency within the selected execution mode, not bitwise
|
| 338 |
+
equivalence to a different BF16 reduction/fusion strategy.
|
| 339 |
+
The final CLI compatibility/signature checks passed 103 tests with eight GPU
|
| 340 |
+
fixtures skipped (`pytest-matmul-cli-final.log`). The complete package manifest
|
| 341 |
+
was verified after updating the runner's SHA256/byte count. After the final
|
| 342 |
+
benchmark, the production checkpoint's SHA256 still matched the paused record.
|
ops/OPTIMIZATION_20260912.md
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# RTX training optimization, 2026-09-12
|
| 2 |
+
|
| 3 |
+
The refreshed baseline delivers **30,799 tokens/s during optimizer updates** on
|
| 4 |
+
the two RTX PRO 6000 Blackwell Max-Q GPUs. The experiments below address
|
| 5 |
+
conversion and gradient accumulation, DDP communication, SwiGLU/RMSNorm, and
|
| 6 |
+
repeated core weight casts. The communication-overlap candidate beats the baseline in two full-model
|
| 7 |
+
comparisons. The workstation resume preset selects this mode; isolated kernel
|
| 8 |
+
improvements are not evidence of the same gain in complete training. See
|
| 9 |
+
[MATMUL_PROFILE_20260912.md](MATMUL_PROFILE_20260912.md) for the matrix breakdown.
|
| 10 |
+
|
| 11 |
+
## Protected training state
|
| 12 |
+
|
| 13 |
+
Production training was gracefully paused at step **40,081**, after
|
| 14 |
+
**5,253,496,832 tokens**. The next update is step 40,082. Its full resume checkpoint
|
| 15 |
+
is `work/run/latest.pt`, containing FP32 model parameters, AdamW moments, optimizer
|
| 16 |
+
step 40,081, and both ranks' RNG state. All inspected model and optimizer tensors
|
| 17 |
+
were finite. The model has 2,005,111,424 parameters.
|
| 18 |
+
|
| 19 |
+
The checkpoint is 24,061,689,997 bytes; its SHA256 at the pause was
|
| 20 |
+
`9745d93345dff7c389ac6fa65948dc78ec021c90f56b40e9caed8ddcd84e8c54`.
|
| 21 |
+
The generated verification record is
|
| 22 |
+
[`work/validation/rtx-opt-20260912/paused-checkpoint.json`](../work/validation/rtx-opt-20260912/paused-checkpoint.json).
|
| 23 |
+
|
| 24 |
+
The step-8,000, step-16,000, and step-40,000 archives under `work/archives/` are
|
| 25 |
+
immutable inputs. These experiments do not replace, re-export, or edit them.
|
| 26 |
+
The benchmark reads the production checkpoint through a read-only CPU mapping,
|
| 27 |
+
loads independent CUDA model/optimizer tensors, and writes no checkpoints.
|
| 28 |
+
Benchmark updates do not advance the production token cursor or RNG state.
|
| 29 |
+
|
| 30 |
+
Production remains paused while disposable experiments run. A later production
|
| 31 |
+
resume must use the retained full checkpoint and record any selected execution
|
| 32 |
+
changes through the runner's explicit reconfiguration mechanism.
|
| 33 |
+
|
| 34 |
+
## Baseline and profile method
|
| 35 |
+
|
| 36 |
+
The baseline resumes the full AdamW state from step 40,081 and uses deterministic
|
| 37 |
+
training-token offsets. It measures 20 updates after three warmup updates:
|
| 38 |
+
|
| 39 |
+
| Setting | Baseline |
|
| 40 |
+
| --- | --- |
|
| 41 |
+
| PyTorch / CUDA | `2.14.0+cu130` / `13.0` |
|
| 42 |
+
| Devices | Two RTX PRO 6000 Blackwell Max-Q GPUs |
|
| 43 |
+
| Sequence length / per-rank microbatch | 2,048 / 4 |
|
| 44 |
+
| Accumulation / global tokens per update | 8 / 131,072 |
|
| 45 |
+
| Core execution | `torch.compile`, default mode |
|
| 46 |
+
| Loss / residual activations | Compiled full-vocabulary loss / BF16 |
|
| 47 |
+
| Activation checkpointing | Off |
|
| 48 |
+
| DDP | FP32 gradients, 128 MiB buckets, bucket views |
|
| 49 |
+
| Optimizer | Fused AdamW, original FP32 state |
|
| 50 |
+
| Measured throughput | 30,799.17 tokens/s |
|
| 51 |
+
| Peak allocated / reserved memory, rank 0 | 65.49 / 66.44 GiB |
|
| 52 |
+
|
| 53 |
+
This throughput includes gradient zeroing, forward/backward, accumulation,
|
| 54 |
+
gradient clipping, and optimizer updates. It excludes periodic production
|
| 55 |
+
evaluation/checkpoint pauses and initial compilation. It must not be presented
|
| 56 |
+
as sustained wall-clock throughput over a long training run.
|
| 57 |
+
|
| 58 |
+
The separate profile records one complete update after two warmups. That
|
| 59 |
+
instrumented update took **4.1781 seconds**. Use the unprofiled 20-update baseline
|
| 60 |
+
for speed comparisons; the shorter profiled run has a different warmup history
|
| 61 |
+
and instrumentation overhead.
|
| 62 |
+
|
| 63 |
+
| GPU activity | Rank 0 summed milliseconds | Rank 1 summed milliseconds |
|
| 64 |
+
| --- | ---: | ---: |
|
| 65 |
+
| Matrix multiplication | 3,166.45 | 3,049.22 |
|
| 66 |
+
| Compiled elementwise operations and reductions | 512.12 | 515.05 |
|
| 67 |
+
| NCCL communication | 422.56 | 532.72 |
|
| 68 |
+
| Other tensor kernels | 206.16 | 206.27 |
|
| 69 |
+
| Local attention | 99.41 | 98.82 |
|
| 70 |
+
| Fused AdamW | 38.50 | 38.46 |
|
| 71 |
+
| Explicit copies and fills | 2.96 | 3.08 |
|
| 72 |
+
|
| 73 |
+
Matrix multiplication accounts for 71.19% and 68.62% of summed GPU activity.
|
| 74 |
+
The totals exceed wall time because streams overlap. Rank 0's GPU activity union
|
| 75 |
+
is 4.1257 seconds; its summed durations are 4.4482 seconds. Name-based kernel
|
| 76 |
+
categories are attribution aids, not hardware utilization or bandwidth counters.
|
| 77 |
+
Operator and kernel tables describe the same work and must not be added together.
|
| 78 |
+
|
| 79 |
+
The explicit copy category excludes memory traffic inside arithmetic kernels.
|
| 80 |
+
It therefore does not measure total VRAM bandwidth consumption. This profile
|
| 81 |
+
does not establish a hardware bandwidth ceiling.
|
| 82 |
+
|
| 83 |
+
## Work outside matrix multiplication
|
| 84 |
+
|
| 85 |
+
The following rank-0 costs are subsets of the preceding categories, measured
|
| 86 |
+
over all eight accumulation microbatches and 28 cores:
|
| 87 |
+
|
| 88 |
+
| Candidate work | Summed milliseconds per update |
|
| 89 |
+
| --- | ---: |
|
| 90 |
+
| SwiGLU forward / backward | 41.156 / 87.166 |
|
| 91 |
+
| Core gradient BF16-to-FP32 conversion / FP32 addition | 39.749 / 89.327 |
|
| 92 |
+
| Repeated core weight casts | 54.006 |
|
| 93 |
+
| Normalization forward / backward | 12.087 / 50.377 |
|
| 94 |
+
| Two groups of backward attention/projection transforms | 82.969 / 32.730 |
|
| 95 |
+
| BF16 attention layout copy | 14.036 |
|
| 96 |
+
| DDP parameter/bucket scaling, 256 calls | Approximately 7.884 |
|
| 97 |
+
|
| 98 |
+
The conversion-plus-accumulation budget is **129.076 ms/update**. Its attribution
|
| 99 |
+
uses CUDA external IDs mapped to CPU operator input dimensions to distinguish
|
| 100 |
+
parameter gradients from activation gradients. Weight casts are the three
|
| 101 |
+
compiled transpose/copy families with durations 16.138, 14.107, and 23.761 ms.
|
| 102 |
+
|
| 103 |
+
NCCL overlaps substantial computation. Intersecting communication intervals with
|
| 104 |
+
the union of other GPU activity leaves approximately **100.05 ms on rank 0** and
|
| 105 |
+
**215.80 ms on rank 1** containing only communication. These exposed intervals
|
| 106 |
+
are not a guaranteed speedup: moving bucket boundaries can alter overlap and
|
| 107 |
+
contention elsewhere in the update.
|
| 108 |
+
|
| 109 |
+
The current compiler already fuses many operations. A custom normalization
|
| 110 |
+
kernel may save local kernel time while preventing a larger surrounding fusion.
|
| 111 |
+
The candidate budgets therefore cannot be summed into a projected speedup.
|
| 112 |
+
|
| 113 |
+
## Implemented execution options
|
| 114 |
+
|
| 115 |
+
The runner and disposable benchmark expose the following options. Defaults
|
| 116 |
+
retain the prior execution, and the runner's resume signature includes the new
|
| 117 |
+
settings. Legacy checkpoints normalize to these defaults; a selected change
|
| 118 |
+
requires explicit resume reconfiguration while preserving the objective, global
|
| 119 |
+
token batch, training cursor, optimizer, and schedule.
|
| 120 |
+
|
| 121 |
+
| Option | Default | Implementation and scope |
|
| 122 |
+
| --- | --- | --- |
|
| 123 |
+
| `--gradient-mode eager\|compiled-autograd\|compiled-autograd-overlap` | `eager` | Captures backward through leaf accumulation so Inductor can fuse BF16 gradient conversion with FP32 addition. |
|
| 124 |
+
| `--activation-mode baseline\|swiglu\|rmsnorm\|all` | `baseline` | Optional Triton activation/normalization forwards and registered autograd backwards. |
|
| 125 |
+
| `--cache-core-weight-casts` | Off | Reuses differentiable BF16 core linear-weight views for one accumulation update. |
|
| 126 |
+
| `--ddp-bucket-cap-mb` | `128` | Changes communication grouping while retaining FP32 reduction. |
|
| 127 |
+
| `--ddp-batched-grad-copy` | Off | Batches gradient copying/scaling in the C++ DDP reducer. |
|
| 128 |
+
|
| 129 |
+
Compiled autograd uses the Python DDP reducer and cannot be combined with the
|
| 130 |
+
C++ reducer's batched gradient copy option. The installed PyTorch can mishandle
|
| 131 |
+
a saved CPU scalar in compiled CUDA backward; `scale_loss()` materializes that
|
| 132 |
+
same scaling coefficient on the loss device for this mode. Unsupported requested
|
| 133 |
+
DDP options fail explicitly. The helper also supports capacity lists for
|
| 134 |
+
controlled programmatic experiments; the current CLI exposes a uniform capacity.
|
| 135 |
+
|
| 136 |
+
The weight cast cache creates fresh views at the start of each optimizer update
|
| 137 |
+
and clears them on normal or exceptional exit, before clipping/optimizer,
|
| 138 |
+
evaluation, or checkpoint operations. FP32 master identities and version counters
|
| 139 |
+
are checked. Its current scope requires one use of each core per forward,
|
| 140 |
+
activation checkpointing off, and unaliased trainable linear weights. It does not
|
| 141 |
+
cache the tied embedding/output weight. Extra BF16 views consume VRAM; a complete
|
| 142 |
+
model measurement must establish their memory cost and benefit.
|
| 143 |
+
|
| 144 |
+
These options retain BF16 tensor-core computation and BF16 activations. FP32
|
| 145 |
+
master parameters, parameter gradients, normalization reductions, DDP gradient
|
| 146 |
+
communication, and AdamW moments retain their original representation. No FP8,
|
| 147 |
+
FP16, quantized weights, or reduced-precision gradient communication is introduced.
|
| 148 |
+
Canonical checkpoint keys and the default CPU chat/evaluation paths are unchanged.
|
| 149 |
+
|
| 150 |
+
## Numerical checks and isolated measurements
|
| 151 |
+
|
| 152 |
+
The activation suite passed **22 tests**, including eager and compiled CUDA
|
| 153 |
+
forward/backward comparisons, BF16 residual integration, FP32 AdamW, canonical
|
| 154 |
+
state reload, and unchanged CPU/evaluation behavior. Its final log is
|
| 155 |
+
`work/validation/rtx-opt-20260912/activation-tests-final.log`.
|
| 156 |
+
|
| 157 |
+
The activation implementation distinguishes eager rounding from current compiled
|
| 158 |
+
arithmetic. Inductor can eliminate intermediate BF16 cast pairs; the compiled
|
| 159 |
+
variant follows that behavior instead of forcing additional eager truncations.
|
| 160 |
+
The comparison does not promise bitwise equality for every reduction shape or
|
| 161 |
+
future compiler configuration.
|
| 162 |
+
|
| 163 |
+
On GPU 1, five timing trials of 20 iterations after five warmup iterations used
|
| 164 |
+
the actual activation shapes, seed 951, BF16 random upstream gradients, and
|
| 165 |
+
compiled forward plus backward. Compilation used `fullgraph=True, dynamic=False`;
|
| 166 |
+
CUDA events timed each trial:
|
| 167 |
+
|
| 168 |
+
| Primitive | Compiled baseline | Custom | Numerical comparison |
|
| 169 |
+
| --- | ---: | ---: | --- |
|
| 170 |
+
| SwiGLU, packed `8192 x 14336` | 0.61335 ms | 0.59965 ms | Outputs and input gradients bitwise equal in this fixture. |
|
| 171 |
+
| RMSNorm, `8192 x 2048` | 0.43256 ms | 0.39248 ms | Relative RMS error: output `1.64e-6`, input gradient `1.17e-5`, FP32 weight gradient `0.001604`. |
|
| 172 |
+
|
| 173 |
+
These are approximately 2.23% and 9.27% primitive improvements, not complete
|
| 174 |
+
training gains. RMSNorm's large-shape baseline can retain a final BF16 rounding
|
| 175 |
+
at a compiler reduction boundary; the custom compiled reduction retains FP32.
|
| 176 |
+
That explains the approximately 0.16% weight-gradient discrepancy and requires
|
| 177 |
+
whole-model numerical screening before selection.
|
| 178 |
+
|
| 179 |
+
A separate large-linear gradient fixture verifies that compiled autograd can
|
| 180 |
+
replace repeated conversion/addition work with fused kernels while preserving
|
| 181 |
+
the accumulated gradient exactly in that fixture. Its measured cycle was
|
| 182 |
+
44.668 ms eager versus 41.013 ms compiled autograd. The fixture is not a complete
|
| 183 |
+
training update. Small-model comparisons show some changed BF16 rounding in
|
| 184 |
+
embedding, depth modulation, and final normalization, so exact linear gradients
|
| 185 |
+
do not establish bitwise equality for the whole model.
|
| 186 |
+
|
| 187 |
+
DDP tests exercise repeated eight-microbatch accumulation and optimizer updates,
|
| 188 |
+
including multiple buckets. They preserve FP32 reduction and optimizer state.
|
| 189 |
+
Full-model two-GPU tests remain the criterion for selecting an execution change.
|
| 190 |
+
|
| 191 |
+
## Whole-model candidate results
|
| 192 |
+
|
| 193 |
+
All screens start from the same step-40,081 full checkpoint and token cursor,
|
| 194 |
+
with three warmups and ten measured updates. The initial baseline measures 20
|
| 195 |
+
updates. The benchmark holds the checkpoint's learning rate fixed for every
|
| 196 |
+
candidate; production continues its existing schedule. The measurements below
|
| 197 |
+
exclude compilation, evaluation and saving.
|
| 198 |
+
|
| 199 |
+
| Candidate / artifact directory | Tokens/s | Peak reserved/GPU | Maximum absolute loss difference from baseline |
|
| 200 |
+
| --- | ---: | ---: | ---: |
|
| 201 |
+
| Baseline, 20 updates (`baseline`) | 30,799.17 | 66.44 GiB | 0 |
|
| 202 |
+
| Batched DDP copies, 64 MiB (`ddp-copy64`) | 30,896.39 | 67.42 GiB | 0 |
|
| 203 |
+
| Compiled autograd (`compiled-autograd`) | 30,498.20 | 66.45 GiB | 0.00009394 |
|
| 204 |
+
| Cached core casts (`cache`) | 29,956.94 | 66.44 GiB | 0.00008345 |
|
| 205 |
+
| Custom SwiGLU (`swiglu`) | 30,749.24 | 66.44 GiB | 0 |
|
| 206 |
+
| Custom RMSNorm (`rmsnorm`) | 30,737.22 | 66.14 GiB | 0.00012708 |
|
| 207 |
+
| Compiled autograd with wait sinking (`ca-overlap`) | **31,331.80** | 66.98 GiB | 0.00016069 |
|
| 208 |
+
|
| 209 |
+
A second pair of complete, unprofiled **20-update** measurements confirms the
|
| 210 |
+
communication fix: `ca-overlap-confirm` reaches **31,045.42 tokens/s**, versus
|
| 211 |
+
**30,578.17 tokens/s** for the subsequent `baseline-confirm` (**1.53% faster**).
|
| 212 |
+
The initial ten-update screen was 1.73% above its baseline. These are modest,
|
| 213 |
+
repeatable gains, not a claim of 50K throughput or long-run model quality.
|
| 214 |
+
|
| 215 |
+
Custom activation/normalization kernels and cached casts remain opt-in
|
| 216 |
+
experiments. No complete-model gain justifies selecting them in this preset.
|
| 217 |
+
The cache's BF16 views also stop DDP's compiler partitioner from counting those
|
| 218 |
+
inputs as `nn.Parameter`s. Generated layouts match baseline, but whole-core
|
| 219 |
+
backward regions can delay leaf-gradient readiness and change fusion boundaries.
|
| 220 |
+
Do not attribute that regression to a confirmed matrix-transpose problem.
|
| 221 |
+
|
| 222 |
+
### Why fusion alone lost, and the communication fix
|
| 223 |
+
|
| 224 |
+
Compiled autograd removes nearly all native gradient-add kernels. Rank 0's
|
| 225 |
+
other-kernel category falls from 206.17 to 51.36 ms while compiled elementwise
|
| 226 |
+
work rises from 512.12 to 585.32 ms, saving roughly 82 ms across those categories.
|
| 227 |
+
However, its default schedule puts each wait and gradient copy immediately after
|
| 228 |
+
the collective, before independent backward GEMMs. The 57 gradient collectives
|
| 229 |
+
carry the same 8,020,445,696 FP32 bytes.
|
| 230 |
+
|
| 231 |
+
| Communication time/update | Baseline | Compiled autograd | With wait sinking |
|
| 232 |
+
| --- | ---: | ---: | ---: |
|
| 233 |
+
| Overlapping compute, rank 0 | 322.51 ms | 0 ms | **190.74 ms** |
|
| 234 |
+
| Overlapping compute, rank 1 | 316.91 ms | 0 ms | **190.58 ms** |
|
| 235 |
+
| Communication only, rank 0 | 100.05 ms | 285.17 ms | **166.64 ms** |
|
| 236 |
+
| Communication only, rank 1 | 215.80 ms | 407.35 ms | **267.61 ms** |
|
| 237 |
+
|
| 238 |
+
`--gradient-mode compiled-autograd-overlap` enables Inductor's
|
| 239 |
+
`sink_waits_iterative` scheduler. It moves waits together with their dependent
|
| 240 |
+
copies past independent computation, retaining collective launch order and
|
| 241 |
+
requiring no increase in the scheduler's **estimated** peak memory. It does not
|
| 242 |
+
permit unsafe collective reordering or moves past subsequent collectives.
|
| 243 |
+
Actual allocator peaks remain measured separately.
|
| 244 |
+
|
| 245 |
+
The two CA profiles contain identical kernel counts: 5,400 matmuls, 59
|
| 246 |
+
communication events, 9,577 compiled pointwise/reduction kernels and 1,531 other
|
| 247 |
+
kernels per rank. Profiled update time changes from 4.25901 to 4.12683 seconds;
|
| 248 |
+
the gain is restored overlap, not less arithmetic. Compared with baseline,
|
| 249 |
+
52–67 ms more communication remains exposed. That is potential scheduling work,
|
| 250 |
+
not a guaranteed further saving.
|
| 251 |
+
|
| 252 |
+
Cold first-update compilation took approximately 195–196 seconds in these
|
| 253 |
+
full-model CA screens. A cached second process still incurs graph capture and
|
| 254 |
+
compiler setup; restarts can spend minutes before regular updates appear.
|
| 255 |
+
|
| 256 |
+
Whole-backward compilation can fuse a tied-head BF16 matrix product and gradient
|
| 257 |
+
addition into BF16 `addmm`, changing an intermediate rounding boundary. Original
|
| 258 |
+
FP32 master weights, gradients, AdamW state and communication remain FP32; GEMM
|
| 259 |
+
operands and outputs remain BF16. Short loss/gradient comparisons use explicit
|
| 260 |
+
numerical tolerances rather than claiming a bitwise baseline trajectory.
|
| 261 |
+
|
| 262 |
+
The CLI rejects weight cast caching with either compiled-autograd mode, and
|
| 263 |
+
rejects either feature with `--compile-mode reduce-overhead`. Cache plus
|
| 264 |
+
activation checkpointing is rejected before loading; it requires
|
| 265 |
+
`--no-activation-checkpointing`. Batched DDP copying uses the C++ reducer and is
|
| 266 |
+
incompatible with both compiled-autograd modes' Python reducer.
|
| 267 |
+
|
| 268 |
+
### Validation and selected preset
|
| 269 |
+
|
| 270 |
+
The completed CPU suite passed **159 tests** (28 CUDA tests skipped in that
|
| 271 |
+
CPU-only invocation). Separate activation, gradient and cast-cache CUDA checks
|
| 272 |
+
passed. The final two-GPU runner suite passed **both** continuation cases in
|
| 273 |
+
136.79 seconds: combined custom activations/cached casts/batched DDP copies, and
|
| 274 |
+
compiled autograd with communication overlap. Each compares uninterrupted
|
| 275 |
+
training against an asynchronous stop/resume, including an entirely masked
|
| 276 |
+
final rank. Model tensors, full FP32 optimizer state and both ranks' RNG states
|
| 277 |
+
match **bitwise** between continuous and resumed execution of the same mode.
|
| 278 |
+
This is separate from the bounded numerical differences versus baseline.
|
| 279 |
+
|
| 280 |
+
`train-local-resume.sh` selects `--gradient-mode compiled-autograd-overlap`.
|
| 281 |
+
Activation kernels, external weight-cast caching and batched DDP copies remain
|
| 282 |
+
at their baseline settings. BF16 compute, FP32 state, the global token batch,
|
| 283 |
+
context, data, optimizer schedule and save/evaluation intervals are preserved.
|
| 284 |
+
Production remains paused at step 40,081 until resumption is requested.
|
| 285 |
+
|
| 286 |
+
Follow-up scheduling controls and paired measurements are documented in
|
| 287 |
+
[COMMUNICATION_EXPLORATION_20260912.md](COMMUNICATION_EXPLORATION_20260912.md).
|
| 288 |
+
The controls are opt-in; the preset above keeps its conservative defaults.
|
| 289 |
+
The separate [backward layout experiment](BACKWARD_LAYOUT_EXPERIMENT.md)
|
| 290 |
+
measures a fused rotary-gradient and projection-packing region without changing
|
| 291 |
+
production model execution.
|
| 292 |
+
|
| 293 |
+
Test logs are `work/validation/rtx-opt-20260912/pytest-communication-final.log`
|
| 294 |
+
and `work/validation/rtx-opt-20260912/pytest-optimized-resume.log`.
|
| 295 |
+
|
| 296 |
+
## Reproducing and locating results
|
| 297 |
+
|
| 298 |
+
All paths above are relative to `rtx6000_40b/`. Source the workspace environment
|
| 299 |
+
before any test or benchmark. GPU experiments require training paused and an
|
| 300 |
+
exclusive device window. Run each candidate in a fresh process and a new output
|
| 301 |
+
directory because compiler/reducer settings are process-wide.
|
| 302 |
+
|
| 303 |
+
The baseline can be reproduced with:
|
| 304 |
+
|
| 305 |
+
```bash
|
| 306 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 307 |
+
source ops/training-env.sh
|
| 308 |
+
.server-venv/bin/torchrun --standalone --nproc-per-node=2 \
|
| 309 |
+
ops/benchmark_training.py \
|
| 310 |
+
--micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 311 |
+
--no-activation-checkpointing --autocast-cache \
|
| 312 |
+
--compile-cores --compile-mode default --loss-mode compiled-full \
|
| 313 |
+
--bf16-residuals --optimizer-state resume --vram-limit-gib 88 \
|
| 314 |
+
--warmup-updates 3 --measured-updates 20 \
|
| 315 |
+
--output-dir work/benchmark-opt-20260912/baseline-recheck
|
| 316 |
+
```
|
| 317 |
+
|
| 318 |
+
Use a new output directory and append the candidate flags for each comparison.
|
| 319 |
+
For the profile, use two warmup updates, one measured update, and `--profile`.
|
| 320 |
+
The original run artifacts are:
|
| 321 |
+
|
| 322 |
+
- `work/benchmark-opt-20260912/baseline/summary.json`: configuration, per-rank
|
| 323 |
+
update measurements, source hashes, checkpoint identity, and memory peaks.
|
| 324 |
+
- `work/benchmark-opt-20260912/profile-baseline/`: both ranks' Chrome traces,
|
| 325 |
+
operator tables, kernel TSV files, and summary/category JSON.
|
| 326 |
+
- `work/validation/rtx-opt-20260912/activation/actual-shape-compiled-arithmetic.json`:
|
| 327 |
+
final primitive timings and random-gradient differences. The earlier
|
| 328 |
+
`actual-shape-kernels.json` tested eager rounding against compiled math and is
|
| 329 |
+
superseded for comparing the current implementation.
|
| 330 |
+
- `work/validation/rtx-opt-20260912/gradient/cuda-linear-proof.json`: isolated
|
| 331 |
+
accumulation timing, gradient equality, and kernel trace counts.
|
| 332 |
+
- `work/validation/rtx-opt-20260912/gradient/cuda-gradient-deltas.json`: initial
|
| 333 |
+
small-model per-parameter differences for compiled autograd.
|
| 334 |
+
|
| 335 |
+
The source implementations are `activation_execution.py`,
|
| 336 |
+
`gradient_execution.py`, `weight_cast_execution.py`, and `distributed_execution.py`
|
| 337 |
+
in this directory. Their focused tests live at the package root. Generated
|
| 338 |
+
benchmarks, traces, validation outputs, data, and checkpoints remain ignored in
|
| 339 |
+
Git; this document records methods and findings for source tracking.
|
| 340 |
+
|
| 341 |
+
The subsequent paired attention-backward integration and separate gradient
|
| 342 |
+
producer-scheduling experiment are documented in
|
| 343 |
+
[BACKWARD_LAYOUT_INTEGRATION.md](BACKWARD_LAYOUT_INTEGRATION.md), with gradient
|
| 344 |
+
and checkpoint-continuation checks in
|
| 345 |
+
[BACKWARD_LAYOUT_TRAINING_VALIDATION.md](BACKWARD_LAYOUT_TRAINING_VALIDATION.md).
|
| 346 |
+
The completed integration selects `--backward-layout paired` in
|
| 347 |
+
`train-local-resume.sh`: repeated update throughput improves from 31,001.57 to
|
| 348 |
+
31,199.03 tokens/second (+0.637%). The separate producer scheduler remains
|
| 349 |
+
disabled after a smaller, less consistent gain. Production remains paused.
|
ops/OPTIMIZED_TRAINING.md
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training with both GPUs available
|
| 2 |
+
|
| 3 |
+
**Currently paused at the user's request.** Training and benchmarks are
|
| 4 |
+
stopped so another session can resume VORTEX. The latest full checkpoint is
|
| 5 |
+
step 448 / 58,720,256 tokens. Start commands below are for a future
|
| 6 |
+
user-requested resume after checking GPU availability.
|
| 7 |
+
|
| 8 |
+
The selected configuration measured **29,490 training tokens/second** on two
|
| 9 |
+
NVIDIA RTX PRO 6000 Blackwell Max-Q GPUs, with approximately **68.8 GiB reserved
|
| 10 |
+
and 25 GiB free per GPU**. The measurements below were taken on September 9,
|
| 11 |
+
2026, after VORTEX and the previous WarpState training process had stopped.
|
| 12 |
+
|
| 13 |
+
See [PERFORMANCE.md](PERFORMANCE.md) for the subsequent bottleneck analysis,
|
| 14 |
+
GPU profiling method, and the compute requirements of a 100K-token/second
|
| 15 |
+
target.
|
| 16 |
+
|
| 17 |
+
## Selected configuration
|
| 18 |
+
|
| 19 |
+
[`train-fast.sh`](train-fast.sh) launches two workers through
|
| 20 |
+
[`launch_training.py`](launch_training.py). Each worker holds the complete
|
| 21 |
+
2,005,111,424-parameter model and AdamW state; DDP synchronizes gradients.
|
| 22 |
+
|
| 23 |
+
| Setting | Value |
|
| 24 |
+
| --- | --- |
|
| 25 |
+
| GPUs / workers | 2, one worker per GPU |
|
| 26 |
+
| Sequence length | 2,048 tokens |
|
| 27 |
+
| Microbatch per GPU | 4 sequences |
|
| 28 |
+
| Gradient accumulation | 8 microbatches per GPU |
|
| 29 |
+
| Global tokens per optimizer update | **2 × 4 × 2,048 × 8 = 131,072** |
|
| 30 |
+
| Parameters, gradients, AdamW moments | FP32 on each GPU |
|
| 31 |
+
| Matrix computation | BF16 autocast, weight cache enabled |
|
| 32 |
+
| Compilation | Each core's training forward, full graph, fixed shapes |
|
| 33 |
+
| Core activation checkpointing | Disabled |
|
| 34 |
+
| Loss projection chunk | 1,024 tokens; loss projection recomputation remains enabled |
|
| 35 |
+
| DDP gradient buckets | 128 MiB target, gradients use bucket views |
|
| 36 |
+
| PyTorch allocator cap | 88 GiB per GPU |
|
| 37 |
+
| Additional free memory required at startup | 3 GiB per GPU |
|
| 38 |
+
| Validation | 8 batches of 1 sequence, initially and every 100 updates |
|
| 39 |
+
| Checkpoint | Every 50 updates, on graceful stop, and at completion |
|
| 40 |
+
|
| 41 |
+
The global batch, sequence length, data cursor, optimizer state, and learning
|
| 42 |
+
rate schedule are preserved when moving from the previous shared-GPU run.
|
| 43 |
+
The launcher's `--reconfigure` allows execution changes while the runner
|
| 44 |
+
checks those invariants and records the transition in `execution_history`.
|
| 45 |
+
Compilation and changed batch grouping can change floating-point rounding;
|
| 46 |
+
this is not a promise of bitwise equivalence across configurations.
|
| 47 |
+
|
| 48 |
+
The allocator cap excludes CUDA context and library allocations. The startup
|
| 49 |
+
headroom check does not reserve memory for another process. This configuration
|
| 50 |
+
assumes VORTEX remains paused and both GPUs are available.
|
| 51 |
+
|
| 52 |
+
## Measured configuration grid
|
| 53 |
+
|
| 54 |
+
All rows use sequence length 2,048, 131,072 global tokens per update, BF16
|
| 55 |
+
autocast with its cache enabled, FP32 parameters and full fused AdamW state.
|
| 56 |
+
Memory is the largest steady reserved allocation on any participating GPU.
|
| 57 |
+
The single-GPU trials used a 90 GiB allocator cap; DDP trials used 88 GiB.
|
| 58 |
+
|
| 59 |
+
| GPUs | Microbatch/GPU | Core execution | Core checkpointing | Loss chunk | Tokens/s | Reserved/GPU |
|
| 60 |
+
| ---: | ---: | --- | --- | ---: | ---: | ---: |
|
| 61 |
+
| 1 | 2 | Eager | Off | 1,024 | 12,180 | 51.72 GiB |
|
| 62 |
+
| 1 | 4 | Eager | Off | 1,024 | 12,252 | 68.83 GiB |
|
| 63 |
+
| 1 | 8 | Eager | Off | 1,024 | Out of memory | Exceeded 90 GiB cap |
|
| 64 |
+
| 1 | 8 | Eager | On | 1,024 | 9,249 | 45.13 GiB |
|
| 65 |
+
| 1 | 4 | Compiled cores | Off | 1,024 | 15,142 | 60.44 GiB |
|
| 66 |
+
| 1 | 8 | Compiled cores | Off | 1,024 | 15,014 | 86.08 GiB |
|
| 67 |
+
| 1 | 4 | Compiled cores | Off | 8,192 | 15,321 | 62.89 GiB |
|
| 68 |
+
| **2** | **4** | **Compiled cores** | **Off** | **1,024** | **29,490** | **68.76 GiB** |
|
| 69 |
+
| 2 | 2 | Compiled cores | Off | 1,024 | 28,140 | 55.33 GiB |
|
| 70 |
+
|
| 71 |
+
Compiled cores improved single-GPU throughput by about 24% over eager
|
| 72 |
+
microbatch 4 and reduced reserved memory by about 8.4 GiB. Increasing the
|
| 73 |
+
compiled microbatch to 8 consumed another 25.6 GiB without improving speed.
|
| 74 |
+
Activation checkpointing saved memory but repeated enough computation to
|
| 75 |
+
make the tested eager microbatch-8 configuration slower.
|
| 76 |
+
|
| 77 |
+
The 8,192-token loss chunk improved single-GPU throughput by only 1.2%, used
|
| 78 |
+
another 2.5 GiB, and produced slightly different gradient rounding. Retaining
|
| 79 |
+
1,024 keeps the existing loss computation closer without a material measured
|
| 80 |
+
speed penalty. Among the two-GPU trials, microbatch 4 was 4.8% faster than
|
| 81 |
+
microbatch 2 and still left substantial memory available.
|
| 82 |
+
|
| 83 |
+
`nvidia-smi topo -m` reported a **SYS** connection between these GPUs: traffic
|
| 84 |
+
crosses PCIe and the CPU interconnect between NUMA nodes. There is no NVLink
|
| 85 |
+
connection between this pair. Despite that topology, the measured compiled
|
| 86 |
+
microbatch-4 DDP run achieved **1.95×** the corresponding single-GPU throughput.
|
| 87 |
+
|
| 88 |
+
## Measurement method and limits
|
| 89 |
+
|
| 90 |
+
[`benchmark_training.py`](benchmark_training.py) loaded the trusted local
|
| 91 |
+
checkpoint `work/run/before-optimization-step-237.pt`, an immutable hardlink
|
| 92 |
+
to the checkpoint at step 237 and 31,064,064 training tokens. It mmap-loaded
|
| 93 |
+
the checkpoint on the CPU, copied model parameters and every optimizer
|
| 94 |
+
tensor into separate CUDA storage, and optimized those disposable copies.
|
| 95 |
+
It read the prepared corpus in place and never saved a training checkpoint.
|
| 96 |
+
|
| 97 |
+
Each case performed complete forward/backward, gradient clipping, and fused
|
| 98 |
+
AdamW updates over 131,072 global tokens. Timings include data batching and
|
| 99 |
+
device transfer. DDP timings include gradient communication and use the
|
| 100 |
+
slowest rank's elapsed time. Compilation and the first update were excluded
|
| 101 |
+
as warmup; DDP excluded two warmup updates to allow gradient bucket rebuilding.
|
| 102 |
+
Every successful result aggregates **three measured updates**. The full
|
| 103 |
+
14.94 GiB of FP32 Adam moments was present on each GPU, and all measured
|
| 104 |
+
losses and gradient norms were finite.
|
| 105 |
+
|
| 106 |
+
These are short steady-compute samples, not long-run throughput guarantees.
|
| 107 |
+
They exclude checkpoint loading/saving, validation, process startup, and
|
| 108 |
+
initial compilation. At an unchanged 29,490 tokens/second, 40B tokens take
|
| 109 |
+
about 15.7 days of training compute; operational overhead adds time.
|
| 110 |
+
|
| 111 |
+
Generated evidence remains ignored by Git:
|
| 112 |
+
|
| 113 |
+
- `work/benchmark-grid-summary.json`: all nine cases and throughput ranges.
|
| 114 |
+
- `work/benchmark-*/summary.json`: exact arguments, source hashes, checkpoint
|
| 115 |
+
identity, per-update times, and first-update/steady memory peaks.
|
| 116 |
+
- `work/benchmark-*/rank-*.jsonl`: individual worker measurements.
|
| 117 |
+
- `work/benchmark-eager-mb8/failure.json`: the bounded allocation failure.
|
| 118 |
+
|
| 119 |
+
The source hashes in each result identify the files present for that run.
|
| 120 |
+
Runner source evolved during this optimization task; the full-model benchmark
|
| 121 |
+
used its own update loop and the same batching/model computation throughout.
|
| 122 |
+
|
| 123 |
+
## Live continuation and validation
|
| 124 |
+
|
| 125 |
+
The production run resumed at step 237 with 31,064,064 tokens consumed and
|
| 126 |
+
completed twelve updates using the selected configuration. Steady updates
|
| 127 |
+
ran at approximately 29.0–29.7K tokens/second, with 67.75 GiB allocated,
|
| 128 |
+
68.64 GiB reserved, and 25.31 GiB physically free per GPU. The previous
|
| 129 |
+
shared settings measured 8.5K tokens/second after VORTEX stopped and
|
| 130 |
+
5.5–6K while VORTEX was training.
|
| 131 |
+
|
| 132 |
+
A real `datactl.sh stop warpstate-train` completed the current update and
|
| 133 |
+
saved step 249 at 32,636,928 tokens. Both workers exited and released GPU
|
| 134 |
+
memory. Inspection of the 22.41 GiB checkpoint confirmed all 256 canonical
|
| 135 |
+
model tensors, full FP32 Adam moments and step counters, both rank random
|
| 136 |
+
states, the original initialization metadata, and the execution transition
|
| 137 |
+
at step 237. Restart restored validation loss 5.56880 from that checkpoint.
|
| 138 |
+
The original step-237 checkpoint remains at
|
| 139 |
+
`work/run/before-optimization-step-237.pt`.
|
| 140 |
+
|
| 141 |
+
Verification also passed 64 CPU tests and four CUDA integration tests
|
| 142 |
+
(two original single-GPU cases and two distributed cases). These cover
|
| 143 |
+
exact resume within one configuration, one-to-two-GPU continuation,
|
| 144 |
+
the final partial update with an empty rank, and graceful/forced worker
|
| 145 |
+
cleanup. A full-model eager-versus-compiled comparison had a maximum
|
| 146 |
+
loss difference of 0.000012 and relative gradient-norm difference below
|
| 147 |
+
0.072% across three updates. Initial validation across the two execution
|
| 148 |
+
configurations was 5.63291 versus 5.63279, consistent with small rounding
|
| 149 |
+
differences.
|
| 150 |
+
|
| 151 |
+
The generated live checkpoint verification is recorded in
|
| 152 |
+
`work/optimization-resume-validation.json`.
|
| 153 |
+
|
| 154 |
+
## Production controls
|
| 155 |
+
|
| 156 |
+
In optimized mode, the `warpstate-train` program in
|
| 157 |
+
[`supervisord.conf`](supervisord.conf) must point to `ops/train-fast.sh`.
|
| 158 |
+
Use the project-private Supervisor through `datactl.sh`:
|
| 159 |
+
|
| 160 |
+
```bash
|
| 161 |
+
cd /workspace/WarpState
|
| 162 |
+
rtx6000_40b/ops/datactl.sh status
|
| 163 |
+
rtx6000_40b/ops/datactl.sh start warpstate-train
|
| 164 |
+
tail -n 10 rtx6000_40b/work/logs/train.log
|
| 165 |
+
tail -n 3 rtx6000_40b/work/run/metrics.jsonl
|
| 166 |
+
```
|
| 167 |
+
|
| 168 |
+
To stop and preserve progress:
|
| 169 |
+
|
| 170 |
+
```bash
|
| 171 |
+
rtx6000_40b/ops/datactl.sh stop warpstate-train
|
| 172 |
+
```
|
| 173 |
+
|
| 174 |
+
The launcher receives the stop signal and creates
|
| 175 |
+
`work/control/training-stop-request`. Both workers finish the current update;
|
| 176 |
+
the runner saves the model, full optimizer state, progress, and rank random
|
| 177 |
+
states before exiting. The launcher allows up to 820 seconds for this,
|
| 178 |
+
followed by 45 seconds for termination if workers fail to exit. Supervisor's
|
| 179 |
+
stop timeout is 900 seconds. Use the launcher/Supervisor control rather than
|
| 180 |
+
signalling an individual DDP worker. Starting the same program resumes
|
| 181 |
+
`work/run/latest.pt`; do not delete the checkpoint to change execution options.
|
| 182 |
+
|
| 183 |
+
Keep at least 53 GiB free for a full checkpoint and atomic replacement.
|
| 184 |
+
Training artifacts, caches, and runtime control files remain inside
|
| 185 |
+
`/workspace/WarpState` and are excluded from Git.
|
| 186 |
+
|
| 187 |
+
## Reproducing benchmarks
|
| 188 |
+
|
| 189 |
+
**Stop training first and wait until both GPUs are free.** A benchmark uses
|
| 190 |
+
disposable model state, but it still competes for GPU memory and computation.
|
| 191 |
+
VORTEX must also remain paused. From the repository root:
|
| 192 |
+
|
| 193 |
+
```bash
|
| 194 |
+
rtx6000_40b/ops/datactl.sh stop warpstate-train
|
| 195 |
+
rtx6000_40b/ops/datactl.sh status
|
| 196 |
+
nvidia-smi
|
| 197 |
+
rtx6000_40b/ops/datactl.sh start warpstate-benchmark
|
| 198 |
+
tail -n 10 rtx6000_40b/work/logs/benchmark.log
|
| 199 |
+
```
|
| 200 |
+
|
| 201 |
+
[`benchmark-supervisord.conf`](benchmark-supervisord.conf) specifies the
|
| 202 |
+
winning two-GPU configuration and sets `autostart=false` and
|
| 203 |
+
`autorestart=false`. Its command, expressed with environment variables, is:
|
| 204 |
+
|
| 205 |
+
```bash
|
| 206 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 207 |
+
WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 \
|
| 208 |
+
TORCHINDUCTOR_COMPILE_THREADS=8 ops/benchmark.sh \
|
| 209 |
+
--checkpoint /workspace/WarpState/rtx6000_40b/work/run/before-optimization-step-237.pt \
|
| 210 |
+
--micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 211 |
+
--no-activation-checkpointing --compile-cores --autocast-cache \
|
| 212 |
+
--loss-chunk-size 1024 --vram-limit-gib 88 \
|
| 213 |
+
--warmup-updates 2 --measured-updates 3
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
Use Supervisor for routine runs; the command above documents its effective
|
| 217 |
+
arguments. `benchmark.sh` sources `training-env.sh` and launches `torchrun`.
|
| 218 |
+
Each invocation automatically receives a fresh
|
| 219 |
+
`work/benchmark-<TORCHELASTIC_RUN_ID>` directory. An explicit `--output-dir`
|
| 220 |
+
must name a new directory under `work/benchmark-*`; existing rank log files
|
| 221 |
+
are not overwritten. If reproducing against another trusted checkpoint,
|
| 222 |
+
change the configured checkpoint path and record which checkpoint was used.
|
| 223 |
+
|
| 224 |
+
For a different candidate, edit only the benchmark program's command or
|
| 225 |
+
environment, then apply it with `ops/datactl.sh reread` and
|
| 226 |
+
`ops/datactl.sh update warpstate-benchmark` from the package directory.
|
| 227 |
+
Starting that program runs the selected finite benchmark. Once it has exited,
|
| 228 |
+
check `nvidia-smi` and restart `warpstate-train` to resume production.
|
ops/PERFORMANCE.md
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WarpState training bottlenecks
|
| 2 |
+
|
| 3 |
+
For the current single-B200 measurements and selected preset, see
|
| 4 |
+
[B200_PERFORMANCE.md](B200_PERFORMANCE.md). This document records the original
|
| 5 |
+
workstation experiments.
|
| 6 |
+
|
| 7 |
+
**Paused at the user's request:** production and benchmarks are stopped,
|
| 8 |
+
with the full production checkpoint saved at step 448 / 58,720,256 tokens.
|
| 9 |
+
Another session will resume VORTEX. Do not start the reproduction commands
|
| 10 |
+
until the user requests further WarpState work and the GPUs are available.
|
| 11 |
+
|
| 12 |
+
Measurements and analysis from September 9, 2026, using the configuration in
|
| 13 |
+
[OPTIMIZED_TRAINING.md](OPTIMIZED_TRAINING.md): two RTX PRO 6000 Blackwell
|
| 14 |
+
Max-Q GPUs, 300 W each, compiled cores, BF16 matrix computation, FP32 AdamW,
|
| 15 |
+
microbatch 4 per GPU, accumulation 8, sequence length 2,048.
|
| 16 |
+
|
| 17 |
+
## Sustained throughput
|
| 18 |
+
|
| 19 |
+
Production updates 251–315 averaged 28,928 training tokens/second inside
|
| 20 |
+
the timed update loop. Their median update took 4.544 seconds for 131,072
|
| 21 |
+
global tokens. The interval between completed updates averaged 26,711
|
| 22 |
+
tokens/second including the validation/checkpoint pause after step 300.
|
| 23 |
+
That pause produced a 28.31-second gap between updates 300 and 301.
|
| 24 |
+
|
| 25 |
+
The GPUs had about 68.64 GiB reserved and 25.31 GiB physically free each.
|
| 26 |
+
They were busy at the 300 W power limit, with power-cap limiting observed
|
| 27 |
+
and no thermal slowdown. Available VRAM does not imply available compute.
|
| 28 |
+
The driver reported a maximum configurable power limit of 325 W; that is
|
| 29 |
+
only 8.3% above the current power allowance, and does not imply an equal
|
| 30 |
+
throughput gain. Hardware settings were left unchanged.
|
| 31 |
+
|
| 32 |
+
## GPU profile
|
| 33 |
+
|
| 34 |
+
One full update was profiled after two warmup updates on disposable copies
|
| 35 |
+
of checkpoint step 357. The update took 4.452 seconds. Rank 0 showed:
|
| 36 |
+
|
| 37 |
+
| GPU activity | Summed activity time |
|
| 38 |
+
| --- | ---: |
|
| 39 |
+
| Matrix multiplication | 3.215 seconds |
|
| 40 |
+
| Compiled pointwise functions and reductions | 0.561 seconds |
|
| 41 |
+
| NCCL communication | 0.439 seconds |
|
| 42 |
+
| Local attention | 0.100 seconds |
|
| 43 |
+
| Fused AdamW kernels | 0.039 seconds |
|
| 44 |
+
| Other kernels and copies | Approximately 0.382 seconds |
|
| 45 |
+
|
| 46 |
+
These are actual CUDA kernel/copy activities, excluding nested GPU annotation
|
| 47 |
+
ranges. They sum to more than elapsed time because streams overlap. Across
|
| 48 |
+
the two ranks, matrix multiplication was approximately 65–68% of summed
|
| 49 |
+
activity time and attention approximately 2.1%. Operator shapes attribute
|
| 50 |
+
1.888 seconds on rank 0 to FFN matrix multiplication, 0.919 seconds to the
|
| 51 |
+
input projection, and 0.141 seconds to the full-vocabulary head's matrix
|
| 52 |
+
multiplications. Normalization, gating, state operations, casts, gradient
|
| 53 |
+
accumulation, and other memory traffic account for much of the remaining
|
| 54 |
+
work. GPU utilization near 100% does not mean peak tensor arithmetic
|
| 55 |
+
throughput.
|
| 56 |
+
|
| 57 |
+
Communication overlaps computation, so its summed time is not the amount
|
| 58 |
+
recoverable by removing it. The previous two-GPU benchmark scaled 1.95 times
|
| 59 |
+
the equivalent single-GPU configuration. Neither more data-loader workers
|
| 60 |
+
nor a different attention implementation addresses the dominant matrix cost.
|
| 61 |
+
|
| 62 |
+
The raw evidence is under `work/benchmark-profile-20260909`. Profiler
|
| 63 |
+
instrumentation was confined to the benchmark; production resumed from
|
| 64 |
+
the unchanged step-357 checkpoint afterward.
|
| 65 |
+
|
| 66 |
+
## Compute required for 100K tokens/second
|
| 67 |
+
|
| 68 |
+
The following counts use two FLOPs per multiply-add. Let `d=2048`,
|
| 69 |
+
`f=7168`, `L=28`, and `V=32768`. Each core has an input projection with
|
| 70 |
+
`5*d*d` weights, an output projection with `d*d` weights, and SwiGLU
|
| 71 |
+
matrices with `3*d*f` weights. Forward and backward together require
|
| 72 |
+
approximately six FLOPs per dense matrix weight per token.
|
| 73 |
+
|
| 74 |
+
| Matrix computation | Billion FLOPs / training token |
|
| 75 |
+
| --- | ---: |
|
| 76 |
+
| FFNs: `6*L*3*d*f` | 7.398752 |
|
| 77 |
+
| Core input/output projections: `6*L*6*d*d` | 4.227858 |
|
| 78 |
+
| Tied output head with forward recomputation: `8*d*V` | 0.536871 |
|
| 79 |
+
| Parallel tensor-memory operations | 0.049545 |
|
| 80 |
+
| Local causal attention, including backward | Approximately 0.044–0.103 |
|
| 81 |
+
| **Total matrix computation** | **Approximately 12.26–12.32** |
|
| 82 |
+
|
| 83 |
+
The attention range reflects causal triangular arithmetic versus tiled work
|
| 84 |
+
and backward recomputation. These counts exclude normalization, nonlinear
|
| 85 |
+
functions, casts, optimizer work, and communication. Core activation
|
| 86 |
+
checkpointing is disabled; the chunked loss projection still recomputes
|
| 87 |
+
its forward during backward.
|
| 88 |
+
|
| 89 |
+
At 100,000 tokens/second, the matrix operations alone require approximately
|
| 90 |
+
1.23 PFLOPS. NVIDIA specifies 438.9 TFLOPS of dense BF16 with FP32
|
| 91 |
+
accumulation per Max-Q GPU, or 877.8 TFLOPS for this pair. Thus the
|
| 92 |
+
advertised ideal ceiling is about 71K tokens/second before other work.
|
| 93 |
+
Actual clocks, kernel efficiency, and memory traffic reduce that ceiling.
|
| 94 |
+
This rules out 100K through ordinary BF16 configuration tuning for the
|
| 95 |
+
unchanged dense computation. NVIDIA's 3,511 AI TOPS figure refers to sparse
|
| 96 |
+
FP4 and does not describe this BF16 training. See the
|
| 97 |
+
[NVIDIA architecture whitepaper, Appendix A, pages 45–46](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1.0.pdf#page=45).
|
| 98 |
+
|
| 99 |
+
## What to investigate next
|
| 100 |
+
|
| 101 |
+
Completed BF16 experiments used disposable copies of the same step-448
|
| 102 |
+
checkpoint, two warmup updates and three measured updates per case:
|
| 103 |
+
|
| 104 |
+
| Configuration | Tokens/second | Peak reserved/GPU |
|
| 105 |
+
| --- | ---: | ---: |
|
| 106 |
+
| Existing compiled-core baseline | 29,396 | 68.64 GiB |
|
| 107 |
+
| Full loss without recomputation | 29,576 | 72.03 GiB |
|
| 108 |
+
| Compiled full loss | 30,315 | 68.99 GiB |
|
| 109 |
+
| Compiled full loss + cuBLASLt / 32 MiB workspace | 30,329 | 68.99 GiB |
|
| 110 |
+
| Compiled full loss + matrix autotuning | 30,403 | 68.99 GiB |
|
| 111 |
+
| Compiled full loss + BF16 residual activations, microbatch 4 | 31,111 | 66.42 GiB |
|
| 112 |
+
|
| 113 |
+
The 50K target was not reached. None of these experimental configurations
|
| 114 |
+
was promoted to production. The microbatch-8 residual trial has no completed
|
| 115 |
+
summary and must not be treated as a measured result. All benchmark losses
|
| 116 |
+
and gradient norms in the completed cases were finite. The compiled-full
|
| 117 |
+
cases differed from baseline by up to 0.00078 loss and 2.47% gradient norm
|
| 118 |
+
after several updates; execution changes are not bitwise equivalent.
|
| 119 |
+
|
| 120 |
+
The added loss and residual helpers passed CPU numerical/parameter-identity
|
| 121 |
+
checks, and the full-model benchmarks exercised their GPU execution.
|
| 122 |
+
The new compiled-full two-GPU checkpoint/resume regression and a production
|
| 123 |
+
continuation/validation trial remain outstanding. Complete those before
|
| 124 |
+
promoting a new execution configuration. The production launcher retains
|
| 125 |
+
its previously validated BF16 settings. No FP8 GPU test or training ran.
|
| 126 |
+
At pause handoff, the combined CPU suite passed 84 tests; five CUDA cases
|
| 127 |
+
were skipped to leave the GPUs available for VORTEX.
|
| 128 |
+
|
| 129 |
+
The next requested target is 50K tokens/second, about 1.73 times the current
|
| 130 |
+
steady rate. It is below the advertised ideal BF16 ceiling, but attaining it
|
| 131 |
+
in BF16 would require unusually high sustained efficiency at this power cap.
|
| 132 |
+
As a rough illustration, halving the observed 3.215 seconds of matrix work
|
| 133 |
+
while leaving everything else fixed would reduce a 4.452-second update to
|
| 134 |
+
about 2.845 seconds, or 46K tokens/second. Overlap and power behavior make
|
| 135 |
+
this an illustration, not a measured prediction. Reaching 50K likely needs
|
| 136 |
+
both faster matrix computation and reductions in the remaining costs.
|
| 137 |
+
|
| 138 |
+
The requested constraint is BF16 matrix computation with FP32 master
|
| 139 |
+
weights and optimizer state. The experiments therefore focus on selecting
|
| 140 |
+
faster BF16 matrix kernels, using the spare memory to remove loss
|
| 141 |
+
recomputation, and reducing activation-memory traffic. Lower-precision
|
| 142 |
+
matrix training is outside this task's scope.
|
| 143 |
+
|
| 144 |
+
A fused linear cross-entropy implementation could reduce repeated output
|
| 145 |
+
projection and intermediate-memory traffic. The current 32,768-way head
|
| 146 |
+
is only 4.4% of the counted dense linear FLOPs, although its memory costs
|
| 147 |
+
can consume a larger fraction of execution time. An implementation such as
|
| 148 |
+
[Cut Cross Entropy](https://github.com/apple/ml-cross-entropy) needs a
|
| 149 |
+
numerical comparison and an end-to-end benchmark; its default gradient
|
| 150 |
+
filtering should not be silently introduced into this pretraining run.
|
| 151 |
+
|
| 152 |
+
Less frequent full checkpoints can recover some wall-clock throughput.
|
| 153 |
+
The current interval is 50 updates, about four minutes of compute. Raising
|
| 154 |
+
that interval increases the amount of work lost after an abrupt failure.
|
| 155 |
+
Checkpoint frequency does not change tokens/second inside the update loop.
|
| 156 |
+
|
| 157 |
+
Reaching 100K could instead use more aggregate GPU compute, or a smaller,
|
| 158 |
+
low-rank, or conditionally activated model. Those architecture changes
|
| 159 |
+
alter the experiment and require their own training/quality comparison.
|
| 160 |
+
VORTEX uses low-rank projections, conditional parameter pages, and a
|
| 161 |
+
hierarchical output head. Its total parameter count does not imply the
|
| 162 |
+
same computation per token as WarpState's 28 dense cores.
|
| 163 |
+
|
| 164 |
+
## Planning for one B200 GPU
|
| 165 |
+
|
| 166 |
+
The user specified one B200 GPU and BF16 training. NVIDIA lists 180 GB
|
| 167 |
+
HBM3e, approximately 7.7 TB/s memory bandwidth, and 2.25 PFLOPS dense
|
| 168 |
+
BF16 for one HGX B200 GPU. The larger 4.5 PFLOPS BF16 number uses
|
| 169 |
+
sparsity. These specifications differ from eight-GPU DGX totals. Sources:
|
| 170 |
+
[NVIDIA Blackwell datasheet](https://dam-cdn.nvd.orangelogic.com/AssetLink/y441155802qub41q118b2852i557jem5.pdf),
|
| 171 |
+
[NVIDIA performance reference](https://github.com/NVIDIA/exemplar-performance/blob/main/README.md#peak-theoretical-throughput).
|
| 172 |
+
|
| 173 |
+
For this custom model, 60–90K BF16 training tokens/second is an engineering
|
| 174 |
+
planning estimate, not a measured range or guarantee. The 75K midpoint
|
| 175 |
+
roughly preserves the current fraction of advertised compute throughput;
|
| 176 |
+
bandwidth, larger useful microbatches, B200 kernel selection, and removal
|
| 177 |
+
of two-GPU gradient synchronization can change the result. The theoretical
|
| 178 |
+
matrix-only ceiling is approximately 183K tokens/second. Reaching 100K
|
| 179 |
+
would need roughly 55% of advertised dense BF16 compute before accounting
|
| 180 |
+
for non-matrix work, so it is a plausible tuning target on B200 that still
|
| 181 |
+
requires an actual benchmark.
|
| 182 |
+
|
| 183 |
+
| Assumed update throughput | Training tokens/day | Compute time for 40B |
|
| 184 |
+
| ---: | ---: | ---: |
|
| 185 |
+
| 60K/s | 5.18 billion | 7.7 days |
|
| 186 |
+
| 75K/s | 6.48 billion | 6.2 days |
|
| 187 |
+
| 90K/s | 7.78 billion | 5.1 days |
|
| 188 |
+
| 100K/s | 8.64 billion | 4.6 days |
|
| 189 |
+
|
| 190 |
+
Checkpointing and validation add elapsed time to these compute estimates.
|
| 191 |
+
The current 50-update checkpoint interval becomes relatively more
|
| 192 |
+
expensive as updates get faster.
|
| 193 |
+
|
| 194 |
+
Storage is independent of GPU choice. In decimal GB, the prepared corpus
|
| 195 |
+
uses 80.002 GB, a complete model/Adam checkpoint uses 24.062 GB, and atomic
|
| 196 |
+
replacement needs another 24.062 GB temporarily. The current Python
|
| 197 |
+
environment is approximately 5.7 GB. Extra retained checkpoints each add
|
| 198 |
+
24.062 GB. A deployment with token data, one checkpoint, and its replacement
|
| 199 |
+
therefore needs roughly 134 GB before OS, cache, and reserve space. The
|
| 200 |
+
runner also checks for about 53 GiB free at startup for checkpoint writing.
|
| 201 |
+
|
| 202 |
+
Allocate **250 GB as a practical minimum**, or **500 GB NVMe SSD for
|
| 203 |
+
comfortable checkpoint retention and working space**. This assumes copying
|
| 204 |
+
the already prepared tokens, not retaining raw source downloads. A 1 TB
|
| 205 |
+
SSD is not required for this 40B-token run. SSD capacity does not cap the
|
| 206 |
+
lifetime number of training tokens: further data can be prepared or
|
| 207 |
+
streamed after old data is released.
|
| 208 |
+
|
| 209 |
+
## Reproducing the GPU profile
|
| 210 |
+
|
| 211 |
+
Stop production gracefully and wait for both GPUs to be free before running
|
| 212 |
+
this finite benchmark. It reads the latest checkpoint into disposable CUDA
|
| 213 |
+
copies and writes no training checkpoint. VORTEX must remain paused.
|
| 214 |
+
|
| 215 |
+
```bash
|
| 216 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 217 |
+
ops/datactl.sh stop warpstate-train
|
| 218 |
+
nvidia-smi
|
| 219 |
+
WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 \
|
| 220 |
+
TORCHINDUCTOR_COMPILE_THREADS=8 ops/benchmark.sh \
|
| 221 |
+
--checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt \
|
| 222 |
+
--micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 223 |
+
--no-activation-checkpointing --compile-cores --autocast-cache \
|
| 224 |
+
--loss-chunk-size 1024 --vram-limit-gib 88 \
|
| 225 |
+
--warmup-updates 2 --measured-updates 1 --profile
|
| 226 |
+
ops/datactl.sh start warpstate-train
|
| 227 |
+
```
|
| 228 |
+
|
| 229 |
+
Only the first measured update is profiled. Each rank exports a Chrome
|
| 230 |
+
trace, an operator table grouped by shape, a kernel table, and a JSON
|
| 231 |
+
summary under its fresh `work/benchmark-*` directory. Profiling collects
|
| 232 |
+
CPU and CUDA activity with shapes, without Python stacks or memory
|
| 233 |
+
tracking. Use unprofiled runs to measure throughput: recording adds
|
| 234 |
+
overhead. CUDA kernels on different streams can overlap, so summed kernel
|
| 235 |
+
time is not the same as elapsed time. Operator and kernel tables describe
|
| 236 |
+
the same work and must not be added together.
|
ops/POINTWISE_HEADROOM_20260913.md
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Remaining pointwise work and GEMM saturation — September 13, 2026
|
| 2 |
+
|
| 3 |
+
This analysis uses the saved September 12 full-model profile of the selected
|
| 4 |
+
paired backward layout. It does not launch a new GPU profile, pause production,
|
| 5 |
+
or change the training recipe. At the initial status check, production was
|
| 6 |
+
running past step 53,854, at approximately 31.2K tokens/second during updates.
|
| 7 |
+
|
| 8 |
+
## Timing scope
|
| 9 |
+
|
| 10 |
+
The profile is one update after eight warmups, with two RTX PRO 6000 Blackwell
|
| 11 |
+
Max-Q GPUs, microbatch four per rank, sequence 2,048, eight accumulation
|
| 12 |
+
microbatches, and 131,072 global tokens. Model computation uses BF16; master
|
| 13 |
+
parameters, accumulated gradients, communication, and AdamW state use FP32.
|
| 14 |
+
The communication preset uses conservative wait sinking and 128 MiB buckets.
|
| 15 |
+
|
| 16 |
+
The profiled update lasts 4.16463 seconds. Repeated unprofiled measurements of
|
| 17 |
+
this preset averaged 4.20116 seconds, or 31,199.03 tokens/second. Profiles include
|
| 18 |
+
instrumentation; neither number includes periodic validation/checkpoint pauses.
|
| 19 |
+
Kernel durations from separate streams can overlap and must not be summed into
|
| 20 |
+
elapsed time without interval accounting. Individual operator totals and their
|
| 21 |
+
kernel totals describe the same work.
|
| 22 |
+
|
| 23 |
+
## Remaining pointwise and reduction work
|
| 24 |
+
|
| 25 |
+
The name-based compiled-pointwise/reduction category contains 475.795 ms on
|
| 26 |
+
rank zero and 479.331 ms on rank one. This includes arithmetic, conversions,
|
| 27 |
+
copies, and reductions fused into generated kernels. It excludes the selected
|
| 28 |
+
custom paired layout, whose two kernels take approximately 78.96 ms per rank.
|
| 29 |
+
Consequently, neither the small explicit memcpy category nor the 476 ms category
|
| 30 |
+
alone describes all memory traffic or all work outside matrix multiplication.
|
| 31 |
+
|
| 32 |
+
The largest individually attributed groups in that category are:
|
| 33 |
+
|
| 34 |
+
| Work | Rank 0 ms/update | Rank 1 ms/update | Current boundary |
|
| 35 |
+
| --- | ---: | ---: | --- |
|
| 36 |
+
| SwiGLU backward | 85.70 | 85.96 | Packed activation-gradient output |
|
| 37 |
+
| Parameter gradient conversion/accumulation | 69.55 | 70.29 | Includes already fused FP32 accumulation |
|
| 38 |
+
| Core normalization/residual backward | 68.42 | 68.10 | Several operations share each kernel |
|
| 39 |
+
| Attention/memory forward transforms | 66.84 | 69.35 | Rotary, gating, and layout work |
|
| 40 |
+
| Core FP32-to-BF16 weight casts | 49.59 | 49.54 | Four weight families, eight microbatches |
|
| 41 |
+
| SwiGLU forward | 49.20 | 49.13 | Already fused SiLU and multiplication |
|
| 42 |
+
| Core normalization forward | 11.20 | 11.18 | Includes fused surrounding work |
|
| 43 |
+
|
| 44 |
+
These are disjoint groups within the compiled category, not the full category.
|
| 45 |
+
Attention/memory backward transforms, residuals, bucket operations, loss, and small
|
| 46 |
+
parameter reductions account for the remainder. Attribution groups profiler
|
| 47 |
+
operators by complete input shapes and checked generated-code bodies. The same
|
| 48 |
+
generic kernel number can represent different operations in different graphs;
|
| 49 |
+
names alone do not establish the mapping. The generated report is
|
| 50 |
+
`work/validation/pointwise-headroom-20260913/semantic-families.json` and retains
|
| 51 |
+
the explicit per-operator mapping, input shapes, source paths, and provenance.
|
| 52 |
+
|
| 53 |
+
Cutting rank zero's 475.795 ms budget by 25% would remove about 119 ms. If every
|
| 54 |
+
saved millisecond reduced elapsed time and other execution stayed unchanged,
|
| 55 |
+
4.20116 seconds would become 4.08221 seconds: approximately 2.9% more throughput.
|
| 56 |
+
This is conditional arithmetic, not a measured candidate or a promised gain.
|
| 57 |
+
|
| 58 |
+
Prior experiments already tested standalone SwiGLU and RMSNorm replacements.
|
| 59 |
+
SwiGLU forward/backward improved from 0.61335 to 0.59965 ms in its isolated
|
| 60 |
+
production-shaped fixture; RMSNorm improved from 0.43256 to 0.39248 ms. Neither
|
| 61 |
+
standalone mode improved the full-model screen. External weight-cast caching
|
| 62 |
+
also regressed overall and changed compiler/DDP partition boundaries. Their
|
| 63 |
+
switches are not supported with the selected compiled-autograd configuration.
|
| 64 |
+
See [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md).
|
| 65 |
+
|
| 66 |
+
The earlier fused SwiGLU-forward/TMA-weight-gradient recipe produced one +0.43%
|
| 67 |
+
full-model comparison after correcting duplicated weight storage. It lacks a
|
| 68 |
+
repeated gain and has not been combined with the selected paired layout.
|
| 69 |
+
The isolated epilogue's 45-microsecond gain predicts only about 10 ms per update
|
| 70 |
+
before interactions. See [MATMUL_TUNING.md](MATMUL_TUNING.md).
|
| 71 |
+
|
| 72 |
+
### Best focused next experiment
|
| 73 |
+
|
| 74 |
+
Start with the **85.7 ms SwiGLU backward region**, using a replacement inside
|
| 75 |
+
the compiled backward graph rather than the previously screened standalone
|
| 76 |
+
activation mode. Its generated kernel traverses a packed `8192 x 14336` output
|
| 77 |
+
and selects between the two derivative halves. The source has five masked
|
| 78 |
+
loads across those branches. A paired-output implementation can load the
|
| 79 |
+
up value, gate value, and upstream derivative once for each logical pair, then
|
| 80 |
+
write both derivative halves. This targets repeated operand access and branch
|
| 81 |
+
work while retaining native packed storage and the existing GEMMs.
|
| 82 |
+
The existing `activation_execution.py` already contains a paired backward
|
| 83 |
+
kernel. First compare that arithmetic and layout directly against this actual
|
| 84 |
+
compiled region; a new primitive implementation is not the starting requirement.
|
| 85 |
+
The integration route would preserve the surrounding compiled-autograd graph.
|
| 86 |
+
|
| 87 |
+
Arithmetic must follow the compiled reference. The up derivative uses
|
| 88 |
+
`g / (1 + libdevice.exp(-g))`, while the gate derivative uses `tl.sigmoid(g)`.
|
| 89 |
+
Sharing one sigmoid value by algebraic rewriting is not automatically equivalent
|
| 90 |
+
to those two implementations. Preserve their evaluation and BF16 output rounding
|
| 91 |
+
unless a separately declared numerical comparison justifies a change.
|
| 92 |
+
|
| 93 |
+
The relevant body is `triton_poi_fused_5` near line 436 of
|
| 94 |
+
`.cache/torchinductor/cw/ccwrp762vinzoemcnamsrl2wt54aoanpxjxxqobh7e7oxxutputc.py`.
|
| 95 |
+
Its synchronized variant is classified separately by matching operand shapes.
|
| 96 |
+
The source suggests an opportunity; it does not prove an instruction-count,
|
| 97 |
+
physical-bandwidth, or elapsed-time saving. A 25% reduction of this region
|
| 98 |
+
would save about 21.4 ms per update, approximately 0.5% more throughput if fully
|
| 99 |
+
realized. No new candidate kernel was timed or selected in this analysis.
|
| 100 |
+
|
| 101 |
+
The next candidate is tuning the **68.4 ms normalization/residual backward
|
| 102 |
+
group as a whole**. The existing kernels already fuse FP32 row reductions,
|
| 103 |
+
parameter-gradient partials, residual additions, and BF16 output stores.
|
| 104 |
+
Tune tiling and intermediate-buffer traffic while preserving those combinations;
|
| 105 |
+
the entire group is not removable normalization overhead.
|
| 106 |
+
One specific subregion accounts for approximately 24.5 ms and loops twice over
|
| 107 |
+
2,048 channels, rereading inputs for the normalization input derivative and
|
| 108 |
+
residual output. A persistent row kernel could retain loaded intermediates,
|
| 109 |
+
but increased register pressure could offset the saved reads. This is a narrower
|
| 110 |
+
test than replacing every normalization operation.
|
| 111 |
+
|
| 112 |
+
Gradient conversion/accumulation is already fused after many matrix products.
|
| 113 |
+
Further integration into a GEMM epilogue must preserve the BF16 product's
|
| 114 |
+
rounding before FP32 accumulation and keep gradient readiness/communication
|
| 115 |
+
ordering. Hoisting weight casts across accumulation microbatches could reduce
|
| 116 |
+
the 49.6 ms cast budget, but must solve the prior autograd/DDP-boundary regression.
|
| 117 |
+
These are more involved follow-ups than the isolated SwiGLU backward replacement.
|
| 118 |
+
|
| 119 |
+
Use the same sequence as the selected layout integration: exact-shape primitive
|
| 120 |
+
comparison, guarded graph matching and runtime dispatch evidence, full gradient/
|
| 121 |
+
optimizer/resume fixtures, then repeated complete-model timing. An isolated
|
| 122 |
+
kernel gain alone is insufficient to change the training preset.
|
| 123 |
+
|
| 124 |
+
## Where matrix time is spent
|
| 125 |
+
|
| 126 |
+
Every one of the 5,400 matrix kernels per rank is joined by its CUDA External ID
|
| 127 |
+
to the originating `aten::mm`, `aten::bmm`, or `aten::addmm` event. Exact operand
|
| 128 |
+
shapes and strides identify families and forward/input-gradient/weight-gradient
|
| 129 |
+
directions. This includes all eight vocabulary-head weight-gradient ADDMM calls.
|
| 130 |
+
The calculation counts 791.597 trillion matrix FLOPs per rank per update.
|
| 131 |
+
All matrix kernels use the same compute stream in these traces.
|
| 132 |
+
|
| 133 |
+
| Matrix family | Rank 0 ms | Rank 1 ms | Rank 0 share of matrix time |
|
| 134 |
+
| --- | ---: | ---: | ---: |
|
| 135 |
+
| FFN up/gate | 1,281.23 | 1,225.96 | 40.05% |
|
| 136 |
+
| Attention/memory input projection | 907.66 | 862.34 | 28.38% |
|
| 137 |
+
| FFN down | 632.02 | 602.74 | 19.76% |
|
| 138 |
+
| Core output projection | 174.37 | 171.46 | 5.45% |
|
| 139 |
+
| Vocabulary head | 103.37 | 98.64 | 3.23% |
|
| 140 |
+
| Small state-memory batched products | 100.09 | 96.61 | 3.13% |
|
| 141 |
+
| Total | 3,198.75 | 3,057.76 | 100% |
|
| 142 |
+
|
| 143 |
+
The first three families account for **88.19% / 88.01% of matrix time** and
|
| 144 |
+
**67.74% / 64.62% of the profiled update** on ranks zero/one. All matrix kernels
|
| 145 |
+
occupy 76.81% / 73.42% of that update. These are duration shares, not measures
|
| 146 |
+
of compute saturation. The generated attribution is
|
| 147 |
+
`work/validation/pointwise-headroom-20260913/paired-matmul-attribution.json`.
|
| 148 |
+
Its source profiles and reviewed shape/stride mapping are hashed in the report.
|
| 149 |
+
|
| 150 |
+
## What the saturation evidence establishes
|
| 151 |
+
|
| 152 |
+
The dominant dense families sustain approximately 252–256 TFLOPS on rank zero
|
| 153 |
+
and 264–268 TFLOPS on rank one in this stored full-model profile. Earlier warmed
|
| 154 |
+
isolated tests measured approximately 243–263 TFLOPS for their individual shapes.
|
| 155 |
+
Relative to an illustrative peak adjusted for their observed clocks, those
|
| 156 |
+
isolated results reached approximately **88–93%**. That estimate scales the
|
| 157 |
+
published 438.9 dense BF16 TFLOPS by observed MHz / 2,280 MHz. The specification
|
| 158 |
+
uses boost clocks; the denominator is not a measured sustained hardware limit.
|
| 159 |
+
[NVIDIA architecture specifications](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46).
|
| 160 |
+
|
| 161 |
+
Those isolated clock samples are snapshots, not time-integrated clocks for the
|
| 162 |
+
new profile. The results support a compute-efficiency focus for large dense
|
| 163 |
+
GEMMs, while small state-memory products have much lower arithmetic intensity.
|
| 164 |
+
They do **not** mean that 88% of matrix time is fully saturated, or that each
|
| 165 |
+
dense kernel spends 88–93% of its lifetime at full utilization. Operand loads,
|
| 166 |
+
Tensor Core instructions, and output work overlap within a pipelined GEMM;
|
| 167 |
+
kernel duration alone cannot partition that execution into saturated and
|
| 168 |
+
unsaturated intervals.
|
| 169 |
+
|
| 170 |
+
The exact requested fraction remains unmeasured. A prior Nsight Compute attempt
|
| 171 |
+
failed with `ERR_NVGPUCTRPERM`; the read-only driver check still reports
|
| 172 |
+
`RmProfilingAdminOnly=1`. No counter-access policy was changed for this analysis.
|
| 173 |
+
The session's effective `CAP_SYS_ADMIN` and `CAP_PERFMON` capabilities are both
|
| 174 |
+
absent. The read-only policy/capability check does not collect GPU counters.
|
| 175 |
+
The prior failure is saved in
|
| 176 |
+
`work/validation/rtx-profile-20260910/ncu-metrics.txt`.
|
| 177 |
+
NVIDIA documents this as restricted access to GPU performance counters.
|
| 178 |
+
[Counter permission reference](https://developer.nvidia.com/nvidia-development-tools-solutions-err_nvgpuctrperm-permission-issue-performance-counters).
|
| 179 |
+
|
| 180 |
+
After counter access is available, a bounded measurement should use disposable,
|
| 181 |
+
warmed copies of the dominant GEMM shapes, with the same BF16 operands, strides,
|
| 182 |
+
power limits, and relevant cache conditions. Collect Tensor-pipeline throughput,
|
| 183 |
+
SM activity, memory throughput, and scheduler statistics. Prefer elapsed-cycle
|
| 184 |
+
metrics when assessing whole-device utilization: active-cycle metrics exclude
|
| 185 |
+
idle cycles. Query the installed tool's supported metrics rather than assuming
|
| 186 |
+
a metric name from another architecture.
|
| 187 |
+
|
| 188 |
+
To answer a literal fraction-of-time question, collect supported PM-sampling
|
| 189 |
+
timelines and declare a near-saturation threshold, for example 90% of the
|
| 190 |
+
appropriate Tensor-pipeline peak. Correlate above-threshold sample intervals
|
| 191 |
+
with the GEMM execution intervals, and report sampling resolution and coverage.
|
| 192 |
+
Per-kernel average utilization alone cannot determine that time fraction.
|
| 193 |
+
Nsight Compute documents both active/elapsed normalization and time-resolved PM
|
| 194 |
+
sampling, including examples of operand-delivery stalls and underfilled tails.
|
| 195 |
+
[Profiling guide](https://docs.nvidia.com/nsight-compute/ProfilingGuide/#pm-sampling).
|
ops/POSTTRAIN.md
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Step 76,294 archive and supervised fine-tuning
|
| 2 |
+
|
| 3 |
+
The SFT stage runs from
|
| 4 |
+
[`../configs/posttrain_10b.json`](../configs/posttrain_10b.json). The user later
|
| 5 |
+
authorized **archiving the full completed SFT state, then automatically running
|
| 6 |
+
DPO**. A separate `warpstate-dpo` Supervisor job implements that continuation;
|
| 7 |
+
see [DPO_HANDOFF.md](DPO_HANDOFF.md). The active SFT plan/source bytes stay fixed
|
| 8 |
+
to preserve their checkpoint/resume signatures. Its original guardian still
|
| 9 |
+
finishes and exits cleanly before the DPO guardian can take over.
|
| 10 |
+
|
| 11 |
+
`prepare_posttrain_data.py` prepares conversations and preference pairs. The
|
| 12 |
+
separate `posttrain.py` runner implements assistant-only supervised training.
|
| 13 |
+
|
| 14 |
+
## Automatic sequence
|
| 15 |
+
|
| 16 |
+
1. `warpstate-posttrain-data` prepares the pinned dataset recipe on CPU now.
|
| 17 |
+
2. `warpstate-posttrain` waits for `warpstate-train` to exit successfully. It
|
| 18 |
+
never changes the base run or resumes an earlier stopped base checkpoint.
|
| 19 |
+
3. Under the base launch/pipeline locks, require the full checkpoint to contain
|
| 20 |
+
step **76,294**, **10,000,007,168 tokens**, and final validation at that same
|
| 21 |
+
boundary. A checkpoint before the boundary remains waiting; an overshoot,
|
| 22 |
+
stale validation, or mismatched tokenizer fails without starting SFT.
|
| 23 |
+
4. Export the original FP32 model using the same `chat.capture_latest` method as
|
| 24 |
+
the 8K, 16K and 40K archives. Compare every exported tensor exactly against the
|
| 25 |
+
stopped checkpoint; copy the exact tokenizer; write provenance/checksums;
|
| 26 |
+
seal files read-only; publish the complete archive atomically without
|
| 27 |
+
replacing an existing directory.
|
| 28 |
+
5. Make a physical, independent copy of the archive as SFT initialization. The
|
| 29 |
+
full pretraining checkpoint, corpus, historical archives and chat selection
|
| 30 |
+
are retained. SFT starts with a fresh AdamW optimizer on its first launch.
|
| 31 |
+
6. Verify prepared SFT data, run a tiny two-GPU BF16 training/resume preflight,
|
| 32 |
+
then start the full SFT model. Failed preflight or data checks stop the
|
| 33 |
+
handoff and retain the raw archive for diagnosis.
|
| 34 |
+
7. Save final SFT state and evaluation results, verify the raw archive and
|
| 35 |
+
initialization remain unchanged, then exit. Neither program automatically
|
| 36 |
+
restarts training after completion or failure. The separately armed DPO job
|
| 37 |
+
then archives this full SFT state and continues according to DPO_HANDOFF.md.
|
| 38 |
+
|
| 39 |
+
The boundary triggers archival, checksum verification, compilation/preflight
|
| 40 |
+
and initialization before the first SFT update; allow several minutes for this
|
| 41 |
+
handoff. GPU preflight is deliberately deferred until base training releases
|
| 42 |
+
the GPUs. Its result is not claimed by the CPU-only setup tests.
|
| 43 |
+
|
| 44 |
+
## Paths
|
| 45 |
+
|
| 46 |
+
All paths below are relative to `/workspace/WarpState/rtx6000_40b`.
|
| 47 |
+
|
| 48 |
+
| Purpose | Path |
|
| 49 |
+
| --- | --- |
|
| 50 |
+
| Untouched full base resume checkpoint | `work/run/latest.pt` |
|
| 51 |
+
| Raw historical model archive | `work/archives/step-00076294/` |
|
| 52 |
+
| Independent SFT initialization | `work/posttrain/step-00076294/initialization/` |
|
| 53 |
+
| Prepared SFT/DPO data and final manifest | `work/posttrain/data/` |
|
| 54 |
+
| SFT checkpoints, settings and metrics | `work/posttrain/step-00076294/run/` |
|
| 55 |
+
| Final evaluation / completion | `run/evaluation.json`, `run/completion.json` under the SFT directory |
|
| 56 |
+
| Handoff state and frozen execution provenance | `work/posttrain/step-00076294/control/` |
|
| 57 |
+
| GPU preflight artifacts | `work/posttrain/step-00076294/preflight/` |
|
| 58 |
+
| Handoff and training log | `work/logs/posttrain.log` |
|
| 59 |
+
| CPU preparation log | `work/logs/posttrain-data.log` |
|
| 60 |
+
|
| 61 |
+
Archives contain model weights only, without Adam or training RNG state. SFT's
|
| 62 |
+
`latest.pt` is a separate full `warpstate-posttrain-v1` checkpoint with optimizer,
|
| 63 |
+
per-rank RNG, deterministic sample cursor, source/data identities and base
|
| 64 |
+
provenance. It is not interchangeable with the base runner's checkpoint.
|
| 65 |
+
|
| 66 |
+
## SFT recipe and validation
|
| 67 |
+
|
| 68 |
+
This first run uses one epoch, two GPUs, sequence length 2,048, microbatch 4 per
|
| 69 |
+
rank and four accumulation microbatches: **32 conversations per update**. There
|
| 70 |
+
is no cross-conversation packing. All parameters train with BF16 compute,
|
| 71 |
+
original FP32 parameters/gradients and fresh FP32 AdamW state; no FP8 is used.
|
| 72 |
+
The learning rate peaks at `5e-6`, with 3% warmup and cosine decay to zero,
|
| 73 |
+
zero weight decay and gradient clipping at 1. Evaluation/checkpoints occur
|
| 74 |
+
every 100 SFT updates and at the end. Initial and final validation use the full
|
| 75 |
+
SFT holdout; intermediate checks use the same fixed 128-example subset.
|
| 76 |
+
Settings and effective epoch length are
|
| 77 |
+
recorded in the SFT run's `settings.json` once data is loaded.
|
| 78 |
+
|
| 79 |
+
The learning rate is a conservative experimental starting point, not a
|
| 80 |
+
validated optimum for this 2B model. This is not a reproduction of Tülu 3's
|
| 81 |
+
SFT schedule: its published learning rate was calibrated using summed token
|
| 82 |
+
loss, and its models/context/batch sizes differ. See the
|
| 83 |
+
[Tülu 3 paper, section 4.3.2](https://arxiv.org/html/2411.15124v3).
|
| 84 |
+
|
| 85 |
+
Role headers use ordinary tokens from the existing SentencePiece vocabulary.
|
| 86 |
+
Only assistant content and the actual EOS ID contribute to the loss. Prompt,
|
| 87 |
+
role-header and padding labels are masked; DDP gradients are normalized by the
|
| 88 |
+
global count of supervised tokens, including uneven final batches. The shared
|
| 89 |
+
renderer is `ops/posttrain_data.py`; its hash is pinned in the data manifest.
|
| 90 |
+
|
| 91 |
+
SFT metrics report actual nonpadding input tokens and supervised assistant
|
| 92 |
+
tokens separately. SFT validation loss/perplexity uses the assistant-token
|
| 93 |
+
objective on the prepared holdout; it is **not directly comparable** with the
|
| 94 |
+
base model's all-token pretraining validation loss. Evaluation reports include
|
| 95 |
+
the exact evaluated sample/token counts, dataset and base-weight identities.
|
| 96 |
+
Behavioral instruction-following and tool-use evaluation is the next manual
|
| 97 |
+
decision after the full authorized SFT/DPO sequence pauses; a lower SFT loss
|
| 98 |
+
alone does not establish those capabilities. The final SFT archive preserves
|
| 99 |
+
the exact model and its evaluation for comparison with DPO.
|
| 100 |
+
|
| 101 |
+
## Monitor, pause and resume
|
| 102 |
+
|
| 103 |
+
From the repository root:
|
| 104 |
+
|
| 105 |
+
```bash
|
| 106 |
+
rtx6000_40b/ops/datactl.sh status warpstate-train warpstate-posttrain-data warpstate-posttrain
|
| 107 |
+
rtx6000_40b/ops/posttrain-handoff.sh --check
|
| 108 |
+
tmux attach -t warpstate-posttrain
|
| 109 |
+
tail -f rtx6000_40b/work/logs/posttrain.log
|
| 110 |
+
```
|
| 111 |
+
|
| 112 |
+
The tmux session displays the durable Supervisor log. Detaching tmux does not
|
| 113 |
+
stop the handoff or training. `RUNNING` on the posttraining program can mean
|
| 114 |
+
waiting for the base boundary; inspect `--check` or its phase log.
|
| 115 |
+
|
| 116 |
+
To pause the handoff or gracefully checkpoint an active SFT run:
|
| 117 |
+
|
| 118 |
+
```bash
|
| 119 |
+
rtx6000_40b/ops/datactl.sh stop warpstate-posttrain
|
| 120 |
+
```
|
| 121 |
+
|
| 122 |
+
To explicitly re-enable the unchanged handoff or resume an interrupted SFT run:
|
| 123 |
+
|
| 124 |
+
```bash
|
| 125 |
+
rtx6000_40b/ops/datactl.sh start warpstate-posttrain
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
The saved plan/source identity must still match. A completed run stays paused
|
| 129 |
+
even if the program is started again. Do not remove its completion state to
|
| 130 |
+
repeat SFT. Use a separately reviewed plan and output directory for a new
|
| 131 |
+
experiment. Do not clear lock files or launch the base trainer manually while
|
| 132 |
+
SFT workers are alive: every SFT rank holds a shared base pipeline lock, and
|
| 133 |
+
the base/handoff require its exclusive lock. The launcher also retains its
|
| 134 |
+
exclusion locks across child launch and unexpected watcher death.
|
| 135 |
+
|
| 136 |
+
SFT Supervisor includes are in `ops/posttrain-supervisord.conf`; both programs
|
| 137 |
+
have `autostart=false` and `autorestart=false`. Setup explicitly starts the
|
| 138 |
+
authorized jobs. A whole-machine/Supervisor restart requires an explicit
|
| 139 |
+
restart of the appropriate jobs; there is no hidden reboot training launch.
|
| 140 |
+
|
| 141 |
+
To cancel automatic continuation into DPO while allowing SFT to finish, stop
|
| 142 |
+
the separate waiting job with `rtx6000_40b/ops/datactl.sh stop warpstate-dpo`.
|
| 143 |
+
Stopping SFT itself leaves the DPO guardian waiting for verified completion.
|
| 144 |
+
|
| 145 |
+
## Rebuild the prepared inputs
|
| 146 |
+
|
| 147 |
+
The recipe in `configs/posttrain_data.json` pins immutable dataset revisions,
|
| 148 |
+
the tokenizer identity, sample counts/seed, split rule, renderer and required
|
| 149 |
+
preparation packages. Preparation records downloaded file hashes, schemas,
|
| 150 |
+
source/licensing metadata, row/token counts, rejections and output hashes.
|
| 151 |
+
Prompt groups use the same deterministic partition across SFT and DPO; exact
|
| 152 |
+
duplicate conversations are excluded. Overlength examples and DPO branches
|
| 153 |
+
are rejected rather than truncated. Gated xLAM data is recorded as skipped
|
| 154 |
+
without automatically accepting terms or requiring interactive input.
|
| 155 |
+
|
| 156 |
+
To rebuild the isolated CPU environment, source `ops/training-env.sh`, create
|
| 157 |
+
`work/posttrain-data-venv` with Python 3.12, then install
|
| 158 |
+
`ops/requirements-posttrain-data.lock` into that environment. Do not change the
|
| 159 |
+
active training environment. `ops/prepare-posttrain-data.sh` runs the recipe;
|
| 160 |
+
do not regenerate/replace data while SFT is running. The completed manifest is
|
| 161 |
+
published only after every selected output has been written and hashed.
|
| 162 |
+
|
| 163 |
+
Source/configuration/tests/dependency metadata belong in Git. Downloaded data,
|
| 164 |
+
token caches, generated manifests/reports, checkpoints, archives, environments
|
| 165 |
+
and logs under `work/` remain ignored.
|
ops/RESTART_HANDOFF.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# RTX workstation restart handoff
|
| 2 |
+
|
| 3 |
+
Prepared on 2026-09-15. All paths below are relative to `/workspace/WarpState`.
|
| 4 |
+
Training, posttraining, and benchmarks have finished; leave them stopped.
|
| 5 |
+
Use the existing environments and prepared data. BF16 compute remains required.
|
| 6 |
+
|
| 7 |
+
## Saved model state
|
| 8 |
+
|
| 9 |
+
| Stage | Final step | Full resumable checkpoint | Immutable model archive |
|
| 10 |
+
| --- | ---: | --- | --- |
|
| 11 |
+
| Raw pretraining | 76,294 | `rtx6000_40b/work/run/latest.pt` | `rtx6000_40b/work/archives/step-00076294/` |
|
| 12 |
+
| SFT | 10,312 | `rtx6000_40b/work/archives/step-00076294-sft-final/checkpoint.pt` | `rtx6000_40b/work/archives/step-00076294-sft-final/` |
|
| 13 |
+
| DPO | 2,334 | `rtx6000_40b/work/archives/step-00076294-dpo-final/checkpoint.pt` | `rtx6000_40b/work/archives/step-00076294-dpo-final/` |
|
| 14 |
+
|
| 15 |
+
Raw training reached 10,000,007,168 tokens with validation loss 2.3496886492.
|
| 16 |
+
SFT and DPO checkpoints both record `completed=true`. Their validation losses
|
| 17 |
+
are 1.1157395900 and 0.5790014537 respectively, for different objectives.
|
| 18 |
+
The earlier step 8,000, 16,000, and 40,000 archives are also retained.
|
| 19 |
+
Do not modify archived files. Chat uses an independent copy in
|
| 20 |
+
`rtx6000_40b/work/chat/dpo-final/`.
|
| 21 |
+
|
| 22 |
+
With user approval, five byte-identical working weight files were removed
|
| 23 |
+
after comparison with their retained archives, reclaiming 72,184,948,679 bytes.
|
| 24 |
+
These are the SFT and DPO working `run/latest.pt` and `initialization/model.pt`
|
| 25 |
+
files, plus `work/chat/step-8000-s536yem8/model.pt` (under `rtx6000_40b/`).
|
| 26 |
+
All archives and the independent current chat copy remain intact.
|
| 27 |
+
|
| 28 |
+
Before any future posttraining continuation or rerun of completion checks,
|
| 29 |
+
restore independent working files from the following sources. Paths in this
|
| 30 |
+
table are relative to `rtx6000_40b/work/`:
|
| 31 |
+
|
| 32 |
+
| Archived source | Working destination |
|
| 33 |
+
| --- | --- |
|
| 34 |
+
| `archives/step-00076294/model.pt` | `posttrain/step-00076294/initialization/model.pt` |
|
| 35 |
+
| `archives/step-00076294-sft-final/checkpoint.pt` | `posttrain/step-00076294/run/latest.pt` |
|
| 36 |
+
| `archives/step-00076294-sft-final/model.pt` | `posttrain/step-00076294-dpo/initialization/model.pt` |
|
| 37 |
+
| `archives/step-00076294-dpo-final/checkpoint.pt` | `posttrain/step-00076294-dpo/run/latest.pt` |
|
| 38 |
+
|
| 39 |
+
Verify the archive's `SHA256SUMS`, copy only to an absent destination using
|
| 40 |
+
`cp --reflink=auto --no-clobber`, and compare the copy with its source before
|
| 41 |
+
use. Never train against an archive path or a hardlink to archived weights.
|
| 42 |
+
Restoring these files does not authorize starting training.
|
| 43 |
+
|
| 44 |
+
Prepared pretraining tokens remain in `rtx6000_40b/work/data/`; posttraining
|
| 45 |
+
data remain in `rtx6000_40b/work/posttrain/data/`. Benchmark results are in
|
| 46 |
+
`rtx6000_40b/work/validation/nord-dpo/` and
|
| 47 |
+
`rtx6000_40b/work/validation/nord-stage-comparison/COMPARISON.md`.
|
| 48 |
+
|
| 49 |
+
## After reboot
|
| 50 |
+
|
| 51 |
+
Start in `/workspace/WarpState` and read `AGENTS.md` and this handoff.
|
| 52 |
+
Check `nvidia-smi` and `git status --short --branch`.
|
| 53 |
+
To restore post-DPO chat in its own tmux session:
|
| 54 |
+
|
| 55 |
+
```bash
|
| 56 |
+
cd /workspace/WarpState
|
| 57 |
+
tmux has-session -t warpstate-chat 2>/dev/null || tmux new-session -d -s warpstate-chat -n dpo-chat -c /workspace/WarpState 'rtx6000_40b/ops/chat-dpo.sh'
|
| 58 |
+
tmux attach -t warpstate-chat
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Ctrl-C cancels typing or generation and returns to the next input. `/reset`
|
| 62 |
+
clears conversation history, `/quit` exits, and Ctrl-B then D detaches tmux.
|
| 63 |
+
Tmux sessions and in-memory chat history do not survive reboot; captured
|
| 64 |
+
terminal history is available in the restart record below.
|
| 65 |
+
|
| 66 |
+
All workspace Supervisor jobs have `autostart=false` and `autorestart=false`.
|
| 67 |
+
If job management is needed, check `rtx6000_40b/ops/datactl.sh status` first.
|
| 68 |
+
Initialize the workspace Supervisor only when it is absent:
|
| 69 |
+
|
| 70 |
+
```bash
|
| 71 |
+
/usr/local/bin/supervisord -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
This starts no training. The raw launcher still stops at step 76,294, and
|
| 75 |
+
posttraining has completed. Further training requires a new user request and
|
| 76 |
+
an explicitly chosen continuation configuration; do not simply restart jobs.
|
| 77 |
+
Do not use the B200 launcher on this RTX workstation.
|
| 78 |
+
|
| 79 |
+
Jupyter currently uses HTTPS port 8080 with the existing authentication.
|
| 80 |
+
After reboot, check the platform-managed Jupyter service before launching
|
| 81 |
+
another server. Runtime files and logs from the last standalone launch are
|
| 82 |
+
under `rtx6000_40b/work/control/jupyter/`; treat them as private because they
|
| 83 |
+
can contain authentication data. Never commit them or expose their contents.
|
| 84 |
+
The raw model download path on the Jupyter host is
|
| 85 |
+
`/files/workspace/WarpState/rtx6000_40b/work/archives/step-00076294/model.pt?download=1`.
|
| 86 |
+
For post-DPO weights, substitute `step-00076294-dpo-final`.
|
| 87 |
+
|
| 88 |
+
## Restart record
|
| 89 |
+
|
| 90 |
+
`rtx6000_40b/work/restart/LATEST` identifies the private local restart record.
|
| 91 |
+
It contains available tmux history, checkpoint metadata, file sizes, archive
|
| 92 |
+
checksum results, job state, and a Git bundle of committed sources.
|
| 93 |
+
The earlier checkpoint inventory is a pre-cleanup snapshot; the current paths
|
| 94 |
+
are listed above. `rtx6000_40b/work/restart/duplicate-weight-deletion.json`
|
| 95 |
+
records the approved removals and their retained counterparts.
|
| 96 |
+
Artifacts and this bundle are on the same machine, not an off-machine backup.
|
| 97 |
+
Keep datasets, weights, runtime files, and this generated record out of Git.
|
ops/RTX_ASYNC_TRAINING.md
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# RTX BF16 execution and overlapping checkpoints
|
| 2 |
+
|
| 3 |
+
The local launcher selects the fastest complete-update experiment from
|
| 4 |
+
[RTX_PROFILE_20260910.md](RTX_PROFILE_20260910.md): compiled cores in default
|
| 5 |
+
mode, compiled full vocabulary loss, BF16 residual activations, microbatch 4
|
| 6 |
+
per GPU, and accumulation 8 across two RTX PRO 6000 GPUs. The measured update
|
| 7 |
+
rate was 30,707 tokens/second versus 29,081 for the previous configuration,
|
| 8 |
+
a 5.59% increase. These measurements exclude checkpoint and evaluation pauses.
|
| 9 |
+
The larger microbatch experiments and standalone GEMM/gradient prototypes
|
| 10 |
+
are not enabled in the production launcher.
|
| 11 |
+
|
| 12 |
+
The model, tokenizer, token corpus, 131,072-token global batch, context length,
|
| 13 |
+
optimizer and learning-rate schedule are preserved. Forward/backward compute
|
| 14 |
+
uses BF16. Master parameters, gradients and Adam moments remain FP32 on the
|
| 15 |
+
GPU. BF16 residual execution changes activation rounding; the first validation
|
| 16 |
+
at an unchanged checkpoint measures that numerical difference, not learning.
|
| 17 |
+
|
| 18 |
+
## Checkpoint lifecycle
|
| 19 |
+
|
| 20 |
+
`--async-checkpoint` works with one GPU or DDP. Every save captures random
|
| 21 |
+
states from every rank at a completed optimizer update. Only rank zero
|
| 22 |
+
constructs the checkpoint and owns the background writer.
|
| 23 |
+
|
| 24 |
+
1. Finish the previous write, if it is still pending.
|
| 25 |
+
2. Copy all checkpoint tensors and mutable metadata into an immutable CPU
|
| 26 |
+
snapshot. Both ranks wait; later updates cannot mutate this snapshot.
|
| 27 |
+
3. Submit serialization to one CPU thread and let both GPUs resume training.
|
| 28 |
+
4. Write `latest.pt.tmp`, synchronize the file, atomically replace `latest.pt`,
|
| 29 |
+
and synchronize its directory. The previous published checkpoint stays
|
| 30 |
+
available until replacement.
|
| 31 |
+
|
| 32 |
+
One full snapshot costs approximately 22.4 GiB of host RAM for this model and
|
| 33 |
+
optimizer. There is no additional full checkpoint clone on either GPU, and
|
| 34 |
+
at most one snapshot/write is outstanding. The local launcher now saves and
|
| 35 |
+
evaluates every 100 updates. The initial promotion used a 50-update save
|
| 36 |
+
interval; increasing it to 100 reduces time spent staging checkpoints.
|
| 37 |
+
|
| 38 |
+
Step lines display validation loss only when that step has a fresh result,
|
| 39 |
+
together with its evaluation step. Metrics and checkpoints still record
|
| 40 |
+
`val_step` and `val_tokens` alongside `val_loss`, retaining the latest measured
|
| 41 |
+
values between evaluations. This avoids repeated validation values in the
|
| 42 |
+
terminal and adds no GPU work. Startup evaluation and final evaluation and
|
| 43 |
+
saving on a graceful stop remain enabled.
|
| 44 |
+
|
| 45 |
+
The main training threads coordinate initialization, construction, staging,
|
| 46 |
+
completed background errors and final close. The writer thread performs no
|
| 47 |
+
CUDA or distributed operations. Every update first checks the prior writer's
|
| 48 |
+
status on both ranks. A failure observed there is raised on both ranks before
|
| 49 |
+
the next update; a failure occurring during an update is observed at the next
|
| 50 |
+
check or save. Normal completion and graceful stop join the final write.
|
| 51 |
+
Exceptional exit joins only the local writer, avoiding extra collectives
|
| 52 |
+
when another worker may already have failed.
|
| 53 |
+
|
| 54 |
+
`Staged step ...` identifies a frozen snapshot, not durable progress. A later
|
| 55 |
+
`Saved ... (step ..., tokens=...; background write ...s)` identifies completed
|
| 56 |
+
publication. GPU-to-CPU staging and any wait for a previous save remain
|
| 57 |
+
synchronous; only disk serialization and publication overlap training.
|
| 58 |
+
|
| 59 |
+
## Validation and reproduction
|
| 60 |
+
|
| 61 |
+
The promotion checks passed 112 CPU tests and two CUDA continuation tests
|
| 62 |
+
(one GPU and two GPUs). The full-model canary started from the verified
|
| 63 |
+
production checkpoint at step 5,618 / 736,362,496 tokens. Its initial
|
| 64 |
+
validation remained 2.97925 to five decimal places. After startup, the first
|
| 65 |
+
three measured updates ran at 31,637, 31,482 and 31,450 tokens/second with
|
| 66 |
+
66.57 GiB reserved per GPU. This short canary validates operation; the
|
| 67 |
+
20-update comparison above remains the controlled performance experiment.
|
| 68 |
+
|
| 69 |
+
The canary staged step 5,620 in 11.41 seconds, then completed updates 5,621
|
| 70 |
+
and 5,622 before that checkpoint's 10.62-second background write finished.
|
| 71 |
+
It published the final step-5,622 full checkpoint before process exit.
|
| 72 |
+
This confirms real overlap and also shows that the CPU snapshot remains a
|
| 73 |
+
material pause. The initial production save interval was 50 updates; the
|
| 74 |
+
canary used two updates to exercise overlap and queue backpressure quickly.
|
| 75 |
+
|
| 76 |
+
Reloading that full checkpoint completed two further updates and published
|
| 77 |
+
step 5,624 / 737,148,928 tokens. These six canary updates belong only to the
|
| 78 |
+
disposable run; production resumes its original step-5,618 checkpoint. The
|
| 79 |
+
canary's logs, exact launcher arguments and integrity report remain in
|
| 80 |
+
`work/validation/async-promotion/`.
|
| 81 |
+
|
| 82 |
+
The distributed writer tests inject failures during state construction, CPU
|
| 83 |
+
snapshot allocation, previous-save waits, background checks, flush and close.
|
| 84 |
+
Two CPU/Gloo processes must both report each failure without hanging. Tests
|
| 85 |
+
also verify immutable snapshots, rank-zero-only writing, file and directory
|
| 86 |
+
synchronization, and cleanup without collectives on exceptional exit.
|
| 87 |
+
|
| 88 |
+
The CUDA execution regression compares continuous synchronous training with
|
| 89 |
+
an asynchronous split/resume run using compiled full loss and BF16 residuals.
|
| 90 |
+
Model tensors, Adam state, RNG state, validation loss and progress must match
|
| 91 |
+
exactly. It exercises periodic saves, final flush and a partially filled last
|
| 92 |
+
update where one rank has no valid tokens. This verifies exact continuation
|
| 93 |
+
for the test model and selected execution configuration; it does not establish
|
| 94 |
+
bitwise equivalence to the previous FP32-residual configuration.
|
| 95 |
+
|
| 96 |
+
Run from `rtx6000_40b` with both GPUs available:
|
| 97 |
+
|
| 98 |
+
```bash
|
| 99 |
+
source ops/training-env.sh
|
| 100 |
+
mkdir -p work/validation/async-promotion
|
| 101 |
+
.server-venv/bin/python -m pytest -q \
|
| 102 |
+
test_checkpoint_writer.py test_distributed_checkpoint_writer.py \
|
| 103 |
+
--basetemp="$PWD/work/validation/async-promotion/pytest-writer"
|
| 104 |
+
.server-venv/bin/python -m pytest -q test_training_execution.py \
|
| 105 |
+
-k 'two_gpu_bf16_residual_async_full_state_resume_and_empty_final_rank or max-autotune-no-cudagraphs-async' \
|
| 106 |
+
--basetemp="$PWD/work/validation/async-promotion/pytest-gpu"
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
Full-model validation uses a separate work directory with read-only access to
|
| 110 |
+
the prepared data and an initial hardlink to the paused checkpoint. Saving
|
| 111 |
+
atomically replaces only the disposable checkpoint path. Source checkpoint
|
| 112 |
+
integrity, canary logs and generated verification reports are retained under
|
| 113 |
+
`work/validation/async-promotion/`, outside Git. Production controls and status
|
| 114 |
+
commands are in [LOCAL_RESUME.md](LOCAL_RESUME.md).
|
ops/RTX_PROFILE_20260910.md
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# RTX training profile, 2026-09-10
|
| 2 |
+
|
| 3 |
+
This report records the experiments before production promotion. The selected
|
| 4 |
+
BF16 configuration and subsequent checkpoint-overlap implementation are
|
| 5 |
+
documented in [RTX_ASYNC_TRAINING.md](RTX_ASYNC_TRAINING.md); use
|
| 6 |
+
[LOCAL_RESUME.md](LOCAL_RESUME.md) for the current launcher and status controls.
|
| 7 |
+
|
| 8 |
+
The production run was gracefully paused to measure training bottlenecks and
|
| 9 |
+
compare disposable execution experiments. Production remains paused while the
|
| 10 |
+
experiments run. Benchmark updates do not advance the saved training run.
|
| 11 |
+
|
| 12 |
+
## Preserved training state
|
| 13 |
+
|
| 14 |
+
The completed checkpoint was verified at 18:12:57 UTC:
|
| 15 |
+
|
| 16 |
+
| Field | Value |
|
| 17 |
+
| --- | --- |
|
| 18 |
+
| Checkpoint | `work/run/latest.pt` |
|
| 19 |
+
| Step | 5,618 |
|
| 20 |
+
| Tokens consumed | 736,362,496 |
|
| 21 |
+
| Validation loss at pause | 2.979250341653824 |
|
| 22 |
+
| Bytes | 24,061,700,237 (22.409 GiB) |
|
| 23 |
+
| Saved optimizer entries | 256 |
|
| 24 |
+
| Saved RNG ranks | 2 |
|
| 25 |
+
| SHA256 | `0f2e2a47f13b9d91d0049a0e99109c289399fe593c244bab046f120c1f7ea835` |
|
| 26 |
+
|
| 27 |
+
The benchmark loads this full checkpoint, including optimizer state, into
|
| 28 |
+
disposable processes. It does not write production checkpoints. Its configuration
|
| 29 |
+
and final summary record the checkpoint file identity before and after the run.
|
| 30 |
+
The verification record is generated at
|
| 31 |
+
`work/validation/rtx-profile-20260910/paused-checkpoint.json`.
|
| 32 |
+
|
| 33 |
+
## Configuration and measurement method
|
| 34 |
+
|
| 35 |
+
The hardware is two NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition
|
| 36 |
+
GPUs, each with approximately 96 GB VRAM. The 2,005,111,424-parameter model uses
|
| 37 |
+
BF16 autocast with FP32 master parameters, gradients, and full fused AdamW state.
|
| 38 |
+
No FP8 or reduced-precision optimizer state is used.
|
| 39 |
+
|
| 40 |
+
The production configuration uses sequence length 2,048, microbatch 4 per GPU,
|
| 41 |
+
and eight accumulation microsteps. The global batch is therefore 131,072 tokens
|
| 42 |
+
per optimizer update. It compiles the model cores in `default` mode, uses the
|
| 43 |
+
checkpointed vocabulary loss with 1,024-token chunks, enables the autocast cache,
|
| 44 |
+
and disables activation checkpointing for the model blocks. Production saves
|
| 45 |
+
every 50 updates and evaluates every 100 updates.
|
| 46 |
+
|
| 47 |
+
The unprofiled baseline restores the checkpoint's optimizer state and runs two
|
| 48 |
+
warmup updates followed by 20 measured updates. Its aggregate throughput was
|
| 49 |
+
**29,081.0 tokens/s**, with a median of **28,996.8 tokens/s**. Peak allocated VRAM
|
| 50 |
+
was **67.747 GiB** and reserved VRAM **68.643 GiB** per GPU, with a minimum of
|
| 51 |
+
**25.311 GiB** free. The short baseline excludes evaluation and checkpoint writes
|
| 52 |
+
and has a transient speed change as the GPUs warm up. Sustained production
|
| 53 |
+
measurements below provide the wall throughput reference.
|
| 54 |
+
|
| 55 |
+
Profiler runs identify operators and timing relationships. Their instrumented
|
| 56 |
+
update durations must not be presented as unprofiled training throughput.
|
| 57 |
+
Summed CUDA operator durations may overlap; they do not automatically partition
|
| 58 |
+
wall time. Candidate speed comparisons must use unprofiled measurements with
|
| 59 |
+
the same global token batch and full optimizer state.
|
| 60 |
+
|
| 61 |
+
Baseline artifacts are in `work/benchmark-rtx-20260910/baseline/`. Telemetry is
|
| 62 |
+
in `work/validation/rtx-profile-20260910/baseline-gpu-telemetry.json`.
|
| 63 |
+
|
| 64 |
+
## Production overhead
|
| 65 |
+
|
| 66 |
+
The last 200 completed production intervals end at step 5,618 and start just
|
| 67 |
+
after step 5,418. This interval stays within the restart segment and excludes
|
| 68 |
+
the shutdown gap. It includes four periodic checkpoints and two evaluations;
|
| 69 |
+
the final pause-time evaluation and save occur after its last update timestamp.
|
| 70 |
+
|
| 71 |
+
| Measurement | Result |
|
| 72 |
+
| --- | ---: |
|
| 73 |
+
| Tokens processed | 26,214,400 |
|
| 74 |
+
| Sum of measured optimizer-update time | 909.490 s |
|
| 75 |
+
| Elapsed time between metric timestamps | 1,000.906 s |
|
| 76 |
+
| Throughput within updates | **28,823.2 tokens/s** |
|
| 77 |
+
| Throughput including recurring overhead | **26,190.7 tokens/s** |
|
| 78 |
+
| Wall time outside updates | **9.133%** |
|
| 79 |
+
| Median ordinary inter-update overhead | **2.10 ms** |
|
| 80 |
+
|
| 81 |
+
For each consecutive metric pair, outside-update time is the difference between
|
| 82 |
+
record timestamps minus the later record's update duration. Metrics are emitted
|
| 83 |
+
before periodic evaluation and saving, so the gap following a save step captures
|
| 84 |
+
that save's stall.
|
| 85 |
+
|
| 86 |
+
| After step | Operations before next update | Outside-update time |
|
| 87 |
+
| --- | --- | ---: |
|
| 88 |
+
| 5,450 | Checkpoint | 22.730 s |
|
| 89 |
+
| 5,500 | Validation and checkpoint | 23.026 s |
|
| 90 |
+
| 5,550 | Checkpoint | 22.505 s |
|
| 91 |
+
| 5,600 | Validation and checkpoint | 22.743 s |
|
| 92 |
+
|
| 93 |
+
Checkpoint-only gaps average 22.618 s. The difference between grouped means
|
| 94 |
+
suggests approximately 0.27 s for validation in this window, but validation was
|
| 95 |
+
not separately timed and checkpoint duration also varies. The wider restart
|
| 96 |
+
segment gives approximately 22.49 s per checkpoint and a 0.35 s grouped
|
| 97 |
+
validation difference. Both windows show that synchronous checkpoint saving
|
| 98 |
+
dominates the approximately 9% operational overhead.
|
| 99 |
+
|
| 100 |
+
These gaps combine GPU-to-host staging, serialization, filesystem writes,
|
| 101 |
+
durability synchronization, and waiting. Dividing checkpoint bytes by the pause
|
| 102 |
+
gives approximately 1.06 GB/s for the combined path; this is not an isolated SSD
|
| 103 |
+
bandwidth measurement. Raw interval calculations are recorded in
|
| 104 |
+
`work/validation/rtx-profile-20260910/production-overhead.json`.
|
| 105 |
+
|
| 106 |
+
### Checkpoint cadence model
|
| 107 |
+
|
| 108 |
+
Holding compute time and evaluation cadence fixed, amortizing approximately
|
| 109 |
+
22.6 s per checkpoint gives the following estimates. These are predictions,
|
| 110 |
+
not measured candidate runs; production cadence has not been changed.
|
| 111 |
+
|
| 112 |
+
| Save every | Estimated wall throughput | Approximate gain | Maximum replay after a crash |
|
| 113 |
+
| --- | ---: | ---: | --- |
|
| 114 |
+
| 50 updates, current | 26.2K tokens/s | Reference | 6.55M tokens, 4.2 min |
|
| 115 |
+
| 100 updates | 27.4K tokens/s | 4.7% | 13.11M tokens, 8.0 min |
|
| 116 |
+
| 200 updates | 28.1K tokens/s | 7.3% | 26.21M tokens, 15.5 min |
|
| 117 |
+
| 500 updates | 28.5K tokens/s | 8.9% | 65.54M tokens, 38.4 min |
|
| 118 |
+
|
| 119 |
+
Eliminating all measured outside-update time would raise this production window
|
| 120 |
+
from 26.19K to 28.82K tokens/s: an approximately **10.1% upper bound** for
|
| 121 |
+
operational improvements alone. Substantially higher throughput requires faster
|
| 122 |
+
optimizer updates as well.
|
| 123 |
+
|
| 124 |
+
`run_40b.py` currently saves model and optimizer GPU tensors synchronously on
|
| 125 |
+
rank 0 and then waits at a distributed barrier. Its `--async-checkpoint` option
|
| 126 |
+
explicitly rejects a world size other than one. The existing
|
| 127 |
+
`ops/checkpoint_writer.py` stages an immutable CPU snapshot before overlapping
|
| 128 |
+
serialization and durable publication with training. Adapting this path to DDP
|
| 129 |
+
requires coordinated error handling and validation; the flag cannot simply be
|
| 130 |
+
enabled for the current two-GPU run. CPU snapshot staging would still block,
|
| 131 |
+
and approximately one full checkpoint of additional host memory is required.
|
| 132 |
+
|
| 133 |
+
## GPU profile
|
| 134 |
+
|
| 135 |
+
One complete update, after two warmup updates, took 4.4745 seconds with profiler
|
| 136 |
+
instrumentation. Actual CUDA activity occupied approximately 98.8% of that
|
| 137 |
+
interval. This is the union of kernel/copy intervals, not tensor-core utilization.
|
| 138 |
+
|
| 139 |
+
| GPU activity | Rank 0 summed time | Rank 1 summed time |
|
| 140 |
+
| --- | ---: | ---: |
|
| 141 |
+
| Matrix multiplication | 3.231 s | 3.122 s |
|
| 142 |
+
| Compiled pointwise functions and reductions | 0.561 s | 0.563 s |
|
| 143 |
+
| NCCL communication | 0.448 s | 0.545 s |
|
| 144 |
+
| Other kernels | 0.378 s | 0.378 s |
|
| 145 |
+
| Local attention | 0.100 s | 0.099 s |
|
| 146 |
+
| Fused AdamW | 0.0385 s | 0.0384 s |
|
| 147 |
+
| CUDA copies and fills | 0.0033 s | 0.0031 s |
|
| 148 |
+
|
| 149 |
+
Matrices represent 65.7–67.9% of summed GPU activity. Streams overlap, so this
|
| 150 |
+
table must not be added up as a partition of elapsed time. Rank 0 communication
|
| 151 |
+
overlaps other activity for 337 ms; its communication-only intervals total
|
| 152 |
+
111 ms. Rank 1 has 215 ms of communication-only activity. Those intervals are
|
| 153 |
+
not guaranteed recoverable by removing communication.
|
| 154 |
+
|
| 155 |
+
The eight data batches require only 5.65/5.12 ms of host time for construction,
|
| 156 |
+
pinning and transfer submission. Increasing loader workers does not address a
|
| 157 |
+
measured bottleneck. Gradient clipping's 280/380 ms host range mostly waits for
|
| 158 |
+
previously queued GPU work during the nonfinite check. Actual clipping-related
|
| 159 |
+
CUDA work is approximately 16 ms. Optimizer and attention replacements likewise
|
| 160 |
+
have small direct budgets compared with dense projections.
|
| 161 |
+
|
| 162 |
+
Raw traces and operator/kernel tables are under
|
| 163 |
+
`work/benchmark-rtx-20260910/profile/`. The generated interpretation is
|
| 164 |
+
`work/validation/rtx-profile-20260910/gpu-attribution.json`.
|
| 165 |
+
|
| 166 |
+
## What the matrix multiplication contains
|
| 167 |
+
|
| 168 |
+
For a linear projection with input X and weight W, training computes three
|
| 169 |
+
matrix products: `Y = X @ W.T`, `dX = dY @ W`, and `dW = dY.T @ X`. Each product
|
| 170 |
+
costs approximately `2*M*N*K` FLOPs. Thus backward contributes approximately
|
| 171 |
+
twice the dense arithmetic of forward, before other backward operations.
|
| 172 |
+
|
| 173 |
+
Each rank processes 65,536 tokens per update as eight 8,192-token microbatches.
|
| 174 |
+
Each core projection therefore executes 224 times: 28 cores times eight
|
| 175 |
+
microbatches. The following rank-0 times sum all calls within the profiled update.
|
| 176 |
+
|
| 177 |
+
| Projection | Forward input/output width | Forward | Input gradient | Weight gradient |
|
| 178 |
+
| --- | --- | ---: | ---: | ---: |
|
| 179 |
+
| FFN up and gate | 2,048 → 14,336 | 407.4 ms | 437.1 ms | 431.5 ms |
|
| 180 |
+
| FFN down | 7,168 → 2,048 | 222.2 ms | 210.5 ms | 195.5 ms |
|
| 181 |
+
| Q/K/V/gate/memory input | 2,048 → 10,240 | 291.0 ms | 314.9 ms | 309.1 ms |
|
| 182 |
+
| Core output | 2,048 → 2,048 | 57.8 ms | 57.4 ms | 57.5 ms |
|
| 183 |
+
| Vocabulary head | 2,048 → 32,768 | 71.2 ms | 39.8 ms | 31.2 ms |
|
| 184 |
+
|
| 185 |
+
Head forward includes recomputation: 64 original and 64 recomputed projections
|
| 186 |
+
of 1,024 tokens, with 64 calls for each derivative. Core activation checkpointing
|
| 187 |
+
is disabled; head checkpointing still repeats the head's forward projection.
|
| 188 |
+
State-memory batched matrix operations add another 96.5 ms on rank 0.
|
| 189 |
+
|
| 190 |
+
The dense linear products alone account for 797.146 trillion FLOPs per rank
|
| 191 |
+
per update, or 12.1635 billion FLOPs per training token. Their summed execution
|
| 192 |
+
time is 3.1342 s, approximately 254.3 TFLOPS during those operations. FFNs alone
|
| 193 |
+
take 1.905 s, and the five-way input projection takes 0.916 s. These are the
|
| 194 |
+
primary matrix optimization targets.
|
| 195 |
+
|
| 196 |
+
The large forward GEMMs have ideal one-pass BF16 operand arithmetic intensities
|
| 197 |
+
of roughly 910–1,470 FLOPs/byte. This is an operand-footprint calculation, not
|
| 198 |
+
measured device-memory traffic: tile reloads, caches, casts and accumulation
|
| 199 |
+
change actual traffic. It supports investigating compute efficiency before
|
| 200 |
+
assuming that the large GEMMs are starved for external memory bandwidth.
|
| 201 |
+
|
| 202 |
+
Shapes, strides, dtype, counts, FLOPs and matching kernel symbols are recorded
|
| 203 |
+
in `work/validation/rtx-profile-20260910/matmul-detail.json`. The dominant kernel
|
| 204 |
+
names begin `cutlass_80_tensorop_bf16`, with 128×256 or 256×128 tiles, K32 and
|
| 205 |
+
three pipeline stages. The symbol identifies an implementation family; it does
|
| 206 |
+
not establish that the GPU is executing in an obsolete architecture mode.
|
| 207 |
+
|
| 208 |
+
## Compute ceiling at the measured power limit
|
| 209 |
+
|
| 210 |
+
NVIDIA specifies **438.9 TFLOPS of dense BF16 with FP32 accumulation per Max-Q
|
| 211 |
+
GPU**, based on a **2,280 MHz boost clock**, with **300 W board power** and
|
| 212 |
+
**1,792 GB/s memory bandwidth**. The larger sparsity figure is not applicable
|
| 213 |
+
to this dense model. See Appendix A of the
|
| 214 |
+
[NVIDIA RTX Blackwell architecture whitepaper](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46).
|
| 215 |
+
|
| 216 |
+
During loaded baseline samples, median SM clocks were approximately 1,537 and
|
| 217 |
+
1,571 MHz. The software power cap was active; thermal throttling was not.
|
| 218 |
+
Scaling the advertised rate by those clocks gives roughly 296–302 TFLOPS per
|
| 219 |
+
GPU as an illustrative instantaneous ceiling, substantially below boost-rated
|
| 220 |
+
peak. This calculation assumes arithmetic capacity scales linearly with SM
|
| 221 |
+
clock; it is not a hardware-counter utilization measurement.
|
| 222 |
+
|
| 223 |
+
The isolated GEMM tests below sustain approximately 244–263 TFLOPS with
|
| 224 |
+
observed clocks of 1,417–1,537 MHz and approximately 300 W. Relative to the
|
| 225 |
+
clock-scaled specification, these correspond to approximately **88–94%**.
|
| 226 |
+
Telemetry is sampled after each timed case and clocks vary within a case, so
|
| 227 |
+
this percentage is approximate. Nevertheless, the evidence indicates that the
|
| 228 |
+
large library GEMMs already use much of the compute available at this power
|
| 229 |
+
limit. Whole-update efficiency is lower because normalization, activations,
|
| 230 |
+
gradient handling, memory operations and communication still consume time.
|
| 231 |
+
|
| 232 |
+
Nsight Compute 2025.1.1 is installed, but its hardware-counter query was rejected
|
| 233 |
+
with `ERR_NVGPUCTRPERM` by the host driver. No driver permissions or power settings
|
| 234 |
+
were changed. Profiler occupancy fields reporting zero for active kernels were
|
| 235 |
+
not treated as measured occupancy. The conclusion above uses CUDA timings,
|
| 236 |
+
shape-derived FLOPs and clock observations, with those limits made explicit.
|
| 237 |
+
|
| 238 |
+
## Isolated GEMM algorithm experiments
|
| 239 |
+
|
| 240 |
+
[`benchmark_gemm.py`](benchmark_gemm.py) generates deterministic BF16 operands
|
| 241 |
+
with the projection's actual transpose strides. It preserves the production
|
| 242 |
+
BF16 reduction flag, excludes compilation, warms each case for at least two
|
| 243 |
+
seconds, then measures five CUDA-event trials of 30 products. The inputs are
|
| 244 |
+
reused and no autograd, communication, optimizer or training checkpoint is
|
| 245 |
+
involved. These are kernel comparisons, not training throughput measurements.
|
| 246 |
+
|
| 247 |
+
| Product | cuBLAS | cuBLASLt preference | Default Triton-only autotune |
|
| 248 |
+
| --- | ---: | ---: | ---: |
|
| 249 |
+
| FFN up/gate forward | 1.828 ms | 1.838 ms | 1.981 ms |
|
| 250 |
+
| FFN up/gate dX | 1.885 ms | 1.885 ms | 2.057 ms |
|
| 251 |
+
| FFN up/gate dW | 1.871 ms | 1.777 ms | 1.869 ms |
|
| 252 |
+
| Input projection forward | 1.326 ms | 1.314 ms | 1.441 ms |
|
| 253 |
+
| Input projection dX | 1.389 ms | 1.376 ms | 1.474 ms |
|
| 254 |
+
| Input projection dW | 1.351 ms | 1.336 ms | 1.350 ms |
|
| 255 |
+
| FFN down forward | 0.984 ms | 0.965 ms | 1.076 ms |
|
| 256 |
+
| FFN down dX | 0.914 ms | 0.902 ms | 0.993 ms |
|
| 257 |
+
| FFN down dW | 0.935 ms | 0.955 ms | 0.993 ms |
|
| 258 |
+
|
| 259 |
+
ATEN-only compilation was essentially equal to eager cuBLAS. Default
|
| 260 |
+
Triton-only selection was usually slower; swapping backends globally is not an
|
| 261 |
+
established improvement. cuBLASLt gave mixed shape-specific results. Its 5.3%
|
| 262 |
+
up/gate dW gain coincided with a higher observed clock, so it must not be
|
| 263 |
+
attributed solely to a better algorithm. Some Triton input-gradient cases
|
| 264 |
+
differed from the BF16 cuBLAS reference by about 0.30% relative RMS; finite
|
| 265 |
+
outputs do not establish numerical equivalence.
|
| 266 |
+
|
| 267 |
+
Enabling `ENABLE_PERSISTENT_TMA_MATMUL=1` added persistent TMA candidates and
|
| 268 |
+
selected them for the three up/gate products. Measured times were 1.798 ms
|
| 269 |
+
forward, 1.890 ms dX and 1.810 ms dW: approximately 268, 255 and 266 TFLOPS.
|
| 270 |
+
This closes the default Triton gap and gives small shape-specific improvements
|
| 271 |
+
over cuBLAS, rather than a large untapped acceleration. Autotuner trial times
|
| 272 |
+
were faster than the subsequent warmed measurements; the latter are reported.
|
| 273 |
+
|
| 274 |
+
Evidence is under `work/benchmark-gemm-20260910/`,
|
| 275 |
+
`work/benchmark-gemm-cublaslt-20260910/`, and
|
| 276 |
+
`work/benchmark-gemm-tma-20260910/`.
|
| 277 |
+
|
| 278 |
+
## Fusion opportunities
|
| 279 |
+
|
| 280 |
+
The two FFN input projections already share one GEMM. Q/K/V/gate/memory likewise
|
| 281 |
+
share one projection. The compiled core already fuses residual addition,
|
| 282 |
+
RMSNorm and BF16 conversion, as well as SwiGLU split/SiLU/multiply and several
|
| 283 |
+
attention transforms. Generated wrappers still use `extern_kernels.mm` between
|
| 284 |
+
these fused regions.
|
| 285 |
+
|
| 286 |
+
The FFN computes `(X @ W_up * silu(X @ W_gate)) @ W_down`. The nonlinearity
|
| 287 |
+
prevents collapsing both projection stages into a single ordinary linear map.
|
| 288 |
+
A custom fused implementation still performs both matrix stages.
|
| 289 |
+
|
| 290 |
+
| Remaining target | Rank-0 measured kernel-time budget | Practical constraint |
|
| 291 |
+
| --- | ---: | --- |
|
| 292 |
+
| Weight-gradient conversion and FP32 accumulation | 130.5 ms | Preserve BF16 rounding and DDP gradient hooks |
|
| 293 |
+
| SwiGLU backward | 88.0 ms | Paired-element derivative can reuse loads and sigmoid |
|
| 294 |
+
| Repeated core weight casts | 56.4 ms | Cached BF16 copies need optimizer-step invalidation and correct autograd |
|
| 295 |
+
| SwiGLU forward | 41.1 ms | Up/gate columns are 7,168 apart; backward still needs saved values |
|
| 296 |
+
| Forward norms | 29.6 ms | Row reductions complicate fusion into tiled GEMMs |
|
| 297 |
+
| Forward Q/K/V/U transforms | 43.9 ms | Multiple attention and memory consumers need compatible layouts |
|
| 298 |
+
|
| 299 |
+
These are budgets, not promised savings, and are not additive end-to-end
|
| 300 |
+
speedups. A slower custom GEMM can erase the benefit of its fused epilogue.
|
| 301 |
+
The complete SwiGLU forward/backward pointwise cost is only 129 ms, about 2.9%
|
| 302 |
+
of the profiled update. Recomputing the up/gate projection to avoid saving its
|
| 303 |
+
activations would add approximately 407 ms, exceeding its forward fusion budget.
|
| 304 |
+
|
| 305 |
+
[`benchmark_gradient_accumulation.py`](benchmark_gradient_accumulation.py)
|
| 306 |
+
compares existing BF16 dW followed by FP32 accumulation, fused conversion/add,
|
| 307 |
+
and direct FP32-output `addmm`. The direct variant changes the original
|
| 308 |
+
BF16-round-before-add semantics. Neither helper implements production autograd
|
| 309 |
+
or DDP integration, and neither has been enabled in the training run.
|
| 310 |
+
|
| 311 |
+
The final accumulation test uses eight distinct deterministic input pairs. It
|
| 312 |
+
creates the first gradient and then performs seven additions, matching the
|
| 313 |
+
production accumulation order. Each case uses two seconds of burn-in and five
|
| 314 |
+
timing trials, each containing ten eight-gradient cycles.
|
| 315 |
+
|
| 316 |
+
| Weight-gradient family | Separate BF16 GEMM/cast/add | Fused cast/add | Direct FP32 GEMM accumulation |
|
| 317 |
+
| --- | ---: | ---: | ---: |
|
| 318 |
+
| FFN up/gate | 16.928 ms | 15.850 ms | 15.438 ms |
|
| 319 |
+
| Input projection | 11.804 ms | 11.308 ms | 11.620 ms |
|
| 320 |
+
| FFN down | 8.141 ms | 7.802 ms | 7.873 ms |
|
| 321 |
+
|
| 322 |
+
Fused conversion/add improves these complete gradient cycles by **4.3–6.8%**,
|
| 323 |
+
with zero output difference on all three tested fixtures. Scaling the measured
|
| 324 |
+
cycle differences over 28 blocks suggests approximately 54 ms per update,
|
| 325 |
+
around **1.2%** of profiled wall time. This is a hypothesis for integration,
|
| 326 |
+
not a measured whole-model improvement; autograd, DDP hooks and overlap remain
|
| 327 |
+
to be tested. Direct FP32 accumulation is not consistently faster than the
|
| 328 |
+
rounding-preserving fusion and differs by approximately 0.166–0.203% relative
|
| 329 |
+
RMS from the baseline on these synthetic gradients.
|
| 330 |
+
|
| 331 |
+
Final evidence is in `work/benchmark-gradient-distinct-20260910/`. The earlier
|
| 332 |
+
`work/benchmark-gradient-20260910/` experiment used eight identical inputs and
|
| 333 |
+
a zero-reset-plus-eight-add cycle; it is retained as exploratory evidence and
|
| 334 |
+
is not used for the table or integration estimate above.
|
| 335 |
+
|
| 336 |
+
## Complete training-update experiments
|
| 337 |
+
|
| 338 |
+
Each successful case uses the same checkpoint and token cursor, two warmup
|
| 339 |
+
updates and 20 measured updates, full FP32 optimizer state and BF16 computation.
|
| 340 |
+
Global batch and context are unchanged. The table excludes validation and saves.
|
| 341 |
+
|
| 342 |
+
| Execution | Tokens/s | Gain over baseline | Peak reserved/GPU |
|
| 343 |
+
| --- | ---: | ---: | ---: |
|
| 344 |
+
| Production baseline, microbatch 4 | 29,081 | Reference | 68.64 GiB |
|
| 345 |
+
| Compiled full vocabulary loss, microbatch 4 | 30,050 | 3.3% | 68.99 GiB |
|
| 346 |
+
| Compiled full loss + BF16 residuals, microbatch 4 | 30,707 | 5.6% | 66.42 GiB |
|
| 347 |
+
| Chunked loss + BF16 residuals, microbatch 8 | 30,394 | 4.5% | 89.49 GiB |
|
| 348 |
+
| Compiled full loss + BF16 residuals, microbatch 8 | Out of memory | — | Exceeded 91 GiB cap |
|
| 349 |
+
| Baseline repeated after the other experiments | 29,073 | -0.03% | 68.64 GiB |
|
| 350 |
+
|
| 351 |
+
The repeated baseline also measured 20 updates and returned within 0.03% of
|
| 352 |
+
the initial aggregate throughput, supporting the small measured candidate gains.
|
| 353 |
+
|
| 354 |
+
The full-loss microbatch-8 trial failed while allocating its 1 GiB logits
|
| 355 |
+
buffer, before completing a warmup update. Retaining chunked loss made that
|
| 356 |
+
microbatch fit, but it was slower than microbatch-4 compiled-full with BF16
|
| 357 |
+
residuals. Available VRAM did not translate into a larger useful speed gain.
|
| 358 |
+
|
| 359 |
+
Against baseline's corresponding 20 updates, maximum absolute loss differences
|
| 360 |
+
were 0.000210 for compiled-full, 0.000235 for compiled-full plus BF16 residuals,
|
| 361 |
+
and 0.000146 for microbatch-8 chunked residuals. Maximum relative gradient-norm
|
| 362 |
+
differences were 0.236%, 0.384% and 0.362%, respectively. All completed updates
|
| 363 |
+
had finite losses and norms. These short comparisons do not establish long-run
|
| 364 |
+
quality or distributed checkpoint/resume equivalence for an experimental path.
|
| 365 |
+
|
| 366 |
+
The immediate measured compute improvement is approximately 3–6%. A further
|
| 367 |
+
implementation effort should prioritize gradient conversion/accumulation,
|
| 368 |
+
selective GEMM tuning, and checkpoint staging/overlap. Preserving the current
|
| 369 |
+
save frequency is especially useful after the recent abrupt shutdown. Production
|
| 370 |
+
remains paused at step 5,618; no experiment has been promoted to its launcher.
|
| 371 |
+
|
| 372 |
+
## Reproduction and paused-state controls
|
| 373 |
+
|
| 374 |
+
These commands target the two-GPU workstation. The dedicated B200 host instead
|
| 375 |
+
uses `ops/b200ctl.sh` and its own handoff instructions.
|
| 376 |
+
|
| 377 |
+
Start at the repository root and inspect the production job. If it is running,
|
| 378 |
+
request a graceful stop and wait for `STOPPED` and the final checkpoint save
|
| 379 |
+
before starting a benchmark. A stopped job may make the status command return a
|
| 380 |
+
nonzero shell exit status; inspect its output.
|
| 381 |
+
|
| 382 |
+
```bash
|
| 383 |
+
cd /workspace/WarpState
|
| 384 |
+
source rtx6000_40b/ops/training-env.sh
|
| 385 |
+
rtx6000_40b/ops/datactl.sh status warpstate-train
|
| 386 |
+
# Only when the production job is running:
|
| 387 |
+
rtx6000_40b/ops/datactl.sh stop warpstate-train
|
| 388 |
+
rtx6000_40b/ops/datactl.sh status warpstate-train
|
| 389 |
+
tail -n 12 rtx6000_40b/work/logs/train.log
|
| 390 |
+
```
|
| 391 |
+
|
| 392 |
+
Use a fresh output directory for every disposable run. The benchmark restores
|
| 393 |
+
full optimizer state, preserves the global batch, and writes no checkpoint:
|
| 394 |
+
|
| 395 |
+
```bash
|
| 396 |
+
cd /workspace/WarpState
|
| 397 |
+
source rtx6000_40b/ops/training-env.sh
|
| 398 |
+
rtx_profile_run=$(date -u +%Y%m%dT%H%M%S)
|
| 399 |
+
WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 TORCHINDUCTOR_COMPILE_THREADS=8 \
|
| 400 |
+
rtx6000_40b/ops/benchmark.sh \
|
| 401 |
+
--checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt \
|
| 402 |
+
--data /workspace/WarpState/rtx6000_40b/work/data/train.bin \
|
| 403 |
+
--output-dir "/workspace/WarpState/rtx6000_40b/work/benchmark-rtx-${rtx_profile_run}/baseline" \
|
| 404 |
+
--micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \
|
| 405 |
+
--compile-cores --compile-mode default \
|
| 406 |
+
--no-activation-checkpointing --autocast-cache \
|
| 407 |
+
--loss-mode checkpointed --loss-chunk-size 1024 \
|
| 408 |
+
--optimizer-state resume --warmup-updates 2 --measured-updates 20 \
|
| 409 |
+
--vram-limit-gib 88 --threads 4 --ddp-bucket-cap-mb 128
|
| 410 |
+
```
|
| 411 |
+
|
| 412 |
+
For the operator trace, use another fresh output directory, add `--profile`, and
|
| 413 |
+
`--diagnose-stalls`, and use `--measured-updates 1`. Only the first measured update is traced. Inspect
|
| 414 |
+
the output summary and both ranks' profile artifacts, then compare throughput
|
| 415 |
+
using separate unprofiled runs. Leave production paused after the experiments.
|
| 416 |
+
|
| 417 |
+
Candidate overrides, each with a new output directory, are `--loss-mode
|
| 418 |
+
compiled-full`; then additionally `--bf16-residuals`; and, for the larger batch,
|
| 419 |
+
`--micro-batch 8 --vram-limit-gib 91`. The successful larger-batch case retains
|
| 420 |
+
`--loss-mode checkpointed`. Do not run these GPU experiments concurrently.
|
| 421 |
+
|
| 422 |
+
The isolated matrix and gradient comparisons use no training data or checkpoint:
|
| 423 |
+
|
| 424 |
+
```bash
|
| 425 |
+
cd /workspace/WarpState
|
| 426 |
+
source rtx6000_40b/ops/training-env.sh
|
| 427 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 428 |
+
export TORCHINDUCTOR_COMPILE_THREADS=8
|
| 429 |
+
rtx6000_40b/.server-venv/bin/python rtx6000_40b/ops/benchmark_gemm.py \
|
| 430 |
+
--backends eager aten triton --warmup 10 --burnin-seconds 2 --repeats 30 --trials 5
|
| 431 |
+
rtx6000_40b/.server-venv/bin/python rtx6000_40b/ops/benchmark_gemm.py \
|
| 432 |
+
--backends eager --blas-backend cublaslt
|
| 433 |
+
ENABLE_PERSISTENT_TMA_MATMUL=1 rtx6000_40b/.server-venv/bin/python \
|
| 434 |
+
rtx6000_40b/ops/benchmark_gemm.py --families ffn_up_gate --backends triton
|
| 435 |
+
rtx6000_40b/.server-venv/bin/python \
|
| 436 |
+
rtx6000_40b/ops/benchmark_gradient_accumulation.py
|
| 437 |
+
```
|
| 438 |
+
|
| 439 |
+
Each invocation defaults to a fresh timestamped output directory and refuses
|
| 440 |
+
to overwrite an existing directory. `--list-shapes` validates and lists the
|
| 441 |
+
selected layouts on CPU. GPU comparisons save source hashes, arguments,
|
| 442 |
+
precision settings, numerical errors and timing samples. The two new helpers
|
| 443 |
+
passed CPU syntax, layout/FLOP and accumulation-control checks, and actual BF16
|
| 444 |
+
GPU comparisons; they do not alter the production model or launcher.
|
| 445 |
+
|
| 446 |
+
When the user requests production resume, use the existing verified checkpoint
|
| 447 |
+
and `rtx6000_40b/ops/datactl.sh start warpstate-train`. The tmux session named
|
| 448 |
+
`warpstate-train` is the monitoring session; Supervisor controls the training
|
| 449 |
+
process. Never use a from-scratch launcher to resume this run.
|
| 450 |
+
|
| 451 |
+
Raw profiles, benchmark summaries, telemetry, and verification reports remain
|
| 452 |
+
ignored under `work/`. This source document records the recipe and interpretation
|
| 453 |
+
without adding generated artifacts or checkpoints to Git.
|
ops/TRAINING.md
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training environment and shared GPU configuration
|
| 2 |
+
|
| 3 |
+
This document records the original two-GPU workstation configuration. Its
|
| 4 |
+
Supervisor and launchers are not the controls for the single-B200 host.
|
| 5 |
+
Current B200 settings, checkpoint and pause/resume controls are in
|
| 6 |
+
[B200_HANDOFF.md](B200_HANDOFF.md). For workstation history, see
|
| 7 |
+
[OPTIMIZED_TRAINING.md](OPTIMIZED_TRAINING.md); the shared GPU settings below
|
| 8 |
+
preceded that workstation preset.
|
| 9 |
+
|
| 10 |
+
The completed corpus in `work/data` contains exactly 40,000,000,000 training
|
| 11 |
+
tokens and 1,048,576 validation tokens. Its generated manifest records the
|
| 12 |
+
SHA256 checksums. The model has 2,005,111,424 parameters; 40B is the token
|
| 13 |
+
budget, not the parameter count.
|
| 14 |
+
|
| 15 |
+
No initial WarpState checkpoint was supplied. The shared-GPU launcher uses
|
| 16 |
+
`--from-scratch` with seed 1337 for its first launch. Once `work/run/latest.pt`
|
| 17 |
+
exists, subsequent launches restore its model, full AdamW state, token cursor,
|
| 18 |
+
learning-rate schedule, and random states. `--from-scratch` never overwrites
|
| 19 |
+
an existing run. To initialize another run from trusted weights, invoke
|
| 20 |
+
`run_40b.py --weights ...` with a separate work directory instead.
|
| 21 |
+
|
| 22 |
+
## Shared GPU settings
|
| 23 |
+
|
| 24 |
+
`train-shared.sh` selects physical GPU 1 by its UUID. VORTEX uses both GPUs,
|
| 25 |
+
so concurrent training shares GPU compute and can slow both jobs.
|
| 26 |
+
On another machine, set `WARPSTATE_CUDA_DEVICE` to the intended GPU UUID or
|
| 27 |
+
index before launching.
|
| 28 |
+
|
| 29 |
+
| Setting | Value |
|
| 30 |
+
| --- | --- |
|
| 31 |
+
| Sequence length | 2,048 tokens |
|
| 32 |
+
| Microbatch | 1 sequence |
|
| 33 |
+
| Gradient accumulation | 64 microbatches |
|
| 34 |
+
| Tokens per optimizer update | 131,072 |
|
| 35 |
+
| Parameters, gradients, AdamW moments | FP32 on the GPU |
|
| 36 |
+
| Forward/backward matrix operations | BF16 autocast |
|
| 37 |
+
| Activation checkpointing | Enabled |
|
| 38 |
+
| Autocast weight cache | Disabled to reduce retained weight copies |
|
| 39 |
+
| Loss projection chunks | 256 tokens |
|
| 40 |
+
| PyTorch allocator limit | 33.5 GiB |
|
| 41 |
+
| Required extra free memory at startup | 3 GiB |
|
| 42 |
+
| Validation | 8 batches initially and every 100 updates |
|
| 43 |
+
| Checkpoint | Every 50 updates, on a graceful stop, and at completion |
|
| 44 |
+
|
| 45 |
+
The September 9, 2026 live trial completed two full updates, saved a
|
| 46 |
+
22.41 GiB checkpoint, and resumed through step 5. Peak memory was 31.54 GiB
|
| 47 |
+
allocated and 32.02 GiB reserved, leaving about 4.7 GiB physically free.
|
| 48 |
+
Observed training throughput was approximately 5,500–6,000 tokens/second.
|
| 49 |
+
At that unchanged rate, 40B tokens would take roughly 77–84 days before
|
| 50 |
+
checkpoint/evaluation overhead; this is an early measurement, not an ETA
|
| 51 |
+
that accounts for future GPU availability. VORTEX continued advancing,
|
| 52 |
+
with overlapping updates around 68–69K tokens/second versus roughly 100K
|
| 53 |
+
before the trial.
|
| 54 |
+
|
| 55 |
+
The fixed FP32 parameters, gradients, and two AdamW moments alone occupy
|
| 56 |
+
about 29.88 GiB. Activations and allocator overhead must also fit the cap.
|
| 57 |
+
The launcher refuses to start if the cap plus startup headroom exceeds
|
| 58 |
+
available memory. This is a check, not a reservation for VORTEX.
|
| 59 |
+
|
| 60 |
+
PyTorch's [allocator memory limit](https://docs.pytorch.org/docs/2.14/generated/torch.cuda.memory.set_per_process_memory_fraction.html)
|
| 61 |
+
excludes CUDA context and library allocations. Compare both the allocated
|
| 62 |
+
and reserved peaks in `work/run/metrics.jsonl` with physical memory in
|
| 63 |
+
`nvidia-smi`. A different VORTEX workload may need different headroom.
|
| 64 |
+
|
| 65 |
+
Each full checkpoint is approximately 22.4 GiB. Keep at least 53 GiB free
|
| 66 |
+
for checkpoint writing and atomic replacement. Checkpoints, metrics, data,
|
| 67 |
+
and environments stay inside this project and are ignored by Git.
|
| 68 |
+
|
| 69 |
+
## Environment
|
| 70 |
+
|
| 71 |
+
The CUDA training environment is separate from the CPU data-preparation
|
| 72 |
+
environment and from VORTEX's environment. Recreate it with Python 3.12:
|
| 73 |
+
|
| 74 |
+
```bash
|
| 75 |
+
cd /workspace/WarpState
|
| 76 |
+
source rtx6000_40b/ops/training-env.sh
|
| 77 |
+
python3 -m venv rtx6000_40b/.server-venv
|
| 78 |
+
uv pip install --python rtx6000_40b/.server-venv/bin/python \
|
| 79 |
+
-r rtx6000_40b/ops/training-requirements-resolved.txt \
|
| 80 |
+
--extra-index-url https://download.pytorch.org/whl/cu130 \
|
| 81 |
+
--index-strategy unsafe-best-match
|
| 82 |
+
uv pip check --python rtx6000_40b/.server-venv/bin/python
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
The lock pins PyTorch 2.14.0+cu130 and its installed dependencies. Always
|
| 86 |
+
source `training-env.sh` when testing or provisioning so caches and temporary
|
| 87 |
+
files remain under `/workspace/WarpState`. The launcher sets these paths
|
| 88 |
+
itself and uses `--no-bootstrap`; the minimal GPU environment does not
|
| 89 |
+
contain the separate data-preparation dependencies.
|
| 90 |
+
|
| 91 |
+
## Operation
|
| 92 |
+
|
| 93 |
+
The existing project-private Supervisor manages both preparation and training.
|
| 94 |
+
Its training command is now `train-fast.sh`, which needs both GPUs available.
|
| 95 |
+
`datactl.sh` is its control wrapper, including for the training job:
|
| 96 |
+
|
| 97 |
+
```bash
|
| 98 |
+
cd /workspace/WarpState
|
| 99 |
+
rtx6000_40b/ops/datactl.sh status
|
| 100 |
+
rtx6000_40b/ops/datactl.sh start warpstate-train
|
| 101 |
+
tail -n 10 rtx6000_40b/work/logs/train.log
|
| 102 |
+
tail -n 3 rtx6000_40b/work/run/metrics.jsonl
|
| 103 |
+
nvidia-smi
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
To stop, use `rtx6000_40b/ops/datactl.sh stop warpstate-train`. The runner
|
| 107 |
+
finishes its current optimizer update and saves before exiting. Supervisor
|
| 108 |
+
allows up to 15 minutes for that. Starting the same job resumes its latest
|
| 109 |
+
checkpoint. `--reconfigure` permits changes to execution settings while keeping
|
| 110 |
+
the sequence length, global tokens per update, data, and optimizer schedule
|
| 111 |
+
unchanged. Changes to accumulation or GPU count can change floating-point
|
| 112 |
+
rounding; the model and full optimizer state are retained.
|
| 113 |
+
|
| 114 |
+
Training is started explicitly and does not automatically restart after a
|
| 115 |
+
failure, avoiding repeated allocation attempts on a shared GPU. Inspect the
|
| 116 |
+
log and available memory before restarting a failed job. A container restart
|
| 117 |
+
also requires starting the project Supervisor and the training job again.
|
| 118 |
+
|
| 119 |
+
For a bounded trial of the current configuration, run
|
| 120 |
+
`ops/train-fast.sh --max-steps 2` from the package directory while the
|
| 121 |
+
supervised training job is stopped. This saves after two
|
| 122 |
+
additional updates while preserving the full 40B schedule. Only one runner
|
| 123 |
+
can hold `work/pipeline.lock` at a time.
|
| 124 |
+
|
| 125 |
+
To return to the original shared GPU configuration, stop the supervised job
|
| 126 |
+
and run `ops/train-shared.sh --reconfigure` with enough free memory on GPU 1.
|
| 127 |
+
For persistent supervision, update the program command to that launcher plus
|
| 128 |
+
`--reconfigure` and use `stopsignal=TERM` before `datactl.sh reread` and
|
| 129 |
+
`datactl.sh update warpstate-train`. The shared launcher runs the worker
|
| 130 |
+
directly; the optimized launcher relays stop requests to distributed workers.
|
| 131 |
+
Do not run the benchmark or another launcher alongside the production job.
|
| 132 |
+
|
| 133 |
+
## Checks
|
| 134 |
+
|
| 135 |
+
The tests exercise interrupted data preparation, exact final partial updates,
|
| 136 |
+
the memory-headroom refusal, and exact CUDA checkpoint/resume behavior for
|
| 137 |
+
both imported weights and fresh initialization. The CUDA tests use a tiny
|
| 138 |
+
model and a 0.5 GiB allocator cap:
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
cd /workspace/WarpState
|
| 142 |
+
source rtx6000_40b/ops/training-env.sh
|
| 143 |
+
CUDA_VISIBLE_DEVICES=1 rtx6000_40b/.server-venv/bin/python \
|
| 144 |
+
-m pytest rtx6000_40b/test_server.py -q
|
| 145 |
+
```
|
| 146 |
+
|
| 147 |
+
Use `CUDA_VISIBLE_DEVICES=''` to skip CUDA tests when there is insufficient
|
| 148 |
+
free memory. Test artifacts use the project-local `TMPDIR`.
|
ops/activation_execution.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Opt-in BF16 SwiGLU and RMSNorm execution without replacing parameters.
|
| 2 |
+
|
| 3 |
+
Eager execution retains the model's BF16 rounding boundaries. Compiled execution
|
| 4 |
+
uses the current Inductor arithmetic, eliding intermediate BF16 casts while
|
| 5 |
+
retaining BF16 kernel inputs/outputs. Normalization statistics and master
|
| 6 |
+
parameter gradients remain FP32. FP32 arithmetic reductions can differ
|
| 7 |
+
slightly from Inductor's reduction order; this is an execution experiment, not a
|
| 8 |
+
claim of bitwise equivalence. CPU, evaluation and non-BF16 paths stay original.
|
| 9 |
+
|
| 10 |
+
Apply before compiling cores. Checkpoints and inference archives are unchanged.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import triton
|
| 17 |
+
import triton.language as tl
|
| 18 |
+
from triton.language.extra.cuda import libdevice
|
| 19 |
+
from torch.library import triton_op, wrap_triton
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@triton.jit
|
| 23 |
+
def _swiglu_forward_kernel(X, Y, N: tl.constexpr, H: tl.constexpr,
|
| 24 |
+
BLOCK: tl.constexpr, ROUND: tl.constexpr):
|
| 25 |
+
i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
| 26 |
+
mask = i < N
|
| 27 |
+
packed = (i // H) * (2 * H) + i % H
|
| 28 |
+
up = tl.load(X + packed, mask, 0).to(tl.float32)
|
| 29 |
+
gate = tl.load(X + packed + H, mask, 0).to(tl.float32)
|
| 30 |
+
silu = gate / (1.0 + libdevice.exp(-gate))
|
| 31 |
+
if ROUND:
|
| 32 |
+
silu = silu.to(tl.bfloat16).to(tl.float32)
|
| 33 |
+
tl.store(Y + i, silu * up, mask)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@triton.jit
|
| 37 |
+
def _swiglu_backward_kernel(X, DY, DX, N: tl.constexpr, H: tl.constexpr,
|
| 38 |
+
BLOCK: tl.constexpr, ROUND: tl.constexpr):
|
| 39 |
+
i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
| 40 |
+
mask = i < N
|
| 41 |
+
packed = (i // H) * (2 * H) + i % H
|
| 42 |
+
up = tl.load(X + packed, mask, 0).to(tl.float32)
|
| 43 |
+
gate = tl.load(X + packed + H, mask, 0).to(tl.float32)
|
| 44 |
+
dy = tl.load(DY + i, mask, 0).to(tl.float32)
|
| 45 |
+
sigmoid = 1.0 / (1.0 + tl.exp(-gate))
|
| 46 |
+
silu = gate / (1.0 + libdevice.exp(-gate))
|
| 47 |
+
if ROUND:
|
| 48 |
+
silu = silu.to(tl.bfloat16).to(tl.float32)
|
| 49 |
+
# aten's multiply backward rounds before aten's silu backward executes.
|
| 50 |
+
dsilu = dy * up
|
| 51 |
+
if ROUND:
|
| 52 |
+
dsilu = dsilu.to(tl.bfloat16).to(tl.float32)
|
| 53 |
+
dgate = (dsilu * sigmoid) * (1.0 + gate * (1.0 - sigmoid))
|
| 54 |
+
tl.store(DX + packed, dy * silu, mask)
|
| 55 |
+
tl.store(DX + packed + H, dgate, mask)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@triton_op("warpstate::bf16_swiglu", mutates_args={})
|
| 59 |
+
def bf16_swiglu(packed: torch.Tensor, eager_rounding: bool = True) -> torch.Tensor:
|
| 60 |
+
"""SwiGLU of a contiguous BF16 CUDA [*, 2*hidden] projection.
|
| 61 |
+
|
| 62 |
+
Set ``eager_rounding=False`` to match the current compiled core arithmetic.
|
| 63 |
+
"""
|
| 64 |
+
hidden = packed.shape[-1] // 2
|
| 65 |
+
output = torch.empty((*packed.shape[:-1], hidden), device=packed.device,
|
| 66 |
+
dtype=packed.dtype)
|
| 67 |
+
wrap_triton(_swiglu_forward_kernel)[(triton.cdiv(output.numel(), 1024),)](
|
| 68 |
+
packed, output, output.numel(), hidden, 1024, eager_rounding,
|
| 69 |
+
enable_fp_fusion=not eager_rounding)
|
| 70 |
+
return output
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@triton_op("warpstate::bf16_swiglu_backward", mutates_args={})
|
| 74 |
+
def _swiglu_backward(packed: torch.Tensor, grad: torch.Tensor,
|
| 75 |
+
eager_rounding: bool) -> torch.Tensor:
|
| 76 |
+
grad = grad.contiguous()
|
| 77 |
+
output = torch.empty_like(packed)
|
| 78 |
+
wrap_triton(_swiglu_backward_kernel)[(triton.cdiv(grad.numel(), 1024),)](
|
| 79 |
+
packed, grad, output, grad.numel(), packed.shape[-1] // 2, 1024, eager_rounding,
|
| 80 |
+
enable_fp_fusion=not eager_rounding)
|
| 81 |
+
return output
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _swiglu_setup(ctx, inputs, output):
|
| 85 |
+
ctx.save_for_backward(inputs[0])
|
| 86 |
+
ctx.eager_rounding = inputs[1]
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _swiglu_autograd(ctx, grad):
|
| 90 |
+
return _swiglu_backward(ctx.saved_tensors[0], grad, ctx.eager_rounding), None
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
bf16_swiglu.register_autograd(_swiglu_autograd, setup_context=_swiglu_setup)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
@triton.jit
|
| 97 |
+
def _rms_forward_kernel(X, W, Y, R, D: tl.constexpr, EPS: tl.constexpr,
|
| 98 |
+
BLOCK: tl.constexpr, ROUND: tl.constexpr):
|
| 99 |
+
row = tl.program_id(0)
|
| 100 |
+
col = tl.arange(0, BLOCK)
|
| 101 |
+
x = tl.load(X + row * D + col, col < D, 0).to(tl.float32)
|
| 102 |
+
w = tl.load(W + col, col < D, 0)
|
| 103 |
+
if ROUND:
|
| 104 |
+
w = w.to(tl.bfloat16).to(tl.float32)
|
| 105 |
+
r = tl.rsqrt(tl.sum(x * x, 0) / D + EPS)
|
| 106 |
+
z = x * r
|
| 107 |
+
if ROUND:
|
| 108 |
+
z = z.to(tl.bfloat16).to(tl.float32)
|
| 109 |
+
tl.store(Y + row * D + col, z * w, col < D)
|
| 110 |
+
tl.store(R + row, r)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@triton.jit
|
| 114 |
+
def _rms_backward_kernel(X, W, R, DY, DX, DW_PART,
|
| 115 |
+
M: tl.constexpr, D: tl.constexpr,
|
| 116 |
+
ROWS: tl.constexpr, BLOCK: tl.constexpr, ROUND: tl.constexpr):
|
| 117 |
+
group = tl.program_id(0)
|
| 118 |
+
rows = group * ROWS + tl.arange(0, ROWS)
|
| 119 |
+
cols = tl.arange(0, BLOCK)
|
| 120 |
+
mask = (rows[:, None] < M) & (cols[None, :] < D)
|
| 121 |
+
x = tl.load(X + rows[:, None] * D + cols[None, :], mask, 0).to(tl.float32)
|
| 122 |
+
dy = tl.load(DY + rows[:, None] * D + cols[None, :], mask, 0).to(tl.float32)
|
| 123 |
+
w = tl.load(W + cols, cols < D, 0)
|
| 124 |
+
if ROUND:
|
| 125 |
+
w = w.to(tl.bfloat16).to(tl.float32)
|
| 126 |
+
r = tl.load(R + rows, rows < M, 0)
|
| 127 |
+
z = x * r[:, None]
|
| 128 |
+
dz = dy * w[None, :]
|
| 129 |
+
if ROUND:
|
| 130 |
+
z = z.to(tl.bfloat16).to(tl.float32)
|
| 131 |
+
dz = dz.to(tl.bfloat16).to(tl.float32)
|
| 132 |
+
dr = tl.sum(dz * x, 1)
|
| 133 |
+
# Keep the same explicit derivative chain as rsqrt(mean(x**2) + eps).
|
| 134 |
+
dsquare = (-0.5 * dr * (r * r * r)) / D
|
| 135 |
+
dx = dz * r[:, None] + dsquare[:, None] * (2.0 * x)
|
| 136 |
+
dw = dy * z
|
| 137 |
+
if ROUND:
|
| 138 |
+
dw = dw.to(tl.bfloat16).to(tl.float32)
|
| 139 |
+
partial = tl.sum(dw, 0)
|
| 140 |
+
tl.store(DX + rows[:, None] * D + cols[None, :], dx, mask)
|
| 141 |
+
tl.store(DW_PART + group * D + cols, partial, cols < D)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
@triton.jit
|
| 145 |
+
def _rms_weight_reduce_kernel(PART, DW, D: tl.constexpr, GROUPS: tl.constexpr,
|
| 146 |
+
BLOCK_G: tl.constexpr, BLOCK_D: tl.constexpr, ROUND: tl.constexpr):
|
| 147 |
+
cols = tl.program_id(0) * BLOCK_D + tl.arange(0, BLOCK_D)
|
| 148 |
+
groups = tl.arange(0, BLOCK_G)
|
| 149 |
+
partial = tl.load(PART + groups[:, None] * D + cols[None, :],
|
| 150 |
+
(groups[:, None] < GROUPS) & (cols[None, :] < D), 0)
|
| 151 |
+
# Eager weight.to(BF16) makes sum-to-size return BF16 before its gradient is
|
| 152 |
+
# cast back to FP32. The compiled path retains the FP32 reduction; Inductor's
|
| 153 |
+
# own partitioning can retain a final BF16 cast at some large shapes, so this
|
| 154 |
+
# comparison is numerically bounded rather than universally bitwise equal.
|
| 155 |
+
total = tl.sum(partial, 0)
|
| 156 |
+
if ROUND:
|
| 157 |
+
total = total.to(tl.bfloat16).to(tl.float32)
|
| 158 |
+
tl.store(DW + cols, total, cols < D)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
@triton_op("warpstate::bf16_rmsnorm", mutates_args={})
|
| 162 |
+
def bf16_rmsnorm(x: torch.Tensor, weight: torch.Tensor,
|
| 163 |
+
eps: float, eager_rounding: bool = True) -> tuple[torch.Tensor, torch.Tensor]:
|
| 164 |
+
"""BF16 CUDA RMSNorm with FP32 statistics; also return saved row rsqrts."""
|
| 165 |
+
width = x.shape[-1]
|
| 166 |
+
rows = x.numel() // width
|
| 167 |
+
output = torch.empty_like(x)
|
| 168 |
+
rstd = torch.empty((rows,), device=x.device, dtype=torch.float32)
|
| 169 |
+
block = triton.next_power_of_2(width)
|
| 170 |
+
wrap_triton(_rms_forward_kernel)[(rows,)](
|
| 171 |
+
x, weight, output, rstd, width, eps, block, eager_rounding,
|
| 172 |
+
num_warps=4 if block <= 4096 else 8, enable_fp_fusion=not eager_rounding)
|
| 173 |
+
return output, rstd
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
@triton_op("warpstate::bf16_rmsnorm_backward", mutates_args={})
|
| 177 |
+
def _rms_backward(x: torch.Tensor, weight: torch.Tensor, rstd: torch.Tensor,
|
| 178 |
+
grad: torch.Tensor, eager_rounding: bool) -> tuple[torch.Tensor, torch.Tensor]:
|
| 179 |
+
grad = grad.contiguous()
|
| 180 |
+
width = x.shape[-1]
|
| 181 |
+
rows = x.numel() // width
|
| 182 |
+
group_rows = 4
|
| 183 |
+
groups = triton.cdiv(rows, group_rows)
|
| 184 |
+
output = torch.empty_like(x)
|
| 185 |
+
partial = torch.empty((groups, width), device=x.device, dtype=torch.float32)
|
| 186 |
+
weight_grad = torch.empty_like(weight)
|
| 187 |
+
block = triton.next_power_of_2(width)
|
| 188 |
+
wrap_triton(_rms_backward_kernel)[(groups,)](
|
| 189 |
+
x, weight, rstd, grad, output, partial, rows, width, group_rows, block, eager_rounding,
|
| 190 |
+
num_warps=4 if block <= 2048 else 8, enable_fp_fusion=not eager_rounding)
|
| 191 |
+
wrap_triton(_rms_weight_reduce_kernel)[(triton.cdiv(width, 8),)](
|
| 192 |
+
partial, weight_grad, width, groups, triton.next_power_of_2(groups), 8, eager_rounding,
|
| 193 |
+
num_warps=8, enable_fp_fusion=False)
|
| 194 |
+
return output, weight_grad
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _rms_setup(ctx, inputs, output):
|
| 198 |
+
ctx.save_for_backward(inputs[0], inputs[1], output[1])
|
| 199 |
+
ctx.mark_non_differentiable(output[1])
|
| 200 |
+
ctx.eager_rounding = inputs[3]
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _rms_autograd(ctx, grad, _grad_rstd):
|
| 204 |
+
x, weight, rstd = ctx.saved_tensors
|
| 205 |
+
dx, dw = _rms_backward(x, weight, rstd, grad, ctx.eager_rounding)
|
| 206 |
+
return dx, dw, None, None
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
bf16_rmsnorm.register_autograd(_rms_autograd, setup_context=_rms_setup)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def apply_activation_execution(model: torch.nn.Module, mode: str = "baseline") -> None:
|
| 213 |
+
"""Patch training forwards only, retaining canonical FP32 parameter objects.
|
| 214 |
+
|
| 215 |
+
Modes select fused ``swiglu``, ``rmsnorm``, or ``all``; ``baseline`` is a no-op.
|
| 216 |
+
Inputs outside the supported contiguous BF16 CUDA shapes use eager model
|
| 217 |
+
code. Evaluation retains the original path, avoiding validation graph churn.
|
| 218 |
+
Eager invocation preserves eager casts; core graph capture selects the
|
| 219 |
+
current Inductor arithmetic with elided intermediate casts.
|
| 220 |
+
"""
|
| 221 |
+
from warpstate.model import RMSNorm, SwiGLU
|
| 222 |
+
|
| 223 |
+
if mode not in {"baseline", "swiglu", "rmsnorm", "all"}:
|
| 224 |
+
raise ValueError(f"Unknown activation execution mode: {mode}")
|
| 225 |
+
if getattr(model, "_activation_execution_mode", "baseline") != "baseline":
|
| 226 |
+
raise ValueError("Activation execution has already been applied")
|
| 227 |
+
if mode == "baseline":
|
| 228 |
+
return
|
| 229 |
+
if any(p.dtype != torch.float32 for p in model.parameters()):
|
| 230 |
+
raise ValueError("Activation execution requires FP32 master parameters")
|
| 231 |
+
|
| 232 |
+
for module in model.modules():
|
| 233 |
+
original = module.forward
|
| 234 |
+
if isinstance(module, SwiGLU) and mode in {"swiglu", "all"}:
|
| 235 |
+
def swiglu_forward(x, _module=module, _original=original):
|
| 236 |
+
if (_module.training and torch.is_grad_enabled() and x.is_cuda
|
| 237 |
+
and torch.is_autocast_enabled("cuda")
|
| 238 |
+
and torch.get_autocast_dtype("cuda") == torch.bfloat16):
|
| 239 |
+
packed = _module.up_gate(x)
|
| 240 |
+
return _module.down(bf16_swiglu(packed.contiguous(),
|
| 241 |
+
eager_rounding=not torch.compiler.is_compiling()))
|
| 242 |
+
return _original(x)
|
| 243 |
+
module.forward = swiglu_forward
|
| 244 |
+
elif isinstance(module, RMSNorm) and mode in {"rmsnorm", "all"}:
|
| 245 |
+
def rms_forward(x, _module=module, _original=original):
|
| 246 |
+
if (_module.training and torch.is_grad_enabled() and x.is_cuda
|
| 247 |
+
and x.dtype == torch.bfloat16 and x.is_contiguous()
|
| 248 |
+
and 0 < x.shape[-1] <= 8192 and x.numel() > 0):
|
| 249 |
+
return bf16_rmsnorm(x, _module.weight, _module.eps,
|
| 250 |
+
eager_rounding=not torch.compiler.is_compiling())[0]
|
| 251 |
+
return _original(x)
|
| 252 |
+
module.forward = rms_forward
|
| 253 |
+
model._activation_execution_mode = mode
|
ops/analyze_communication_profile.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Summarize compute/communication overlap from saved Chrome traces, without CUDA.
|
| 3 |
+
|
| 4 |
+
Each rank and device is reported separately. All time values use microseconds.
|
| 5 |
+
Noncommunication activity includes kernels, copies, and fills. Logical NCCL
|
| 6 |
+
input payload bytes are distinct from bytes physically transported over a link.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
from collections import Counter
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
import math
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
import re
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
PACKAGE = Path(__file__).resolve().parents[1]
|
| 21 |
+
WORKSPACE = PACKAGE.parent
|
| 22 |
+
ACTIVITY_CATEGORIES = {"kernel", "gpu_memcpy", "gpu_memset"}
|
| 23 |
+
COLLECTIVE_OPERATORS = {
|
| 24 |
+
"c10d::allreduce_", "c10d::allreduce_coalesced_", "c10d::allgather_",
|
| 25 |
+
"c10d::_allgather_base_", "c10d::allgather_into_tensor_coalesced_",
|
| 26 |
+
"c10d::reduce_scatter_", "c10d::_reduce_scatter_base_",
|
| 27 |
+
"c10d::reduce_scatter_tensor_coalesced_", "c10d::broadcast_", "c10d::reduce_",
|
| 28 |
+
"c10d::alltoall_", "c10d::alltoall_base_", "c10d::gather_", "c10d::scatter_",
|
| 29 |
+
"c10d::barrier",
|
| 30 |
+
}
|
| 31 |
+
DTYPE_BYTES = {"float": 4, "float32": 4, "double": 8, "float64": 8,
|
| 32 |
+
"half": 2, "float16": 2, "bfloat16": 2, "long": 8,
|
| 33 |
+
"int64": 8, "int": 4, "int32": 4, "short": 2,
|
| 34 |
+
"int16": 2, "char": 1, "int8": 1, "byte": 1, "uint8": 1,
|
| 35 |
+
"bool": 1}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def merge_intervals(intervals):
|
| 39 |
+
"""Return disjoint intervals, counting overlapping streams only once."""
|
| 40 |
+
merged = []
|
| 41 |
+
for begin, end in sorted(intervals):
|
| 42 |
+
if not math.isfinite(begin) or not math.isfinite(end) or end < begin:
|
| 43 |
+
raise ValueError("Activity intervals must have finite, ordered endpoints")
|
| 44 |
+
if begin == end:
|
| 45 |
+
continue
|
| 46 |
+
if merged and begin <= merged[-1][1]:
|
| 47 |
+
merged[-1] = (merged[-1][0], max(end, merged[-1][1]))
|
| 48 |
+
else:
|
| 49 |
+
merged.append((begin, end))
|
| 50 |
+
return merged
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def interval_duration(intervals):
|
| 54 |
+
return sum(end - begin for begin, end in intervals)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def intersection_duration(left, right):
|
| 58 |
+
"""Measure intersection of two disjoint, ordered interval lists."""
|
| 59 |
+
total = 0.0
|
| 60 |
+
i = j = 0
|
| 61 |
+
while i < len(left) and j < len(right):
|
| 62 |
+
total += max(0.0, min(left[i][1], right[j][1]) - max(left[i][0], right[j][0]))
|
| 63 |
+
if left[i][1] <= right[j][1]:
|
| 64 |
+
i += 1
|
| 65 |
+
else:
|
| 66 |
+
j += 1
|
| 67 |
+
return total
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def collective_metadata(kernels):
|
| 71 |
+
"""Deduplicate channel kernels using process-group name and sequence number.
|
| 72 |
+
|
| 73 |
+
Unknown/inconsistent metadata yields null totals instead of guessed bytes or
|
| 74 |
+
a kernel count mislabeled as a collective count. Scalar metric collectives
|
| 75 |
+
are included; the returned sequence permits separating them from gradients.
|
| 76 |
+
"""
|
| 77 |
+
calls = {}
|
| 78 |
+
missing_identity = 0
|
| 79 |
+
for event in sorted(kernels, key=lambda item: item["ts"]):
|
| 80 |
+
args = event.get("args", {})
|
| 81 |
+
sequence_id = args.get("Seq")
|
| 82 |
+
if (args.get("Process Group Name") is None or not args.get("Collective name")
|
| 83 |
+
or type(sequence_id) not in (int, str) or not str(sequence_id).isdigit()):
|
| 84 |
+
missing_identity += 1
|
| 85 |
+
continue
|
| 86 |
+
key = (str(args["Process Group Name"]), str(args["Seq"]))
|
| 87 |
+
count = args.get("In msg nelems")
|
| 88 |
+
width = DTYPE_BYTES.get(str(args.get("dtype", "")).lower())
|
| 89 |
+
byte_count = count * width if type(count) is int and count >= 0 and width else None
|
| 90 |
+
row = {"process_group": key[0], "sequence": key[1],
|
| 91 |
+
"collective": args["Collective name"], "dtype": args.get("dtype"),
|
| 92 |
+
"input_elements": count, "input_payload_bytes": byte_count}
|
| 93 |
+
previous = calls.get(key)
|
| 94 |
+
if previous is None:
|
| 95 |
+
calls[key] = {**row, "kernel_count": 1, "metadata_consistent": True}
|
| 96 |
+
else:
|
| 97 |
+
previous["kernel_count"] += 1
|
| 98 |
+
if any(previous[name] != value for name, value in row.items()):
|
| 99 |
+
previous["metadata_consistent"] = False
|
| 100 |
+
previous["input_payload_bytes"] = None
|
| 101 |
+
sequence = list(calls.values())
|
| 102 |
+
known_bytes = sum(row["input_payload_bytes"] or 0 for row in sequence)
|
| 103 |
+
complete = not missing_identity and all(row["metadata_consistent"] and
|
| 104 |
+
row["input_payload_bytes"] is not None
|
| 105 |
+
for row in sequence)
|
| 106 |
+
return {"nccl_kernel_count": len(kernels),
|
| 107 |
+
"collective_call_count": len(sequence) if not missing_identity else None,
|
| 108 |
+
"known_collective_call_count": len(sequence),
|
| 109 |
+
"kernels_missing_collective_identity": missing_identity,
|
| 110 |
+
"logical_input_payload_bytes": known_bytes if complete else None,
|
| 111 |
+
"known_logical_input_payload_bytes": known_bytes,
|
| 112 |
+
"collectives": sequence}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def cpu_collective_fingerprint(events):
|
| 116 |
+
"""Structural CPU submission order, never double-counting NCCL annotations.
|
| 117 |
+
|
| 118 |
+
Shapes/types do not identify tensor contents. This can detect many bucket
|
| 119 |
+
order changes but cannot prove equal tensor order for equal-shaped buckets.
|
| 120 |
+
"""
|
| 121 |
+
selected = sorted((event for event in events if event.get("ph") == "X"
|
| 122 |
+
and event.get("cat") == "cpu_op"
|
| 123 |
+
and event.get("name") in COLLECTIVE_OPERATORS),
|
| 124 |
+
key=lambda event: event["ts"])
|
| 125 |
+
sequence = [{"operator": event["name"],
|
| 126 |
+
"input_dims": event.get("args", {}).get("Input Dims"),
|
| 127 |
+
"input_types": event.get("args", {}).get("Input type")}
|
| 128 |
+
for event in selected]
|
| 129 |
+
payload = json.dumps(sequence, sort_keys=True, separators=(",", ":"))
|
| 130 |
+
return {"call_count": len(sequence), "sequence": sequence,
|
| 131 |
+
"sha256": hashlib.sha256(payload.encode()).hexdigest() if sequence else None,
|
| 132 |
+
"all_input_shapes_recorded": all(row["input_dims"] is not None for row in sequence)}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def summarize_trace(trace):
|
| 136 |
+
events = trace.get("traceEvents", [])
|
| 137 |
+
devices = {}
|
| 138 |
+
excluded = 0
|
| 139 |
+
for event in events:
|
| 140 |
+
if event.get("ph") != "X":
|
| 141 |
+
continue
|
| 142 |
+
category = event.get("cat")
|
| 143 |
+
if category == "gpu_user_annotation":
|
| 144 |
+
excluded += 1
|
| 145 |
+
if category not in ACTIVITY_CATEGORIES:
|
| 146 |
+
continue
|
| 147 |
+
begin, duration = event.get("ts"), event.get("dur")
|
| 148 |
+
if (not isinstance(begin, (int, float)) or not isinstance(duration, (int, float))
|
| 149 |
+
or not math.isfinite(begin) or not math.isfinite(duration) or duration < 0):
|
| 150 |
+
raise ValueError("CUDA activities require finite timestamps and nonnegative durations")
|
| 151 |
+
device = str(event.get("args", {}).get("device", event.get("pid", "unknown")))
|
| 152 |
+
group = devices.setdefault(device, {"communication": [], "compute": [], "kernels": []})
|
| 153 |
+
communication = category == "kernel" and "nccl" in event.get("name", "").lower()
|
| 154 |
+
group["communication" if communication else "compute"].append((begin, begin + duration))
|
| 155 |
+
if communication:
|
| 156 |
+
group["kernels"].append(event)
|
| 157 |
+
summaries = []
|
| 158 |
+
for device, group in sorted(devices.items()):
|
| 159 |
+
compute = merge_intervals(group["compute"])
|
| 160 |
+
communication = merge_intervals(group["communication"])
|
| 161 |
+
active = merge_intervals([*compute, *communication])
|
| 162 |
+
overlap = intersection_duration(compute, communication)
|
| 163 |
+
compute_us = interval_duration(compute)
|
| 164 |
+
communication_us = interval_duration(communication)
|
| 165 |
+
span = active[-1][1] - active[0][0] if active else 0.0
|
| 166 |
+
active_us = interval_duration(active)
|
| 167 |
+
summaries.append({"device": device, "activity_span_us": span,
|
| 168 |
+
"active_union_us": active_us, "idle_within_activity_span_us": span - active_us,
|
| 169 |
+
"noncommunication_union_us": compute_us,
|
| 170 |
+
"communication_union_us": communication_us,
|
| 171 |
+
"compute_only_us": max(0.0, compute_us - overlap),
|
| 172 |
+
"compute_communication_overlap_us": overlap,
|
| 173 |
+
"communication_only_us": max(0.0, communication_us - overlap),
|
| 174 |
+
"communication_overlap_fraction": overlap / communication_us if communication_us else None,
|
| 175 |
+
"summed_noncommunication_us": interval_duration(group["compute"]),
|
| 176 |
+
"summed_communication_us": interval_duration(group["communication"]),
|
| 177 |
+
"nccl_kernel_names": dict(Counter(event["name"] for event in group["kernels"])),
|
| 178 |
+
**collective_metadata(group["kernels"])})
|
| 179 |
+
return {"rank": trace.get("distributedInfo", {}).get("rank"),
|
| 180 |
+
"distributed_info": trace.get("distributedInfo"), "devices": summaries,
|
| 181 |
+
"excluded_gpu_annotation_ranges": excluded,
|
| 182 |
+
"cpu_collective_fingerprint": cpu_collective_fingerprint(events)}
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def main(argv=None):
|
| 186 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 187 |
+
parser.add_argument("traces", nargs="+", type=Path)
|
| 188 |
+
parser.add_argument("--output", required=True, type=Path)
|
| 189 |
+
args = parser.parse_args(argv)
|
| 190 |
+
args.traces = [path.expanduser().resolve() for path in args.traces]
|
| 191 |
+
args.output = args.output.expanduser().resolve()
|
| 192 |
+
if any(not path.is_relative_to(WORKSPACE) for path in args.traces):
|
| 193 |
+
parser.error("Trace inputs must stay inside the WarpState workspace")
|
| 194 |
+
if not args.output.is_relative_to(PACKAGE / "work") or args.output.suffix != ".json":
|
| 195 |
+
parser.error("Output must be a JSON file under rtx6000_40b/work")
|
| 196 |
+
if args.output.exists():
|
| 197 |
+
parser.error("Output already exists; select a new report path")
|
| 198 |
+
report = {"format": "warpstate-communication-profile-v1",
|
| 199 |
+
"analyzer_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
| 200 |
+
"ranks": [], "limitations": [
|
| 201 |
+
"Per-rank/device budgets; never sum ranks or devices into a wall-time estimate.",
|
| 202 |
+
"Compute includes all non-NCCL kernels, memory copies, and fills.",
|
| 203 |
+
"GPU annotation ranges are excluded; real CUDA activities alone define intervals.",
|
| 204 |
+
"Communication-only time is an exposed interval, not a guaranteed removable delay.",
|
| 205 |
+
"Payload bytes describe logical collective inputs, not physical transport traffic.",
|
| 206 |
+
"Scalar benchmark metric collectives remain included in counts and bytes.",
|
| 207 |
+
"CPU fingerprints compare operator/shape/type submission sequences, not tensor identities.",
|
| 208 |
+
"Activity spans exclude untraced startup/tail time; profiling adds instrumentation overhead."]}
|
| 209 |
+
for path in args.traces:
|
| 210 |
+
payload = path.read_bytes()
|
| 211 |
+
summary = summarize_trace(json.loads(payload))
|
| 212 |
+
if summary["rank"] is None:
|
| 213 |
+
match = re.search(r"profile-rank-(\d+)", path.name)
|
| 214 |
+
if match:
|
| 215 |
+
summary["rank"] = int(match.group(1))
|
| 216 |
+
report["ranks"].append({"trace": str(path), "trace_sha256": hashlib.sha256(payload).hexdigest(),
|
| 217 |
+
**summary})
|
| 218 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 219 |
+
temporary = args.output.with_name(args.output.name + ".partial")
|
| 220 |
+
with temporary.open("x") as output:
|
| 221 |
+
output.write(json.dumps(report, indent=2, allow_nan=False) + "\n")
|
| 222 |
+
temporary.replace(args.output)
|
| 223 |
+
print(json.dumps({"output": str(args.output), "traces": len(report["ranks"])}))
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
if __name__ == "__main__":
|
| 227 |
+
main()
|
ops/archive_dpo.py
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Seal a completed DPO run as independent, read-only full-state/model archives.
|
| 3 |
+
|
| 4 |
+
Run from /workspace/WarpState after the DPO guardian exits successfully. The
|
| 5 |
+
default destination is work/archives/step-00076294-dpo-final. Existing archives
|
| 6 |
+
are verified and reused; they are never replaced or modified.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import contextlib
|
| 13 |
+
import json
|
| 14 |
+
import math
|
| 15 |
+
import os
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
import stat
|
| 18 |
+
import subprocess
|
| 19 |
+
import sys
|
| 20 |
+
import tempfile
|
| 21 |
+
|
| 22 |
+
PACKAGE = Path(__file__).resolve().parents[1]
|
| 23 |
+
WORKSPACE = PACKAGE.parent
|
| 24 |
+
sys.path.insert(0, str(PACKAGE))
|
| 25 |
+
|
| 26 |
+
import numpy as np
|
| 27 |
+
import torch
|
| 28 |
+
|
| 29 |
+
from dpo import DEFAULTS, identity_digest
|
| 30 |
+
from ops.dpo_handoff import load_plan, read_json, seal, verify_files, verify_dpo_completion
|
| 31 |
+
from ops.launch_training import workspace_path
|
| 32 |
+
from ops.posttrain_handoff import (atomic_json, clean_exit, copy_regular, exclusive_lock,
|
| 33 |
+
process_status, publish_directory, sha256, tensors_equal, utc_now)
|
| 34 |
+
|
| 35 |
+
FILES = ('checkpoint.pt', 'model.pt', 'snapshot.json', 'tokenizer.model', 'plan.json',
|
| 36 |
+
'settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl',
|
| 37 |
+
'dpo-execution.json', 'dpo-handoff.json', 'gpu-preflight.json',
|
| 38 |
+
'data-manifest.json', 'reference-manifest.json', 'reference-train.npy',
|
| 39 |
+
'reference-val.npy', 'PROVENANCE.json', 'ARCHIVE.md')
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def validate_state(checkpoint, evaluation, completion, settings_report, data, reference, plan):
|
| 43 |
+
"""Validate an archive using its own preserved records, without a live run."""
|
| 44 |
+
settings = DEFAULTS | plan['training']
|
| 45 |
+
processes = plan['handoff']['processes']
|
| 46 |
+
train, val = (data['outputs'][name] for name in ('dpo_train.jsonl', 'dpo_val.jsonl'))
|
| 47 |
+
total_steps = math.ceil(train['rows'] / (settings['micro_batch'] * settings['grad_accum'] * processes)) * settings['epochs']
|
| 48 |
+
if settings['max_steps'] is not None:
|
| 49 |
+
raise ValueError('Archive requires a completed whole DPO epoch, without a plan step cap')
|
| 50 |
+
expected = dict(step=total_steps, total_steps=total_steps,
|
| 51 |
+
sample_cursor=train['rows'] * settings['epochs'],
|
| 52 |
+
tokens_seen=(train['tokens'] - 2 * train['rows']) * settings['epochs'],
|
| 53 |
+
response_tokens_seen=train['supervised_tokens'] * settings['epochs'], val_step=total_steps)
|
| 54 |
+
if (checkpoint.get('format') != 'warpstate-dpo-v1' or checkpoint.get('completed') is not True or
|
| 55 |
+
any(checkpoint.get(key) != value for key, value in expected.items()) or
|
| 56 |
+
not math.isfinite(checkpoint.get('val_loss', float('nan')))):
|
| 57 |
+
raise ValueError('DPO checkpoint is not the fully completed, finally evaluated state')
|
| 58 |
+
signature = checkpoint['signature']
|
| 59 |
+
if (signature.get('plan') != plan or signature.get('settings') != settings or
|
| 60 |
+
signature.get('world_size') != processes or
|
| 61 |
+
signature.get('tokenizer_sha256') != plan['sft']['tokenizer_sha256'] or
|
| 62 |
+
signature.get('data_sha256') != plan['handoff']['data_manifest_sha256'] or
|
| 63 |
+
settings_report.get('signature') != signature or
|
| 64 |
+
settings_report.get('total_steps') != total_steps or
|
| 65 |
+
settings_report.get('train_rows') != train['rows'] or
|
| 66 |
+
settings_report.get('val_rows') != val['rows']):
|
| 67 |
+
raise ValueError('DPO checkpoint/settings signatures or dataset counts differ')
|
| 68 |
+
if (data.get('format') != 'warpstate-posttrain-data-v1' or data.get('complete') is not True or
|
| 69 |
+
data.get('tokenizer_sha256') != signature['tokenizer_sha256'] or
|
| 70 |
+
data.get('max_length') != settings['sequence_length']):
|
| 71 |
+
raise ValueError('DPO data provenance differs from its signature')
|
| 72 |
+
origin = checkpoint['origin']
|
| 73 |
+
if (origin.get('stage') != 'sft' or origin.get('step') != plan['sft']['step'] or
|
| 74 |
+
origin.get('weights_sha256') != signature['base_sha256'] or
|
| 75 |
+
origin.get('tokens_seen') != plan['sft']['tokens_seen'] or
|
| 76 |
+
origin.get('origin', {}).get('step') != plan['sft']['base_step'] or
|
| 77 |
+
origin.get('origin', {}).get('tokens_seen') != plan['sft']['base_tokens_seen']):
|
| 78 |
+
raise ValueError('DPO SFT/raw ancestry differs from the authorized plan')
|
| 79 |
+
if (not checkpoint.get('optimizer', {}).get('state') or
|
| 80 |
+
len(checkpoint.get('rng_by_rank', [])) != processes or
|
| 81 |
+
any('torch' not in value or 'cuda' not in value for value in checkpoint['rng_by_rank'])):
|
| 82 |
+
raise ValueError('DPO optimizer or per-rank RNG state is missing')
|
| 83 |
+
for value in checkpoint['optimizer']['state'].values():
|
| 84 |
+
if float(value['step']) != total_steps:
|
| 85 |
+
raise ValueError('DPO Adam step differs from the final checkpoint')
|
| 86 |
+
for name in ('exp_avg', 'exp_avg_sq'):
|
| 87 |
+
if value[name].dtype != torch.float32:
|
| 88 |
+
raise ValueError('DPO Adam state must retain its original FP32 tensors')
|
| 89 |
+
if not checkpoint.get('model'):
|
| 90 |
+
raise ValueError('DPO model tensors are missing')
|
| 91 |
+
for name, tensor in checkpoint['model'].items():
|
| 92 |
+
if tensor.is_floating_point() and tensor.dtype != torch.float32:
|
| 93 |
+
raise ValueError(f'DPO model tensor is not original FP32: {name}')
|
| 94 |
+
counters = ('step', 'total_steps', 'sample_cursor', 'tokens_seen', 'response_tokens_seen', 'val_loss')
|
| 95 |
+
if (completion.get('format') != 'warpstate-dpo-completion-v1' or completion.get('stage') != 'dpo' or
|
| 96 |
+
completion.get('completed') is not True or completion.get('paused_for_evaluation') is not True or
|
| 97 |
+
any(completion.get(key) != checkpoint[key] for key in counters)):
|
| 98 |
+
raise ValueError('DPO completion report differs from its checkpoint')
|
| 99 |
+
if (evaluation.get('stage') != 'dpo' or evaluation.get('step') != total_steps or
|
| 100 |
+
evaluation.get('full_holdout') is not True or evaluation.get('evaluated_samples') != val['rows'] or
|
| 101 |
+
evaluation.get('available_validation_samples') != val['rows'] or
|
| 102 |
+
evaluation.get('objective') != settings['objective'] or
|
| 103 |
+
any(evaluation.get(key) != checkpoint['validation'].get(key) for key in
|
| 104 |
+
('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll', 'evaluated_samples')) or
|
| 105 |
+
any(evaluation.get(key) != signature[key] for key in
|
| 106 |
+
('base_sha256', 'data_sha256', 'tokenizer_sha256', 'reference_sha256')) or
|
| 107 |
+
not all(math.isfinite(evaluation.get(key, float('nan'))) for key in
|
| 108 |
+
('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'))):
|
| 109 |
+
raise ValueError('DPO final full-holdout evaluation differs from its checkpoint')
|
| 110 |
+
if not 0 <= evaluation['reward_accuracy'] <= 1 or evaluation['chosen_nll'] < 0:
|
| 111 |
+
raise ValueError('DPO evaluation contains impossible accuracy/NLL')
|
| 112 |
+
identity = reference.get('identity', {})
|
| 113 |
+
if (reference.get('format') != 'warpstate-dpo-reference-v1' or reference.get('complete') is not True or
|
| 114 |
+
reference.get('identity_sha256') != identity_digest(identity) or
|
| 115 |
+
identity.get('weights_sha256') != signature['base_sha256'] or
|
| 116 |
+
identity.get('data_sha256') != signature['data_sha256'] or
|
| 117 |
+
identity.get('tokenizer_sha256') != signature['tokenizer_sha256'] or
|
| 118 |
+
identity.get('sources') != signature['sources'] or
|
| 119 |
+
identity.get('renderer_sha256') != data.get('renderer_sha256') or
|
| 120 |
+
identity.get('mask_format') != signature['mask'] or
|
| 121 |
+
identity.get('device_type') != signature['device_type'] or
|
| 122 |
+
identity.get('world_size') != processes or
|
| 123 |
+
identity.get('torch_version') != signature['torch_version'] or
|
| 124 |
+
any(value != settings.get(key) for key, value in identity.get('settings', {}).items()) or
|
| 125 |
+
completion.get('reference_sha256') != signature['reference_sha256']):
|
| 126 |
+
raise ValueError('DPO fixed-reference identity differs from its final checkpoint')
|
| 127 |
+
return expected
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def verify_dpo_archive(folder, *, expected_checkpoint_hash=None):
|
| 131 |
+
folder = Path(folder)
|
| 132 |
+
entries = verify_files(folder, FILES)
|
| 133 |
+
if stat.S_IMODE(folder.stat().st_mode) & 0o222 or {path.name for path in folder.iterdir()} != set(FILES) | {'SHA256SUMS'}:
|
| 134 |
+
raise ValueError('DPO archive directory is not sealed or contains unexpected files')
|
| 135 |
+
if expected_checkpoint_hash and entries['checkpoint.pt'] != expected_checkpoint_hash:
|
| 136 |
+
raise ValueError('Archived DPO state differs from the source checkpoint')
|
| 137 |
+
checkpoint = torch.load(folder / 'checkpoint.pt', map_location='cpu', weights_only=False, mmap=True)
|
| 138 |
+
plan, data = read_json(folder / 'plan.json'), read_json(folder / 'data-manifest.json')
|
| 139 |
+
reference = read_json(folder / 'reference-manifest.json')
|
| 140 |
+
validate_state(checkpoint, read_json(folder / 'evaluation.json'), read_json(folder / 'completion.json'),
|
| 141 |
+
read_json(folder / 'settings.json'), data, reference, plan)
|
| 142 |
+
if (entries['data-manifest.json'] != checkpoint['signature']['data_sha256'] or
|
| 143 |
+
entries['reference-manifest.json'] != checkpoint['signature']['reference_sha256'] or
|
| 144 |
+
entries['tokenizer.model'] != checkpoint['signature']['tokenizer_sha256']):
|
| 145 |
+
raise ValueError('Archived data/reference/tokenizer hashes differ from checkpoint signatures')
|
| 146 |
+
for split in ('train', 'val'):
|
| 147 |
+
entry = reference['files'][split + '.npy']
|
| 148 |
+
name = f'reference-{split}.npy'
|
| 149 |
+
rows = data['outputs'][f'dpo_{split}.jsonl']['rows']
|
| 150 |
+
values = np.load(folder / name, allow_pickle=False, mmap_mode='r')
|
| 151 |
+
if (entries[name] != entry['sha256'] or entry['rows'] != rows or
|
| 152 |
+
values.shape != (rows, 2) or values.dtype != np.float64 or
|
| 153 |
+
not np.isfinite(values).all() or not (values <= 0).all()):
|
| 154 |
+
raise ValueError('Archived fixed-reference score array differs')
|
| 155 |
+
metadata = read_json(folder / 'snapshot.json')
|
| 156 |
+
if (metadata.get('format') != 'warpstate-chat-v1' or metadata.get('inference_only') is not True or
|
| 157 |
+
metadata.get('stage') != 'dpo' or metadata.get('config') != checkpoint['config'] or
|
| 158 |
+
any(metadata.get(key) != checkpoint[key] for key in
|
| 159 |
+
('step', 'tokens_seen', 'response_tokens_seen', 'val_loss', 'origin')) or
|
| 160 |
+
metadata.get('tokenizer_sha256') != entries['tokenizer.model'] or
|
| 161 |
+
metadata.get('snapshot_sha256') != entries['model.pt'] or
|
| 162 |
+
metadata.get('snapshot_bytes') != (folder / 'model.pt').stat().st_size or
|
| 163 |
+
metadata.get('source', {}).get('sha256') != entries['checkpoint.pt']):
|
| 164 |
+
raise ValueError('DPO model export metadata differs from the full archive')
|
| 165 |
+
exported = torch.load(folder / 'model.pt', map_location='cpu', weights_only=False, mmap=True)
|
| 166 |
+
if {key: value for key, value in exported.items() if key != 'model'} != {
|
| 167 |
+
key: value for key, value in metadata.items() if key not in ('snapshot_sha256', 'snapshot_bytes')}:
|
| 168 |
+
raise ValueError('DPO exported model metadata differs from snapshot.json')
|
| 169 |
+
tensors_equal(checkpoint['model'], exported['model'])
|
| 170 |
+
return metadata, entries
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def archive_dpo(work, tokenizer, destination, plan, revision):
|
| 174 |
+
"""Caller owns all training exclusion locks; no source files are changed."""
|
| 175 |
+
work, tokenizer, destination = map(Path, (work, tokenizer, destination))
|
| 176 |
+
run, reference_dir = work / 'run', work / 'reference-cache'
|
| 177 |
+
source = run / 'latest.pt'
|
| 178 |
+
if source.is_symlink() or not source.is_file():
|
| 179 |
+
raise ValueError('DPO checkpoint must be a regular completed file')
|
| 180 |
+
before = source.stat()
|
| 181 |
+
checkpoint = torch.load(source, map_location='cpu', weights_only=False, mmap=True)
|
| 182 |
+
data_path = workspace_path(WORKSPACE / plan['handoff']['data_dir']) / 'manifest.json'
|
| 183 |
+
validate_state(checkpoint, read_json(run / 'evaluation.json'), read_json(run / 'completion.json'),
|
| 184 |
+
read_json(run / 'settings.json'), read_json(data_path), read_json(reference_dir / 'manifest.json'), plan)
|
| 185 |
+
full_hash = sha256(source)
|
| 186 |
+
if destination.exists() or destination.is_symlink():
|
| 187 |
+
return verify_dpo_archive(destination, expected_checkpoint_hash=full_hash)
|
| 188 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 189 |
+
staging = Path(tempfile.mkdtemp(prefix='.dpo-final-', dir=destination.parent))
|
| 190 |
+
copy_regular(source, staging / 'checkpoint.pt')
|
| 191 |
+
if sha256(staging / 'checkpoint.pt') != full_hash:
|
| 192 |
+
raise ValueError('DPO full-state archive copy differs from its source')
|
| 193 |
+
metadata = dict(format='warpstate-chat-v1', inference_only=True, stage='dpo',
|
| 194 |
+
config=checkpoint['config'], step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'],
|
| 195 |
+
response_tokens_seen=checkpoint['response_tokens_seen'], val_loss=checkpoint['val_loss'],
|
| 196 |
+
tokenizer_sha256=checkpoint['signature']['tokenizer_sha256'], origin=checkpoint['origin'],
|
| 197 |
+
captured_at_utc=utc_now(), source=dict(path=str(source), device=before.st_dev,
|
| 198 |
+
inode=before.st_ino, size=before.st_size, mtime_ns=before.st_mtime_ns, sha256=full_hash))
|
| 199 |
+
with (staging / 'model.pt').open('xb') as stream:
|
| 200 |
+
torch.save(dict(metadata, model=checkpoint['model']), stream)
|
| 201 |
+
stream.flush()
|
| 202 |
+
os.fsync(stream.fileno())
|
| 203 |
+
metadata.update(snapshot_sha256=sha256(staging / 'model.pt'),
|
| 204 |
+
snapshot_bytes=(staging / 'model.pt').stat().st_size)
|
| 205 |
+
atomic_json(staging / 'snapshot.json', metadata)
|
| 206 |
+
atomic_json(staging / 'plan.json', plan)
|
| 207 |
+
copies = {name: run / name for name in ('settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl')}
|
| 208 |
+
copies.update({'tokenizer.model': tokenizer, 'dpo-execution.json': work / 'control/execution.json',
|
| 209 |
+
'dpo-handoff.json': work / 'control/handoff.json', 'gpu-preflight.json': work / 'preflight/report.json',
|
| 210 |
+
'data-manifest.json': data_path, 'reference-manifest.json': reference_dir / 'manifest.json',
|
| 211 |
+
'reference-train.npy': reference_dir / 'train.npy', 'reference-val.npy': reference_dir / 'val.npy'})
|
| 212 |
+
for name, path in copies.items():
|
| 213 |
+
copy_regular(path, staging / name)
|
| 214 |
+
atomic_json(staging / 'PROVENANCE.json', dict(format='warpstate-dpo-archive-v1',
|
| 215 |
+
repository_revision=revision, archive_command_sha256=sha256(Path(__file__)),
|
| 216 |
+
source_checkpoint_sha256=full_hash, model_sha256=metadata['snapshot_sha256'],
|
| 217 |
+
dpo_step=checkpoint['step'], dpo_input_tokens=checkpoint['tokens_seen'],
|
| 218 |
+
dpo_response_tokens=checkpoint['response_tokens_seen'], dpo_pairs=checkpoint['sample_cursor'],
|
| 219 |
+
sft_step=plan['sft']['step'], sft_model_sha256=checkpoint['signature']['base_sha256'],
|
| 220 |
+
raw_base_step=plan['sft']['base_step'], raw_base_tokens=plan['sft']['base_tokens_seen'],
|
| 221 |
+
signature=checkpoint['signature']))
|
| 222 |
+
note = f'''# KEEP: completed DPO after raw step {plan['sft']['base_step']}
|
| 223 |
+
|
| 224 |
+
User-requested historical archive created {utc_now()}. Preserve during routine
|
| 225 |
+
cleanup, further training and chat refreshes. Delete only on explicit request.
|
| 226 |
+
|
| 227 |
+
- DPO update {checkpoint['step']:,}; {checkpoint['sample_cursor']:,} preference pairs.
|
| 228 |
+
- DPO input tokens {checkpoint['tokens_seen']:,}; response targets {checkpoint['response_tokens_seen']:,}.
|
| 229 |
+
- Parent SFT update {plan['sft']['step']:,}; raw base update {plan['sft']['base_step']:,}.
|
| 230 |
+
- Final full-holdout DPO objective loss: {checkpoint['val_loss']:.12f}.
|
| 231 |
+
- `checkpoint.pt`: byte-identical full checkpoint, including FP32 model and Adam,
|
| 232 |
+
per-rank RNG, sample cursor, model configuration and immutable signatures.
|
| 233 |
+
- `model.pt`: tensor-exact FP32 inference export with the original tokenizer.
|
| 234 |
+
- Plan, data/reference manifests and reference scores, original settings,
|
| 235 |
+
evaluation, completion, metrics, preflight and execution provenance are included.
|
| 236 |
+
- Stage counters are distinct. DPO objective loss is not language-model perplexity.
|
| 237 |
+
- All files are independent physical copies, read-only, without hardlinks/reflinks.
|
| 238 |
+
- Existing raw/SFT archives and the original DPO run remain unchanged.
|
| 239 |
+
- Repository revision at archival time: `{revision}`.
|
| 240 |
+
|
| 241 |
+
Verify with `sha256sum -c SHA256SUMS`, or run `archive_dpo.py --verify-only`
|
| 242 |
+
with this directory as `--destination`. This is an on-device archive.
|
| 243 |
+
'''
|
| 244 |
+
with (staging / 'ARCHIVE.md').open('x') as stream:
|
| 245 |
+
stream.write(note)
|
| 246 |
+
stream.flush()
|
| 247 |
+
os.fsync(stream.fileno())
|
| 248 |
+
after = source.stat()
|
| 249 |
+
if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
|
| 250 |
+
after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns):
|
| 251 |
+
raise ValueError('DPO checkpoint changed during archival')
|
| 252 |
+
seal(staging, FILES)
|
| 253 |
+
result = verify_dpo_archive(staging, expected_checkpoint_hash=full_hash)
|
| 254 |
+
publish_directory(staging, destination)
|
| 255 |
+
return result
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def run(config, destination=None, *, program='warpstate-dpo', verify_only=False):
|
| 259 |
+
config = workspace_path(config)
|
| 260 |
+
plan, paths = load_plan(config)
|
| 261 |
+
work, base = paths['work_dir'], paths['base_work_dir']
|
| 262 |
+
destination = workspace_path(destination or (base / 'archives' / f"step-{plan['sft']['base_step']:08d}-dpo-final"))
|
| 263 |
+
if (destination.parent != base / 'archives' or
|
| 264 |
+
destination == paths['archive_dir']):
|
| 265 |
+
raise ValueError('DPO archive must use a distinct direct child of work/archives')
|
| 266 |
+
if verify_only:
|
| 267 |
+
metadata, checksums = verify_dpo_archive(destination)
|
| 268 |
+
else:
|
| 269 |
+
with contextlib.ExitStack() as locks:
|
| 270 |
+
for path in (work / 'control/archive.lock', work / 'control/handoff.lock',
|
| 271 |
+
paths['sft_work_dir'] / 'control/handoff.lock',
|
| 272 |
+
base / 'control/training-launcher.lock', base / 'pipeline.lock'):
|
| 273 |
+
locks.enter_context(exclusive_lock(path))
|
| 274 |
+
if not clean_exit(process_status(paths['supervisor_socket'], program)):
|
| 275 |
+
raise ValueError('DPO guardian must have exited successfully before archival')
|
| 276 |
+
if read_json(work / 'control/handoff.json').get('phase') != 'complete':
|
| 277 |
+
raise ValueError('DPO guardian has not completed its final integrity checks')
|
| 278 |
+
model_hash = sha256(work / 'initialization/model.pt')
|
| 279 |
+
if model_hash != sha256(paths['archive_dir'] / 'model.pt'):
|
| 280 |
+
raise ValueError('Original SFT archive and DPO initialization differ')
|
| 281 |
+
verify_dpo_completion(work / 'run', plan, model_hash)
|
| 282 |
+
revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=WORKSPACE, text=True).strip()
|
| 283 |
+
metadata, checksums = archive_dpo(work, paths['tokenizer'], destination, plan, revision)
|
| 284 |
+
print(json.dumps(dict(archive=str(destination), step=metadata['step'], stage='dpo',
|
| 285 |
+
checkpoint_sha256=checksums['checkpoint.pt'], model_sha256=checksums['model.pt'],
|
| 286 |
+
tokens_seen=metadata['tokens_seen'], response_tokens_seen=metadata['response_tokens_seen']), indent=2))
|
| 287 |
+
return metadata, checksums
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def main():
|
| 291 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 292 |
+
parser.add_argument('--config', type=Path, default=PACKAGE / 'configs/dpo_10b.json')
|
| 293 |
+
parser.add_argument('--destination', type=Path)
|
| 294 |
+
parser.add_argument('--program', default='warpstate-dpo')
|
| 295 |
+
parser.add_argument('--verify-only', action='store_true')
|
| 296 |
+
args = parser.parse_args()
|
| 297 |
+
torch.set_num_threads(8)
|
| 298 |
+
run(args.config, args.destination, program=args.program, verify_only=args.verify_only)
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
if __name__ == '__main__':
|
| 302 |
+
main()
|
ops/b200-requirements.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r training-requirements-resolved.txt
|
| 2 |
+
supervisor==4.3.0
|
ops/b200-supervisord.conf
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[unix_http_server]
|
| 2 |
+
file=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisor.sock
|
| 3 |
+
chmod=0700
|
| 4 |
+
|
| 5 |
+
[supervisord]
|
| 6 |
+
logfile=/workspace/WarpState/rtx6000_40b/work/logs/b200-supervisor.log
|
| 7 |
+
logfile_maxbytes=10MB
|
| 8 |
+
logfile_backups=3
|
| 9 |
+
pidfile=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisord.pid
|
| 10 |
+
childlogdir=/workspace/WarpState/rtx6000_40b/work/logs
|
| 11 |
+
directory=/workspace/WarpState/rtx6000_40b
|
| 12 |
+
umask=077
|
| 13 |
+
|
| 14 |
+
[rpcinterface:supervisor]
|
| 15 |
+
supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
|
| 16 |
+
|
| 17 |
+
[supervisorctl]
|
| 18 |
+
serverurl=unix:///workspace/WarpState/rtx6000_40b/work/control/b200-supervisor.sock
|
| 19 |
+
history_file=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisorctl-history
|
| 20 |
+
|
| 21 |
+
[program:warpstate-train]
|
| 22 |
+
command=/workspace/WarpState/rtx6000_40b/ops/train-b200.sh
|
| 23 |
+
directory=/workspace/WarpState/rtx6000_40b
|
| 24 |
+
autostart=false
|
| 25 |
+
autorestart=false
|
| 26 |
+
startsecs=5
|
| 27 |
+
startretries=0
|
| 28 |
+
stopsignal=USR1
|
| 29 |
+
stopwaitsecs=900
|
| 30 |
+
stopasgroup=true
|
| 31 |
+
killasgroup=true
|
| 32 |
+
stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/b200-train.log
|
| 33 |
+
stdout_logfile_maxbytes=20MB
|
| 34 |
+
stdout_logfile_backups=5
|
| 35 |
+
redirect_stderr=true
|
ops/b200ctl.sh
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
source /workspace/WarpState/rtx6000_40b/ops/training-env.sh
|
| 4 |
+
exec /workspace/WarpState/rtx6000_40b/.server-venv/bin/supervisorctl \
|
| 5 |
+
-c /workspace/WarpState/rtx6000_40b/ops/b200-supervisord.conf "$@"
|
ops/backward_layout.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Isolated BF16 attention-backward layout experiment, not a training adapter.
|
| 2 |
+
|
| 3 |
+
The reference boundary is the September 12 compiled backward's rotary kernel
|
| 4 |
+
followed by Q/K/V/gate/U packing. Both BF16 intermediate roundings are retained.
|
| 5 |
+
No reductions, GEMMs, parameter gradients or optimizer state are changed here.
|
| 6 |
+
All saved tensors use [batch,chunks,heads,chunk_size,head_dim] contiguous layout,
|
| 7 |
+
except transposed K-write gradients and token-major output gradients.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from typing import NamedTuple
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
import triton
|
| 16 |
+
import triton.language as tl
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
PROFILE_SHAPE = (4, 16, 32, 128, 64)
|
| 20 |
+
ALGORITHMS = ("profile", "paired", "single")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class LayoutInputs(NamedTuple):
|
| 24 |
+
q_memory: torch.Tensor
|
| 25 |
+
q_local: torch.Tensor
|
| 26 |
+
sin: torch.Tensor
|
| 27 |
+
cos: torch.Tensor
|
| 28 |
+
k_write: torch.Tensor
|
| 29 |
+
tanh_k: torch.Tensor
|
| 30 |
+
k_local: torch.Tensor
|
| 31 |
+
v_local: torch.Tensor
|
| 32 |
+
grad_y: torch.Tensor
|
| 33 |
+
memory: torch.Tensor
|
| 34 |
+
local: torch.Tensor
|
| 35 |
+
gate: torch.Tensor
|
| 36 |
+
u_write: torch.Tensor
|
| 37 |
+
tanh_u: torch.Tensor
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def validate_inputs(inputs: LayoutInputs, *, cuda=False):
|
| 41 |
+
if not isinstance(inputs, LayoutInputs):
|
| 42 |
+
raise TypeError("Expected named LayoutInputs")
|
| 43 |
+
shape = tuple(inputs.q_memory.shape)
|
| 44 |
+
if len(shape) != 5 or min(shape) < 1 or shape[-1] % 2:
|
| 45 |
+
raise ValueError("Expected positive [B,N,H,C,D] with an even head dimension")
|
| 46 |
+
b, n, h, c, d = shape
|
| 47 |
+
device = inputs.q_memory.device
|
| 48 |
+
if cuda and device.type != "cuda":
|
| 49 |
+
raise ValueError("Triton layout kernels require CUDA")
|
| 50 |
+
for name, tensor in zip(inputs._fields, inputs, strict=True):
|
| 51 |
+
wanted = shape
|
| 52 |
+
dtype = torch.bfloat16
|
| 53 |
+
if name in ("sin", "cos"):
|
| 54 |
+
wanted, dtype = (c, d), torch.float32
|
| 55 |
+
elif name == "k_write":
|
| 56 |
+
wanted = (b, n, h, d, c)
|
| 57 |
+
elif name == "grad_y":
|
| 58 |
+
wanted = (b, n * c, h * d)
|
| 59 |
+
if tuple(tensor.shape) != wanted or tensor.dtype != dtype:
|
| 60 |
+
raise ValueError(f"{name} must have shape {wanted} and dtype {dtype}")
|
| 61 |
+
if tensor.device != device or not tensor.is_contiguous():
|
| 62 |
+
raise ValueError(f"{name} must be contiguous on {device}")
|
| 63 |
+
return shape
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def make_inputs(shape=PROFILE_SHAPE, *, device="cpu", seed=817) -> LayoutInputs:
|
| 67 |
+
"""Deterministic synthetic saved activations/gradients, no model/data reads."""
|
| 68 |
+
b, n, h, c, d = shape
|
| 69 |
+
generator = torch.Generator(device=device).manual_seed(seed)
|
| 70 |
+
|
| 71 |
+
def random(dims=shape, scale=0.1):
|
| 72 |
+
return torch.randn(dims, device=device, dtype=torch.bfloat16,
|
| 73 |
+
generator=generator) * scale
|
| 74 |
+
|
| 75 |
+
from warpstate.model import rope_cos_sin
|
| 76 |
+
cos, sin = rope_cos_sin(c, d, torch.device(device), torch.float32)
|
| 77 |
+
return LayoutInputs(random(), random(), sin, cos, random((b, n, h, d, c)),
|
| 78 |
+
random(scale=1).float().tanh().bfloat16(), random(), random(),
|
| 79 |
+
random((b, n * c, h * d)), random(), random(),
|
| 80 |
+
random(scale=1).float().sigmoid().bfloat16(), random(),
|
| 81 |
+
random(scale=1).float().tanh().bfloat16())
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def reference_layout(inputs: LayoutInputs) -> torch.Tensor:
|
| 85 |
+
"""Independent tensor expression for both stages, including BF16 boundaries.
|
| 86 |
+
|
| 87 |
+
FP32 operation contraction may differ between eager PyTorch and Triton;
|
| 88 |
+
use the tracked profile kernels as the exact GPU rounding oracle.
|
| 89 |
+
"""
|
| 90 |
+
b, n, h, c, d = validate_inputs(inputs)
|
| 91 |
+
q = inputs.q_memory.float() + inputs.q_local.float()
|
| 92 |
+
k = (inputs.k_write.transpose(-2, -1).float()
|
| 93 |
+
* (1 - inputs.tanh_k.float().square()) + inputs.k_local.float())
|
| 94 |
+
|
| 95 |
+
def inverse_rotate(value):
|
| 96 |
+
return torch.stack((value[..., 1::2], -value[..., ::2]), dim=-1).flatten(-2)
|
| 97 |
+
|
| 98 |
+
q_rounded = (inverse_rotate(q * inputs.sin) + q * inputs.cos).bfloat16()
|
| 99 |
+
k_rotated_rounded = inverse_rotate(k * inputs.sin).bfloat16()
|
| 100 |
+
k_packed = (k_rotated_rounded.float() + k * inputs.cos).bfloat16()
|
| 101 |
+
dy = inputs.grad_y.view(b, n, c, h, d).permute(0, 1, 3, 2, 4).float()
|
| 102 |
+
gate = inputs.gate.float()
|
| 103 |
+
dg = ((-(dy * (inputs.memory.float() * d**-0.5))
|
| 104 |
+
+ dy * inputs.local.float()) * (gate * (1 - gate))).bfloat16()
|
| 105 |
+
du = (inputs.u_write.float() * (1 - inputs.tanh_u.float().square())).bfloat16()
|
| 106 |
+
|
| 107 |
+
def token_major(value):
|
| 108 |
+
return value.permute(0, 1, 3, 2, 4).reshape(b, n * c, h * d)
|
| 109 |
+
|
| 110 |
+
return torch.cat([token_major(value) for value in
|
| 111 |
+
(q_rounded, k_packed, inputs.v_local, dg, du)], dim=-1)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
@triton.jit
|
| 115 |
+
def _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT,
|
| 116 |
+
ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
|
| 117 |
+
D: tl.constexpr, BLOCK: tl.constexpr, program):
|
| 118 |
+
pairs = program * BLOCK + tl.arange(0, BLOCK)
|
| 119 |
+
even = pairs * 2
|
| 120 |
+
odd = even + 1
|
| 121 |
+
mask = even < ELEMENTS
|
| 122 |
+
token = (even // D) % C
|
| 123 |
+
channel = even % D
|
| 124 |
+
group = even // (C * D)
|
| 125 |
+
rope = token * D + channel
|
| 126 |
+
s0 = tl.load(SIN + rope, mask, 0)
|
| 127 |
+
s1 = tl.load(SIN + rope + 1, mask, 0)
|
| 128 |
+
c0 = tl.load(COS + rope, mask, 0)
|
| 129 |
+
c1 = tl.load(COS + rope + 1, mask, 0)
|
| 130 |
+
q0 = (tl.load(QM + even, mask, 0).to(tl.float32)
|
| 131 |
+
+ tl.load(QL + even, mask, 0).to(tl.float32))
|
| 132 |
+
q1 = (tl.load(QM + odd, mask, 0).to(tl.float32)
|
| 133 |
+
+ tl.load(QL + odd, mask, 0).to(tl.float32))
|
| 134 |
+
# The profiled interleave sums each active rotary lane with an inactive +0.
|
| 135 |
+
# Retain that IEEE signed-zero boundary as well as its BF16 store below.
|
| 136 |
+
q_even_rot = q1 * s1 + 0.0
|
| 137 |
+
q_odd_rot = (-q0 * s0) + 0.0
|
| 138 |
+
q_even = (q_even_rot + q0 * c0).to(tl.bfloat16)
|
| 139 |
+
q_odd = (q_odd_rot + q1 * c1).to(tl.bfloat16)
|
| 140 |
+
kw_offset = group * C * D + channel * C + token
|
| 141 |
+
kw0 = tl.load(KW + kw_offset, mask, 0).to(tl.float32)
|
| 142 |
+
kw1 = tl.load(KW + kw_offset + C, mask, 0).to(tl.float32)
|
| 143 |
+
tk0 = tl.load(TK + even, mask, 0).to(tl.float32)
|
| 144 |
+
tk1 = tl.load(TK + odd, mask, 0).to(tl.float32)
|
| 145 |
+
k0 = kw0 * (1.0 - tk0 * tk0) + tl.load(KL + even, mask, 0).to(tl.float32)
|
| 146 |
+
k1 = kw1 * (1.0 - tk1 * tk1) + tl.load(KL + odd, mask, 0).to(tl.float32)
|
| 147 |
+
# Stage 17 stores these separately, before adding the cosine term in 18.
|
| 148 |
+
# Retain the truncation even though the fused implementation has no scratch.
|
| 149 |
+
k_even_rot = (k1 * s1 + 0.0).to(tl.bfloat16).to(tl.float32)
|
| 150 |
+
k_odd_rot = ((-k0 * s0) + 0.0).to(tl.bfloat16).to(tl.float32)
|
| 151 |
+
k_even = k_even_rot + k0 * c0
|
| 152 |
+
k_odd = k_odd_rot + k1 * c1
|
| 153 |
+
out = ((group // H) * C + token) * (5 * H * D) + (group % H) * D + channel
|
| 154 |
+
tl.store(OUT + out, q_even, mask)
|
| 155 |
+
tl.store(OUT + out + 1, q_odd, mask)
|
| 156 |
+
tl.store(OUT + out + H * D, k_even, mask)
|
| 157 |
+
tl.store(OUT + out + H * D + 1, k_odd, mask)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@triton.jit
|
| 161 |
+
def _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
|
| 162 |
+
ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
|
| 163 |
+
D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr, program):
|
| 164 |
+
index = program * BLOCK + tl.arange(0, BLOCK)
|
| 165 |
+
mask = index < ELEMENTS
|
| 166 |
+
token = (index // D) % C
|
| 167 |
+
channel = index % D
|
| 168 |
+
group = index // (C * D)
|
| 169 |
+
token_row = (group // H) * C + token
|
| 170 |
+
feature = (group % H) * D + channel
|
| 171 |
+
dy = tl.load(DY + token_row * H * D + feature, mask, 0).to(tl.float32)
|
| 172 |
+
memory = tl.load(MEMORY + index, mask, 0).to(tl.float32)
|
| 173 |
+
local = tl.load(LOCAL + index, mask, 0).to(tl.float32)
|
| 174 |
+
gate = tl.load(GATE + index, mask, 0).to(tl.float32)
|
| 175 |
+
# Keep the original chain rather than algebraically factoring dy.
|
| 176 |
+
dgate = (-(dy * (memory * SCALE)) + dy * local) * (gate * (1.0 - gate))
|
| 177 |
+
uw = tl.load(UW + index, mask, 0).to(tl.float32)
|
| 178 |
+
tu = tl.load(TU + index, mask, 0).to(tl.float32)
|
| 179 |
+
du = uw * (1.0 - tu * tu)
|
| 180 |
+
out = token_row * 5 * H * D + feature
|
| 181 |
+
tl.store(OUT + out + 2 * H * D, tl.load(VL + index, mask, 0), mask)
|
| 182 |
+
tl.store(OUT + out + 3 * H * D, dgate, mask)
|
| 183 |
+
tl.store(OUT + out + 4 * H * D, du, mask)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
@triton.jit
|
| 187 |
+
def _paired_kernel(QM, QL, SIN, COS, KW, TK, KL, VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
|
| 188 |
+
ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
|
| 189 |
+
D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr,
|
| 190 |
+
ROTARY: tl.constexpr):
|
| 191 |
+
if ROTARY:
|
| 192 |
+
_rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT,
|
| 193 |
+
ELEMENTS, H, C, D, BLOCK, tl.program_id(0))
|
| 194 |
+
else:
|
| 195 |
+
_other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
|
| 196 |
+
ELEMENTS, H, C, D, SCALE, BLOCK, tl.program_id(0))
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
@triton.jit
|
| 200 |
+
def _single_kernel(QM, QL, SIN, COS, KW, TK, KL, VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
|
| 201 |
+
ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr,
|
| 202 |
+
D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr):
|
| 203 |
+
rotary_programs = tl.cdiv(ELEMENTS // 2, BLOCK)
|
| 204 |
+
program = tl.program_id(0)
|
| 205 |
+
if program < rotary_programs:
|
| 206 |
+
_rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT,
|
| 207 |
+
ELEMENTS, H, C, D, BLOCK, program)
|
| 208 |
+
else:
|
| 209 |
+
_other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT,
|
| 210 |
+
ELEMENTS, H, C, D, SCALE, BLOCK, program - rotary_programs)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def prepare_layout(inputs: LayoutInputs, algorithm="paired", *, block=512, warps=4,
|
| 214 |
+
output=None, scratch=None):
|
| 215 |
+
"""Validate once and return an allocation-free launch callable.
|
| 216 |
+
|
| 217 |
+
Callers must keep the inputs/output/scratch metadata and storage stable.
|
| 218 |
+
No GPU kernel executes until the returned callable is invoked.
|
| 219 |
+
"""
|
| 220 |
+
b, n, h, c, d = validate_inputs(inputs, cuda=True)
|
| 221 |
+
if algorithm not in ALGORITHMS:
|
| 222 |
+
raise ValueError(f"Unknown backward layout algorithm: {algorithm}")
|
| 223 |
+
if block not in (128, 256, 512, 1024) or warps not in (4, 8):
|
| 224 |
+
raise ValueError("Use block 128/256/512/1024 with 4/8 warps")
|
| 225 |
+
shape = (b, n * c, 5 * h * d)
|
| 226 |
+
if output is None:
|
| 227 |
+
output = torch.empty(shape, dtype=torch.bfloat16, device=inputs.q_memory.device)
|
| 228 |
+
if (tuple(output.shape) != shape or output.dtype != torch.bfloat16
|
| 229 |
+
or output.device != inputs.q_memory.device or not output.is_contiguous()):
|
| 230 |
+
raise ValueError("Output must be a contiguous BF16 packed gradient on the input device")
|
| 231 |
+
if any(output.untyped_storage().data_ptr() == value.untyped_storage().data_ptr()
|
| 232 |
+
for value in inputs):
|
| 233 |
+
raise ValueError("Output may not alias a saved input")
|
| 234 |
+
if algorithm == "profile":
|
| 235 |
+
if (b, n, h, c, d) != PROFILE_SHAPE:
|
| 236 |
+
raise ValueError("The exact profiled baseline supports PROFILE_SHAPE only")
|
| 237 |
+
from ops.backward_layout_baseline import _profile_pack, _profile_rotary
|
| 238 |
+
if scratch is None:
|
| 239 |
+
scratch = (torch.empty_like(inputs.q_memory), torch.empty_like(inputs.q_memory))
|
| 240 |
+
if (len(scratch) != 2 or any(value.shape != inputs.q_memory.shape
|
| 241 |
+
or value.dtype != torch.bfloat16 or value.device != output.device
|
| 242 |
+
or not value.is_contiguous() for value in scratch)):
|
| 243 |
+
raise ValueError("The profile baseline needs two contiguous BF16 scratch tensors")
|
| 244 |
+
q, kr = scratch
|
| 245 |
+
storages = [value.untyped_storage().data_ptr() for value in (*inputs, output)]
|
| 246 |
+
if (q.untyped_storage().data_ptr() in storages
|
| 247 |
+
or kr.untyped_storage().data_ptr() in storages
|
| 248 |
+
or q.untyped_storage().data_ptr() == kr.untyped_storage().data_ptr()):
|
| 249 |
+
raise ValueError("Scratch tensors may not alias inputs, output, or one another")
|
| 250 |
+
|
| 251 |
+
def operation():
|
| 252 |
+
_profile_rotary[(1, 16384)](
|
| 253 |
+
inputs.q_memory, inputs.q_local, inputs.sin, inputs.cos,
|
| 254 |
+
inputs.k_write, inputs.tanh_k, inputs.k_local, q, kr,
|
| 255 |
+
262144, 64, YBLOCK=16, XBLOCK=64, num_warps=4, enable_fp_fusion=True)
|
| 256 |
+
_profile_pack[(160, 128)](
|
| 257 |
+
q, kr, inputs.k_write, inputs.tanh_k, inputs.k_local, inputs.cos,
|
| 258 |
+
inputs.v_local, inputs.grad_y, inputs.memory, inputs.local, inputs.gate,
|
| 259 |
+
inputs.u_write, inputs.tanh_u, output, 8192, 10240,
|
| 260 |
+
YBLOCK=64, XBLOCK=64, num_warps=8, enable_fp_fusion=True)
|
| 261 |
+
return output
|
| 262 |
+
else:
|
| 263 |
+
elements = inputs.q_memory.numel()
|
| 264 |
+
args = (*inputs, output, elements, h, c, d, d**-0.5, block)
|
| 265 |
+
|
| 266 |
+
def operation():
|
| 267 |
+
if algorithm == "paired":
|
| 268 |
+
_paired_kernel[(triton.cdiv(elements // 2, block),)](
|
| 269 |
+
*args, True, num_warps=warps, enable_fp_fusion=True)
|
| 270 |
+
_paired_kernel[(triton.cdiv(elements, block),)](
|
| 271 |
+
*args, False, num_warps=warps, enable_fp_fusion=True)
|
| 272 |
+
else:
|
| 273 |
+
_single_kernel[(triton.cdiv(elements // 2, block) + triton.cdiv(elements, block),)](
|
| 274 |
+
*args, num_warps=warps, enable_fp_fusion=True)
|
| 275 |
+
return output
|
| 276 |
+
return operation
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def launch_layout(inputs: LayoutInputs, algorithm="paired", *, block=512, warps=4,
|
| 280 |
+
output=None, scratch=None) -> torch.Tensor:
|
| 281 |
+
"""One checked launch; timed loops should use prepare_layout once instead."""
|
| 282 |
+
return prepare_layout(inputs, algorithm, block=block, warps=warps,
|
| 283 |
+
output=output, scratch=scratch)()
|
ops/backward_layout_baseline.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tracked reference kernels from the September 12 CA-overlap backward profile.
|
| 2 |
+
|
| 3 |
+
The bodies below reproduce the exact profiled arithmetic/indexing at shape
|
| 4 |
+
B=4, chunks=16, heads=32, chunk_size=128, head_dim=64. They are kept as source,
|
| 5 |
+
not imported from generated compiler caches. Names 17/18 in that trace consumed
|
| 6 |
+
31.907/77.576 ms per rank-0 update. Their measured launch choices were
|
| 7 |
+
XBLOCK=64/YBLOCK=16/4 warps and XBLOCK=64/YBLOCK=64/8 warps respectively.
|
| 8 |
+
|
| 9 |
+
Provenance: ops/model sources and profile-ca-overlap trace map these bodies to
|
| 10 |
+
cache hashes cq665jkyfy6uac7wrzvwa5aigwids6rzt4n7vzip6xvufai4ajjw and
|
| 11 |
+
cbr6gfx3vjhnzjbykfch3pg5peunfivhgdrfvl5hnzolhvfjfgdy. The isolated launcher
|
| 12 |
+
preallocates both scratch tensors; production reused a dead buffer for one of
|
| 13 |
+
them. Compare kernel time and traffic, not an inferred peak-allocation saving.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import triton
|
| 17 |
+
import triton.language as tl
|
| 18 |
+
from torch._inductor.runtime import triton_helpers
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@triton.jit
|
| 22 |
+
def _profile_rotary(in_ptr0, in_ptr1, in_ptr2, in_ptr3, in_ptr4, in_ptr5, in_ptr6, out_ptr0, out_ptr1, ynumel, xnumel, YBLOCK : tl.constexpr, XBLOCK : tl.constexpr):
|
| 23 |
+
ynumel = 262144
|
| 24 |
+
xnumel = 64
|
| 25 |
+
yoffset = (tl.program_id(1) + tl.program_id(2) * tl.num_programs(1)) * YBLOCK
|
| 26 |
+
yindex = yoffset + tl.arange(0, YBLOCK)[:, None]
|
| 27 |
+
ymask = yindex < ynumel
|
| 28 |
+
xoffset = tl.program_id(0) * XBLOCK
|
| 29 |
+
xindex = xoffset + tl.arange(0, XBLOCK)[None, :]
|
| 30 |
+
xmask = xindex < xnumel
|
| 31 |
+
x2 = xindex
|
| 32 |
+
y3 = yindex
|
| 33 |
+
y0 = (yindex % 128)
|
| 34 |
+
y1 = yindex // 128
|
| 35 |
+
tmp33 = tl.load(in_ptr0 + (x2 + 64*y3), xmask & ymask).to(tl.float32)
|
| 36 |
+
tmp34 = tl.load(in_ptr1 + (x2 + 64*y3), xmask & ymask).to(tl.float32)
|
| 37 |
+
tmp36 = tl.load(in_ptr3 + (x2 + 64*y0), xmask & ymask, eviction_policy='evict_last')
|
| 38 |
+
tmp0 = (x2).to(tl.int64)
|
| 39 |
+
tmp1 = (tmp0).to(tl.int64)
|
| 40 |
+
tmp2 = tl.full([1, 1], 1, tl.int64)
|
| 41 |
+
tmp3 = tmp1 >= tmp2
|
| 42 |
+
tmp4 = ((((-1) + x2) % 2)).to(tl.int64)
|
| 43 |
+
tmp5 = (tmp4).to(tl.int64)
|
| 44 |
+
tmp6 = tl.full([1, 1], 0, tl.int64)
|
| 45 |
+
tmp7 = tmp5 == tmp6
|
| 46 |
+
tmp8 = tmp3 & tmp7
|
| 47 |
+
tmp9 = tl.load(in_ptr0 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 48 |
+
tmp10 = tl.load(in_ptr1 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 49 |
+
tmp11 = tmp9 + tmp10
|
| 50 |
+
tmp12 = tl.load(in_ptr2 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y0), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0)
|
| 51 |
+
tmp13 = tmp12.to(tl.float32)
|
| 52 |
+
tmp14 = tmp11 * tmp13
|
| 53 |
+
tmp15 = -tmp14
|
| 54 |
+
tmp16 = tl.full(tmp15.shape, 0.0, tmp15.dtype)
|
| 55 |
+
tmp17 = tl.where(tmp8, tmp15, tmp16)
|
| 56 |
+
tmp18 = tl.full([1, 1], 0.0, tl.float32)
|
| 57 |
+
tmp19 = tl.where(tmp8, tmp17, tmp18)
|
| 58 |
+
tmp20 = ((x2 % 2)).to(tl.int64)
|
| 59 |
+
tmp21 = (tmp20).to(tl.int64)
|
| 60 |
+
tmp22 = tmp21 == tmp6
|
| 61 |
+
tmp23 = tl.load(in_ptr0 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 62 |
+
tmp24 = tl.load(in_ptr1 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 63 |
+
tmp25 = tmp23 + tmp24
|
| 64 |
+
tmp26 = tl.load(in_ptr2 + (1 + 2*(x2 // 2) + 64*y0), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0)
|
| 65 |
+
tmp27 = tmp26.to(tl.float32)
|
| 66 |
+
tmp28 = tmp25 * tmp27
|
| 67 |
+
tmp29 = tl.full(tmp28.shape, 0.0, tmp28.dtype)
|
| 68 |
+
tmp30 = tl.where(tmp22, tmp28, tmp29)
|
| 69 |
+
tmp31 = tl.where(tmp22, tmp30, tmp18)
|
| 70 |
+
tmp32 = tmp19 + tmp31
|
| 71 |
+
tmp35 = tmp33 + tmp34
|
| 72 |
+
tmp37 = tmp36.to(tl.float32)
|
| 73 |
+
tmp38 = tmp35 * tmp37
|
| 74 |
+
tmp39 = tmp32 + tmp38
|
| 75 |
+
tmp40 = tl.load(in_ptr4 + (y0 + 256*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 8192*y1), tmp8 & xmask & ymask, other=0.0).to(tl.float32)
|
| 76 |
+
tmp41 = tmp40.to(tl.float32)
|
| 77 |
+
tmp42 = tl.load(in_ptr5 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 78 |
+
tmp43 = tmp42.to(tl.float32)
|
| 79 |
+
tmp44 = tmp43 * tmp43
|
| 80 |
+
tmp45 = tl.full([1, 1], 1.0, tl.float32)
|
| 81 |
+
tmp46 = tmp45 - tmp44
|
| 82 |
+
tmp47 = tmp41 * tmp46
|
| 83 |
+
tmp48 = tmp47.to(tl.float32)
|
| 84 |
+
tmp49 = tl.load(in_ptr6 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 85 |
+
tmp50 = tmp48 + tmp49
|
| 86 |
+
tmp51 = tmp50 * tmp13
|
| 87 |
+
tmp52 = -tmp51
|
| 88 |
+
tmp53 = tl.full(tmp52.shape, 0.0, tmp52.dtype)
|
| 89 |
+
tmp54 = tl.where(tmp8, tmp52, tmp53)
|
| 90 |
+
tmp55 = tl.where(tmp8, tmp54, tmp18)
|
| 91 |
+
tmp56 = tl.load(in_ptr4 + (128 + y0 + 256*(x2 // 2) + 8192*y1), tmp22 & xmask & ymask, other=0.0).to(tl.float32)
|
| 92 |
+
tmp57 = tmp56.to(tl.float32)
|
| 93 |
+
tmp58 = tl.load(in_ptr5 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 94 |
+
tmp59 = tmp58.to(tl.float32)
|
| 95 |
+
tmp60 = tmp59 * tmp59
|
| 96 |
+
tmp61 = tl.full([1, 1], 1.0, tl.float32)
|
| 97 |
+
tmp62 = tmp61 - tmp60
|
| 98 |
+
tmp63 = tmp57 * tmp62
|
| 99 |
+
tmp64 = tmp63.to(tl.float32)
|
| 100 |
+
tmp65 = tl.load(in_ptr6 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 101 |
+
tmp66 = tmp64 + tmp65
|
| 102 |
+
tmp67 = tmp66 * tmp27
|
| 103 |
+
tmp68 = tl.full(tmp67.shape, 0.0, tmp67.dtype)
|
| 104 |
+
tmp69 = tl.where(tmp22, tmp67, tmp68)
|
| 105 |
+
tmp70 = tl.where(tmp22, tmp69, tmp18)
|
| 106 |
+
tmp71 = tmp55 + tmp70
|
| 107 |
+
tl.store(out_ptr0 + (x2 + 64*y3), tmp39, xmask & ymask)
|
| 108 |
+
tl.store(out_ptr1 + (x2 + 64*y3), tmp71, xmask & ymask)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
@triton.jit
|
| 112 |
+
def _profile_pack(in_ptr0, in_ptr1, in_ptr2, in_ptr3, in_ptr4, in_ptr5, in_ptr6, in_ptr7, in_ptr8, in_ptr9, in_ptr10, in_ptr11, in_ptr12, out_ptr0, ynumel, xnumel, YBLOCK : tl.constexpr, XBLOCK : tl.constexpr):
|
| 113 |
+
ynumel = 8192
|
| 114 |
+
xnumel = 10240
|
| 115 |
+
yoffset = tl.program_id(1) * YBLOCK
|
| 116 |
+
yindex = yoffset + tl.arange(0, YBLOCK)[:, None]
|
| 117 |
+
ymask = tl.full([YBLOCK], True, tl.int1)[:, None]
|
| 118 |
+
xoffset = tl.program_id(0) * XBLOCK
|
| 119 |
+
xindex = xoffset + tl.arange(0, XBLOCK)[None, :]
|
| 120 |
+
xmask = xindex < xnumel
|
| 121 |
+
x2 = xindex
|
| 122 |
+
y0 = (yindex % 2048)
|
| 123 |
+
y1 = yindex // 2048
|
| 124 |
+
y3 = yindex
|
| 125 |
+
tmp0 = (x2).to(tl.int32)
|
| 126 |
+
tmp1 = tl.full([1, 1], 0, tl.int64)
|
| 127 |
+
tmp2 = tmp0 >= tmp1
|
| 128 |
+
tmp3 = (x2).to(tl.int64)
|
| 129 |
+
tmp4 = (tmp3).to(tl.int64)
|
| 130 |
+
tmp5 = tl.full([1, 1], 2048, tl.int64)
|
| 131 |
+
tmp6 = tmp4 < tmp5
|
| 132 |
+
tmp7 = tl.load(in_ptr0 + (64*((y0 % 128)) + 8192*((((x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + (((x2) % 64))), tmp6 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 133 |
+
tmp8 = tmp4 >= tmp5
|
| 134 |
+
tmp9 = tl.full([1, 1], 4096, tl.int64)
|
| 135 |
+
tmp10 = tmp4 < tmp9
|
| 136 |
+
tmp11 = tmp8 & tmp10
|
| 137 |
+
tmp12 = tl.load(in_ptr1 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 138 |
+
tmp13 = tl.load(in_ptr2 + (128*((((-2048) + x2) % 2048)) + 262144*(y0 // 128) + 4194304*y1 + ((y0 % 128))), tmp11 & xmask, other=0.0).to(tl.float32)
|
| 139 |
+
tmp14 = tmp13.to(tl.float32)
|
| 140 |
+
tmp15 = tl.load(in_ptr3 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 141 |
+
tmp16 = tmp15.to(tl.float32)
|
| 142 |
+
tmp17 = tmp16 * tmp16
|
| 143 |
+
tmp18 = tl.full([1, 1], 1.0, tl.float32)
|
| 144 |
+
tmp19 = tmp18 - tmp17
|
| 145 |
+
tmp20 = tmp14 * tmp19
|
| 146 |
+
tmp21 = tmp20.to(tl.float32)
|
| 147 |
+
tmp22 = tl.load(in_ptr4 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 148 |
+
tmp23 = tmp21 + tmp22
|
| 149 |
+
tmp24 = tl.load(in_ptr5 + (64*((y0 % 128)) + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0)
|
| 150 |
+
tmp25 = tmp24.to(tl.float32)
|
| 151 |
+
tmp26 = tmp23 * tmp25
|
| 152 |
+
tmp27 = tmp12 + tmp26
|
| 153 |
+
tmp28 = tl.full(tmp27.shape, 0.0, tmp27.dtype)
|
| 154 |
+
tmp29 = tl.where(tmp11, tmp27, tmp28)
|
| 155 |
+
tmp30 = tmp4 >= tmp9
|
| 156 |
+
tmp31 = tl.full([1, 1], 6144, tl.int64)
|
| 157 |
+
tmp32 = tmp4 < tmp31
|
| 158 |
+
tmp33 = tmp30 & tmp32
|
| 159 |
+
tmp34 = tl.load(in_ptr6 + (64*((y0 % 128)) + 8192*(((((-4096) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-4096) + x2) % 64))), tmp33 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 160 |
+
tmp35 = tmp4 >= tmp31
|
| 161 |
+
tmp36 = tl.full([1, 1], 8192, tl.int64)
|
| 162 |
+
tmp37 = tmp4 < tmp36
|
| 163 |
+
tmp38 = tmp35 & tmp37
|
| 164 |
+
tmp39 = tl.load(in_ptr7 + (2048*y3 + ((((-6144) + x2) % 2048))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 165 |
+
tmp40 = tl.load(in_ptr8 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 166 |
+
tmp41 = tl.full([1, 1], 0.125, tl.float32)
|
| 167 |
+
tmp42 = tmp40 * tmp41
|
| 168 |
+
tmp43 = tmp39 * tmp42
|
| 169 |
+
tmp44 = -tmp43
|
| 170 |
+
tmp45 = tl.load(in_ptr9 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 171 |
+
tmp46 = tmp39 * tmp45
|
| 172 |
+
tmp47 = tmp44 + tmp46
|
| 173 |
+
tmp48 = tmp47.to(tl.float32)
|
| 174 |
+
tmp49 = tl.load(in_ptr10 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 175 |
+
tmp50 = tmp49.to(tl.float32)
|
| 176 |
+
tmp51 = tl.full([1, 1], 1.0, tl.float32)
|
| 177 |
+
tmp52 = tmp51 - tmp50
|
| 178 |
+
tmp53 = tmp50 * tmp52
|
| 179 |
+
tmp54 = tmp48 * tmp53
|
| 180 |
+
tmp55 = tmp54.to(tl.float32)
|
| 181 |
+
tmp56 = tl.full(tmp55.shape, 0.0, tmp55.dtype)
|
| 182 |
+
tmp57 = tl.where(tmp38, tmp55, tmp56)
|
| 183 |
+
tmp58 = tmp0 >= tmp36
|
| 184 |
+
tmp59 = tl.full([1, 1], 10240, tl.int64)
|
| 185 |
+
tmp60 = tmp0 < tmp59
|
| 186 |
+
tmp61 = tl.load(in_ptr11 + (64*((y0 % 128)) + 8192*(((((-8192) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-8192) + x2) % 64))), tmp58 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 187 |
+
tmp62 = tmp61.to(tl.float32)
|
| 188 |
+
tmp63 = tl.load(in_ptr12 + (64*((y0 % 128)) + 8192*(((((-8192) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-8192) + x2) % 64))), tmp58 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32)
|
| 189 |
+
tmp64 = tmp63.to(tl.float32)
|
| 190 |
+
tmp65 = tmp64 * tmp64
|
| 191 |
+
tmp66 = tl.full([1, 1], 1.0, tl.float32)
|
| 192 |
+
tmp67 = tmp66 - tmp65
|
| 193 |
+
tmp68 = tmp62 * tmp67
|
| 194 |
+
tmp69 = tmp68.to(tl.float32)
|
| 195 |
+
tmp70 = tl.full(tmp69.shape, 0.0, tmp69.dtype)
|
| 196 |
+
tmp71 = tl.where(tmp58, tmp69, tmp70)
|
| 197 |
+
tmp72 = tl.where(tmp38, tmp57, tmp71)
|
| 198 |
+
tmp73 = tl.where(tmp33, tmp34, tmp72)
|
| 199 |
+
tmp74 = tl.where(tmp11, tmp29, tmp73)
|
| 200 |
+
tmp75 = tl.where(tmp6, tmp7, tmp74)
|
| 201 |
+
tl.store(out_ptr0 + (x2 + 10240*y3), tmp75, xmask)
|
ops/backward_layout_benchmark.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Reproduce isolated BF16 rotary/packed-gradient traffic measurements on one GPU."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import datetime as dt
|
| 8 |
+
import hashlib
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
import os
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
import statistics
|
| 14 |
+
import sys
|
| 15 |
+
import time
|
| 16 |
+
|
| 17 |
+
PACKAGE = Path(__file__).resolve().parents[1]
|
| 18 |
+
WORKSPACE = PACKAGE.parent
|
| 19 |
+
sys.path.insert(0, str(PACKAGE))
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def arguments(argv=None):
|
| 23 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 24 |
+
parser.add_argument("--algorithms", nargs="+", choices=["paired", "single"], default=["paired", "single"])
|
| 25 |
+
parser.add_argument("--blocks", nargs="+", type=int, choices=[128, 256, 512, 1024], default=[256, 512, 1024])
|
| 26 |
+
parser.add_argument("--warps", nargs="+", type=int, choices=[4, 8], default=[4, 8])
|
| 27 |
+
parser.add_argument("--trials", type=int, default=5)
|
| 28 |
+
parser.add_argument("--repeats", type=int, default=40)
|
| 29 |
+
parser.add_argument("--warmup", type=int, default=20)
|
| 30 |
+
parser.add_argument("--burnin-seconds", type=float, default=0.5)
|
| 31 |
+
parser.add_argument("--seed", type=int, default=817)
|
| 32 |
+
parser.add_argument("--output-dir", type=Path, required=True)
|
| 33 |
+
args = parser.parse_args(argv)
|
| 34 |
+
for name in ("trials", "repeats", "warmup"):
|
| 35 |
+
if getattr(args, name) < 1:
|
| 36 |
+
parser.error(name + " must be positive")
|
| 37 |
+
if not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0:
|
| 38 |
+
parser.error("burnin-seconds must be finite and nonnegative")
|
| 39 |
+
args.output_dir = args.output_dir.expanduser().resolve()
|
| 40 |
+
work = PACKAGE / "work"
|
| 41 |
+
if (not args.output_dir.is_relative_to(work) or args.output_dir == work
|
| 42 |
+
or not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")):
|
| 43 |
+
parser.error("output-dir must be below rtx6000_40b/work/benchmark-*")
|
| 44 |
+
for name in ("TMPDIR", "XDG_CACHE_HOME", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"):
|
| 45 |
+
if not os.environ.get(name) or not Path(os.environ[name]).resolve().is_relative_to(WORKSPACE):
|
| 46 |
+
parser.error("Source ops/training-env.sh; " + name + " must stay inside the workspace")
|
| 47 |
+
return args
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def output_errors(actual, expected):
|
| 51 |
+
"""Check each projection independently so a bad derivative cannot hide."""
|
| 52 |
+
import torch
|
| 53 |
+
result = {}
|
| 54 |
+
for name, value, wanted in zip(("q", "k", "v", "gate", "u"),
|
| 55 |
+
actual.chunk(5, -1), expected.chunk(5, -1), strict=True):
|
| 56 |
+
delta = value.float() - wanted.float()
|
| 57 |
+
finite = bool(torch.isfinite(value).all())
|
| 58 |
+
result[name] = {"finite": finite,
|
| 59 |
+
"unequal": int((value != wanted).sum()),
|
| 60 |
+
"bitwise_unequal": int((value.view(torch.int16) != wanted.view(torch.int16)).sum()),
|
| 61 |
+
"relative_rms": (float(delta.norm() / wanted.float().norm().clamp_min(1e-30))
|
| 62 |
+
if finite else None),
|
| 63 |
+
"max_abs": float(delta.abs().max()) if finite else None}
|
| 64 |
+
return result
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def main(argv=None):
|
| 68 |
+
args = arguments(argv)
|
| 69 |
+
import torch
|
| 70 |
+
from ops.backward_layout import PROFILE_SHAPE, make_inputs, prepare_layout, reference_layout
|
| 71 |
+
if not torch.cuda.is_available() or torch.cuda.device_count() != 1:
|
| 72 |
+
raise RuntimeError("Reserve exactly one GPU with CUDA_VISIBLE_DEVICES")
|
| 73 |
+
torch.cuda.set_device(0)
|
| 74 |
+
inputs = make_inputs(PROFILE_SHAPE, device="cuda", seed=args.seed)
|
| 75 |
+
baseline_output = torch.empty((4, 2048, 10240), device="cuda", dtype=torch.bfloat16)
|
| 76 |
+
candidate_output = torch.empty_like(baseline_output)
|
| 77 |
+
scratch = tuple(torch.empty_like(inputs.q_memory) for _ in range(2))
|
| 78 |
+
baseline = prepare_layout(inputs, "profile", output=baseline_output, scratch=scratch)
|
| 79 |
+
baseline()
|
| 80 |
+
independent = reference_layout(inputs)
|
| 81 |
+
baseline_errors = output_errors(baseline_output, independent)
|
| 82 |
+
del independent
|
| 83 |
+
if any(not v["finite"] or v["relative_rms"] > 0.002 for v in baseline_errors.values()):
|
| 84 |
+
raise RuntimeError("Tracked profile arithmetic disagrees with independent tensor reference")
|
| 85 |
+
args.output_dir.mkdir(parents=True, exist_ok=False)
|
| 86 |
+
sources = ["ops/backward_layout.py", "ops/backward_layout_baseline.py",
|
| 87 |
+
"ops/backward_layout_benchmark.py", "warpstate/model.py"]
|
| 88 |
+
report = {"format": "warpstate-backward-layout-v1", "started_utc": dt.datetime.now(dt.UTC).isoformat(),
|
| 89 |
+
"arguments": {k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()},
|
| 90 |
+
"shape_b_n_h_c_d": PROFILE_SHAPE, "dtype": "bfloat16", "rope_dtype": "float32",
|
| 91 |
+
"torch": torch.__version__, "cuda": torch.version.cuda,
|
| 92 |
+
"device": torch.cuda.get_device_name(0), "cuda_visible_devices": os.environ["CUDA_VISIBLE_DEVICES"],
|
| 93 |
+
"source_sha256": {name: hashlib.sha256((PACKAGE / name).read_bytes()).hexdigest() for name in sources},
|
| 94 |
+
"baseline_reference_errors": baseline_errors,
|
| 95 |
+
"static_removed_scratch_store_and_read_bytes_per_call": inputs.q_memory.numel() * 2 * 4,
|
| 96 |
+
"limitations": ["Isolated backward region, not a measured training speedup.",
|
| 97 |
+
"Preallocated outputs and scratch; CUDA event times include launch scheduling.",
|
| 98 |
+
"Synthetic saved inputs are reused; no model/checkpoint/token files are read.",
|
| 99 |
+
"Static removed scratch traffic is not a measured bandwidth counter.",
|
| 100 |
+
"Production reuses one dead buffer; this is not a claim of two saved allocations.",
|
| 101 |
+
"Integration must preserve separate local/memory gradients and rounding joins."], "results": []}
|
| 102 |
+
|
| 103 |
+
def save():
|
| 104 |
+
path = args.output_dir / "results.json"
|
| 105 |
+
temporary = path.with_suffix(".json.partial")
|
| 106 |
+
temporary.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
|
| 107 |
+
temporary.replace(path)
|
| 108 |
+
|
| 109 |
+
def elapsed_ms(operation):
|
| 110 |
+
begin, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
|
| 111 |
+
begin.record()
|
| 112 |
+
for _ in range(args.repeats):
|
| 113 |
+
operation()
|
| 114 |
+
end.record()
|
| 115 |
+
end.synchronize()
|
| 116 |
+
return begin.elapsed_time(end) / args.repeats
|
| 117 |
+
|
| 118 |
+
for algorithm in args.algorithms:
|
| 119 |
+
for block in args.blocks:
|
| 120 |
+
for warps in args.warps:
|
| 121 |
+
row = {"algorithm": algorithm, "block": block, "warps": warps}
|
| 122 |
+
try:
|
| 123 |
+
candidate = prepare_layout(inputs, algorithm, block=block, warps=warps,
|
| 124 |
+
output=candidate_output)
|
| 125 |
+
candidate()
|
| 126 |
+
row["errors"] = output_errors(candidate_output, baseline_output)
|
| 127 |
+
if any(not v["finite"] for v in row["errors"].values()):
|
| 128 |
+
raise RuntimeError("Nonfinite candidate output")
|
| 129 |
+
row["bitwise_equal"] = all(v["bitwise_unequal"] == 0 for v in row["errors"].values())
|
| 130 |
+
for _ in range(args.warmup):
|
| 131 |
+
baseline()
|
| 132 |
+
candidate()
|
| 133 |
+
torch.cuda.synchronize()
|
| 134 |
+
begin = time.monotonic()
|
| 135 |
+
while time.monotonic() - begin < args.burnin_seconds:
|
| 136 |
+
for _ in range(10):
|
| 137 |
+
baseline()
|
| 138 |
+
candidate()
|
| 139 |
+
torch.cuda.synchronize()
|
| 140 |
+
trials = []
|
| 141 |
+
for _ in range(args.trials):
|
| 142 |
+
trials.append({"baseline_before_ms": elapsed_ms(baseline),
|
| 143 |
+
"candidate_ms": elapsed_ms(candidate),
|
| 144 |
+
"baseline_after_ms": elapsed_ms(baseline)})
|
| 145 |
+
gains = [((t["baseline_before_ms"] + t["baseline_after_ms"]) / 2
|
| 146 |
+
/ t["candidate_ms"] - 1) * 100 for t in trials]
|
| 147 |
+
row.update(status="ok", trials=trials,
|
| 148 |
+
median_candidate_ms=statistics.median(t["candidate_ms"] for t in trials),
|
| 149 |
+
median_baseline_ms=statistics.median((t["baseline_before_ms"]
|
| 150 |
+
+ t["baseline_after_ms"]) / 2 for t in trials),
|
| 151 |
+
median_gain_percent=statistics.median(gains), minimum_gain_percent=min(gains))
|
| 152 |
+
except Exception as error:
|
| 153 |
+
row.update(status="error", error=type(error).__name__ + ": " + str(error)[:4000])
|
| 154 |
+
report["results"].append(row)
|
| 155 |
+
save()
|
| 156 |
+
print(json.dumps(row), flush=True)
|
| 157 |
+
report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat()
|
| 158 |
+
save()
|
| 159 |
+
if not any(row["status"] == "ok" for row in report["results"]):
|
| 160 |
+
raise RuntimeError("No candidate completed")
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
if __name__ == "__main__":
|
| 164 |
+
main()
|
ops/backward_layout_execution.py
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Opt-in replacement of the measured, normalized attention backward subgraph.
|
| 2 |
+
|
| 3 |
+
The boundary is the existing compiled BF16 kernel arithmetic, not eager BF16
|
| 4 |
+
ATen arithmetic: Inductor normally retains FP32 intermediates within a fused
|
| 5 |
+
kernel. Forward/autograd ownership, leaf accumulation and communication remain
|
| 6 |
+
unchanged. Unsupported graphs are retained; callers must require actual runtime
|
| 7 |
+
dispatches in opt-in validation. Configure once before compiling in a fresh process.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from collections import Counter
|
| 13 |
+
from functools import lru_cache
|
| 14 |
+
import hashlib
|
| 15 |
+
import json
|
| 16 |
+
import operator
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from torch._inductor.custom_graph_pass import CustomGraphPass
|
| 21 |
+
from torch.fx import Node
|
| 22 |
+
from torch.library import custom_op
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
BACKWARD_LAYOUT_MODES = ("baseline", "inspect", "paired")
|
| 26 |
+
_SHAPE = (4, 16, 32, 128, 64)
|
| 27 |
+
_NAMES = ("q_memory", "q_local", "sin", "cos", "k_write", "tanh_k", "k_local",
|
| 28 |
+
"v_local", "grad_y", "memory", "local", "gate", "u_write", "tanh_u")
|
| 29 |
+
_NATIVE_SHAPES = (
|
| 30 |
+
(2048, 128, 64), (64, 32, 128, 64), (128, 64), (128, 64),
|
| 31 |
+
(2048, 64, 128), _SHAPE, (64, 32, 128, 64), (64, 32, 128, 64),
|
| 32 |
+
(8192, 2048), (2048, 128, 64), (64, 32, 128, 64), _SHAPE,
|
| 33 |
+
(2048, 128, 64), _SHAPE,
|
| 34 |
+
)
|
| 35 |
+
_COUNTS = Counter()
|
| 36 |
+
_MISSES = Counter()
|
| 37 |
+
_ACTIVE_MODE = "baseline"
|
| 38 |
+
_ACTIVE_PASS = None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _aten_boundary(qm, ql, sin, cos, kw, tk, kl, vl, dy, memory, local, gate, uw, tu):
|
| 42 |
+
"""Exact normalized ATen dataflow of the real production backward boundary.
|
| 43 |
+
|
| 44 |
+
This recipe is traced on fake tensors once to construct a structural pattern.
|
| 45 |
+
Explicit operations retain every original cast, view, projection position,
|
| 46 |
+
and zero-filled scatter; arbitrary algebraic equivalents are not accepted.
|
| 47 |
+
It is never called to execute training or to provide a numeric GPU oracle.
|
| 48 |
+
"""
|
| 49 |
+
a = torch.ops.aten
|
| 50 |
+
cvt = torch.ops.prims.convert_element_type.default
|
| 51 |
+
reshape = a.reshape.default
|
| 52 |
+
head = lambda x: reshape(x, list(_SHAPE))
|
| 53 |
+
sin = reshape(cvt(sin, torch.bfloat16), [1, 1, 1, 128, 64])
|
| 54 |
+
cos = reshape(cvt(cos, torch.bfloat16), [1, 1, 1, 128, 64])
|
| 55 |
+
zero = a.full.default(list(_SHAPE), 0, dtype=torch.bfloat16,
|
| 56 |
+
layout=torch.strided, device=qm.device, pin_memory=False)
|
| 57 |
+
|
| 58 |
+
def rotary(value):
|
| 59 |
+
lanes = reshape(a.mul.Tensor(value, sin), [4, 16, 32, 128, 32, 2])
|
| 60 |
+
odd = a.slice_scatter.default(zero, a.neg.default(a.select.int(lanes, -1, 0)),
|
| 61 |
+
4, 1, 9223372036854775807, 2)
|
| 62 |
+
even = a.slice_scatter.default(zero, a.select.int(lanes, -1, 1),
|
| 63 |
+
4, 0, 9223372036854775807, 2)
|
| 64 |
+
return a.add.Tensor(a.add.Tensor(odd, even), a.mul.Tensor(value, cos))
|
| 65 |
+
|
| 66 |
+
def tanh_backward(gradient, saved):
|
| 67 |
+
gradient = cvt(gradient, torch.float32)
|
| 68 |
+
saved = cvt(saved, torch.float32)
|
| 69 |
+
return cvt(a.mul.Tensor(gradient, a.sub.Tensor(1, a.mul.Tensor(saved, saved))),
|
| 70 |
+
torch.bfloat16)
|
| 71 |
+
|
| 72 |
+
q = rotary(a.add.Tensor(head(qm), head(ql)))
|
| 73 |
+
kw = a.permute.default(reshape(kw, [4, 16, 32, 64, 128]), [0, 1, 2, 4, 3])
|
| 74 |
+
k = rotary(a.add.Tensor(tanh_backward(kw, tk), head(kl)))
|
| 75 |
+
v = head(vl)
|
| 76 |
+
dy = a.permute.default(reshape(reshape(dy, [4, 2048, 2048]),
|
| 77 |
+
[4, 16, 128, 32, 64]), [0, 1, 3, 2, 4])
|
| 78 |
+
dg = a.add.Tensor(a.neg.default(a.mul.Tensor(dy, a.div.Tensor(head(memory), 8.0))),
|
| 79 |
+
a.mul.Tensor(dy, head(local)))
|
| 80 |
+
gf = cvt(gate, torch.float32)
|
| 81 |
+
dg = cvt(a.mul.Tensor(cvt(dg, torch.float32),
|
| 82 |
+
a.mul.Tensor(gf, a.sub.Tensor(1, gf))), torch.bfloat16)
|
| 83 |
+
u = tanh_backward(head(uw), tu)
|
| 84 |
+
|
| 85 |
+
def pack(value, clone=True):
|
| 86 |
+
value = a.permute.default(value, [0, 1, 3, 2, 4])
|
| 87 |
+
if clone:
|
| 88 |
+
value = a.clone.default(value, memory_format=torch.contiguous_format)
|
| 89 |
+
return reshape(reshape(value, [4, 2048, 32, 64]), [4, 2048, 2048])
|
| 90 |
+
|
| 91 |
+
return a.cat.default([pack(q), pack(k), pack(v), pack(dg, False), pack(u)], 2)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
@lru_cache(maxsize=1)
|
| 95 |
+
def _pattern():
|
| 96 |
+
from torch._subclasses.fake_tensor import FakeTensorMode
|
| 97 |
+
from torch.fx.experimental.proxy_tensor import make_fx
|
| 98 |
+
|
| 99 |
+
with FakeTensorMode():
|
| 100 |
+
inputs = [torch.empty(shape, dtype=torch.float32 if name in ("sin", "cos")
|
| 101 |
+
else torch.bfloat16, device="cuda:0")
|
| 102 |
+
for name, shape in zip(_NAMES, _NATIVE_SHAPES, strict=True)]
|
| 103 |
+
gm = make_fx(_aten_boundary, tracing_mode="fake")(*inputs)
|
| 104 |
+
# make_fx normalizes explicit reshape to view; this hook runs after
|
| 105 |
+
# Inductor's view-to-reshape normalization.
|
| 106 |
+
for node in gm.graph.nodes:
|
| 107 |
+
if node.target == torch.ops.aten.view.default:
|
| 108 |
+
node.target = torch.ops.aten.reshape.default
|
| 109 |
+
return gm
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _metadata(value):
|
| 113 |
+
if not isinstance(value, torch.Tensor):
|
| 114 |
+
return None
|
| 115 |
+
# Symbolic/unsupported dimensions are rejected, not specialized implicitly.
|
| 116 |
+
if any(type(x) is not int for x in (*value.shape, *value.stride())):
|
| 117 |
+
return None
|
| 118 |
+
return (tuple(value.shape), tuple(value.stride()), value.dtype, value.device.type)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
class _Mismatch(Exception):
|
| 122 |
+
pass
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _match(candidate, pattern):
|
| 126 |
+
binding, reverse = {}, {}
|
| 127 |
+
|
| 128 |
+
def walk(actual, expected):
|
| 129 |
+
if isinstance(expected, Node):
|
| 130 |
+
if not isinstance(actual, Node):
|
| 131 |
+
raise _Mismatch("node kind")
|
| 132 |
+
if expected in binding:
|
| 133 |
+
if binding[expected] is not actual:
|
| 134 |
+
raise _Mismatch("shared dataflow")
|
| 135 |
+
return
|
| 136 |
+
if actual in reverse:
|
| 137 |
+
raise _Mismatch("distinct dataflow")
|
| 138 |
+
actual_meta, expected_meta = _metadata(actual.meta.get("val")), _metadata(expected.meta.get("val"))
|
| 139 |
+
if actual_meta is None or actual_meta != expected_meta:
|
| 140 |
+
raise _Mismatch("tensor metadata")
|
| 141 |
+
if expected.op != "placeholder":
|
| 142 |
+
if actual.op != expected.op or actual.target != expected.target:
|
| 143 |
+
raise _Mismatch("operator")
|
| 144 |
+
walk(actual.args, expected.args)
|
| 145 |
+
walk(actual.kwargs, expected.kwargs)
|
| 146 |
+
binding[expected] = actual
|
| 147 |
+
reverse[actual] = expected
|
| 148 |
+
elif isinstance(expected, (tuple, list)):
|
| 149 |
+
if not isinstance(actual, (tuple, list)) or len(actual) != len(expected):
|
| 150 |
+
raise _Mismatch("argument list")
|
| 151 |
+
for av, ev in zip(actual, expected, strict=True):
|
| 152 |
+
walk(av, ev)
|
| 153 |
+
elif isinstance(expected, dict):
|
| 154 |
+
if not isinstance(actual, dict) or set(actual) != set(expected):
|
| 155 |
+
raise _Mismatch("keyword arguments")
|
| 156 |
+
for key in expected:
|
| 157 |
+
walk(actual[key], expected[key])
|
| 158 |
+
elif isinstance(expected, torch.device):
|
| 159 |
+
if not isinstance(actual, torch.device) or actual.type != expected.type:
|
| 160 |
+
raise _Mismatch("device")
|
| 161 |
+
elif type(actual) is not type(expected) or actual != expected:
|
| 162 |
+
raise _Mismatch("literal")
|
| 163 |
+
|
| 164 |
+
output = next(node for node in pattern.graph.nodes if node.op == "output").args[0]
|
| 165 |
+
walk(candidate, output)
|
| 166 |
+
inputs = tuple(binding[node] for node in pattern.graph.nodes if node.op == "placeholder")
|
| 167 |
+
interior = {actual for expected, actual in binding.items() if expected.op != "placeholder"}
|
| 168 |
+
retained = set()
|
| 169 |
+
|
| 170 |
+
def retain_view(node):
|
| 171 |
+
if node not in interior or node in retained:
|
| 172 |
+
return
|
| 173 |
+
# AOT CSE can share the exact zero-filled scatter base across cores.
|
| 174 |
+
# Its complete literal recipe was matched above; preserve it while
|
| 175 |
+
# another region still uses it, without retaining computed arithmetic.
|
| 176 |
+
if node.target == torch.ops.aten.full.default and not node.all_input_nodes:
|
| 177 |
+
retained.add(node)
|
| 178 |
+
return
|
| 179 |
+
if node.target not in (torch.ops.aten.reshape.default, torch.ops.aten.permute.default):
|
| 180 |
+
raise _Mismatch("external intermediate consumer")
|
| 181 |
+
retained.add(node)
|
| 182 |
+
for parent in node.all_input_nodes:
|
| 183 |
+
retain_view(parent)
|
| 184 |
+
|
| 185 |
+
for node in interior - {candidate}:
|
| 186 |
+
if any(user not in interior for user in node.users):
|
| 187 |
+
# The raw output-gradient's head view also feeds the independent
|
| 188 |
+
# attention/memory VJPs. Retain that metadata-only chain in place;
|
| 189 |
+
# the opaque op takes its original contiguous MM result directly.
|
| 190 |
+
retain_view(node)
|
| 191 |
+
if len({value.meta["val"].device for value in inputs}) != 1:
|
| 192 |
+
raise _Mismatch("mixed input devices")
|
| 193 |
+
_validate_producers(inputs)
|
| 194 |
+
return inputs, interior - retained
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _validate_producers(inputs):
|
| 198 |
+
"""The local branches must belong to one causal dropout-zero Flash VJP."""
|
| 199 |
+
qm, ql, _sin, _cos, kw, _tk, kl, vl, dy, memory, local, _gate, uw, _tu = inputs
|
| 200 |
+
flash = None
|
| 201 |
+
for node, index in ((ql, 0), (kl, 1), (vl, 2)):
|
| 202 |
+
if (node.op != "call_function" or node.target is not operator.getitem
|
| 203 |
+
or len(node.args) != 2 or node.args[1] != index):
|
| 204 |
+
raise _Mismatch("Flash gradient ordering")
|
| 205 |
+
if flash is None:
|
| 206 |
+
flash = node.args[0]
|
| 207 |
+
elif node.args[0] is not flash:
|
| 208 |
+
raise _Mismatch("separate Flash VJPs")
|
| 209 |
+
if (not isinstance(flash, Node)
|
| 210 |
+
or flash.target != torch.ops.aten._scaled_dot_product_flash_attention_backward.default
|
| 211 |
+
or len(flash.args) != 14 or flash.args[8:12] != (128, 128, 0.0, True)
|
| 212 |
+
or dict(flash.kwargs) != {"scale": 0.125} or flash.args[4] is not local):
|
| 213 |
+
raise _Mismatch("Flash attention settings")
|
| 214 |
+
for node in (qm, kw, uw):
|
| 215 |
+
if node.op != "call_function" or node.target != torch.ops.aten.bmm.default:
|
| 216 |
+
raise _Mismatch("native backward GEMM producer")
|
| 217 |
+
if dy.op != "call_function" or dy.target != torch.ops.aten.mm.default:
|
| 218 |
+
raise _Mismatch("native output gradient producer")
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
@custom_op("warpstate::paired_backward_layout", mutates_args=())
|
| 222 |
+
def _paired_backward_layout(q_memory: torch.Tensor, q_local: torch.Tensor,
|
| 223 |
+
sin: torch.Tensor, cos: torch.Tensor, k_write: torch.Tensor,
|
| 224 |
+
tanh_k: torch.Tensor, k_local: torch.Tensor, v_local: torch.Tensor,
|
| 225 |
+
grad_y: torch.Tensor, memory: torch.Tensor, local: torch.Tensor,
|
| 226 |
+
gate: torch.Tensor, u_write: torch.Tensor,
|
| 227 |
+
tanh_u: torch.Tensor) -> torch.Tensor:
|
| 228 |
+
"""Opaque pure launch using native input storage, with no layout copies.
|
| 229 |
+
|
| 230 |
+
Fake/compile guards already require native shapes and strides. The entrypoint
|
| 231 |
+
is internal to this strict rewrite; shape views and fresh output allocation
|
| 232 |
+
are the only tensor operations. Runtime evidence remains valid on cache hits.
|
| 233 |
+
"""
|
| 234 |
+
from ops.backward_layout import _paired_kernel
|
| 235 |
+
|
| 236 |
+
args = (q_memory, q_local, sin, cos, k_write, tanh_k, k_local, v_local,
|
| 237 |
+
grad_y, memory, local, gate, u_write, tanh_u)
|
| 238 |
+
_validate_native_inputs(args)
|
| 239 |
+
output = torch.empty((4, 2048, 10240), dtype=torch.bfloat16, device=q_memory.device)
|
| 240 |
+
# All contiguous native shapes have exactly the raw layouts consumed by the
|
| 241 |
+
# prototype kernel; passing native pointers needs no view or contiguous call.
|
| 242 |
+
launch = (*args, output, 16777216, 32, 128, 64, 0.125, 256)
|
| 243 |
+
_paired_kernel[(32768,)](*launch, True, num_warps=8, enable_fp_fusion=True)
|
| 244 |
+
_paired_kernel[(65536,)](*launch, False, num_warps=8, enable_fp_fusion=True)
|
| 245 |
+
_COUNTS["runtime_calls"] += 1
|
| 246 |
+
return output
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
@_paired_backward_layout.register_fake
|
| 250 |
+
def _paired_backward_layout_fake(*inputs):
|
| 251 |
+
_validate_native_inputs(inputs)
|
| 252 |
+
return inputs[0].new_empty((4, 2048, 10240))
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _validate_native_inputs(inputs):
|
| 256 |
+
for name, value, shape in zip(_NAMES, inputs, _NATIVE_SHAPES, strict=True):
|
| 257 |
+
dtype = torch.float32 if name in ("sin", "cos") else torch.bfloat16
|
| 258 |
+
if (tuple(value.shape) != shape or value.dtype != dtype or not value.is_contiguous()
|
| 259 |
+
or value.device != inputs[0].device or value.device.type != "cuda"):
|
| 260 |
+
raise ValueError(f"Unsupported paired backward input {name}")
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
class BackwardLayoutPass(CustomGraphPass):
|
| 264 |
+
def __init__(self, mode, identity):
|
| 265 |
+
self.mode, self.identity = mode, identity
|
| 266 |
+
|
| 267 |
+
def uuid(self):
|
| 268 |
+
return self.identity
|
| 269 |
+
|
| 270 |
+
def __call__(self, graph):
|
| 271 |
+
_COUNTS["graphs"] += 1
|
| 272 |
+
candidates = [node for node in graph.nodes if node.op == "call_function"
|
| 273 |
+
and node.target == torch.ops.aten.cat.default
|
| 274 |
+
and isinstance(node.args[0], (tuple, list)) and len(node.args[0]) == 5]
|
| 275 |
+
for candidate in candidates:
|
| 276 |
+
_COUNTS["candidates"] += 1
|
| 277 |
+
try:
|
| 278 |
+
inputs, interior = _match(candidate, _pattern())
|
| 279 |
+
except _Mismatch as error:
|
| 280 |
+
_MISSES[str(error)] += 1
|
| 281 |
+
continue
|
| 282 |
+
_COUNTS["matches"] += 1
|
| 283 |
+
if self.mode == "inspect":
|
| 284 |
+
continue
|
| 285 |
+
with graph.inserting_before(candidate):
|
| 286 |
+
replacement = graph.call_function(torch.ops.warpstate.paired_backward_layout.default, inputs)
|
| 287 |
+
replacement.meta = dict(candidate.meta)
|
| 288 |
+
candidate.replace_all_uses_with(replacement)
|
| 289 |
+
for node in reversed(list(graph.nodes)):
|
| 290 |
+
if node in interior:
|
| 291 |
+
graph.erase_node(node)
|
| 292 |
+
_COUNTS["replacements"] += 1
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _identity(mode):
|
| 296 |
+
directory = Path(__file__).resolve().parent
|
| 297 |
+
sources = {"ops/" + name: hashlib.sha256((directory / name).read_bytes()).hexdigest()
|
| 298 |
+
for name in ("backward_layout_execution.py", "backward_layout.py")}
|
| 299 |
+
metadata = {"format": "warpstate-backward-layout-v1", "mode": mode,
|
| 300 |
+
"source_sha256": sources, "torch_version": torch.__version__,
|
| 301 |
+
"shape": list(_SHAPE), "algorithm": "paired", "block": 256, "warps": 8}
|
| 302 |
+
metadata["identity"] = hashlib.sha256(json.dumps(metadata, sort_keys=True).encode()).hexdigest()
|
| 303 |
+
return metadata
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def validate_backward_layout_config(cfg):
|
| 307 |
+
"""Reject unsupported model recipes before allocation or resume mutation."""
|
| 308 |
+
required = {"d_model": 2048, "n_heads": 32, "chunk_size": 128,
|
| 309 |
+
"parallel_memory": True, "dropout": 0.0}
|
| 310 |
+
for name, value in required.items():
|
| 311 |
+
if getattr(cfg, name, None) != value:
|
| 312 |
+
raise ValueError(f"Paired backward layout requires {name}={value}")
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def configure_backward_layout(mode="baseline"):
|
| 316 |
+
"""Configure once before compilation, returning checkpoint/benchmark identity.
|
| 317 |
+
|
| 318 |
+
A pre-existing custom pass is rejected, never replaced or silently composed;
|
| 319 |
+
composition could change the arithmetic boundary. Use a fresh process when
|
| 320 |
+
changing modes, matching Compiled Autograd's process-wide configuration.
|
| 321 |
+
"""
|
| 322 |
+
global _ACTIVE_MODE, _ACTIVE_PASS
|
| 323 |
+
if mode not in BACKWARD_LAYOUT_MODES:
|
| 324 |
+
raise ValueError(f"Unknown backward layout mode: {mode}")
|
| 325 |
+
from torch._inductor import config
|
| 326 |
+
if _ACTIVE_PASS is not None:
|
| 327 |
+
raise RuntimeError("Backward layout is already configured; use a fresh process")
|
| 328 |
+
if mode == "baseline":
|
| 329 |
+
return None
|
| 330 |
+
if config.post_grad_custom_post_pass is not None:
|
| 331 |
+
raise RuntimeError("A post-gradient custom pass already exists; refusing to replace it")
|
| 332 |
+
if (config.emulate_precision_casts or not config.triton.codegen_upcast_to_fp32
|
| 333 |
+
or config.eager_numerics.disable_ftz):
|
| 334 |
+
raise ValueError("Paired backward layout requires the profiled FP32 pointwise fusion settings")
|
| 335 |
+
metadata = _identity(mode)
|
| 336 |
+
_ACTIVE_MODE = mode
|
| 337 |
+
_ACTIVE_PASS = BackwardLayoutPass(mode, metadata["identity"])
|
| 338 |
+
config.post_grad_custom_post_pass = _ACTIVE_PASS
|
| 339 |
+
return metadata
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def backward_layout_stats():
|
| 343 |
+
"""Process-local compilation and runtime evidence; no device synchronization."""
|
| 344 |
+
return {"mode": _ACTIVE_MODE, **{name: _COUNTS[name] for name in
|
| 345 |
+
("graphs", "candidates", "matches", "replacements", "runtime_calls")},
|
| 346 |
+
"miss_reasons": dict(_MISSES)}
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def require_backward_layout_dispatch(min_calls=1):
|
| 350 |
+
if isinstance(min_calls, bool) or not isinstance(min_calls, int) or min_calls < 1:
|
| 351 |
+
raise ValueError("min_calls must be a positive integer")
|
| 352 |
+
if _ACTIVE_MODE != "paired" or _COUNTS["runtime_calls"] < min_calls:
|
| 353 |
+
raise RuntimeError(f"Paired backward layout expected at least {min_calls} runtime calls; "
|
| 354 |
+
f"observed {backward_layout_stats()}")
|
| 355 |
+
return backward_layout_stats()
|
ops/benchmark-supervisord.conf
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[program:warpstate-benchmark]
|
| 2 |
+
command=/workspace/WarpState/rtx6000_40b/ops/benchmark.sh --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt --micro-batch 4 --no-activation-checkpointing --compile-cores --autocast-cache --loss-chunk-size 1024 --vram-limit-gib 88 --warmup-updates 2 --measured-updates 3
|
| 3 |
+
directory=/workspace/WarpState/rtx6000_40b
|
| 4 |
+
autostart=false
|
| 5 |
+
autorestart=false
|
| 6 |
+
startsecs=5
|
| 7 |
+
startretries=0
|
| 8 |
+
stopsignal=TERM
|
| 9 |
+
stopwaitsecs=30
|
| 10 |
+
stopasgroup=true
|
| 11 |
+
killasgroup=true
|
| 12 |
+
stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/benchmark.log
|
| 13 |
+
stdout_logfile_maxbytes=20MB
|
| 14 |
+
stdout_logfile_backups=3
|
| 15 |
+
redirect_stderr=true
|
| 16 |
+
environment=TORCHINDUCTOR_COMPILE_THREADS="8",WARPSTATE_BENCH_DEVICES="0,1",WARPSTATE_BENCH_PROCESSES="2"
|
ops/benchmark.sh
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
ulimit -c 0
|
| 4 |
+
source /workspace/WarpState/rtx6000_40b/ops/training-env.sh
|
| 5 |
+
cd /workspace/WarpState/rtx6000_40b
|
| 6 |
+
export CUDA_VISIBLE_DEVICES="${WARPSTATE_BENCH_DEVICES:-0}"
|
| 7 |
+
if [[ "${WARPSTATE_BENCH_PROCESSES:-1}" == 1 ]]; then
|
| 8 |
+
exec .server-venv/bin/python -u ops/benchmark_training.py "$@"
|
| 9 |
+
else
|
| 10 |
+
exec .server-venv/bin/torchrun --standalone \
|
| 11 |
+
--nproc-per-node="$WARPSTATE_BENCH_PROCESSES" \
|
| 12 |
+
--log-dir=work/logs/torchrun-benchmark ops/benchmark_training.py "$@"
|
| 13 |
+
fi
|
ops/benchmark_gemm.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Time isolated BF16 GEMMs with the dominant WarpState projection layouts.
|
| 3 |
+
|
| 4 |
+
Source ops/training-env.sh first. Only deterministic random fixtures are used;
|
| 5 |
+
this program never opens a checkpoint or token dataset. Compilation/autotuning
|
| 6 |
+
are excluded from CUDA-event timings. These timings exclude communication,
|
| 7 |
+
optimizer updates and most Python overhead, so are not training throughput.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import contextlib
|
| 13 |
+
import datetime as dt
|
| 14 |
+
import hashlib
|
| 15 |
+
import json
|
| 16 |
+
import math
|
| 17 |
+
import os
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
import statistics
|
| 20 |
+
import subprocess
|
| 21 |
+
import time
|
| 22 |
+
|
| 23 |
+
PACKAGE = Path(__file__).resolve().parents[1]
|
| 24 |
+
WORKSPACE = PACKAGE.parent
|
| 25 |
+
GIB = 1024**3
|
| 26 |
+
FAMILIES = {"ffn_up_gate": (2048, 14336), "in_proj": (2048, 10240),
|
| 27 |
+
"ffn_down": (7168, 2048), "out_proj": (2048, 2048)}
|
| 28 |
+
BACKENDS = {"eager": None, "aten": "ATEN", "triton": "TRITON", "mixed": "ATEN,TRITON"}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def projection_cases(families, rows, operations):
|
| 32 |
+
"""Return M,N,K and storage layouts for Y=X@W.T, dX=dY@W, dW=dY.T@X."""
|
| 33 |
+
cases = []
|
| 34 |
+
for family in families:
|
| 35 |
+
input_dim, output_dim = FAMILIES[family]
|
| 36 |
+
specs = {
|
| 37 |
+
"fwd": (rows, output_dim, input_dim, False, True),
|
| 38 |
+
"dx": (rows, input_dim, output_dim, False, False),
|
| 39 |
+
"dw": (output_dim, input_dim, rows, True, False),
|
| 40 |
+
}
|
| 41 |
+
for operation in operations:
|
| 42 |
+
m, n, k, at, bt = specs[operation]
|
| 43 |
+
cases.append({"name": family + "." + operation, "family": family,
|
| 44 |
+
"operation": operation, "m": m, "n": n, "k": k,
|
| 45 |
+
"a_transposed": at, "b_transposed": bt,
|
| 46 |
+
"a_shape": [m, k], "b_shape": [k, n],
|
| 47 |
+
"a_stride": [1, m] if at else [k, 1],
|
| 48 |
+
"b_stride": [1, k] if bt else [n, 1],
|
| 49 |
+
"flops": 2 * m * n * k})
|
| 50 |
+
return cases
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def arguments(argv=None):
|
| 54 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 55 |
+
parser.add_argument("--families", nargs="+", choices=FAMILIES,
|
| 56 |
+
default=["ffn_up_gate", "in_proj", "ffn_down"])
|
| 57 |
+
parser.add_argument("--operations", nargs="+", choices=["fwd", "dx", "dw"],
|
| 58 |
+
default=["fwd", "dx", "dw"])
|
| 59 |
+
parser.add_argument("--rows", type=int, default=8192,
|
| 60 |
+
help="Micro-batch times sequence length; production uses 8192")
|
| 61 |
+
parser.add_argument("--backends", nargs="+", choices=BACKENDS,
|
| 62 |
+
default=["eager", "aten", "triton"])
|
| 63 |
+
parser.add_argument("--warmup", type=int, default=10)
|
| 64 |
+
parser.add_argument("--repeats", type=int, default=30)
|
| 65 |
+
parser.add_argument("--trials", type=int, default=5)
|
| 66 |
+
parser.add_argument("--burnin-seconds", type=float, default=2.0,
|
| 67 |
+
help="Sustain GEMM load before timing to reduce cold boost effects")
|
| 68 |
+
parser.add_argument("--seed", type=int, default=1337)
|
| 69 |
+
parser.add_argument("--device", type=int, default=0,
|
| 70 |
+
help="CUDA device index after CUDA_VISIBLE_DEVICES filtering")
|
| 71 |
+
parser.add_argument("--blas-backend", choices=["default", "cublas", "cublaslt"],
|
| 72 |
+
default="default")
|
| 73 |
+
parser.add_argument("--allow-reduced-precision-reduction", action=argparse.BooleanOptionalAction,
|
| 74 |
+
default=True,
|
| 75 |
+
help="Allow BF16 intermediate reductions, matching the current production setting")
|
| 76 |
+
parser.add_argument("--list-shapes", action="store_true",
|
| 77 |
+
help="Print selected shapes/strides and exit without importing torch")
|
| 78 |
+
parser.add_argument("--output-dir", type=Path,
|
| 79 |
+
default=PACKAGE / "work" / ("benchmark-gemm-" +
|
| 80 |
+
dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%S")))
|
| 81 |
+
args = parser.parse_args(argv)
|
| 82 |
+
for key in ("rows", "warmup", "repeats", "trials"):
|
| 83 |
+
if getattr(args, key) < 1:
|
| 84 |
+
parser.error("--" + key + " must be positive")
|
| 85 |
+
if args.device < 0:
|
| 86 |
+
parser.error("--device must be nonnegative")
|
| 87 |
+
if not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0:
|
| 88 |
+
parser.error("--burnin-seconds must be finite and nonnegative")
|
| 89 |
+
for key in ("families", "operations", "backends"):
|
| 90 |
+
if len(getattr(args, key)) != len(set(getattr(args, key))):
|
| 91 |
+
parser.error("--" + key + " must not contain duplicates")
|
| 92 |
+
args.output_dir = args.output_dir.expanduser().resolve()
|
| 93 |
+
work = PACKAGE / "work"
|
| 94 |
+
if (not args.output_dir.is_relative_to(work) or
|
| 95 |
+
not args.output_dir.relative_to(work).parts or
|
| 96 |
+
not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")):
|
| 97 |
+
parser.error("--output-dir must be below rtx6000_40b/work/benchmark-*")
|
| 98 |
+
if not args.list_shapes:
|
| 99 |
+
for key in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH",
|
| 100 |
+
"TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"):
|
| 101 |
+
if not os.environ.get(key) or not Path(os.environ[key]).resolve().is_relative_to(WORKSPACE):
|
| 102 |
+
parser.error("Source ops/training-env.sh first; " + key + " must stay in the workspace")
|
| 103 |
+
return args
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def gpu_telemetry():
|
| 107 |
+
try:
|
| 108 |
+
result = subprocess.run(
|
| 109 |
+
["nvidia-smi", "--query-gpu=index,uuid,name,power.draw,power.limit,clocks.sm,"
|
| 110 |
+
"clocks.mem,temperature.gpu,utilization.gpu,memory.used", "--format=csv,noheader"],
|
| 111 |
+
capture_output=True, text=True, timeout=10, check=False)
|
| 112 |
+
return {"csv": result.stdout.strip(), "returncode": result.returncode}
|
| 113 |
+
except (OSError, subprocess.TimeoutExpired) as error:
|
| 114 |
+
return {"error": str(error)}
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def main():
|
| 118 |
+
args = arguments()
|
| 119 |
+
cases = projection_cases(args.families, args.rows, args.operations)
|
| 120 |
+
if args.list_shapes:
|
| 121 |
+
print(json.dumps(cases, indent=2))
|
| 122 |
+
return
|
| 123 |
+
# Reject an existing directory to preserve previous measurements.
|
| 124 |
+
args.output_dir.mkdir(parents=True, exist_ok=False)
|
| 125 |
+
import torch
|
| 126 |
+
from torch._inductor import config
|
| 127 |
+
|
| 128 |
+
if not torch.cuda.is_available():
|
| 129 |
+
raise SystemExit("CUDA is required for timings; --list-shapes is CPU-only")
|
| 130 |
+
torch.cuda.set_device(args.device)
|
| 131 |
+
device = torch.device("cuda", args.device)
|
| 132 |
+
if not torch.cuda.is_bf16_supported():
|
| 133 |
+
raise SystemExit("This GPU does not support BF16")
|
| 134 |
+
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = args.allow_reduced_precision_reduction
|
| 135 |
+
torch.backends.cuda.matmul.allow_tf32 = False
|
| 136 |
+
torch.backends.cuda.preferred_blas_library(args.blas_backend)
|
| 137 |
+
properties = torch.cuda.get_device_properties(device)
|
| 138 |
+
report = {
|
| 139 |
+
"started_utc": dt.datetime.now(dt.UTC).isoformat(),
|
| 140 |
+
"source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
| 141 |
+
"arguments": {key: str(value) if isinstance(value, Path) else value
|
| 142 |
+
for key, value in vars(args).items()},
|
| 143 |
+
"torch_version": torch.__version__, "cuda_version": torch.version.cuda,
|
| 144 |
+
"dtype": "bfloat16", "output_dtype": "bfloat16",
|
| 145 |
+
"allow_bf16_reduced_precision_reduction": args.allow_reduced_precision_reduction,
|
| 146 |
+
"compile_mode": "max-autotune-no-cudagraphs",
|
| 147 |
+
"autotune_fallback_to_aten": False,
|
| 148 |
+
"device": {"name": properties.name, "capability": list(torch.cuda.get_device_capability(device)),
|
| 149 |
+
"multiprocessors": properties.multi_processor_count,
|
| 150 |
+
"total_memory_gib": properties.total_memory / GIB},
|
| 151 |
+
"environment": {key: os.environ.get(key) for key in
|
| 152 |
+
("CUDA_VISIBLE_DEVICES", "CUBLAS_WORKSPACE_CONFIG", "OMP_NUM_THREADS",
|
| 153 |
+
"TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR", "PYTORCH_ALLOC_CONF",
|
| 154 |
+
"TORCHINDUCTOR_COMPILE_THREADS", "ENABLE_PERSISTENT_TMA_MATMUL",
|
| 155 |
+
"TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_BACKENDS",
|
| 156 |
+
"TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_SEARCH_SPACE", "CUTLASS_EPILOGUE_FUSION")},
|
| 157 |
+
"telemetry_before": gpu_telemetry(), "results": [],
|
| 158 |
+
"notes": ["Shapes represent isolated linear algebra, not an end-to-end backward graph.",
|
| 159 |
+
"Each repeat allocates a BF16 output, as torch.mm normally does.",
|
| 160 |
+
"Timings reuse inputs and omit compilation/autotuning; compare to full training separately.",
|
| 161 |
+
"Error is against eager cuBLAS BF16 output, not an exact FP32 reference.",
|
| 162 |
+
"Telemetry is a snapshot, not a synchronized power integral."],
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
def save():
|
| 166 |
+
target = args.output_dir / "results.json"
|
| 167 |
+
temporary = args.output_dir / "results.json.partial"
|
| 168 |
+
temporary.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
|
| 169 |
+
temporary.replace(target)
|
| 170 |
+
|
| 171 |
+
save()
|
| 172 |
+
with torch.inference_mode():
|
| 173 |
+
for case_index, case in enumerate(cases):
|
| 174 |
+
generator = torch.Generator(device=device).manual_seed(args.seed + case_index)
|
| 175 |
+
|
| 176 |
+
def fixture(rows, columns, transposed):
|
| 177 |
+
shape = (columns, rows) if transposed else (rows, columns)
|
| 178 |
+
tensor = torch.randn(shape, device=device, dtype=torch.bfloat16, generator=generator)
|
| 179 |
+
return tensor.T if transposed else tensor
|
| 180 |
+
|
| 181 |
+
a = fixture(case["m"], case["k"], case["a_transposed"])
|
| 182 |
+
b = fixture(case["k"], case["n"], case["b_transposed"])
|
| 183 |
+
assert list(a.stride()) == case["a_stride"]
|
| 184 |
+
assert list(b.stride()) == case["b_stride"]
|
| 185 |
+
reference = torch.mm(a, b)
|
| 186 |
+
torch.cuda.synchronize(device)
|
| 187 |
+
for backend in args.backends:
|
| 188 |
+
row = {**case, "backend": backend, "status": "running",
|
| 189 |
+
"max_autotune_gemm_backends": BACKENDS[backend],
|
| 190 |
+
"telemetry_before": gpu_telemetry()}
|
| 191 |
+
print(json.dumps({"case": case["name"], "backend": backend, "status": "starting"}), flush=True)
|
| 192 |
+
try:
|
| 193 |
+
torch._dynamo.reset()
|
| 194 |
+
settings = ({"max_autotune_gemm_backends": BACKENDS[backend],
|
| 195 |
+
"autotune_fallback_to_aten": False}
|
| 196 |
+
if BACKENDS[backend] else {})
|
| 197 |
+
with config.patch(settings) if settings else contextlib.nullcontext():
|
| 198 |
+
operation = torch.mm if backend == "eager" else torch.compile(
|
| 199 |
+
lambda x, y: torch.mm(x, y), fullgraph=True, dynamic=False,
|
| 200 |
+
mode="max-autotune-no-cudagraphs")
|
| 201 |
+
compile_start = time.perf_counter()
|
| 202 |
+
output = operation(a, b)
|
| 203 |
+
torch.cuda.synchronize(device)
|
| 204 |
+
row["first_call_seconds"] = time.perf_counter() - compile_start
|
| 205 |
+
difference = output.float().sub_(reference.float())
|
| 206 |
+
reference_rms = reference.float().square_().mean().sqrt().item()
|
| 207 |
+
row["max_abs_error"] = difference.abs().max().item()
|
| 208 |
+
row["rms_error"] = difference.square_().mean().sqrt().item()
|
| 209 |
+
row["relative_rms_error"] = row["rms_error"] / max(reference_rms, 1e-30)
|
| 210 |
+
if not all(math.isfinite(row[key]) for key in
|
| 211 |
+
("max_abs_error", "rms_error", "relative_rms_error")):
|
| 212 |
+
raise RuntimeError("Nonfinite output error")
|
| 213 |
+
del difference, output
|
| 214 |
+
for _ in range(args.warmup):
|
| 215 |
+
output = operation(a, b)
|
| 216 |
+
del output
|
| 217 |
+
torch.cuda.synchronize(device)
|
| 218 |
+
burnin_start = time.perf_counter()
|
| 219 |
+
burnin_iterations = 0
|
| 220 |
+
while time.perf_counter() - burnin_start < args.burnin_seconds:
|
| 221 |
+
for _ in range(10):
|
| 222 |
+
output = operation(a, b)
|
| 223 |
+
del output
|
| 224 |
+
torch.cuda.synchronize(device)
|
| 225 |
+
burnin_iterations += 10
|
| 226 |
+
row["burnin_elapsed_seconds"] = time.perf_counter() - burnin_start
|
| 227 |
+
row["burnin_iterations"] = burnin_iterations
|
| 228 |
+
torch.cuda.reset_peak_memory_stats(device)
|
| 229 |
+
row["baseline_allocated_gib"] = torch.cuda.memory_allocated(device) / GIB
|
| 230 |
+
trial_ms = []
|
| 231 |
+
for _ in range(args.trials):
|
| 232 |
+
start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
|
| 233 |
+
start.record()
|
| 234 |
+
for _ in range(args.repeats):
|
| 235 |
+
output = operation(a, b)
|
| 236 |
+
end.record()
|
| 237 |
+
end.synchronize()
|
| 238 |
+
trial_ms.append(start.elapsed_time(end) / args.repeats)
|
| 239 |
+
del output
|
| 240 |
+
row["trial_ms_per_gemm"] = trial_ms
|
| 241 |
+
row["median_ms"] = statistics.median(trial_ms)
|
| 242 |
+
row["min_ms"] = min(trial_ms)
|
| 243 |
+
row["tflops"] = case["flops"] / (row["median_ms"] * 1e9)
|
| 244 |
+
row["peak_allocated_gib"] = torch.cuda.max_memory_allocated(device) / GIB
|
| 245 |
+
row["peak_reserved_gib"] = torch.cuda.max_memory_reserved(device) / GIB
|
| 246 |
+
row["telemetry_after"] = gpu_telemetry()
|
| 247 |
+
row["status"] = "ok"
|
| 248 |
+
except Exception as error:
|
| 249 |
+
row["status"] = "error"
|
| 250 |
+
row["error"] = type(error).__name__ + ": " + str(error)[:4000]
|
| 251 |
+
report["results"].append(row)
|
| 252 |
+
save()
|
| 253 |
+
print(json.dumps({key: row[key] for key in
|
| 254 |
+
("name", "backend", "status", "median_ms", "tflops",
|
| 255 |
+
"max_abs_error", "rms_error", "error") if key in row}), flush=True)
|
| 256 |
+
del a, b, reference
|
| 257 |
+
torch.cuda.empty_cache()
|
| 258 |
+
report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat()
|
| 259 |
+
report["telemetry_after"] = gpu_telemetry()
|
| 260 |
+
save()
|
| 261 |
+
print("Results: " + str(args.output_dir / "results.json"), flush=True)
|
| 262 |
+
if any(row["status"] != "ok" for row in report["results"]):
|
| 263 |
+
raise SystemExit(1)
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
if __name__ == "__main__":
|
| 267 |
+
main()
|