diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..154d66226bb2c8d0869e42ac299aab988f0aea82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.venv/ +.server-venv/ +venv/ + +# Model / training outputs +work/ +checkpoints/ +*.pt +*.pth +*.ckpt +*.safetensors +*.bin +*.log +*.pid + +# Secrets and local environment +.env +.env.* +!.env.example diff --git a/DATA_NOTICE.md b/DATA_NOTICE.md new file mode 100644 index 0000000000000000000000000000000000000000..90a4de2c09fcaa746755482d67f020a4b7e4afa5 --- /dev/null +++ b/DATA_NOTICE.md @@ -0,0 +1,14 @@ +# Dataset attribution + +Default upstream dataset: **FineWeb-Edu**, Hugging Face / HuggingFaceFW. +Repository: https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu +Configuration: sample-100BT. Language: English. Dataset card lists ODC-BY. +Read the upstream dataset card and license for applicable attribution and +underlying web-content terms. This ZIP distributes no dataset texts. +The immutable source revision is recorded in work/data/manifest.json at preparation. + +The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale: +https://arxiv.org/abs/2406.17557 + +The WarpState code and tokenizer are copied from the user's existing project. +No new ownership or license over those user-provided assets is asserted here. diff --git a/PACKAGE_MANIFEST.json b/PACKAGE_MANIFEST.json new file mode 100644 index 0000000000000000000000000000000000000000..9ae2e6bab4b260477c5354e0af845b8b821793be --- /dev/null +++ b/PACKAGE_MANIFEST.json @@ -0,0 +1,42 @@ +{ + "run_40b.py": { + "sha256": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0", + "bytes": 50988 + }, + "README_UK.md": { + "sha256": "b401dc313e25aea48ab1a0bd24945a9923cd98adcb254c77cb1e130dd16c1335", + "bytes": 13562 + }, + "DATA_NOTICE.md": { + "sha256": "6b11284648bcb6fe9d1c74204d54e64d228b05d78749330c5caab119806b2a79", + "bytes": 735 + }, + "test_server.py": { + "sha256": "66ebfd733f8d71a462108c5d99dc469a99d1023bb7a0cd611b501a58b9269bff", + "bytes": 6351 + }, + "tokenizer.model": { + "sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e", + "bytes": 545080 + }, + "configs/warpstate_2b.json": { + "sha256": "884ba9e523af2a065ff0e11a4fd86eddfdc5ef656196dc318f94ec455be63009", + "bytes": 269 + }, + "warpstate/__init__.py": { + "sha256": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b", + "bytes": 113 + }, + "warpstate/config.py": { + "sha256": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad", + "bytes": 1187 + }, + "warpstate/model.py": { + "sha256": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161", + "bytes": 15500 + }, + "VALIDATION.json": { + "sha256": "168bb8233b04db144021f185b6a3890c06c137c3c5c29c1f18464ee8474ce1b8", + "bytes": 614 + } +} diff --git a/README.md b/README.md index 32897cd3e640101ba184f8c4ccd896981de3804a..aa409a185ce850321090964c8dd8fa46e8c7c7e2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,282 @@ +
+ WarpState — memory-native language modeling + +

⚡ WarpState

+

A memory-native language model built around local attention, fast/slow temporal states, and depth-wise computation.

+

Experimental architecture · Original 2.005B-parameter research run · Pretraining → SFT → DPO

+ +

+ Research + Documented model size + Tokens trained + PyTorch +

+

Created by Volodymyr Makarenko · Co-authored & sponsored by @ykim078

+

+ Overview · + Architecture · + Results · + Quick start · + Team & sponsor +

+
+ --- -license: mit + +## 🧠 What is WarpState? + +**WarpState** is an experimental autoregressive language model exploring a different way to balance *local precision* and *long-range continuity*. Instead of building a full sequence-wide attention matrix at every layer, it combines: + +- **Causal local attention** within fixed **128-token chunks**. +- **Two decaying temporal memory states** (fast and slow) that carry compressed information from completed chunks. +- **Learned gating** that blends locally attended features with historical memory. +- **Depth-wise computation** through reusable model cores with depth-specific modulation. +- A full language-model training stack, including **BF16 computation / FP32 optimizer state**, checkpoint-resumable pretraining, supervised fine-tuning (SFT), and direct preference optimization (DPO). + +The architecture is implemented in [`warpstate/model.py`](warpstate/model.py) and [`warpstate/config.py`](warpstate/config.py). **WarpState is not Project Nord's SNN architecture**: the supplied Project Nord README was used as a *presentation reference*, not as a source of WarpState features or benchmark claims. + +> [!IMPORTANT] +> **Release contents:** The supplied source archives include Python code, the SentencePiece tokenizer, tests, and operational documentation, **but not pretrained weights, training data, or the `configs/` directory** referenced by several launchers. This README documents the historical research run described in the included records; the uploaded source package alone is **not** a one-command pretrained-model download. A complete runnable release must include the correct configuration files and access to trusted checkpoints. + +## ✨ At a glance + +| Research dimension | Documented state | +| :--- | :--- | +| **Model** | 2,005,111,424 parameters in the recorded full-size training run | +| **Tokenizer** | SentencePiece, 32,768-token vocabulary | +| **Training sequence** | 2,048 tokens; not equivalent to an exact 2,048-token persistent memory | +| **Local attention** | Causal SDPA, 128 tokens per chunk | +| **Persistent state** | Fast + slow fixed-size matrix state per attention head and core execution | +| **Training corpus prepared** | 40,000,000,000 train tokens + 1,048,576 validation tokens | +| **Pretraining completed** | 10,000,007,168 tokens, at step 76,294, per the September 2026 handoff | +| **Post-training** | 10,312 SFT updates; 2,334 DPO updates, reported complete | +| **Compute format** | BF16 forward/backward; FP32 parameters, gradients, and AdamW states | +| **Project stage** | Research implementation with historical training logs and tests; no independently verified general-purpose LLM benchmark in this package | + +Numbers are sourced from the included source files and operational handoff documents, primarily `ops/RESTART_HANDOFF.md`, `ops/TRAINING.md`, `ops/DPO_HANDOFF.md`, and `ops/DPO_CHAT.md`. Training outcomes are historical records, not fresh measurements performed for this README. + --- + +## 🧬 Architecture + +
+ WarpState: token input, local causal attention, fast and slow memory, gated fusion, repeated cores and language modeling head +
+ +### How one WarpState core works + +1. **Normalize and project.** RMSNorm feeds one fused projection into queries, keys, values, a gate, and memory-write features. +2. **Attend locally.** Rotary position embeddings and causal scaled dot-product attention operate within each 128-token tile. +3. **Read compressed history.** Queries read from two memory matrices containing information written by *previously completed* tiles. +4. **Fuse the paths.** A learned sigmoid gate mixes local attention and fast/slow historical memory. +5. **Update memory.** Bounded, outer-product-like writes update the fast/slow states with separate learned decay coefficients. +6. **Refine the representation.** Residual projection, RMSNorm, and SwiGLU complete the block; the language model reuses cores across logical depth passes. + +### Memory, not magic + +The streaming path retains compact fast/slow state instead of an ever-growing full-attention key/value history. That does **not** imply infinite context, lossless recall, perfect retrieval, or constant runtime in every implementation mode. The optional parallel-memory path materializes tile-level interactions during training, and end-to-end cost depends on the kernel and execution configuration. + +### Implementation highlights + +| Module | What it does | Source | +| :--- | :--- | :--- | +| **WarpStateCore** | Local attention + two-timescale associative memory | [`warpstate/model.py`](warpstate/model.py) | +| **WarpStateLM** | Token embeddings, depth modulation, shared cores, tied output head | [`warpstate/model.py`](warpstate/model.py) | +| **Training runner** | Streaming data preparation, AdamW, validation, checkpoint/resume | [`run_40b.py`](run_40b.py) | +| **Distributed and kernel paths** | DDP, execution optimizations, matmul and backward experiments | [`ops/`](ops/) | +| **SFT** | Assistant-token-masked supervised fine-tuning | [`posttrain.py`](posttrain.py) | +| **DPO** | Preference optimization with a fixed reference model | [`dpo.py`](dpo.py) | +| **Inference probes** | CPU base-model chat and archived DPO inference tooling | [`chat.py`](chat.py), [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) | + +--- + +## 📈 Training & evaluation + +### Documented experimental milestones + +| Phase | Recorded output | Interpretation | +| :--- | :--- | :--- | +| **Raw pretraining** | **10,000,007,168 tokens** · **76,294 updates** · validation loss **2.3497** | Pretraining token-level cross-entropy; **not** a claim that all 40B prepared tokens were trained | +| **SFT** | **10,312 updates** · **329,973 examples** · validation loss **1.1157** | Assistant-token-masked fine-tuning; its loss uses a **different objective** | +| **DPO** | **2,334 updates** · **74,661 preference pairs** · held-out DPO loss **0.5790** | Preference objective; **not** model perplexity or accuracy | + +The included DPO record also reports positive relative reward margins for **245 / 339 held-out pairs (72.27%)**. This is a *margin statistic*, **not** an end-user generation win rate. Source: [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md). + +### Selected historical throughput observations + +| Setup | Recorded observation | Scope / caveat | +| :--- | :--- | :--- | +| 2 × RTX PRO 6000 Blackwell Max-Q | **31,199 tokens/s** | September 13 analysis of a repeated 20-update, non-profiled configuration; excludes validation/checkpoint pauses | +| 1 × NVIDIA B200 | **20,752 sustained tokens/s**; **72,499 median tokens/s** | Specific 20-update autotuned benchmark with provider scheduling waits; excludes initialization, validation and saves | + +These measurements describe **training throughput on the named server GPUs**, not inference speed, consumer GPU performance, or a reproducible speed promise. See [`ops/POINTWISE_HEADROOM_20260913.md`](ops/POINTWISE_HEADROOM_20260913.md) and [`ops/B200_PERFORMANCE.md`](ops/B200_PERFORMANCE.md) for test conditions and the discrepancy between median and sustained rates. + +**What remains to be demonstrated publicly:** standardized language tasks, instruction following, multilingual quality, reasoning, independent reproducibility, and memory-retention tests. A low training loss alone cannot establish any of these capabilities. + +--- + +## 🚀 Getting started + +### 1. Assemble the source tree + +The distributed archives contain complementary portions of the same project. Combine their contents under **one project root**, preserving `warpstate/`, `ops/`, root Python files, and `tokenizer.model`: + +```text +WarpState/ +├── README.md # This document +├── assets/ # README visuals +├── warpstate/ +│ ├── __init__.py +│ ├── config.py +│ └── model.py +├── ops/ # Training, optimization, SFT/DPO tooling +├── configs/ # REQUIRED for full training; not in supplied ZIPs +├── run_40b.py # Pretraining entry point +├── posttrain.py # SFT runner +├── dpo.py # DPO runner +├── chat.py # CPU base-model probe +├── tokenizer.model +├── README_UK.md # Existing Ukrainian server guide +└── test_*.py # Test suite +``` + +> [!WARNING] +> The 2.005B training configuration and the SFT/DPO plans under `configs/` were **not present** in the supplied archives. `WarpStateConfig()` has smaller development defaults and does **not** instantiate the documented 2.005B run. Restore the original configuration/plan files from your trusted project copy before attempting to reproduce the historical results. + +### 2. Set up a Python environment + +Use **Linux, Python 3.12, a supported NVIDIA GPU and a CUDA-compatible PyTorch build** for the documented full training workflow. The original server setup used high-memory RTX PRO 6000 or B200-class hardware; the 2B full-optimizer recipe is **not** an 8 GB VRAM training preset. + +```bash +cd WarpState +python3.12 -m venv .venv +source .venv/bin/activate + +# Select a PyTorch/CUDA build compatible with YOUR GPU and driver: +# https://pytorch.org/get-started/locally/ +python -m pip install torch sentencepiece numpy +``` + +The historical automatic bootstrap is implemented in `run_40b.py`. Detailed, pinned deployment requirements are in `ops/training-requirements-resolved.txt`, `ops/B200_SETUP.md`, and `README_UK.md`; package resolution depends on your actual host. + +### 3. Inspect the architecture without a checkpoint + +```python +from warpstate import WarpStateConfig, WarpStateLM + +config = WarpStateConfig() # Development-sized defaults; NOT the historical 2.005B config +model = WarpStateLM(config) +print(f"Parameters: {model.num_parameters():,}") +``` + +This inspects randomly initialized code only. It does **not** provide a trained model. + +### 4. Run a trusted base-model checkpoint + +With the original compatible tokenizer and full `warpstate-server-v1` checkpoint available: + +```bash +python3.12 chat.py \ + --checkpoint /path/to/trusted/latest.pt \ + --tokenizer ./tokenizer.model \ + --prompt "Explain how memory works." \ + --temperature 0 +``` + +`chat.py` is a **CPU FP32 base-pretraining probe**. The DPO-tuned assistant uses a **different interface and archived snapshot format**; follow [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) and [`ops/RESTART_HANDOFF.md`](ops/RESTART_HANDOFF.md) on a host containing the original archives. Some operational scripts embed the historical `/workspace/WarpState/rtx6000_40b` layout and require path adaptation elsewhere. + +### 5. Start or resume training (advanced) + +Only after restoring the **matching** 2.005B config, tokenizer, trusted weights or explicit from-scratch initialization, GPU environment, disk space and dataset permissions: + +```bash +# Example for a NEW run initialized from an existing trusted WarpState checkpoint +python3.12 run_40b.py \ + --weights /path/to/trusted/latest.pt \ + --config ./configs/warpstate_2b.json \ + --work-dir /path/to/warpstate-work +``` + +Alternatively, `--from-scratch` requests random initialization; it is **not** a way to load the trained 2B model. By default, the runner prepares **40B train tokens (~80 GB of `uint16` token data)** before full training. Even `--max-steps 10` does **not** skip full corpus preparation. Preserve checkpoint, tokenizer and data provenance when resuming. Never load untrusted `.pt`/pickle files. + +Read [`README_UK.md`](README_UK.md), [`ops/TRAINING.md`](ops/TRAINING.md), and [`ops/LOCAL_RESUME.md`](ops/LOCAL_RESUME.md) before using production launchers. + +--- + +## 🔍 Reproducibility & engineering + +- **Deterministic corpus recipe:** FineWeb-Edu streaming, SentencePiece token counting, train/holdout split, provenance and SHA-256 manifests (see [`README_UK.md`](README_UK.md)). +- **Recoverable checkpoints:** complete FP32 model and AdamW state, RNG state, training cursor, run signature and guarded resume. +- **Precision discipline:** BF16 compute with FP32 master weights and optimizer; kernel changes require numerical comparisons. +- **Multiple execution modes:** compiled cores, gradient/communication overlap experiments, distributed training, asynchronous checkpoint writing and optional kernel tuning. +- **Testing:** unit and integration tests for training, checkpointing, transfer, post-training and distributed execution are included. Historical validation reports are not a substitute for rerunning them on a new host. + +The repository also contains dated operational documents. **They describe specific machines and historical run states**, not universally safe commands to execute automatically on a new machine. + +### Data and model safety + +The original base data used [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu). Respect upstream dataset licenses and documentation, keep private checkpoints/logs out of Git, and treat PyTorch pickle checkpoints as trusted-code inputs. No datasets or trained weights are embedded in these README assets. + +--- + +## 🗺️ Roadmap + +- [x] Implement chunked causal attention with fast/slow temporal memory. +- [x] Implement depth-modulated reusable cores and tied language-model head. +- [x] Build resumable pretraining and GPU execution/optimization tooling. +- [x] Record a 10B-token pretraining milestone for the 2.005B experiment. +- [x] Record completed SFT and DPO stages in historical run documentation. +- [ ] Publish complete, reproducible configs and a verified release manifest. +- [ ] Publish licensed, integrity-checked downloadable checkpoints (if distribution is authorized). +- [ ] Add standardized benchmarks, memory-retention evaluations and representative generations. +- [ ] Consolidate deployment scripts and add CI for supported hardware paths. +- [ ] Document scaling, memory behavior and real inference performance across devices. + +--- + +## 🤝 Project leadership & sponsorship + + + + + + +
+ 🧠 Project creator & lead developer
+ Volodymyr Makarenko
+ Core research direction, architecture and development +
+ 💜 Project sponsor & co-author
+ @ykim078
+ Recognized as both sponsor and co-author of WarpState +
+ +**Special thanks to [@ykim078](https://github.com/ykim078)** for supporting the project and participating as a **co-author**. Sponsorship and co-authorship are distinct roles; both are intentionally credited here at the project owner's request. Specific financial terms and individual code contributions are not claimed. + +Contributions are welcome through issues and pull requests once the repository is published with a contribution policy and clear license. + +## 📚 Documentation + +| Guide | Purpose | +| :--- | :--- | +| [`README_UK.md`](README_UK.md) | Ukrainian source-package and GPU training guide | +| [`ops/RESTART_HANDOFF.md`](ops/RESTART_HANDOFF.md) | Recorded completed raw / SFT / DPO run and archive state | +| [`ops/B200_PERFORMANCE.md`](ops/B200_PERFORMANCE.md) | B200 benchmark conditions and results | +| [`ops/OPTIMIZED_TRAINING.md`](ops/OPTIMIZED_TRAINING.md) | Dual RTX PRO 6000 training measurements | +| [`ops/POSTTRAIN.md`](ops/POSTTRAIN.md) | SFT execution and data provenance | +| [`ops/DPO_HANDOFF.md`](ops/DPO_HANDOFF.md) | Preference optimization design | +| [`ops/DPO_CHAT.md`](ops/DPO_CHAT.md) | Post-DPO inference instructions and caveats | + +## 📄 License & citation + +**License:** No root `LICENSE` file was included in the supplied WarpState archives. **Do not assume an open-source license or reuse permission** until the repository owner publishes an explicit license. The Project Nord reference's Apache-2.0 label **does not automatically apply** to WarpState. + +If citing the project, link to the **actual WarpState repository URL** once published; do not cite the unrelated Nord repository as its code source. Credit the creator and co-author as shown above. + +--- + +
+ ⚡ WarpState
+ Local attention for immediate detail. Temporal memory for continuity.

+ Built as an experimental research system. Documented results, explicit limitations, reproducible engineering. +
diff --git a/README_UK.md b/README_UK.md new file mode 100644 index 0000000000000000000000000000000000000000..eefff2915521e6841333be5ce3e9defa87e8aaf9 --- /dev/null +++ b/README_UK.md @@ -0,0 +1,170 @@ +# WarpState 2B: один запуск на RTX PRO 6000 Blackwell 96 ГБ + +Архів містить код вашої WarpState, точний наявний токенізатор і єдиний +керівний скрипт `run_40b.py`. Ваг моделі та 80 ГБ даних в архіві немає. +Ваги беруться з вашого сервера, дані готуються на сервері автоматично. + +## Запуск + +Потрібні Linux x86_64, **Python 3.12–3.14 з модулем venv**, робочий NVIDIA +драйвер, сумісний із CUDA 13.0, одна RTX PRO 6000 Blackwell 96 ГБ, +інтернет і рекомендовано **250 ГБ вільного SSD/NVMe**. Бажано 128 ГБ RAM; +фактичні потреби залежать від завантаження checkpoint і підготовки даних. +Скрипт не встановлює системний драйвер і не змінює інші Python-середовища. + +Розпакуйте ZIP, перейдіть у його папку та підставте справжній шлях до ваг: + +```bash +cd rtx6000_40b +python3.12 run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b +``` + +Ця **одна команда**: + +1. Створює `.server-venv` та встановлює CUDA PyTorch і залежності. +2. Перевіряє GPU, формат/архітектуру ваг, розмір словника й доступне місце. +3. Потоково читає FineWeb-Edu та готує **рівно 40 000 000 000 train-токенів** + вашим токенізатором плюс 1 048 576 validation-токенів. +4. Запускає повне навчання всіх параметрів із GPU AdamW та BF16-обчисленнями. +5. Пише метрики, validation і checkpoint з можливістю відновлення. + +Для запуску з від’єднанням від SSH використовуйте вже встановлений tmux +або стандартний nohup. Наприклад: + +```bash +nohup python3.12 -u run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b > server.log 2>&1 & +echo $! > server.pid +tail -f server.log +``` + +Завершення через Ctrl+C у передньому плані або `kill -TERM ` для процесу +скрипта зберігає прогрес. Дочекайтеся `Saved ...latest.pt` та виходу процесу. +У підготовці корпусу зберігається позиція читання й записані токени. +Не використовуйте `kill -9`, коли потрібні останні незбережені оновлення. +Повторна **та сама команда** відновлює запуск. Одночасний запуск у тому +самому `--work-dir` захищено блокуванням. + +## Які ваги підходять + +Це код **вашої WarpState 2B**, а не універсальний тренер для Llama/Qwen. +Очікується ваш PyTorch `latest.pt` з полями `model` і `config`. +Для старого CPU-offload checkpoint використовуються точні FP32 master-ваги, +якщо вони є. Токенізатор в архіві — той самий `tokenizer.model`, що у проєкті. +Не замінюйте його на інший зі схожим розміром словника: перевірка розміру +не доводить однакової відповідності ID словам. + +Перше завантаження через `--weights` — **початок нового 40B-запуску з ваших +ваг**: старий 300M-розклад і старі моменти AdamW не переносяться. +Записується походження ваг і попередній лічильник токенів. +У наступних перезапусках `work/run/latest.pt` відновлюються ваги, +AdamW, RNG, позиція корпусу, лічильник та розклад нового запуску. +Оригінальний файл `--weights` не перезаписується. + +## Дані та очікування якості + +Джерело: [HuggingFaceFW/fineweb-edu, sample-100BT](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu). +Це відфільтрований **англомовний** корпус для попереднього навчання. +Число токенів у його назві пораховане іншим токенізатором; скрипт сам +відміряє 40 млрд саме вашим SentencePiece. Ці 40 млрд — додаткові токени +нового запуску, незалежно від кількості токенів у початкових вагах. + +У кожен документ додається EOS. Незмінний SHA256 тексту визначає train +або holdout; усі holdout-документи виключаються з train навіть після +заповнення validation. Повтор того самого тексту потрапляє в той самий +розділ; це не додаткова глобальна дедуплікація майже однакових документів. +Ревізія джерела фіксується як commit SHA. Підготовка відновлюється через +стан HF IterableDataset; незавершений хвіст вихідного файла відкочується +до останньої узгодженої точки. Якщо джерело закінчиться раніше, скрипт +зупиниться, а не доповнюватиме корпус нескінченними повтореннями. + +Два uint16-файли займають приблизно **80,0 ГБ / 74,5 GiB**. Обсяг мережевого +трафіку більший; вихідні тексти не завантажуються до ZIP. Підготовка може +зайняти години або дні. Це не вже готовий 40B-корпус у маленькому архіві. +Наприкінці обчислюються SHA256 файлів. Повна повторна перевірка даних: + +```bash +python3.12 run_40b.py --stage train --work-dir /srv/warpstate40b --weights /srv/models/latest.pt --verify-data +``` + +40 млрд токенів — суттєвий бюджет для експерименту з 2B, але кількість токенів +не гарантує якість. Тут експериментальна архітектура; оцінюйте validation, +мовні/фактологічні тести й реальні відповіді. Англомовний pretraining сам +по собі не робить модель якісним українським чат-асистентом. Для цього +потрібні відповідні українські дані, instruction tuning та окрема оцінка; +вони не підмінені випадковою сумішшю в цьому пакеті. + +## Параметри й продуктивність + +За замовчуванням: sequence **2048**, micro-batch **8**, накопичення **8**, +**131 072 токени/оновлення**, повний FP32 GPU AdamW, BF16 autocast, +activation checkpointing, TF32 для відповідних FP32 матричних операцій, +loss блоками 1024, clipping 1,0, betas 0,9/0,95, weight decay 0,1. +LR 0,00015 → 0,000015, warmup 1000 кроків, cosine decay. Приблизно +305 176 оновлень; останнє маскується, щоб не перевищити рівно 40 млрд токенів. +Циклічне читання проходить корпус послідовно, а не вибирає випадкові +послідовності з повторенням. На останній межі останній токен передбачає перший. + +Локальний блок архітектури залишається 128 токенів; 2048 — довжина +послідовності навчання, а не обіцянка точного зберігання всього контексту. +Тривала пам’ять лишається стисненими fast/slow станами вашої архітектури. + +Перед довгим навчанням можна обмежити перший запуск десятьма оновленнями: + +```bash +python3.12 run_40b.py --weights /srv/models/latest.pt --work-dir /srv/warpstate40b --max-steps 10 +``` + +Підготовка все одно створить повний 40B-корпус. Потім приберіть `--max-steps` +і повторіть команду: продовжиться той самий 40B-розклад. +Опційний `--compile` вмикає Linux Inductor; потрібні сумісні компілятори, +перший запуск довший. Вибирайте його **до першого навчального запуску**. +За замовчуванням він вимкнений, оскільки повний compile цього сценарію на +RTX PRO 6000 тут не перевірявся. Так само не виміряно оптимальний micro-batch. +За OOM зменшіть micro-batch і відповідно збільште накопичення **до початку +нового запуску**; відновлення навмисно не дозволяє мовчки змінювати налаштування. + +Фактична швидкість виводиться після кожного кроку й включає forward, +backward, clipping, AdamW та отримання пакета. Вона не включає підготовку +корпусу, validation, компіляцію й запис checkpoint. **10 000 токенів/с не +гарантуються**: повна модель на цій серверній карті тут не запускалась. +Для масштабу, суто арифметично: 40B при 10 000 токенів/с — 46,3 доби +чистого навчання, при 20 000 — 23,1 доби, без додаткових витрат. + +Validation виконується кожні 500 кроків на незмінних позиціях; checkpoint — +кожні 1000 та при штатній зупинці. Одне збереження моделі з GPU AdamW +займає приблизно 24 ГБ; атомарна заміна потребує місця для двох файлів. +Довге навчання означає великий сумарний обсяг записів на диск. + +## Файли й перевірки + +- `work/data/train.bin`, `val.bin`, `manifest.json`, `prepare-state.pkl` — дані та їхній прогрес. +- `work/run/latest.pt` — відновлюваний серверний checkpoint. +- `work/run/metrics.jsonl`, `validation.jsonl`, `settings.json` — журнал і параметри. +- `.server-venv/requirements-resolved.txt` — точний список встановлених залежностей. +- `PACKAGE_MANIFEST.json` — контрольні суми файлів архіву. + +Код підготовки й навчання локально перевірений на маленькій моделі: +маска останнього пакета, відновлення підготовки з відкиданням незбереженого +хвоста, точний збіг ваг і станів AdamW між безперервним і відновленим +CUDA-навчанням, відхилення зміненого LR. Це не серверний бенчмарк 2B/96 ГБ. +Також перевірено справжній HF streaming на 10 000 train + 64 validation +токенах: відновлення дає побайтово ті самі файли, SHA256 збігаються. +Локальна перевірка виконувалася на Windows/RTX 5070; Linux bootstrap та +повне навчання 2B на 96 ГБ потрібно перевірити на вашому сервері. + +Для відтворення тестів у встановленому середовищі: + +```bash +.server-venv/bin/python -m pip install pytest==9.1.1 +.server-venv/bin/python -m pytest test_server.py -q +``` + +Лише trusted локальні checkpoint/state-файли: PyTorch та pickle можуть +виконувати код під час читання. Не підставляйте невідомі файли з інтернету. + +## Джерела + +- [FineWeb-Edu: опис, англійська мова, sample-100BT, ODC-BY](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu). +- [Hugging Face: streaming та відновлення IterableDataset](https://huggingface.co/docs/datasets/stream). +- [NVIDIA: RTX PRO 6000 Blackwell 96 ГБ](https://www.nvidia.com/en-sg/data-center/rtx-pro-6000-blackwell-server-edition/). +- [CUDA-версія PyTorch](https://download.pytorch.org/whl/cu130/torch/). diff --git a/SOURCE_PACKAGE_NOTES.md b/SOURCE_PACKAGE_NOTES.md new file mode 100644 index 0000000000000000000000000000000000000000..d692aa6fc56550514478b3064a0074cd2872ae4f --- /dev/null +++ b/SOURCE_PACKAGE_NOTES.md @@ -0,0 +1,29 @@ +# WarpState — assembled source bundle + +This bundle combines the **three source archives you provided** with the newly designed README and SVG assets. Existing source files were preserved byte-for-byte. No Python implementation was rewritten. Python bytecode caches (`__pycache__`, `.pyc`) were omitted. + +## Included + +- `warpstate/model.py`, `warpstate/config.py`, `warpstate/__init__.py` — model implementation +- `run_40b.py` — pretraining +- `posttrain.py` — supervised fine-tuning +- `dpo.py` — preference optimization +- `chat.py` — chat inference/probe +- `ops/*.py` and `ops/*.sh` — execution and training operations +- `test_*.py` — tests +- `tokenizer.model` — supplied tokenizer +- `README.md` and `assets/` — updated GitHub presentation + +## Not supplied in the uploaded archives + +- Trained model checkpoints / weights +- Training datasets +- `configs/` directory, including `configs/warpstate_2b.json` referenced by supplied `PACKAGE_MANIFEST.json` and training documents +- Explicit license file + +**Note:** This is a *complete assembly of the available source*, not a turnkey pretrained model. Before publishing, review all source and documentation, choose a license, restore valid configs, and independently validate environment-specific commands. Scripts with old server-specific paths may require adaptation. Do not upload private secrets or checkpoints. + +Original archives: +- `drive-download-20261026T200043Z-1-001.zip` +- `warpstate-20261026T200055Z-1-001.zip` +- `ops-20261026T200120Z-1-001.zip` diff --git a/VALIDATION.json b/VALIDATION.json new file mode 100644 index 0000000000000000000000000000000000000000..66ce480d84098312173910d40da8385cc947819c --- /dev/null +++ b/VALIDATION.json @@ -0,0 +1,19 @@ +{ + "unit_tests": 3, + "unit_test_status": "passed", + "tests": [ + "partial final token batch", + "preparation interruption and rollback", + "small GPU full-optimizer resume equals uninterrupted run; changed LR rejected" + ], + "live_fineweb_test": { + "train_tokens": 10000, + "val_tokens": 64, + "resume_byte_identical": true, + "sha256_verified": true + }, + "test_platform": "Windows / RTX 5070 Laptop / torch 2.14.0+cu130", + "target_platform": "Linux / one RTX PRO 6000 Blackwell 96 GB", + "full_target_hardware_tested": false, + "full_40b_corpus_prepared_locally": false +} diff --git a/assets/warpstate-architecture.svg b/assets/warpstate-architecture.svg new file mode 100644 index 0000000000000000000000000000000000000000..984714a2cf1b8bf1678b2c2a2dc21aa105968546 --- /dev/null +++ b/assets/warpstate-architecture.svg @@ -0,0 +1,48 @@ + + + + + + + + + + Architecture overview + One core execution: local evidence + compressed history → gated fusion + + + + + + + Input + Token embeddings + RMSNorm + QKVGU + + Local Attention + Causal SDPA + RoPE + 128-token chunks + + Gated Fusion + Local × gate + Memory × (1 − gate) + + Core Refinement + Residual + RMSNorm + SwiGLU + Repeated at logical depth + + Temporal Memory + + FAST STATE + + SLOW STATE + State updated after each completed local chunk + + LM Head + Tied embedding projection + Next-token logits + + Conceptual figure · not a claim of infinite context or lossless memory. + + + \ No newline at end of file diff --git a/assets/warpstate-banner.svg b/assets/warpstate-banner.svg new file mode 100644 index 0000000000000000000000000000000000000000..11161228bae70a14062026cee284fd2163c66563 --- /dev/null +++ b/assets/warpstate-banner.svg @@ -0,0 +1,29 @@ + + + + + + + + + + + + + EXPERIMENTAL NEURAL SYSTEMS + WARPSTATE + + Memory-native language modeling + 2.005B PARAMETERS / FAST + SLOW MEMORY / SFT + DPO + + + + + + + + + + + + \ No newline at end of file diff --git a/chat.py b/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..bf79eab49efc39a29bd5802306e8b5653ff24a95 --- /dev/null +++ b/chat.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""CPU-only chat/completion probe of an independent WarpState weight snapshot. + +Only load trusted local torch checkpoints. This does not stop training, touch +CUDA, load Adam into a model, or write to the training checkpoint directory. +""" +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import math +import os +from pathlib import Path +import stat +import tempfile +import time + +import sentencepiece as spm +import torch + +from warpstate import WarpStateConfig, WarpStateLM + +ROOT = Path(__file__).resolve().parent +FORMAT = 'warpstate-chat-v1' + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(8 * 1024**2), b''): + digest.update(block) + return digest.hexdigest() + + +def capture_latest(source, tokenizer, snapshot_root): + """Pin one atomically published inode, export only its original FP32 model. + + latest.pt is published by rename, never updated in place. An open descriptor + remains bound to that complete version even if the trainer publishes again. + No hardlink to the production checkpoint is created. + """ + source, tokenizer, snapshot_root = map(Path, (source, tokenizer, snapshot_root)) + if source.suffix != '.pt' or source.is_symlink(): + raise ValueError('Select a completed regular .pt checkpoint, not a temporary file') + tokenizer_sha = sha256(tokenizer) + fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + raise ValueError('Checkpoint must be a regular file') + checkpoint = torch.load(f'/proc/self/fd/{fd}', map_location='cpu', + weights_only=False, mmap=True) + if checkpoint.get('format') == FORMAT: + raise ValueError( + f'Chat snapshot requires its original snapshot.json beside {source}. ' + 'Restore that metadata or export a fresh snapshot from a training checkpoint.') + if checkpoint.get('format') != 'warpstate-server-v1': + raise ValueError('Expected a full warpstate-server-v1 training checkpoint') + recorded_tokenizer = checkpoint['signature']['data']['signature']['tokenizer_sha256'] + if recorded_tokenizer != tokenizer_sha: + raise ValueError('Tokenizer SHA256 differs from training checkpoint') + snapshot_root.mkdir(parents=True, exist_ok=True) + folder = Path(tempfile.mkdtemp(prefix=f'step-{checkpoint["step"]}-', dir=snapshot_root)) + destination = folder / 'model.pt' + temporary = folder / 'model.pt.tmp' + metadata = dict(format=FORMAT, inference_only=True, config=checkpoint['config'], + step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'], + val_loss=checkpoint.get('val_loss'), tokenizer_sha256=tokenizer_sha, + captured_at_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + source=dict(path=str(source.resolve()), device=before.st_dev, + inode=before.st_ino, size=before.st_size, + mtime_ns=before.st_mtime_ns)) + print(f'Capturing completed step {metadata["step"]:,} / ' + f'{metadata["tokens_seen"]:,} tokens; model weights only...', flush=True) + with temporary.open('xb') as stream: + torch.save(dict(metadata, model=checkpoint['model']), stream) + stream.flush() + os.fsync(stream.fileno()) + after = os.fstat(fd) + # Atomic unlink/rename may change ctime/nlink without changing contents. + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns): + raise RuntimeError('Pinned checkpoint contents changed; snapshot not published') + os.chmod(temporary, 0o444) + os.replace(temporary, destination) + metadata['snapshot_sha256'] = sha256(destination) + metadata['snapshot_bytes'] = destination.stat().st_size + with (folder / 'snapshot.json').open('x') as stream: + json.dump(metadata, stream, indent=2) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + directory = os.open(folder, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + return destination + finally: + os.close(fd) + + +def resolve_snapshot(source, tokenizer, snapshot_root): + """Reuse a sealed export, or safely capture a full training checkpoint. + + Missing metadata is not permission to load unverified inference weights: + capture_latest accepts only the embedded training-checkpoint format and + pins its inode across atomic replacements by the trainer. + """ + source = Path(source) + if not source.is_file(): + raise ValueError(f'Checkpoint file does not exist or is not a regular file: {source}') + if source.with_name('snapshot.json').exists(): + return source + return capture_latest(source, tokenizer, snapshot_root) + + +def load_model(snapshot, tokenizer): + snapshot, tokenizer = Path(snapshot), Path(tokenizer) + proof_path = snapshot.with_name('snapshot.json') + try: + proof = json.loads(proof_path.read_text()) + except FileNotFoundError as exc: + raise ValueError( + f'Missing snapshot metadata: {proof_path}. ' + 'Use --checkpoint to export training weights, or restore the original metadata.') from exc + if not isinstance(proof, dict) or not isinstance(proof.get('snapshot_sha256'), str): + raise ValueError(f'Invalid snapshot metadata: {proof_path}') + if proof['snapshot_sha256'] != sha256(snapshot): + raise ValueError('Inference snapshot SHA256 mismatch') + checkpoint = torch.load(snapshot, map_location='cpu', weights_only=False, mmap=True) + if checkpoint.get('format') != FORMAT or not checkpoint.get('inference_only'): + raise ValueError('Use the independent inference snapshot, not live latest.pt') + if checkpoint['tokenizer_sha256'] != sha256(tokenizer): + raise ValueError('Tokenizer SHA256 mismatch') + config = WarpStateConfig(**checkpoint['config']) + tokenizer_model = spm.SentencePieceProcessor(model_file=str(tokenizer)) + if tokenizer_model.vocab_size() != config.vocab_size: + raise ValueError('Tokenizer vocabulary differs from model') + with torch.device('meta'): + model = WarpStateLM(config) + model.load_state_dict(checkpoint['model'], strict=True, assign=True) + model.eval().requires_grad_(False) + if any(p.device.type != 'cpu' or p.dtype != torch.float32 for p in model.parameters()): + raise ValueError('This CPU probe requires the original FP32 model parameters') + metadata = {key: value for key, value in checkpoint.items() if key != 'model'} + del checkpoint + gc.collect() + return model, tokenizer_model, metadata + + +def banned_ngram_tokens(tokens, size): + """Block completions of n-grams already present in this reply, not its prompt.""" + if size == 0 or len(tokens) + 1 < size: + return set() + prefix = tuple(tokens[-(size - 1):]) if size > 1 else () + return {tokens[start + size - 1] for start in range(len(tokens) - size + 1) + if tuple(tokens[start:start + size - 1]) == prefix} + + +def sample_token(logits, temperature, top_k, top_p, generator, *, + previous_tokens=(), repetition_penalty=1.0, banned_tokens=()): + logits = logits.float().clone() + if not torch.isfinite(logits).all(): + raise ValueError('Non-finite generation logits') + if not math.isfinite(temperature) or temperature < 0 or not 0 < top_p <= 1 or top_k < 0: + raise ValueError('Invalid sampling settings') + if not math.isfinite(repetition_penalty) or repetition_penalty < 1: + raise ValueError('Require a finite repetition-penalty >= 1') + if previous_tokens and repetition_penalty != 1: + seen = list(set(previous_tokens)) + scores = logits[seen] + # Multiplying negative scores and dividing positive ones lowers both. + logits[seen] = torch.where(scores < 0, scores * repetition_penalty, + scores / repetition_penalty) + if banned_tokens: + logits[list(banned_tokens)] = -torch.inf + if not torch.isfinite(logits).any(): + return None # No legal continuation; never sample a masked token. + if temperature == 0: + return int(logits.argmax()) + values = logits / temperature + if top_k: + threshold = values.topk(min(top_k, values.numel())).values[-1] + values = values.masked_fill(values < threshold, -torch.inf) + ordered, indices = values.sort(descending=True) + probability = ordered.softmax(dim=-1) + # Keep the token crossing the cumulative threshold, including at least one. + ordered = ordered.masked_fill(probability.cumsum(-1) - probability > top_p, -torch.inf) + picked = torch.multinomial(ordered.softmax(dim=-1), 1, generator=generator) + return int(indices[picked].item()) + + +@torch.inference_mode() +def generate(model, sp, prompt, *, max_new_tokens=256, temperature=0.8, + top_k=40, top_p=0.9, seed=1337, max_context=2048, + repetition_penalty=1.2, no_repeat_ngram_size=4): + if max_new_tokens < 1 or max_context <= max_new_tokens: + raise ValueError('Require 0 < max-new-tokens < max-context') + if not math.isfinite(repetition_penalty) or repetition_penalty < 1: + raise ValueError('Require a finite repetition-penalty >= 1') + if not isinstance(no_repeat_ngram_size, int) or no_repeat_ngram_size < 0: + raise ValueError('Require an integer no-repeat-ngram-size >= 0') + ids = sp.encode(prompt, out_type=int) + if not ids: + raise ValueError('Prompt contains no tokens') + ids = ids[-(max_context - max_new_tokens):] + caches = model.init_generation_cache(1, torch.device('cpu'), torch.float32) + token = torch.empty((1, 1), dtype=torch.long) + for value in ids: + token.fill_(value) + logits = model.step(token, caches)[0, -1] + generator = torch.Generator(device='cpu').manual_seed(seed) + generated = [] + for index in range(max_new_tokens): + value = sample_token(logits, temperature, top_k, top_p, generator, + previous_tokens=generated, repetition_penalty=repetition_penalty, + banned_tokens=banned_ngram_tokens(generated, no_repeat_ngram_size)) + if value is None or value == sp.eos_id(): + break + generated.append(value) + yield value + if index + 1 < max_new_tokens: + token.fill_(value) + logits = model.step(token, caches)[0, -1] + + +def reply(model, sp, prompt, args): + ids, emitted = [], '' + begin, first = time.perf_counter(), None + for value in generate(model, sp, prompt, max_new_tokens=args.max_new_tokens, + temperature=args.temperature, top_k=args.top_k, top_p=args.top_p, + seed=args.seed, max_context=args.max_context, + repetition_penalty=args.repetition_penalty, + no_repeat_ngram_size=args.no_repeat_ngram_size): + if first is None: + first = time.perf_counter() + ids.append(value) + text = sp.decode(ids) + # Delay incomplete UTF-8 byte-fallback characters until they are complete. + safe = text.split('\ufffd', 1)[0] + if safe.startswith(emitted): + print(safe[len(emitted):], end='', flush=True) + emitted = safe + text = sp.decode(ids) + if text.startswith(emitted): + print(text[len(emitted):], end='', flush=True) + elapsed = time.perf_counter() - begin + ttft = '-' if first is None else f'{first - begin:.2f}s' + print(f'\n[{len(ids)} tokens; {elapsed:.1f}s total; TTFT {ttft}; CPU FP32]', flush=True) + return text + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--checkpoint', type=Path, default=ROOT / 'work/run/latest.pt') + parser.add_argument('--snapshot', type=Path, + help='Reuse a sealed model-only snapshot, or export a training checkpoint') + parser.add_argument('--snapshot-root', type=Path, default=ROOT / 'work/chat') + parser.add_argument('--tokenizer', type=Path, default=ROOT / 'tokenizer.model') + parser.add_argument('--capture-only', action='store_true') + parser.add_argument('--prompt', help='One-shot probe instead of interactive input') + parser.add_argument('--raw', action='store_true', help='Plain text completion instead of Q/A labels') + parser.add_argument('--threads', type=int, default=16) + parser.add_argument('--max-new-tokens', type=int, default=256) + parser.add_argument('--max-context', type=int, default=2048) + parser.add_argument('--temperature', type=float, default=0.8, help='0 = greedy') + parser.add_argument('--top-k', type=int, default=40) + parser.add_argument('--top-p', type=float, default=0.9) + parser.add_argument('--repetition-penalty', type=float, default=1.2, + help='Lower scores of tokens already generated in this reply; 1 disables') + parser.add_argument('--no-repeat-ngram-size', type=int, default=4, + help='Block repeated token sequences within a reply; 0 disables') + parser.add_argument('--seed', type=int, default=1337) + args = parser.parse_args() + if args.threads < 1 or not 0 < args.max_new_tokens < args.max_context: + parser.error('Require threads >= 1 and 0 < max-new-tokens < max-context') + if not math.isfinite(args.temperature) or args.temperature < 0 or args.top_k < 0 or not 0 < args.top_p <= 1: + parser.error('Invalid temperature / top-k / top-p') + if not math.isfinite(args.repetition_penalty) or args.repetition_penalty < 1 or args.no_repeat_ngram_size < 0: + parser.error('Require finite repetition-penalty >= 1 and no-repeat-ngram-size >= 0') + if args.prompt is not None and not args.prompt.strip(): + parser.error('A one-shot prompt must not be empty') + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + try: + if args.snapshot is not None: + snapshot = resolve_snapshot(args.snapshot, args.tokenizer, args.snapshot_root) + else: + snapshot = capture_latest(args.checkpoint, args.tokenizer, args.snapshot_root) + print(f'Snapshot: {snapshot}', flush=True) + if args.capture_only: + return + model, sp, metadata = load_model(snapshot, args.tokenizer) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + print(f'WarpState {model.num_parameters() / 1e9:.3f}B | CPU FP32 | ' + f'step {metadata["step"]:,} | {metadata["tokens_seen"]:,} training tokens', flush=True) + print('Base pretrained model, not instruction-tuned. Q/A labels are only a probe.\n' + 'Commands: /new, /raw, /chat, /info, /help, /exit. Ctrl-C cancels a reply.\n' + 'Each turn replays up to the latest context tokens; long prompts take time on CPU.', flush=True) + print(f'Repetition protection: penalty={args.repetition_penalty:g}, ' + f'no-repeat-ngram-size={args.no_repeat_ngram_size} (per reply).', flush=True) + history = '' + while True: + try: + user = args.prompt if args.prompt is not None else input('\nYou: ').strip() + if user is None or (args.prompt is None and user in ('/exit', '/quit')): + break + if not user: + continue + if args.prompt is None and user in ('/new', '/raw', '/chat'): + history = '' + if user != '/new': + args.raw = user == '/raw' + print('Context cleared. Mode: ' + ('raw completion' if args.raw else 'Q/A probe')) + continue + if args.prompt is None and user == '/info': + print(json.dumps(metadata, indent=2)) + continue + if args.prompt is None and user == '/help': + print('/new clears context; /raw plain completion; /chat Q/A; /info snapshot; /exit quit') + continue + prompt = history + (user if args.raw else f'Question: {user}\nAnswer:') + print('Model: ', end='', flush=True) + answer = reply(model, sp, prompt, args) + history = prompt + ('' if args.raw else ' ') + answer + '\n\n' + if args.prompt is not None: + break + except EOFError: + break + except KeyboardInterrupt: + print('\nReply cancelled; training was not affected.', flush=True) + if args.prompt is not None: + break + + +if __name__ == '__main__': + main() diff --git a/dpo.py b/dpo.py new file mode 100644 index 0000000000000000000000000000000000000000..9365404732c2e4eb12e13c0b4bfcf0d132e591ad --- /dev/null +++ b/dpo.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python3 +"""BF16 DPO with an immutable SFT reference, exact resume and response-only loss. + +The reference is scored once, before allocating the policy optimizer. Its cache +is bound to weights, data, masking, execution and source hashes. Policy training +uses fresh FP32 Adam state and saves only into its own output directory. +""" + +from __future__ import annotations + +import argparse +import contextlib +import gc +import hashlib +import json +import math +import os +from pathlib import Path +import random +import signal +import time +import types + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint as activation_checkpoint + +from posttrain import (WORKSPACE, atomic_json, base_pipeline_lock, digest, + restore_rng, rng_state, update_indices, validate_paths) +from ops.dpo_data import MASK_FORMAT, collate_pairs, load_data +from ops.posttrain_data import RENDER_FORMAT, renderer_sha256 + +FORMAT = 'warpstate-dpo-v1' +CACHE_FORMAT = 'warpstate-dpo-reference-v1' +STOP = False +DEFAULTS = dict(sequence_length=2048, micro_batch=2, grad_accum=8, lr=5e-7, + min_lr=0., warmup_ratio=.1, epochs=1, max_steps=None, + weight_decay=0., seed=42, eval_every=100, save_every=100, + eval_batches=32, compile_core=True, bf16_residuals=True, + gradient_checkpointing=False, workers=4, deterministic=True, + clip_grad_norm=1., beta=5., objective='length-normalized-dpo', + head_chunk_size=256) + + +def load_plan(path): + plan = json.loads(Path(path).read_text()) + if plan.get('format') != 'warpstate-dpo-plan-v1' or plan.get('stage') != 'dpo': + raise ValueError('Expected warpstate-dpo-plan-v1 stage dpo') + supplied = plan.get('training', {}) + unknown = set(supplied) - set(DEFAULTS) + if unknown: + raise ValueError(f'Unknown DPO training settings: {sorted(unknown)}') + settings = DEFAULTS | supplied + for key in ('sequence_length', 'micro_batch', 'grad_accum', 'epochs', 'eval_every', + 'save_every', 'eval_batches', 'workers', 'head_chunk_size'): + if type(settings[key]) is not int or settings[key] < 1: + raise ValueError(f'{key} must be a positive integer') + if settings['max_steps'] is not None and (type(settings['max_steps']) is not int or settings['max_steps'] < 1): + raise ValueError('max_steps must be null or a positive integer') + for key in ('lr', 'min_lr', 'warmup_ratio', 'weight_decay', 'clip_grad_norm', 'beta'): + value = settings[key] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise ValueError(f'{key} must be finite and nonnegative') + if not 0 <= settings['warmup_ratio'] < 1 or not 0 <= settings['min_lr'] <= settings['lr']: + raise ValueError('Invalid LR schedule') + if not settings['lr'] or not settings['clip_grad_norm'] or not settings['beta']: + raise ValueError('lr, clip_grad_norm and beta must be positive') + for key in ('compile_core', 'bf16_residuals', 'gradient_checkpointing', 'deterministic'): + if type(settings[key]) is not bool: + raise ValueError(f'{key} must be a boolean') + if type(settings['seed']) is not int or settings['seed'] < 0: + raise ValueError('seed must be a nonnegative integer') + if settings['objective'] not in ('length-normalized-dpo', 'summed-dpo'): + raise ValueError('Unsupported DPO objective') + return plan, settings + + +def source_identity(): + from posttrain import source_identity as shared_sources + root = Path(__file__).parent + return shared_sources() | {str(path.relative_to(root)): digest(path) + for path in (Path(__file__), root / 'ops/dpo_data.py')} + + +def identity_digest(identity): + return hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(',', ':'), + allow_nan=False).encode()).hexdigest() + + +def learning_rate(step, total_steps, settings): + """Fixed linear warmup/decay schedule, independent of a pause boundary.""" + warmup = min(math.ceil(settings['warmup_ratio'] * total_steps), max(0, total_steps - 1)) + if step < warmup: + return settings['lr'] * (step + 1) / warmup + progress = min(1., (step - warmup) / max(1, total_steps - warmup - 1)) + return settings['min_lr'] + (settings['lr'] - settings['min_lr']) * (1. - progress) + + +def reference_identity(weights_hash, manifest_hash, tokenizer_hash, settings, device, world): + return dict(format=CACHE_FORMAT, weights_sha256=weights_hash, data_sha256=manifest_hash, + tokenizer_sha256=tokenizer_hash, renderer_sha256=renderer_sha256(), + mask_format=MASK_FORMAT, sources=source_identity(), settings={key: settings[key] + for key in ('sequence_length', 'micro_batch', 'head_chunk_size', 'compile_core', + 'bf16_residuals', 'deterministic', 'workers')}, + dropout=0., logps='response-token-sum-float32', dtype='bfloat16', + device_type=device.type, world_size=world, torch_version=torch.__version__) + + +def validate_cache(directory, identity, sizes): + directory = Path(directory) + manifest = json.loads((directory / 'manifest.json').read_text()) + if (manifest.get('format') != CACHE_FORMAT or manifest.get('complete') is not True or + manifest.get('identity') != identity or manifest.get('identity_sha256') != identity_digest(identity)): + raise ValueError('Reference cache identity/completion differs') + result = [] + for name, size in zip(('train.npy', 'val.npy'), sizes): + path = directory / name + info = manifest.get('files', {}).get(name, {}) + if not path.is_file() or path.is_symlink() or digest(path) != info.get('sha256') or info.get('rows') != size: + raise ValueError(f'Reference cache file changed: {name}') + values = np.load(path, allow_pickle=False, mmap_mode='r') + if values.shape != (size, 2) or values.dtype != np.float64 or not np.isfinite(values).all(): + raise ValueError(f'Invalid reference log probabilities: {name}') + result.append(values) + return *result, digest(directory / 'manifest.json') + + +def apply_sequence_logps(model, chunk_size): + """Return per-sequence log-probability sums without retaining vocabulary logits. + +Vocabulary matmuls use BF16 autocast; cross-entropy reductions use FP32. Each +head chunk is recomputed in backward, so DPO keeps a vector of branch scores. +The backbone modules and canonical parameter/state names are unchanged. +""" + def forward(self, input_ids, targets=None, *, return_logits=False): + if targets is None or return_logits: + raise ValueError('DPO forward requires response targets and no returned logits') + hidden = self.embed(input_ids) + for depth in range(self.cfg.logical_depth): + hidden = hidden * (1. + self.depth_scale[depth]) + self.depth_bias[depth] + core = self.cores[depth % self.cfg.n_cores] + if self.training and self.cfg.gradient_checkpointing: + hidden = activation_checkpoint(core, hidden, use_reentrant=False) + else: + hidden = core(hidden) + hidden = self.final_norm(hidden) + + def score(h, weight, labels): + logits = F.linear(h, weight).float() + losses = F.cross_entropy(logits.reshape(-1, logits.size(-1)), labels.reshape(-1), + reduction='none', ignore_index=-100) + return -losses.reshape(labels.shape).sum(dim=1) + + logps = hidden.new_zeros((hidden.shape[0],), dtype=torch.float32) + for start in range(0, hidden.shape[1], chunk_size): + args = (hidden[:, start:start + chunk_size], self.embed.weight, + targets[:, start:start + chunk_size]) + value = activation_checkpoint(score, *args, use_reentrant=False) if torch.is_grad_enabled() else score(*args) + logps = logps + value + return None, logps + model.forward = types.MethodType(forward, model) + + +def preference_metrics(policy, reference, counts, beta, objective): + """Pair vectors, retaining gradients only through policy log probabilities.""" + reference = reference.detach().to(policy.dtype) + if objective == 'length-normalized-dpo': + policy = policy / counts.clamp_min(1) + reference = reference / counts.clamp_min(1) + elif objective != 'summed-dpo': + raise ValueError('Unsupported DPO objective') + rewards = beta * (policy - reference) + margin = rewards[:, 0] - rewards[:, 1] + return -F.logsigmoid(margin), margin, (margin > 0).to(policy.dtype) + + +def stop_requested(args, device, world): + value = STOP or bool(args.stop_file and Path(args.stop_file).exists()) + if world > 1: + flag = torch.tensor(int(value), device=device) + dist.all_reduce(flag, op=dist.ReduceOp.MAX) + value = bool(flag.item()) + return value + + +def load_model(checkpoint, settings, device, tokenizer): + from warpstate import WarpStateConfig, WarpStateLM + from run_40b import compile_training_cores + from ops.residual_experiment import apply_bf16_residuals + cfg = WarpStateConfig(**checkpoint['config']) + cfg.gradient_checkpointing = settings['gradient_checkpointing'] + # DPO compares a deterministic policy/reference scoring rule, with dropout disabled. + cfg.dropout = 0. + cfg.validate() + if cfg.vocab_size != tokenizer.vocab_size() or settings['sequence_length'] % cfg.chunk_size: + raise ValueError('Model/tokenizer vocabulary or sequence/chunk mismatch') + with torch.device('meta'): + raw = WarpStateLM(cfg) + raw.load_state_dict(checkpoint['model'], strict=True, assign=True) + # Clone on CPU as well: mmap initialization is immutable and must not alias training. + raw.to(device=device, dtype=torch.float32) + if device.type == 'cpu': + for parameter in raw.parameters(): + parameter.data = parameter.data.clone() + if settings['compile_core']: + compile_training_cores(raw) + apply_sequence_logps(raw, settings['head_chunk_size']) + if settings['bf16_residuals']: + apply_bf16_residuals(raw) + return raw, cfg + + +def atomic_array(path, array): + temporary = Path(str(path) + '.tmp') + with temporary.open('wb') as stream: + np.save(stream, array, allow_pickle=False) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + + +def prepare_reference(args, identity, records, settings, device, rank, world, tokenizer): + """Create/reuse distributed resumable score shards, then seal a checked cache.""" + directory = Path(args.reference_cache_dir) + if (directory / 'manifest.json').exists(): + return validate_cache(directory, identity, [len(rows) for rows in records]) + directory.mkdir(parents=True, exist_ok=True) + marker = directory / 'identity.json' + if marker.exists() and json.loads(marker.read_text()) != identity: + raise ValueError('Partial reference cache identity differs') + if rank == 0: + atomic_json(marker, identity) + if world > 1: + dist.barrier() + checkpoint = torch.load(args.weights, map_location='cpu', weights_only=False, mmap=True) + raw, _ = load_model(checkpoint, settings, device, tokenizer) + raw.requires_grad_(False).eval() + del checkpoint + pad_id = tokenizer.pad_id() if tokenizer.pad_id() >= 0 else tokenizer.eos_id() + started = time.monotonic() + try: + for split, rows in zip(('train', 'val'), records): + indices = list(range(rank * settings['micro_batch'], len(rows), world * settings['micro_batch'])) + shard = directory / f'{split}-rank-{rank}.npz' + done, scored_indices, scores = 0, [], [] + if shard.exists(): + with np.load(shard, allow_pickle=False) as old: + if str(old['identity']) != identity_digest(identity): + raise ValueError('Partial reference shard identity differs') + done = int(old['batches']) + scored_indices = old['indices'].tolist() + scores = old['scores'].tolist() + expected_indices = [i for start in indices[:done] + for i in range(start, min(start + settings['micro_batch'], len(rows)))] + if scored_indices != expected_indices or len(scores) != len(scored_indices): + raise ValueError('Partial reference shard cursor differs') + + def save_shard(): + temporary = Path(str(shard) + '.tmp') + with temporary.open('wb') as stream: + np.savez(stream, identity=identity_digest(identity), batches=done, + indices=np.asarray(scored_indices, dtype=np.int64), + scores=np.asarray(scores, dtype=np.float64).reshape(-1, 2)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, shard) + + # Every rank executes equal stop/barrier counts, even on uneven final shards. + max_batches = math.ceil(len(rows) / (world * settings['micro_batch'])) + for batch in range(max_batches): + if stop_requested(args, device, world): + save_shard() + return None + if batch < done or batch >= len(indices): + continue + start = indices[batch] + selected = list(range(start, min(start + settings['micro_batch'], len(rows)))) + padded = [rows[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected)) + x, y, _ = collate_pairs(padded, settings['sequence_length'], pad_id) + with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): + _, logps = raw(x.to(device), y.to(device), return_logits=False) + values = logps.reshape(-1, 2).double().cpu().numpy()[:len(selected)] + if not np.isfinite(values).all(): + raise RuntimeError('Non-finite frozen reference scores') + scored_indices.extend(selected) + scores.extend(values.tolist()) + done = batch + 1 + if done % 128 == 0 or done == len(indices): + save_shard() + print(json.dumps(dict(event='reference_progress', rank=rank, split=split, + batches=done, total_batches=len(indices), scored_pairs=len(scores), + elapsed_seconds=time.monotonic() - started)), flush=True) + save_shard() + if world > 1: + dist.barrier() + if rank == 0: + files = {} + for split, rows in zip(('train', 'val'), records): + combined = np.full((len(rows), 2), np.nan, dtype=np.float64) + seen = np.zeros(len(rows), dtype=np.bool_) + for owner in range(world): + with np.load(directory / f'{split}-rank-{owner}.npz', allow_pickle=False) as shard: + if str(shard['identity']) != identity_digest(identity): + raise ValueError('Reference shard identity differs') + index = shard['indices'] + if seen[index].any(): + raise ValueError('Duplicate reference indices') + combined[index] = shard['scores'] + seen[index] = True + if not seen.all() or not np.isfinite(combined).all(): + raise ValueError('Reference cache is incomplete') + path = directory / f'{split}.npy' + atomic_array(path, combined) + files[path.name] = dict(sha256=digest(path), rows=len(rows)) + atomic_json(directory / 'manifest.json', dict(format=CACHE_FORMAT, complete=True, + identity=identity, identity_sha256=identity_digest(identity), files=files)) + if world > 1: + dist.barrier() + return validate_cache(directory, identity, [len(rows) for rows in records]) + finally: + del raw + # Patched bound forwards and functional BF16 wrappers form cycles; + # collect them before allocating the policy and FP32 Adam state. + gc.collect() + if device.type == 'cuda': + torch.cuda.empty_cache() + + +def training_update(model, optimizer, microbatches, global_pairs, world, device, settings): + optimizer.zero_grad(set_to_none=True) + totals = torch.zeros(5, device=device, dtype=torch.float64) + for index, (x, y, active, reference) in enumerate(microbatches): + sync = model.no_sync() if world > 1 and index < len(microbatches) - 1 else contextlib.nullcontext() + x, y, active, reference = (value.to(device) for value in (x, y, active, reference)) + counts = y.ne(-100).sum(dim=1).reshape(-1, 2) + with sync: + with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): + _, logps = model(x, y, return_logits=False) + logps = logps.reshape(-1, 2) + losses, margins, accuracy = preference_metrics(logps, reference, counts, + settings['beta'], settings['objective']) + if not torch.isfinite(losses).all(): + raise RuntimeError('Non-finite DPO training loss') + # DDP averages rank gradients; this is the true global pair mean, + # including a partial final update and entirely empty rank slots. + (losses * active).sum().mul(world / global_pairs).backward() + totals += torch.stack(((losses.detach() * active).double().sum(), + (margins.detach() * active).double().sum(), + (accuracy.detach() * active).double().sum(), + ((-logps[:, 0].detach() / counts[:, 0].clamp_min(1)) * active).double().sum(), + active.double().sum())) + norm = torch.nn.utils.clip_grad_norm_(model.parameters(), settings['clip_grad_norm'], error_if_nonfinite=True) + optimizer.step() + if world > 1: + dist.all_reduce(totals) + if int(totals[-1]) != global_pairs: + raise RuntimeError('Distributed DPO pair count differs') + values = (totals[:4] / totals[-1]).tolist() + return dict(zip(('loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)), float(norm) + + +def evaluate(model, records, reference, settings, device, pad_id, *, full=False): + state = rng_state(device) + was_training = model.training + model.eval() + totals = torch.zeros(4, dtype=torch.float64, device=device) + count = 0 + limit = len(records) if full else min(len(records), settings['eval_batches'] * settings['micro_batch']) + try: + with torch.no_grad(): + for start in range(0, limit, settings['micro_batch']): + selected = records[start:min(start + settings['micro_batch'], limit)] + n = len(selected) + selected += [None] * (settings['micro_batch'] - n) + x, y, active = collate_pairs(selected, settings['sequence_length'], pad_id) + x, y, active = x.to(device), y.to(device), active.to(device) + ref = torch.zeros((settings['micro_batch'], 2), device=device) + ref[:n] = torch.tensor(np.array(reference[start:start + n]), device=device) + with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): + _, logps = model(x, y, return_logits=False) + logps = logps.reshape(-1, 2) + counts = y.ne(-100).sum(dim=1).reshape(-1, 2) + losses, margins, accuracy = preference_metrics(logps, ref, counts, + settings['beta'], settings['objective']) + totals += torch.stack(((losses * active).double().sum(), (margins * active).double().sum(), + (accuracy * active).double().sum(), + ((-logps[:, 0] / counts[:, 0].clamp_min(1)) * active).double().sum())) + count += n + finally: + model.train(was_training) + restore_rng(state, device) + values = (totals / count).tolist() + if not all(math.isfinite(value) for value in values): + raise RuntimeError('Non-finite DPO validation') + return dict(zip(('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)) | dict(evaluated_samples=count) + + +def validate_resume(checkpoint, signature, world): + if checkpoint.get('format') != FORMAT or checkpoint.get('signature') != signature: + raise ValueError('DPO resume signature differs; use a new output directory') + if len(checkpoint.get('rng_by_rank', [])) != world: + raise ValueError('DPO checkpoint lacks exact per-rank RNG state') + + +def validate_reference_path(args): + cache = Path(args.reference_cache_dir).resolve() + paths = [Path(path).resolve() for path in (args.work_dir, args.data_dir, args.weights.parent)] + if not cache.is_relative_to(WORKSPACE) or cache == WORKSPACE: + raise ValueError('Reference cache must stay inside the workspace') + if any(cache == path or cache.is_relative_to(path) or path.is_relative_to(cache) for path in paths): + raise ValueError('Reference cache must be separate from training outputs and inputs') + return cache + + +def train(args): + import sentencepiece as spm + from ops.checkpoint_writer import DistributedCheckpointWriter + work, data_dir, weights, tokenizer_path, config = validate_paths( + args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) + cache = validate_reference_path(args) + plan, settings = load_plan(config) + rank = int(os.environ.get('RANK', '0')) + world = int(os.environ.get('WORLD_SIZE', '1')) + local_rank = int(os.environ.get('LOCAL_RANK', '0')) + device = torch.device('cuda', local_rank) if args.device == 'cuda' else torch.device('cpu') + if device.type == 'cuda': + torch.cuda.set_device(device) + if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): + raise RuntimeError('BF16-capable CUDA GPU required') + torch.set_num_threads(settings['workers']) + torch.use_deterministic_algorithms(settings['deterministic']) + random.seed(settings['seed'] + rank) + np.random.seed(settings['seed'] + rank) + torch.manual_seed(settings['seed'] + rank) + tokenizer_hash, base_hash = digest(tokenizer_path), digest(weights) + sp = spm.SentencePieceProcessor(model_file=str(tokenizer_path)) + train_data, val_data, manifest, manifest_hash = load_data(data_dir, sp, tokenizer_hash, settings['sequence_length']) + source = torch.load(weights, map_location='cpu', weights_only=False, mmap=True) + if (source.get('format') != 'warpstate-chat-v1' or source.get('inference_only') is not True or + source.get('stage') != 'sft' or source.get('tokenizer_sha256') != tokenizer_hash): + raise ValueError('Initialize DPO from an independent, tokenizer-matched SFT model export') + expected = plan.get('sft', plan.get('base', {})) + for key in ('step', 'tokens_seen', 'tokenizer_sha256'): + if key in expected and expected[key] != source.get(key): + raise ValueError(f'SFT source {key} differs from plan') + ancestor = source.get('origin', {}) + for key in ('step', 'tokens_seen'): + expected_key = 'base_' + key + value = ancestor.get(key, ancestor.get(expected_key)) + if expected_key in expected and expected[expected_key] != value: + raise ValueError(f'SFT source {expected_key} differs from plan') + origin = {key: value for key, value in source.items() if key != 'model'} + origin.update(weights_sha256=base_hash, weights_path=str(weights)) + del source + identity = reference_identity(base_hash, manifest_hash, tokenizer_hash, settings, device, world) + if world > 1: + dist.init_process_group(backend='nccl' if device.type == 'cuda' else 'gloo') + try: + reference = prepare_reference(args, identity, (train_data, val_data), settings, device, rank, world, sp) + if reference is None: + print(json.dumps(dict(event='reference_paused', rank=rank)), flush=True) + return + reference_train, reference_val, reference_hash = reference + if getattr(args, 'reference_only', False): + print(json.dumps(dict(event='reference_complete', reference_sha256=reference_hash)), flush=True) + return + signature = dict(plan=plan, settings=settings, data_sha256=manifest_hash, + tokenizer_sha256=tokenizer_hash, renderer=RENDER_FORMAT, mask=MASK_FORMAT, + base_sha256=base_hash, reference_sha256=reference_hash, + world_size=world, device_type=device.type, torch_version=torch.__version__, sources=source_identity()) + latest = work / 'latest.pt' + resuming = latest.exists() + saved = torch.load(latest if resuming else weights, map_location='cpu', weights_only=False, mmap=True) + if resuming: + validate_resume(saved, signature, world) + origin = saved['origin'] + raw, cfg = load_model(saved, settings, device, sp) + raw.requires_grad_(True) + optimizer = torch.optim.AdamW(raw.parameters(), lr=settings['lr'], betas=(.9, .95), + eps=1e-8, weight_decay=settings['weight_decay'], fused=device.type == 'cuda') + step = cursor = tokens_seen = response_tokens_seen = 0 + if resuming: + optimizer.load_state_dict(saved['optimizer']) + step, cursor = saved['step'], saved['sample_cursor'] + tokens_seen, response_tokens_seen = saved['tokens_seen'], saved['response_tokens_seen'] + resume_rng = saved['rng_by_rank'][rank] if resuming else None + del saved + if world > 1: + model = torch.nn.parallel.DistributedDataParallel(raw, + device_ids=[local_rank] if device.type == 'cuda' else None, + broadcast_buffers=False, gradient_as_bucket_view=True, bucket_cap_mb=128) + else: + model = raw + if resume_rng is not None: + restore_rng(resume_rng, device) + global_batch = settings['micro_batch'] * settings['grad_accum'] * world + total_steps = math.ceil(len(train_data) / global_batch) * settings['epochs'] + if settings['max_steps'] is not None: + total_steps = min(total_steps, settings['max_steps']) + end_step = min(total_steps, step + args.max_steps) if args.max_steps is not None else total_steps + work.mkdir(parents=True, exist_ok=True) + if rank == 0: + atomic_json(work / 'settings.json', dict(signature=signature, origin=origin, + total_steps=total_steps, train_rows=len(train_data), val_rows=len(val_data))) + if step < total_steps: + (work / 'completion.json').unlink(missing_ok=True) + pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id() + last_validation = None + last_validation_step = None + + def validation(full=False): + values = [evaluate(raw, val_data, reference_val, settings, device, pad_id, full=full) if rank == 0 else None] + if world > 1: + dist.broadcast_object_list(values, src=0, device=device) + return values[0] + + def metric(values): + if rank == 0: + values.update(timestamp=time.time(), stage='dpo', step=step, tokens_seen=tokens_seen, + response_tokens_seen=response_tokens_seen, sample_cursor=cursor, + objective=settings['objective']) + with (work / 'metrics.jsonl').open('a') as stream: + stream.write(json.dumps(values, allow_nan=False) + '\n') + if values.get('event') == 'validation': + atomic_json(work / 'evaluation.json', dict(values, data_sha256=manifest_hash, + base_sha256=base_hash, reference_sha256=reference_hash, + tokenizer_sha256=tokenizer_hash, mask_format=MASK_FORMAT, + available_validation_samples=len(val_data))) + print(json.dumps(values, allow_nan=False), flush=True) + + def save(writer): + state = rng_state(device) + states = [None] * world + if world > 1: + dist.all_gather_object(states, state) + else: + states = [state] + writer.submit(state_factory=lambda: dict(format=FORMAT, signature=signature, origin=origin, + config=vars(cfg), model=raw.state_dict(), optimizer=optimizer.state_dict(), step=step, + sample_cursor=cursor, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen, + val_loss=last_validation['val_loss'], validation=last_validation, + val_step=last_validation_step, rng_by_rank=states, completed=step >= total_steps, + total_steps=total_steps)) + + with DistributedCheckpointWriter(latest, rank=rank, device=device, distributed=world > 1) as writer: + last_validation = validation(full=True) + last_validation_step = step + metric(dict(event='validation', full_holdout=True, **last_validation)) + last_saved_step = step if resuming else -1 + model.train() + while step < end_step and not stop_requested(args, device, world): + writer.check() + indices, next_cursor = update_indices(len(train_data), settings['seed'], cursor, + settings['epochs'], global_batch) + if not indices: + break + microbatches = [] + for micro in range(settings['grad_accum']): + offset = (micro * world + rank) * settings['micro_batch'] + selected = indices[offset:offset + settings['micro_batch']] + padded = [train_data[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected)) + x, y, active = collate_pairs(padded, settings['sequence_length'], pad_id) + ref = torch.zeros((settings['micro_batch'], 2), dtype=torch.float32) + if selected: + ref[:len(selected)] = torch.tensor(np.array(reference_train[selected]), dtype=torch.float32) + microbatches.append((x, y, active, ref)) + lr = learning_rate(step, total_steps, settings) + for group in optimizer.param_groups: + group['lr'] = lr + started = time.perf_counter() + values, norm = training_update(model, optimizer, microbatches, len(indices), world, device, settings) + if device.type == 'cuda': + torch.cuda.synchronize(device) + seconds = time.perf_counter() - started + step += 1 + cursor = next_cursor + actual_tokens = sum(len(train_data[i][branch]['input_ids']) - 1 + for i in indices for branch in ('chosen', 'rejected')) + tokens_seen += actual_tokens + response_tokens_seen += sum(sum(value != -100 for value in train_data[i][branch]['labels'][1:]) + for i in indices for branch in ('chosen', 'rejected')) + metric(dict(event='train', **values, lr=lr, grad_norm=norm, update_seconds=seconds, + tokens_per_second=actual_tokens / seconds, update_samples=len(indices))) + if step % settings['eval_every'] == 0 or step == end_step: + full = step >= total_steps + last_validation = validation(full=full) + last_validation_step = step + metric(dict(event='validation', full_holdout=full, **last_validation)) + if step % settings['save_every'] == 0 or step == end_step: + save(writer) + last_saved_step = step + if last_saved_step != step: + save(writer) + writer.flush() + if rank == 0 and step >= total_steps: + atomic_json(work / 'completion.json', dict(format='warpstate-dpo-completion-v1', + stage='dpo', completed=True, paused_for_evaluation=True, step=step, + total_steps=total_steps, sample_cursor=cursor, tokens_seen=tokens_seen, + response_tokens_seen=response_tokens_seen, val_loss=last_validation['val_loss'], + evaluation_file=str(work / 'evaluation.json'), checkpoint=str(latest), + reference_sha256=reference_hash, timestamp=time.time())) + metric(dict(event='complete' if step >= total_steps else 'paused', total_steps=total_steps, + checkpoint=str(latest))) + validate_cache(cache, identity, (len(train_data), len(val_data))) + if digest(weights) != base_hash: + raise RuntimeError('Frozen SFT initialization changed during DPO') + finally: + if world > 1 and dist.is_initialized(): + dist.destroy_process_group() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('config', 'work-dir', 'data-dir', 'weights', 'tokenizer', 'reference-cache-dir'): + parser.add_argument('--' + name, required=True, type=Path) + parser.add_argument('--max-steps', type=int) + parser.add_argument('--stop-file', type=Path) + parser.add_argument('--device', choices=('cuda', 'cpu'), default='cuda') + parser.add_argument('--reference-only', action='store_true') + args = parser.parse_args() + if args.max_steps is not None and args.max_steps < 1: + parser.error('--max-steps must be positive') + if args.stop_file is None: + args.stop_file = args.work_dir / 'control/training-stop-request' + if not args.stop_file.resolve().is_relative_to(WORKSPACE): + parser.error('--stop-file must stay inside the workspace') + def stop(signum, frame): + global STOP + STOP = True + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + signal.signal(signum, stop) + from run_40b import process_lock + validate_paths(args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) + validate_reference_path(args) + args.work_dir.mkdir(parents=True, exist_ok=True) + args.reference_cache_dir.mkdir(parents=True, exist_ok=True) + plan, _ = load_plan(args.config) + rank = int(os.environ.get('RANK', '0')) + with process_lock(args.work_dir / f'rank-{rank}.lock'), \ + process_lock(args.reference_cache_dir / f'rank-{rank}.lock'), base_pipeline_lock(plan): + train(args) + + +if __name__ == '__main__': + main() diff --git a/ops/B200_HANDOFF.md b/ops/B200_HANDOFF.md new file mode 100644 index 0000000000000000000000000000000000000000..d63ec8ad27bc15a05e482b90a4779e793467f52f --- /dev/null +++ b/ops/B200_HANDOFF.md @@ -0,0 +1,214 @@ +# Continue WarpState on the B200 + +This is the starting point for sessions running directly in +`/workspace/WarpState` on the **single visible B200** instance. Use the current +session state below; the original transfer details remain for provenance. + +**Latest session state, September 10, 2026:** training is **STOPPED** at the +user's request. The graceful stop finished step **680 / 89,128,960 tokens**, +evaluated validation loss **4.39469**, and durably published the full checkpoint +at `work/run/latest.pt` before exit. This is the checkpoint for the next resume. + +Checkpoint SHA256: +`bf689a10a030842699bfa3a25ff62e488cfa2d5b022b2b6e3bfc860359eb8b4f`. +Inspection verified all 2,005,111,424 finite FP32 parameters, 256 full AdamW +entries at step 680, FP32 moments, rank-zero RNG structure, unchanged data and +resume signature, and scheduled optimizer LR **0.000102**. The full inspection +is `work/validation/b200/paused-checkpoint-step-680.json`. The final CPU staging +took 4.93 seconds and background publication took 17.74 seconds; shutdown +waited for publication. No training GPU process remains. + +Earlier snapshots `work/run/paused-step-488.pt` and +`work/run/validated-step-500.pt` remain preserved. The step-500 verification is +historical; use the newer step-680 report for the current resume state. +Resume only after the user requests it, using the controls below. + +The B200 launcher now uses microbatch 16 / accumulation 4, compiled full loss, +BF16 kernel autotuning, and asynchronous full checkpoints. See +[B200_PERFORMANCE.md](B200_PERFORMANCE.md) for all measurements and validation, +and [B200_SCHEDULING.md](B200_SCHEDULING.md) for evidence linking recurring waits +to provider GPU scheduling. Check Supervisor status before starting another +job. The deployment record and handover tables below describe the original +step-448 transfer, not the current training cursor. + +## First commands + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +source rtx6000_40b/ops/workspace-tools.sh +git status --short --branch +git log -3 --oneline +rtx6000_40b/ops/b200ctl.sh status +nvidia-smi +df -h /workspace/WarpState +rtx6000_40b/.server-venv/bin/python -m json.tool \ + rtx6000_40b/work/validation/b200/deployment.json +``` + +The generated `work/validation/b200/deployment.json` is the final deployment +record: use it to check artifact hashes, environment validation, and the +handover state. If it is missing or reports pending work, finish those checks +before training. Reading this document alone does not establish that a large +file transfer completed. Published `train.bin` and `latest.pt` must have +verified full SHA256 hashes; `.uploading` files are incomplete transfers. + +The shell setup keeps caches and temporary files inside the workspace and +adds the project-local Git installation to `PATH`. Source both scripts in +each new shell. All source and Git history have been transferred; imported +history includes `a7b25a9`, followed by the handoff documentation commit. Use +`git log` for the actual current revision. Keep the repository on `main`, +follow the root [AGENTS.md](../../AGENTS.md), and commit completed source +changes using `WarpState Automation `. Generated data, +environments, checkpoints, and validation reports remain ignored. +This is an independent Git copy with no push remote configured; commits on +the B200 are local to that instance until explicitly synchronized elsewhere. + +## Environment and preserved training state + +| Item | Handover state | +| --- | --- | +| Hardware | One NVIDIA B200; Ubuntu 24.04; Python 3.12 | +| Python environment | `/workspace/WarpState/rtx6000_40b/.server-venv` | +| Dependencies | PyTorch `2.14.0+cu130`; pinned [training requirements](training-requirements-resolved.txt) and [B200 requirements](b200-requirements.txt) | +| Model | 2,005,111,424 parameters; original tracked 32,768-token tokenizer | +| Training corpus | `work/data/train.bin`: 40,000,000,000 `uint16` token IDs; 80,000,000,000 bytes | +| Validation corpus | `work/data/val.bin`: 1,048,576 `uint16` token IDs; 2,097,152 bytes | +| Checkpoint | `work/run/latest.pt`: full checkpoint at step **448**, with **58,720,256** tokens consumed | +| Restored state | Model, full FP32 AdamW optimizer, token cursor, learning-rate schedule, and rank-zero random state | +| Precision | **BF16 computation only**, as explicitly required by the user; parameters, gradients, and AdamW state stay FP32 | + +Paths beginning with `work/` above are relative to `rtx6000_40b/`. Preserve +the copied data manifest and preparation state. The original request was +revision `main`; the data recipe pins the immutable upstream revision +separately. Changing the requested revision for this existing work directory +breaks its resume signature. The existing data requires no new download or +tokenization. + +The host's GCC/G++ 13.2 and Python development headers passed actual Inductor +compilation. No additional compiler setup is needed. The launcher also +supports an optional `tmp/remote-setup/toolchain-env.sh` if the toolchain is +changed later. Provisioning details are in [B200_SETUP.md](B200_SETUP.md). + +## Resume and pause controls + +Use [b200ctl.sh](b200ctl.sh) with [b200-supervisord.conf](b200-supervisord.conf). +Both `autostart` and `autorestart` are disabled. Starting the Supervisor alone +leaves training stopped. + +If `b200ctl.sh status` reports that the control socket is absent, first check +whether the daemon is already starting. Initialize it only when absent: + +```bash +mkdir -p rtx6000_40b/work/control rtx6000_40b/work/logs +rtx6000_40b/.server-venv/bin/supervisord \ + -c rtx6000_40b/ops/b200-supervisord.conf +rtx6000_40b/ops/b200ctl.sh status +``` + +After deployment verification and a user request to resume training: + +```bash +rtx6000_40b/ops/b200ctl.sh start warpstate-train +rtx6000_40b/ops/b200ctl.sh status +tail -n 30 rtx6000_40b/work/logs/b200-train.log +tail -n 5 rtx6000_40b/work/run/metrics.jsonl +``` + +Initial core compilation can take several minutes. Confirm the log restores +step 448, or a later checkpoint after subsequent training, and then reports +finite loss and completed optimizer updates. Measure throughput after warmup. +To follow the live log, use `tail -f` on +`rtx6000_40b/work/logs/b200-train.log`. + +For interactive monitoring, the separate `warpstate-b200` tmux session has a +`training` window with the live training log and a second pane showing GPU +usage. Attach from the repository root: + +```bash +rtx6000_40b/ops/tmux-workspace.sh attach -t warpstate-b200 +``` + +Detach with `Ctrl-b`, then `d`. Training remains managed by Supervisor, so +detaching or closing a monitoring pane does not stop training. Continue to +use `b200ctl.sh` for start/stop controls. The wrapper keeps the tmux socket +under `work/control`; workspace-local installation details are recorded in +[tmux-workspace-install.md](tmux-workspace-install.md). + +To pause and preserve the full continuation state: + +```bash +rtx6000_40b/ops/b200ctl.sh stop warpstate-train +rtx6000_40b/ops/b200ctl.sh status +tail -n 30 rtx6000_40b/work/logs/b200-train.log +``` + +Stopping requests a graceful save after the current optimizer update and +allows up to 15 minutes. Verify the saved-checkpoint message and `STOPPED` +status. The next start resumes `latest.pt`. Preserve this checkpoint; +**never replace it with a fresh run or use `--from-scratch` to resume**. + +The original `train-fast.sh` hardcodes the workstation's two GPU UUIDs. +The original `supervisord.conf`/`datactl.sh` belongs to that workstation, +specifies user `root`, and can automatically start preparation. Use the B200 +controls on this instance. Run only one training launcher against `work`. + +## Current single-GPU configuration + +[train-b200.sh](train-b200.sh) selects one GPU and uses `--reconfigure` to +continue the existing full checkpoint with its data, objective, and global batch +unchanged: + +| Setting | Value | +| --- | --- | +| Sequence length | 2,048 | +| Microbatch / accumulation | 16 / 4 | +| Tokens per optimizer update | **131,072** | +| Core compilation | Enabled; `max-autotune-no-cudagraphs` | +| Core activation checkpointing | Disabled | +| Loss | Compiled full vocabulary loss | +| BF16 autocast cache | Enabled | +| Allocator cap | 150 GiB | +| Additional required free VRAM at startup | 3 GiB beyond the cap | +| Required free SSD space at startup | At least 53 GiB beyond existing files | +| Save / evaluation interval | Every 50 / 100 updates | +| Checkpoint writing | Immutable CPU staging, then asynchronous serialization and atomic publication | + +The allocator cap excludes CUDA context and library memory. The selected +preset measured about 72.5K median tokens/s and 139.3 GiB peak reserved memory. +Sustained throughput is much lower and varies with provider scheduling waits; +see [B200_PERFORMANCE.md](B200_PERFORMANCE.md) for complete measurements and +limitations. The dataset, 40B token budget, global batch and LR schedule remain +unchanged. Autotuning can extend initial compilation; eight compiler workers +are used by default. + +Asynchronous saves need approximately 24 GB of additional host memory. CPU +staging finishes before the next update; only serialization and disk I/O overlap +training. One outstanding save bounds memory, failures stop the job, and graceful +exit waits for publication. Wait for the **Saved** message, not just **Staged**. +`--no-async-checkpoint` restores synchronous saves. CUDA graphs are validated +as an optional `--compile-mode reduce-overhead` override with persistent FP32 +gradient buffers; they were not selected as the default. cuBLASLt preference +remains an optional experiment rather than a default override. + +## Original deployment validation + +The remote CPU suite passed **92 tests**, with **5 GPU cases deselected**. +This provider hung during collection with `CUDA_VISIBLE_DEVICES=''`; keep +normal GPU visibility and use the explicit test selection documented in +[B200_SETUP.md](B200_SETUP.md). Do not copy the workstation's empty-mask CPU +test command into this environment. + +Both single-GPU full-state resume cases also passed (**2 passed, 3 +deselected**, 80.37 seconds). Actual BF16 Inductor compilation passed with +one graph and 38 generated kernels; two fused AdamW updates retained finite +FP32 parameters, gradients and moments. These checks used small fixtures; +the production checkpoint was preserved. The large transfers' final hashes, +checkpoint inspection and final readiness state are recorded in +`work/validation/b200/deployment.json`; consult that record for completion. +Multi-GPU tests require more than this instance's one GPU. + +The deployment checks above describe the original transfer. Current source +checks, full-state continuation tests, and production checkpoint validation are +recorded in [B200_PERFORMANCE.md](B200_PERFORMANCE.md). Read the current +checkpoint inspection and confirm job status before any later resume. diff --git a/ops/B200_OPTIMIZATION_PLAN.md b/ops/B200_OPTIMIZATION_PLAN.md new file mode 100644 index 0000000000000000000000000000000000000000..ad9824c8003701aa955caf0a60157dfec227b353 --- /dev/null +++ b/ops/B200_OPTIMIZATION_PLAN.md @@ -0,0 +1,120 @@ +# B200 optimization research, September 10, 2026 + +This records the initial research before implementation. The subsequent +measurements, selected launcher settings, CUDA graph validation, backend +comparison and asynchronous checkpoint implementation are in +[B200_PERFORMANCE.md](B200_PERFORMANCE.md). See +[B200_SCHEDULING.md](B200_SCHEDULING.md) for the completed stall investigation. + +At the time of this initial research, training was paused at step **488**, with +**63,963,136** tokens consumed and validation loss **4.7938699126**. +`work/run/latest.pt` and `work/run/paused-step-488.pt` then referenced +the same verified 24,061,671,905 bytes. Subsequent atomic replacements of +`latest.pt` preserved the step-488 hardlink. Consult [B200_HANDOFF.md](B200_HANDOFF.md) +for the current paused checkpoint. + +Checkpoint SHA256: +`69d37e902c41f541d55a07b3d10dcb7643721240cc6aae032455cd8904c64608`. +Inspection verified 2,005,111,424 FP32 parameters, all 256 AdamW state entries +and their step-488 counters, FP32 moments, rank-zero RNG, token cursor, and +learning rate 0.0000732. Details are in the ignored generated report +`work/validation/b200/paused-checkpoint-step-488.json`. + +The dataset and production execution settings are preserved. The constraints +for all candidates remain BF16 computation, FP32 master parameters and AdamW, +sequence length 2,048, and **131,072 global tokens per optimizer update**. +The original learning-rate schedule, objective, tokenizer, and data order +remain unchanged. No optimization benchmark or production restart was run +for this research request. + +## Measured baseline and first priority + +Production updates 450–488 processed 5,111,808 tokens in 351.161 seconds +inside the update timer: **14,557 sustained tokens/s**. The median individual +update ran at **61,888 tokens/s**. Nine updates took approximately 30–32 +seconds; ordinary updates took approximately 2.12 seconds. The timer excludes +checkpoint writes and evaluation. Source: `work/run/metrics.jsonl`. + +Peak allocation was **59.40 GiB**, with **60.48 GiB reserved**. The B200 reports +182,631 MiB total memory; the current allocator cap is 150 GiB. Raising that +cap alone does not increase useful work. + +During sampled stalls, GPU utilization fell to zero and the trainer's main +thread waited on a futex without accumulating CPU execution time. CPU quota +throttling did not increase; cgroup CPU/I/O pressure averages were zero. +There is no matching periodic branch in the training loop. This does not +identify the cause or eliminate data/pinning, compiler, autograd, driver, +or provider waits. Removing the waits is the largest observed opportunity; +the median rate is evidence of active compute speed, not a promised sustained +rate after a fix. + +Use the benchmark's optional `--diagnose-stalls` to attribute host time to +batch transfer, forward, backward, clipping, AdamW, and final synchronization. +It adds no CUDA synchronization and emits Python stacks every ten seconds +during measured updates. Its timings include implicit waits and are not GPU +kernel timings. Then correlate the waiting call with CUDA and OS-runtime +traces across several updates. The existing `--profile` captures only one +measured update and can miss these stalls. Installed Nsight Systems is +2024.3.2.0, so verify B200/tool compatibility before using options documented +for newer releases. Sources: [NVIDIA Nsight Systems](https://docs.nvidia.com/nsight-systems/UserGuide/index.html#os-runtime-libraries-trace), +[PyTorch profiler](https://docs.pytorch.org/docs/2.14/profiler.html). + +## Candidates, in order + +| Candidate | Concrete experiment | Why it may help and what remains unverified | +| --- | --- | --- | +| Larger microbatch | Compare microbatch/accumulation **4/16**, **8/8**, **16/4** at the same sequence length | Larger GEMMs and fewer microbatch dispatches can improve compute efficiency. Prior workstation memory was about 86 GiB at microbatch 8; roughly 137 GiB at 16 is an extrapolation, not a B200 measurement. Measure startup and steady peaks below the existing cap. [NVIDIA GEMM guidance](https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html) | +| Keep loss activations instead of recomputing | Existing `--loss-mode compiled-full` at the best fitting microbatch | Uses more memory to avoid repeating the vocabulary projection during backward. At microbatch 16, BF16 logits alone are 2 GiB, before FP32 loss intermediates. Earlier workstation gains were about 3%; B200 gains are unmeasured. [PyTorch recomputation tradeoff](https://pytorch.org/blog/activation-checkpointing-techniques/), [local prior results](PERFORMANCE.md) | +| Tune matrix kernels for the chosen shape | Existing `--compile-mode max-autotune-no-cudagraphs` | Profiles candidate implementations while preserving BF16. Compile time increases; exclude it from timing and compare numerical results. [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html) | +| Compare cuBLASLt and workspace choices | Compare `TORCH_BLAS_PREFER_CUBLASLT=1` against the current backend, recording effective workspace | NVIDIA recommends 32 MiB workspace for Blackwell sm10x. Current `CUBLAS_WORKSPACE_CONFIG=:4096:8` already provides 32 MiB. PyTorch's unified workspace can cap cuBLASLt at the cuBLAS allocation; changing only `CUBLASLT_WORKSPACE_SIZE` may have no effect. Library preference may fall back. [NVIDIA cuBLAS](https://docs.nvidia.com/cuda/archive/13.0.0/cublas/index.html#cublassetworkspace), [PyTorch BLAS controls](https://docs.pytorch.org/docs/2.14/backends.html#torch.backends.cuda.preferred_blas_library) | +| CUDA graphs when traces show dispatch overhead | Existing `--compile-mode reduce-overhead` | Can trade additional persistent GPU memory for lower host launch overhead. Static training shapes help, but accumulation, evaluation, and partial final updates require validation. Graphs are not an established remedy for the observed waits. [PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html), [graph constraints](https://docs.pytorch.org/docs/2.14/notes/cuda.html#constraints) | + +The model already uses compiled cores, BF16 autocast caching, fused AdamW, +and disabled core activation checkpointing. Recommending those as new changes +would not explain further gains. The prior workstation profile attributes +most matrix time to FFN and input projections; attention and the output head +are smaller fractions. Those proportions need confirmation on B200 before +investing in custom attention or loss kernels. + +As a later wall-clock improvement, investigate checkpoint staging and +asynchronous serialization. PyTorch documents copying a stable model/optimizer +snapshot into CPU buffers before background writing, with one outstanding +save at a time. This needs approximately another checkpoint's worth of host +memory and careful completion/error handling. It requires integration work +to retain this runner's full-state format, atomic publication and graceful +stop guarantee. It would address save pauses, not the recurring waits inside +the measured updates. Keep the existing save cadence during initial compute +comparisons. [PyTorch asynchronous checkpoint recipe](https://docs.pytorch.org/tutorials/recipes/distributed_async_checkpoint_recipe.html). + +## Comparison and promotion requirements + +Use the preserved step-488 checkpoint for every disposable benchmark. The +existing benchmark copies model and optimizer tensors to separate CUDA +storage, reads prepared data in place, and writes no training checkpoints. +Its learning rate stays fixed at the loaded value and RNG is reseeded; its +updates are timing experiments rather than production continuation. + +Start with two warmup updates and at least twenty measured updates, extending +the window if needed to capture several stalls. Report total tokens divided +by total update time, median update speed, slow-update count, VRAM peaks, +loss and gradient norm. Keep profiling runs separate from throughput runs. +Use fresh `work/benchmark-*` output directories and identical checkpoint/data +inputs for each candidate. Benchmark the baseline again beside the winner +to distinguish configuration effects from changing host conditions. + +Before promoting a candidate, compare loss and gradients with the baseline +on identical tokens and validate full-state continuation. The existing CPU +loss tests preserve objective and parameter identity, but their compiled +case uses an eager backend. The compiled-full resume test currently requires +two GPUs. Add and run a single-GPU CUDA regression for compiled-full before +promoting it: continuous versus interrupted/resumed updates, canonical model +keys, full optimizer and RNG state, token cursor, and the final partial batch. +Microbatch-only changes still require the existing resume/signature checks +and a short production continuation with finite loss and a verified saved +checkpoint. Preserve schedule and data invariants through `--reconfigure`. + +The currently committed diagnostics have only CPU validation: syntax and CLI +parsing, accumulated stage timing including exception cleanup, and watchdog +cancellation. GPU profiling and the candidate grid remain pending. Resume +optimization work when the user asks to continue; keep the B200 job stopped +until then. diff --git a/ops/B200_PERFORMANCE.md b/ops/B200_PERFORMANCE.md new file mode 100644 index 0000000000000000000000000000000000000000..076811dcd96531435e393131378dc520353b3b0c --- /dev/null +++ b/ops/B200_PERFORMANCE.md @@ -0,0 +1,231 @@ +# B200 BF16 execution and checkpoint validation + +Measurements from September 10, 2026 use disposable copies of the full +step-488 checkpoint (63,963,136 tokens), including FP32 AdamW moments. +The prepared dataset, tokenizer, model architecture, sequence length 2,048, +and global batch of 131,072 tokens remain unchanged. No FP8 or FP16 training +is used. Production resumed from step 488 with validation loss 4.79387 after +the user requested continuation; see the production validation section below. +The preserved input is `work/run/paused-step-488.pt`, SHA256 +`69d37e902c41f541d55a07b3d10dcb7643721240cc6aae032455cd8904c64608`. + +## Latest pause + +Training stopped gracefully at the user's request after step **680 / +89,128,960 tokens**, with training loss **4.60232** and validation loss +**4.39469**. The full `work/run/latest.pt` checkpoint was published before the +process exited. Final staging took **4.93 seconds**, followed by **17.74 +seconds** of background writing. All 256 optimizer entries are at step 680; +model/AdamW values are finite FP32, the schedule and data signature are +unchanged, and the saved optimizer LR is **0.000102**. SHA256 and the complete +inspection are recorded in `work/validation/b200/paused-checkpoint-step-680.json` +and the current [handoff](B200_HANDOFF.md). + +The final 100 completed updates (581-680) averaged **16,501 wall-clock +tokens/s**, including between-update overhead, with a median **72,418 tokens/s** +and 18 slow updates. This window ends before the final pause evaluation/save. +Peak reserved memory stayed **139.55 GiB**. The earlier measurements below +remain historical comparisons. Production is now stopped; source settings, +prepared data and earlier preserved checkpoint snapshots remain unchanged. + +## Selected launcher and measured tradeoffs + +[train-b200.sh](train-b200.sh) now selects microbatch 16 / accumulation 4, +compiled full loss, `max-autotune-no-cudagraphs`, and asynchronous full +checkpoints. BF16 autocast, FP32 master weights/gradients/AdamW, the 40B token +budget, original learning-rate schedule, seed, and save/evaluation cadence +remain unchanged. Core activation checkpointing stays disabled and the +allocator cap remains 150 GiB. Eight compiler workers are the default; +initial autotuning can take several minutes. + +Every benchmark below uses two warmup and twenty measured updates. Sustained +throughput divides measured tokens by measured update seconds, including +provider scheduling waits. Median speed describes a typical update; it is +not continuous throughput or a guarantee of exclusive GPU capacity. Loading, +compilation, validation and checkpoint writes are outside these measurements. + +| Microbatch / accumulation | Loss | Core mode / BLAS preference | Sustained tokens/s | Median tokens/s | Peak reserved GiB | +| --- | --- | --- | ---: | ---: | ---: | +| 4 / 16 | Checkpointed | Default; stall diagnostics | 13,808 | 61,853 | 60.44 | +| 4 / 16 | Checkpointed | Final baseline control | 15,344 | 61,839 | 60.44 | +| 8 / 8 | Checkpointed | Default | 16,362 | 65,636 | 86.09 | +| 16 / 4 | Checkpointed | Default | 16,467 | 67,421 | 137.72 | +| 16 / 4 | Compiled full | Default | 16,698 | 71,370 | 141.25 | +| 16 / 4 | Compiled full | Autotune, no graphs | 20,752 | 72,499 | 139.31 | +| 16 / 4 | Compiled full | Default / cuBLASLt preference | 18,718 | 71,309 | 141.25 | +| 16 / 4 | Compiled full | CUDA graphs, corrected timer | 17,432 | 70,660 | 141.31 | + +Autotuning gave the fastest typical update, about 17.2% above the initial +baseline. Peak allocated memory was 138.18 GiB. Its first warmup took 206.16 +seconds including compilation/tuning and scheduling waits, excluded above. +The baseline diagnostic run encountered five slow measured updates, versus +three in the autotune run. Scheduling phase contributes substantially to +aggregate differences, so sustained gains cannot be attributed solely to the +chosen execution settings. The final baseline control repeated a median of +61,839 tokens/s, within 0.03% of the original diagnostic baseline despite +different scheduling waits; the typical-speed gain is repeatable. + +cuBLASLt preference produced exactly the same losses and gradient norms as +the default compiled-full case and no meaningful median speed improvement. +It is available for comparison but is not forced in the B200 launcher. The +recorded preferences were `_BlasBackend.Cublas` and `_BlasBackend.Cublaslt`; +a preference does not guarantee every operation uses that backend. Current `CUBLAS_WORKSPACE_CONFIG=:4096:8` +provides 32 MiB workspace. See [PyTorch backend controls](https://docs.pytorch.org/docs/2.14/backends.html#torch.backends.cuda.preferred_blas_library) +and [NVIDIA workspace guidance](https://docs.nvidia.com/cuda/archive/13.0.0/cublas/index.html#cublassetworkspace). + +## CUDA graph implementation and evidence + +`--compile-mode reduce-overhead` now preallocates FP32 gradient buffers outside +capture and zeros them in place between updates. Without this, the installed +PyTorch rejected accumulation into graph-owned gradient outputs. Other modes +retain `zero_grad(set_to_none=True)`. No per-core step markers were inserted; +completed forward/backward microbatches use PyTorch's iteration tracking. + +The full-model graph run recorded **56 captures**, **zero compiler skips**, +and **4,480 successful nonempty replays during measured updates**. Losses and +gradient norms match the default compiled-full trajectory exactly. The +optional [diagnostic helper](cudagraph_diagnostics.py) counts successful tree +replay calls using private PyTorch 2.14 APIs, restores its wrapper on exit, +and adds no CUDA operations or synchronization. Counts are not GPU timings. + +The first exploratory graph benchmark excluded gradient zeroing from the +timer and logged capture details with `TORCH_LOGS=cudagraphs`. It is retained +as evidence but not used in the table. The corrected run includes gradient +zeroing, matching the production update timer. Future benchmark configurations +all use this timing. First-update peak allocation was 138.15 GiB; steady +allocator peaks of 34.69 GiB during replay do not represent the complete graph +memory footprint. Reserved memory, including graph pools, remained 141.31 GiB. + +Graph mode is validated and available as an override, but was not selected as +the default because it did not improve typical update speed. It captures cores; +embedding, depth modulation, output projection, optimizer and the separately +compiled full-loss reduction are outside those core graphs. See +[PyTorch compile modes](https://docs.pytorch.org/docs/2.14/generated/torch.compile.html) +and [CUDA graph constraints](https://docs.pytorch.org/docs/2.14/notes/cuda.html#constraints). + +## Asynchronous full checkpoints + +`--async-checkpoint` is optional in the general runner and enabled in the B200 +launcher. It retains the existing `warpstate-server-v1` single-file checkpoint, +including model, complete AdamW state, RNG, token cursor, schedule/signature, +configuration and provenance. `--no-async-checkpoint` restores synchronous +saving. The asynchronous option currently requires a single training rank. + +The main thread first creates a fully independent CPU snapshot. GPU-to-CPU +staging completes synchronously before the next optimizer update can mutate +live state. One background writer then serializes, flushes and fsyncs the file, +atomically replaces `latest.pt`, and fsyncs the parent directory. Only +serialization and disk I/O overlap training. Staging needs roughly 24 GB of +additional host memory; it does not allocate a full extra GPU checkpoint. + +A new save waits for the previous writer before allocating another snapshot, +so only one snapshot/write can be outstanding. Background errors surface +between updates and again on flush/close. Normal exit, graceful stop and +training exceptions join any already-staged valid save before the pipeline +lock is released. Exceptions do not snapshot partially updated live weights. +Logs distinguish staging from completed publication and include snapshot +step/tokens. A **Staged** message alone is not a durable checkpoint. +This follows the stable-snapshot and bounded-outstanding-save principles in +[PyTorch's asynchronous checkpoint recipe](https://docs.pytorch.org/tutorials/recipes/distributed_async_checkpoint_recipe.html) +while preserving this repository's existing checkpoint format. + +## Numerical and continuation checks + +All 22 updates in each completed benchmark had finite loss and gradient norm. +Against the microbatch-4 trajectory, maximum absolute loss differences were +0.00006819 for microbatch 8, 0.00006199 for microbatch 16, 0.00031376 for compiled +full loss at microbatch 16, and 0.00006914 for its autotuned version. Maximum +relative gradient-norm differences were 0.2680%, 0.1688%, 1.1087%, and 0.1760%. +Executions preserve the objective but need not be bitwise equal across modes. +Checkpoint and dataset identities remained unchanged throughout the benchmarks. + +The final CPU runner/loss/checkpoint/launcher selection passed **91 tests** +with nine GPU cases deselected. Default and autotuned compiled-full CUDA +continuation cases passed earlier; the graph and asynchronous cases then +passed **2/2** in 84.41 seconds. These regressions check exact model, full +optimizer and RNG state, canonical keys, FP32 state, learning-rate schedule, +finite validation, and the final partial update at token counts 32, 64 and 73. +The asynchronous case compares asynchronous pause/resume against synchronous +continuous training and verifies that the asynchronous path actually ran. +CPU writer tests cover immutable snapshots during AdamW mutation, atomic +publication and failure cleanup, bounded writes, persistent errors and both +normal/exceptional exit flushing. Only one GPU workload ran at a time. + +## Production continuation and first asynchronous save + +After the user requested continuation, production resumed under B200 +Supervisor in tmux session `warpstate-b200`, window `training`, new pane `0.1`. +Initial validation loss matched the pause snapshot at **4.79387**. The first +update, step 489, took 8.11 seconds including cached compilation startup. +Steps **490-500** processed 1,441,792 tokens at **19,223 sustained tokens/s**, +with a median **72,509 tokens/s**, including provider waits in update timing. +Peak allocated/reserved memory was **138.18 / 139.55 GiB**. This window excludes +validation, CPU checkpoint staging and background disk-write overhead. + +At step **500 / 65,536,000 tokens**, training loss was **5.0160** and validation +loss was **4.74471**. The different training batches explain why individual +training losses can be above five while the fixed validation loss is lower. +The complete snapshot took **35.15 seconds** to stage synchronously, followed +by **44.44 seconds** of background serialization/fsync/publication. Steps +**501-507** completed while that writer was active. These staging and write +times are observed wall times on this host; the staging wait was not separately +profiled and should not be attributed entirely to transfer bandwidth. + +The published checkpoint retains all **2,005,111,424 FP32 parameters**, **256 +full AdamW entries at step 500**, finite FP32 moments, one rank's RNG state, +the same data/origin, **131,072 tokens per update**, and scheduled optimizer +LR **0.000075**. The full SHA256 and inspection are recorded in +`work/validation/b200/optimized-checkpoint-step-500.json`. The checkpoint is +preserved as `work/run/validated-step-500.pt`; later saves advanced `latest.pt`. +The original `paused-step-488.pt` remained unchanged. Training continued after +this first-save validation, until the later step-680 pause recorded above. +Preparation, tokenizer, dataset, schedule and precision were preserved. + +## Remaining bottleneck + +The recurring extra 30-second waits are strongly associated with the provider's +HAIShare GPU scheduler, which waits for ownership beneath CUDA launch calls. +Data transfer took about 5-6 ms/update; AdamW and CPU quota throttling did not +account for sampled delays. See [B200_SCHEDULING.md](B200_SCHEDULING.md) for +evidence and uncertainty. No provider scheduling or interposition settings +were changed. Confirm exclusive compute with the provider; full visible VRAM +alone does not establish exclusive allocation. + +## Reproduction + +Use [b200ctl.sh](b200ctl.sh) to stop production gracefully and confirm the GPU +is free before benchmarking. Preserve the pause snapshot. Benchmarks copy +model and optimizer state into disposable CUDA storage and never save a +training checkpoint. Their learning rate stays fixed at the loaded value and +RNG is reseeded, so benchmark updates are timing experiments. + +From `/workspace/WarpState`, choose a new output directory for each run: + +```bash +source rtx6000_40b/ops/training-env.sh +source rtx6000_40b/ops/workspace-tools.sh +export TORCHINDUCTOR_COMPILE_THREADS=8 +rtx6000_40b/ops/benchmark.sh \ + --checkpoint /workspace/WarpState/rtx6000_40b/work/run/paused-step-488.pt \ + --micro-batch 16 --seq-len 2048 --global-tokens-per-update 131072 \ + --compile-cores --compile-mode max-autotune-no-cudagraphs \ + --no-activation-checkpointing --autocast-cache --loss-mode compiled-full \ + --loss-chunk-size 1024 --vram-limit-gib 150 \ + --warmup-updates 2 --measured-updates 20 \ + --output-dir /workspace/WarpState/rtx6000_40b/work/benchmark-b200-reproduction +``` + +For backend comparisons use `--compile-mode default` and select +`--blas-backend default`, `--blas-backend cublas`, or `--blas-backend cublaslt` +in otherwise identical cases. The completed cuBLASLt case used the equivalent +`TORCH_BLAS_PREFER_CUBLASLT=1` environment preference, recorded in its metadata. +For graph comparison use `--compile-mode reduce-overhead --cudagraph-diagnostics`. +The same compile-mode override is accepted by `train-b200.sh` on a later +production launch; the general runner's async flag is independent. + +Raw `summary.json` and rank records are under `work/benchmark-b200-*`; +`work/validation/b200/optimization-grid.json` aggregates comparisons. +Metadata records source hashes, input identities, versions, backend options, +precision and each update's measurement. Shared logs are shown in tmux session +`warpstate-b200`, window `optimization`; attach with +`rtx6000_40b/ops/tmux-workspace.sh attach -t warpstate-b200`. diff --git a/ops/B200_SCHEDULING.md b/ops/B200_SCHEDULING.md new file mode 100644 index 0000000000000000000000000000000000000000..92ecae7ed0038d508fc91644d384e08678d19eb2 --- /dev/null +++ b/ops/B200_SCHEDULING.md @@ -0,0 +1,83 @@ +# B200 scheduling investigation + +On September 10, 2026, a BF16 benchmark reproduced the recurring training +stalls while production remained paused at step 488 / 63,963,136 tokens. +The evidence strongly implicates the provider's HAIShare GPU scheduler. +The current compute entitlement and pool configuration have not been verified. + +## Measured behavior + +The diagnostic benchmark restored the full paused checkpoint, including AdamW +state, and used microbatch 4, accumulation 16, sequence length 2,048, compiled +cores, and checkpointed 1,024-token loss chunks. It performed two warmup and +20 measured updates of 131,072 tokens each, without checkpoint writes. +Generated evidence remains in +`work/benchmark-b200-baseline-diagnostic/rank-0.jsonl` and +`work/logs/b200-optimization.log`, relative to `rtx6000_40b/`. + +Sustained measured throughput was **13,807.94 tokens/s**, versus a median +per-update rate of **61,853.22 tokens/s**. Most updates took about 2.12 seconds. +Five slow updates account for the large gap: + +| Benchmark update, including warmup | Total seconds | Forward wall seconds | +| --- | ---: | ---: | +| 3 | 29.1877 | 27.8748 | +| 8 | 32.1939 | 30.7726 | +| 12 | 32.2163 | 30.8157 | +| 17 | 32.1830 | 30.8250 | +| 22 | 32.1977 | 30.8020 | + +These are host wall times, including implicit synchronization waits, not GPU +kernel durations. Batch preparation and transfer took only 4.6-6.5 milliseconds +on those slow updates. During observed stalls, the main process slept in +`futex_wait_queue` without accumulating CPU work and GPU utilization dropped +to zero. Diagnostic Python stacks stopped inside the existing Triton launcher: +compiled model forward -> Inductor generated call -> Triton runtime `run` +-> generated `launcher`, line 5. This was the steady kernel launch path; +the captured stack did not show active compilation. +The final slow update also produced a stack in ordinary eager depth-modulation +tensor arithmetic, without Triton or Inductor frames. The wait therefore +occurs across CUDA entrypoints rather than only one compiled kernel path. + +## Runtime evidence and interpretation + +Read-only inspection found `/usr/lib/haishare/libhaishare.so` in +`/etc/ld.so.preload` and in both benchmark process maps. `nvidia-smi -q` also +identified a HAIShare wrapper before returning its report. The library exports +`cuLaunchKernel`, `continue_with_lock`, `own_lock`, `own_lock_cv`, and scheduler +state symbols. Disassembly of `continue_with_lock` showed a GPU ownership +check, a lock request, and `pthread_cond_timedwait` on `own_lock_cv`, retried +at five-second intervals until ownership is granted. + +Together, CUDA launch interception and the scheduler's condition-variable +wait provide a concrete mechanism for the observed futex sleep beneath the +Python launcher. A native stack tracing every intermediate frame was not +captured, so the association remains an inference rather than a complete +dynamic call trace. Static symbols alone do not establish the active quota. + +The GPU report showed zero ECC errors, no active thermal or power slowdown, +healthy fabric, and no recovery action. MIG and driver virtualization were +disabled; those settings do not exclude user-space GPU scheduling. + +The provider's [GPUaaS FAQ](https://docs.hosted.ai/user-panel/faqs/gpuaas-faqs) +documents time sharing with a default **30-second time quantum**. Guaranteed +compute depends on the pool's sharing ratio, while idle tenants can allow +others to use more of the GPU. This closely matches the recurring extra +30-second waits. The provider's +[pool configuration documentation](https://docs.hosted.ai/admin-panel/gpuaas/creating-a-gpuaas-pool) +describes sharing ratio, scheduling mode, and time-quantum controls. + +## Follow-up + +Ask the provider to confirm this instance's guaranteed compute percentage, +pool sharing ratio, time quantum, scheduling mode, and exclusive-allocation +status. Correlating scheduler grants and releases with the benchmark timestamps +would confirm the cause and distinguish expected allocation from a scheduler +fault. A full-memory B200 view alone does not establish exclusive compute. + +No provider configuration or interposition settings were changed. Local BF16 +optimization continues within the assigned allocation; benchmark comparisons +must report sustained throughput including waits as well as typical update +speed. See [B200_OPTIMIZATION_PLAN.md](B200_OPTIMIZATION_PLAN.md) for the +performance experiments and [B200_HANDOFF.md](B200_HANDOFF.md) for checkpoint +and production controls. diff --git a/ops/B200_SETUP.md b/ops/B200_SETUP.md new file mode 100644 index 0000000000000000000000000000000000000000..0a005b43dff230cef388fb8a8d6835eee1d8d2d1 --- /dev/null +++ b/ops/B200_SETUP.md @@ -0,0 +1,176 @@ +# Single-B200 training environment + +A new deployment starts **paused**. See [B200_HANDOFF.md](B200_HANDOFF.md) +for the latest checkpoint and current job status. Starting its Supervisor +does not start training. Use this configuration on the B200 host after +verifying the transferred dataset and checkpoint. + +The repository and all environments, caches, data, logs, and temporary files +must remain under `/workspace/WarpState`. The Python 3.12 environment at +`rtx6000_40b/.server-venv` needs +[`training-requirements-resolved.txt`](training-requirements-resolved.txt) +and `supervisor==4.3.0`, combined in +[`b200-requirements.txt`](b200-requirements.txt). Source +[`training-env.sh`](training-env.sh) before provisioning. If present, the launcher also sources the generated +`/workspace/WarpState/tmp/remote-setup/toolchain-env.sh` for the host's +workspace-local compiler toolchain. + +For a fresh environment on Ubuntu 24.04 x86-64 with Python 3.12 and a +CUDA-13-compatible driver: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +python3 -m venv rtx6000_40b/.server-venv +rtx6000_40b/.server-venv/bin/python -m pip install \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ + -r rtx6000_40b/ops/b200-requirements.txt +rtx6000_40b/.server-venv/bin/python -m pip check +``` + +A copy of the existing environment is also usable when the OS architecture, +Python version and absolute environment path match. Verify its dependency +versions and CUDA execution on the destination; omit compiler caches from +the transfer. The system needs GCC/G++, libc and Python development headers +for compilation. Do not install an additional CUDA driver inside the project. + +If the host has no Git, download and extract its distribution package under +the workspace instead of modifying system directories: + +```bash +mkdir -p /workspace/WarpState/tmp/remote-setup /workspace/WarpState/.tools +cd /workspace/WarpState/tmp/remote-setup +apt-get download git +dpkg-deb -x git_*.deb /workspace/WarpState/.tools +cd /workspace/WarpState +source rtx6000_40b/ops/workspace-tools.sh +git --version +``` + +This uses the host's existing shared libraries. Source +[`workspace-tools.sh`](workspace-tools.sh) in subsequent shells to use this +Git installation. Transfer Git history with a bundle of all refs; exclude +private keys, local credentials and generated artifacts from that bundle. + +[`train-b200.sh`](train-b200.sh) uses one visible GPU, defaulting to index 0; +an existing `CUDA_VISIBLE_DEVICES` value takes precedence. It selects +sequence length 2,048, microbatch 16, and accumulation 4, preserving +**131,072 tokens per optimizer update**. Matrix computation uses BF16 +autocast; parameters, gradients, and full AdamW state remain FP32. Core +compilation uses `max-autotune-no-cudagraphs`, core activation checkpointing +is disabled, and full vocabulary loss is compiled. Full checkpoints use +immutable CPU staging followed by asynchronous writing. The 150 GiB allocator +cap requires another 3 GiB free at startup and excludes CUDA context and +library memory. Current measurements and validation are recorded in +[B200_PERFORMANCE.md](B200_PERFORMANCE.md). + +Copy the complete prepared `work/data/` directory, including its original +manifest, and `work/run/latest.pt`. The original transferred checkpoint was +step 448 with 58,720,256 tokens consumed; consult the handoff for the latest +saved step. Keep the manifest's requested revision `main`; its immutable source revision is recorded separately. The runner +restores model, full optimizer, token cursor, learning-rate schedule, and +rank-zero random state. `--reconfigure` permits the transition from two +GPUs to one while checking the unchanged data, objective, and global batch. +Different grouping can produce floating-point rounding differences. + +## Environment checks + +On this host, setting `CUDA_VISIBLE_DEVICES=''` caused pytest collection +to hang. Keep normal device visibility and explicitly deselect the CUDA +training cases when running the CPU suite: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +rtx6000_40b/.server-venv/bin/python -m pytest \ + rtx6000_40b/test_server.py rtx6000_40b/test_training_execution.py \ + rtx6000_40b/test_training_launcher.py rtx6000_40b/test_loss_execution.py \ + rtx6000_40b/test_checkpoint_writer.py \ + rtx6000_40b/test_residual_execution.py rtx6000_40b/test_artifact_transfer.py \ + -k 'not gpu_full_state_resume_and_exact_partial_update and not single_gpu_compiled_full_state_resume and not two_gpu_full_state_resume_and_empty_final_rank and not one_to_two_gpu_reconfiguration_preserves_training_state' \ + --basetemp=/workspace/WarpState/tmp/b200-cpu-tests -q +``` + +This runs the CPU tests and performs device-availability queries during +collection. Small CUDA resume tests can be run separately with the B200 +available, using their own temporary checkpoints: + +```bash +rtx6000_40b/.server-venv/bin/python -m pytest rtx6000_40b/test_server.py \ + -k gpu_full_state_resume_and_exact_partial_update \ + --basetemp=/workspace/WarpState/tmp/b200-cuda-tests -q +``` + +The prepared corpus occupies 80,002,097,152 bytes, and the full checkpoint +about 24.06 GB. Keep at least 53 GiB additionally free for checkpoint writing +and atomic replacement. Generated files remain ignored by Git. + +## Large artifact transfers + +[`transfer_artifacts.py`](transfer_artifacts.py) sends large files over +independent SSH connections when one connection limits throughput. Run it +on the source host with the destination's already verified host-key file: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +python3 rtx6000_40b/ops/transfer_artifacts.py \ + /workspace/WarpState/rtx6000_40b/work/data/train.bin \ + /workspace/WarpState/rtx6000_40b/work/data/train.bin \ + --host ubuntu@HOST --port PORT \ + --key /workspace/WarpState/id_ed25519 \ + --known-hosts /workspace/WarpState/tmp/remote-setup/known_hosts \ + --connections 12 --chunk-mib 512 --compress \ + --expected-sha256 237a9ea6a90831769ec22fb8089c2a26c02fb991492629bb28f2b5b10d6b9114 +``` + +Use the same command with the checkpoint's source/destination paths and +its actual SHA256. Copy the small data manifest, preparation state, +validation tokens and run metadata separately. The uploader retains +`.uploading`, `.upload-state.json`, `.upload-chunks/` and `.upload.lock` +sidecars next to each destination. These are inside ignored `work/`. +Interrupted uploads resume by rerunning the same command; completed chunks +are checked again before reuse. Full source and destination hashes must +match before the final filename appears. An existing different destination +is preserved. Keep training stopped until all artifacts have been verified. + +## Controls on the remote host + +Initialize the dedicated Supervisor once after provisioning: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +mkdir -p rtx6000_40b/work/control rtx6000_40b/work/logs +rtx6000_40b/.server-venv/bin/supervisord \ + -c rtx6000_40b/ops/b200-supervisord.conf +rtx6000_40b/ops/b200ctl.sh status +``` + +It runs as the invoking user and uses its own `b200-*` control files and +logs. It does not manage preparation or benchmarks. Training remains +stopped until explicitly started and never restarts automatically after +failure or a daemon restart. + +When ready to resume on the B200: + +```bash +cd /workspace/WarpState +nvidia-smi +rtx6000_40b/ops/b200ctl.sh start warpstate-train +rtx6000_40b/ops/b200ctl.sh status +tail -n 10 rtx6000_40b/work/logs/b200-train.log +tail -n 3 rtx6000_40b/work/run/metrics.jsonl +``` + +To pause and save a full checkpoint: + +```bash +rtx6000_40b/ops/b200ctl.sh stop warpstate-train +``` + +The stop request waits for the current optimizer update and checkpoint, +with a 15-minute Supervisor timeout. The next start resumes `latest.pt`. +Initial compilation can take several minutes. Use `b200ctl.sh` for this +daemon; the original `datactl.sh` addresses the separate workstation +Supervisor. Do not run two launchers against the same `work` directory. diff --git a/ops/BACKWARD_LAYOUT_EXPERIMENT.md b/ops/BACKWARD_LAYOUT_EXPERIMENT.md new file mode 100644 index 0000000000000000000000000000000000000000..3a6f8c6e9952b3b2870788a8ac4b64b68204f754 --- /dev/null +++ b/ops/BACKWARD_LAYOUT_EXPERIMENT.md @@ -0,0 +1,204 @@ +# Attention backward layout experiment — September 12, 2026 + +The isolated paired-lane implementation reduces this region from **0.519935 ms +to 0.357944 ms** on one RTX PRO 6000 Blackwell Max-Q GPU: **31.16% less time**. +It matches the original profiled kernels byte for byte on the tested production +shape. This isolated experiment did not include a training adapter or establish +a full-model speedup. The subsequent opt-in graph integration and its training +measurements are documented in +[BACKWARD_LAYOUT_INTEGRATION.md](BACKWARD_LAYOUT_INTEGRATION.md). + +## Boundary and provenance + +The existing compiled-autograd overlap profile contains two adjacent kernels: + +| Original kernel | Rank-0 time per update | Calls per update | +| --- | ---: | ---: | +| `triton_poi_fused_17`: rotary backward and Q/K scratch | 31.907 ms | 224 | +| `triton_poi_fused_18`: packed Q/K/V/gate/U gradient | 77.576 ms | 224 | + +The evidence is +`work/benchmark-opt-20260912/profile-ca-overlap/profile-rank-0-kernels.tsv`. +Their combined **109.483 ms** is an existing kernel-time budget, not an amount +already removed from training. The isolated timing uses a different execution +context; do not scale it into a claimed training throughput improvement. + +`ops/backward_layout_baseline.py` retains the exact two profiled arithmetic and +indexing bodies as source, with their original launch dimensions. It records +the original generated source identifiers +`cq665jkyfy6uac7wrzvwa5aigwids6rzt4n7vzip6xvufai4ajjw` and +`cbr6gfx3vjhnzjbykfch3pg5peunfivhgdrfvl5hnzolhvfjfgdy` for provenance. +Running the experiment recompiles these tracked bodies; it does not load +ignored compiler-cache files or depend on preserving them. + +The shape is `[B,N,H,C,D] = [4,16,32,128,64]`: microbatch four, 2,048-token +sequences, 32 heads, and a 64-element head dimension. Saved activations and +gradients are BF16. The original saved rotary sine/cosine tables are FP32. +K-write gradients have `[B,N,H,D,C]` storage; the output gradient is token-major. +The final packed tensor has shape `[4,2048,10240]`, ordered Q/K/V/gate/U. + +## Arithmetic and implementation + +`ops/backward_layout.py` provides an independent tensor reference, deterministic +synthetic saved inputs, the validated `launch_layout` interface, and +`prepare_layout`, which returns an allocation-free callable for timing. + +The new rotary kernel processes even/odd pairs together. It reads the separate +local and memory Q gradients and adds them in FP32, applies the rotary backward +transform, and rounds the final Q result to BF16. It preserves the original +interleave's inactive-lane `+0.0`, including signed-zero behavior. + +K first combines the transposed write gradient through the tanh derivative with +the local gradient in FP32. Its sine/rotary contribution is **rounded to BF16 +before** adding the cosine contribution, as in the original two-kernel path. +That rounded intermediate stays in registers. K then rounds to BF16 on output. + +A second kernel copies V, computes the sigmoid gate derivative in the original +operation order, and applies the U tanh derivative. Both kernels write directly +to their disjoint slices of the packed output. The `single` variant launches +both sets of programs in one grid using a uniform per-program branch. Keeping +two launches was slightly faster in the final comparison. + +There are no reductions, GEMMs, parameter-gradient updates, collectives, or +optimizer operations in this boundary. Model parameters, FP32 gradient +accumulation, FP32 communication, and FP32 optimizer state are untouched. +Inputs, output, and profile scratch have explicit dtype/shape/device/stride +checks; output and scratch storage may not overwrite saved inputs or each other. + +## Measurement and correctness + +The initial screen covered two launch algorithms, blocks 256/512/1024, and +4/8 warps. All twelve configurations matched the original outputs numerically. +The largest single-grid configuration was slightly slower than the baseline, +so fewer launches alone did not establish a win. The initial report's equality +field compared values; a later stricter byte comparison detected a few signed +zeros, prompting the explicit inactive-lane zero addition in the final source. + +The final confirmation used a different seed, block 256, eight warps, nine +trials, and 100 calls per timing interval. Each candidate interval was bracketed +by original-kernel intervals. Inputs, output, and original scratch were +preallocated; both paths were warmed before measurements. CUDA events measure +the queued execution, including possible launch scheduling gaps. + +| Final algorithm | Median original | Median candidate | Time reduction | +| --- | ---: | ---: | ---: | +| Paired, two launches | 0.519935 ms | 0.357944 ms | 31.16% | +| Single grid | 0.520078 ms | 0.358521 ms | 31.06% | + +Both final candidates had **zero byte differences** across every Q/K/V/gate/U +projection: 83,886,080 BF16 output elements per comparison. Production-shape +tests use another seed and strict integer-view byte equality. Ragged-shape +tests compare each projection independently against tensor expressions, use +unequal even/odd rotary coefficients, and verify saved inputs remain unchanged. +CPU checks include the model's actual rotary VJP, gate autograd, packing order, +observable intermediate BF16 rounding, and malformed input rejection. + +The independent eager FP32 tensor expression and original compiled kernels can +have different FMA contraction. In the final run their maximum relative RMS +error was `8.72e-6`; V/gate/U were exactly equal. Therefore the independent +reference establishes mathematical indexing/arithmetic, while the original +compiled bodies supply the exact GPU-rounding oracle. These finite synthetic +fixtures do not prove equivalence for every possible activation distribution. + +The final GPU suite passed **18 tests**. The focused suite also supports CPU-only +execution. Reports remain ignored: + +- `work/benchmark-backward-layout-20260912/results.json`: initial twelve-way screen. +- `work/benchmark-backward-layout-confirm-20260912/results.json`: signed-zero diagnostic. +- `work/benchmark-backward-layout-final-20260912/results.json`: final source, strict byte comparison, and confirmation timings. +- `work/validation/rtx-opt-20260912/backward-layout-tests.log`: final GPU tests. + +The final timing report records source hashes, arguments, Torch `2.14.0+cu130`, +CUDA 13.0, device identity, and per-projection errors. Later test/documentation +additions do not change kernel arithmetic. After measurement, one extra EOF +blank line was removed from `backward_layout_baseline.py` for the Git whitespace +check. Its parsed AST is identical; the original and final source hashes are +recorded in `work/validation/communication-exploration/backward-source-normalization.json`. + +## Traffic and integration limits + +The rewrite avoids the logical stores and subsequent reads of two +16,777,216-element BF16 scratch tensors: **134,217,728 bytes (128 MiB) per +region invocation**. This is a source-level traffic calculation, not a measured +DRAM counter. Cache behavior and other reads also affect the measured timing. +The production compiler reuses a dead buffer for one scratch tensor, so this +does **not** establish two fewer allocations or a corresponding peak-VRAM gain. + +The prototype accepts separate local and memory gradients at the same backward +boundary as the profiled graph. A naive custom forward that returns one shared +BF16 Q tensor can cause autograd to round the local-plus-memory gradient sum +before rotary backward; the current graph adds those contributions in FP32. +Integrating the kernel requires preserving that join and the K rounding point, +then validating real model gradients, optimizer updates, checkpoint resumption, +and full-model timing. No such integration is selected by this experiment. + +## Recommended next integration experiment + +The narrowest route is a **post-gradient graph replacement**, leaving the +forward and autograd ownership intact. The installed PyTorch exposes +`torch._inductor.config.post_grad_custom_post_pass`; its +`fx_passes/post_grad.py` applies this hook to normalized, functionalized graphs +before final topological sorting and fake-tensor metadata updates. Implement a +`CustomGraphPass` that replaces only the existing packed-gradient region with a +pure opaque `torch.library.custom_op` and a matching fake implementation. +`ops/matmul_execution.py` already uses that opaque backward-kernel boundary to +avoid this PyTorch version's Compiled Autograd Triton higher-order tracing issue. +These are version-dependent compiler interfaces; their availability is a +proposed route, not evidence that this replacement has been integrated. + +The pass should match the actual backward dataflow and arithmetic, not generated +kernel names or an inference flag. Require the exact supported shapes, dtypes, +strides, projection ordering, separate Q gradient branches, K rounding boundary, +and gate scale/expression. Verify that removed intermediates have no consumers +outside the replacement. On any mismatch, leave the original graph intact and +report the missed replacement; an opt-in validation run must fail if expected +dispatches are absent. Initially support only the measured parallel-memory, +dropout-zero production shape. Pass existing native-layout tensors into the +opaque operator and reshape views inside it, avoiding new contiguous copies +merely to satisfy a custom-operator input layout. + +Use `CustomGraphPass.uuid()` with hashes of the matcher, kernel, and wrapper +sources plus all specialization settings, so a changed replacement cannot reuse +an incompatible compiled artifact. The installed `custom_graph_pass.py` +documents `get_hash_for_files` for this purpose. Record the same execution +identity in benchmark and checkpoint metadata, with explicit reconfiguration +for resumption changes. A new forward custom-autograd splitter is less suitable: +separate Q outputs address its gradient join, but the complete gate/packing +fusion also needs the later blend's raw output gradient and saved local/memory +values. Capturing that region through a forward boundary would require broader +ownership of the attention/memory computation or extra materialized gradients. + +Begin with a match-only graph inspection, then replace one supported core. +Require generated-kernel/profile evidence that the expected regions actually +dispatch the paired implementation, with no added layout copies. Compare +forward outputs, all parameter gradients, accumulated FP32 gradients and Adam +updates against the same compiled baseline; cover first and later accumulation +microbatches, the final DDP synchronization, and checkpoint continuation. Then +measure repeated full-model runs from the same checkpoint/token cursor, +including peak VRAM and communication overlap. Select the replacement only +after those correctness and full-model performance checks pass. + +## Reproduction + +Reserve one idle GPU and keep training paused before running CUDA commands. +Use a new output directory each time. All source, caches, and artifacts stay +inside `/workspace/WarpState`. + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +CUDA_VISIBLE_DEVICES='' .server-venv/bin/python -B -m pytest -q test_backward_layout.py +CUDA_VISIBLE_DEVICES=1 WARPSTATE_TEST_CUDA=1 \ + .server-venv/bin/python -B -m pytest -q test_backward_layout.py +CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B \ + ops/backward_layout_benchmark.py \ + --output-dir work/benchmark-backward-layout-screen-new +CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B \ + ops/backward_layout_benchmark.py --algorithms paired single \ + --blocks 256 --warps 8 --seed 9929 --trials 9 --repeats 100 \ + --output-dir work/benchmark-backward-layout-confirm-new +``` + +The benchmark needs no dataset or model checkpoint. It reports candidates +even when their bytes differ, so inspect correctness fields before interpreting +timings. It does not generate a training recipe or modify the production run. diff --git a/ops/BACKWARD_LAYOUT_INTEGRATION.md b/ops/BACKWARD_LAYOUT_INTEGRATION.md new file mode 100644 index 0000000000000000000000000000000000000000..69f56248c30db28a8dac6d6cdfa0e8a112425617 --- /dev/null +++ b/ops/BACKWARD_LAYOUT_INTEGRATION.md @@ -0,0 +1,307 @@ +# BF16 backward-layout integration experiment + +The paired attention-backward layout prototype is integrated into the compiled +training graph. Repeated complete-model measurements improve update throughput +by 0.637%, so `train-local-resume.sh` selects `--backward-layout paired` for the +next resumption. The CLI default remains `baseline`. The conservative 128 MiB +communication preset remains selected; the separate producer-scheduling trial +is available but disabled. Production training remains paused at step 40,081. + +## Execution contract + +Both `run_40b.py` and `ops/benchmark_training.py` accept +`--backward-layout baseline|inspect|paired`. `baseline` preserves existing +execution. `inspect` checks the graph without replacing it. `paired` replaces +only the recognized backward region and requires actual kernel dispatches. + +The initial supported envelope is microbatch four, sequence length 2,048, +32 heads of width 64, chunk length 128, parallel memory, and zero dropout. +Use BF16 residuals, compiled cores in default mode, baseline activation and +normalization implementations, no activation checkpointing, and +`compiled-autograd-overlap`. Separate matmul recipes are excluded from this +initial experiment. Master parameters, gradient accumulation, collectives, and +AdamW state retain their existing FP32 representation; model compute uses BF16. + +The graph pass matches normalized ATen operations, constants, tensor metadata, +branch relationships, and external consumers. Generated kernel names are not +match keys. Separate local and memory Q gradients and the K intermediate +rounding boundary are preserved. Unsupported regions retain the original graph; +an opted-in training or benchmark update fails if expected dispatches are absent. + +Checkpoint execution signatures and benchmark metadata contain the implementation +identity and source hashes. A changed identity requires `--reconfigure` on +resumption. Compiler cache identity also includes the pass implementation. +Runtime dispatch counters establish execution even when a compiler cache hit +bypasses the graph matcher. Benchmarks require exactly one replacement per +logical layer per accumulation microbatch in each measured update. + +The opaque operator follows PyTorch's [functional custom-operator contract](https://docs.pytorch.org/tutorials/advanced/python_custom_ops_functional.html): +it mutates no input, returns a fresh output, and registers fake output metadata. +The hook and arithmetic specialization are tied to the installed PyTorch +version. This is a backward graph transformation; it does not replace the +model's forward autograd ownership. + +## Validation and timing protocol + +The isolated prototype's 31% reduction applies only to its two-kernel region. +Roughly 34 ms saved per training update, or 0.8% overall, was the pre-integration +estimate. The measured complete-model result appears below. + +Validation compares the compiled baseline and candidate at production attention +geometry using a smaller number of layers. It covers forward outputs, every +parameter gradient after initial and accumulated backwards, optimizer updates, +and two-rank continuation through a final partly masked update. Continuation +within an execution mode is a separate check from numerical agreement between +different execution modes. + +Complete-model timings use the retained checkpoint and data cursor, microbatch +four per rank, eight accumulation microbatches, and 131,072 global tokens per +update. Both GPUs retain their 300 W power limits. Full resumed AdamW state and +an 88 GiB allocator cap remain enabled. No production checkpoint is written. + +Repeated runs use fresh processes and interleave baseline and candidate order. +Warmup and measured counts are matched. GPU telemetry is sampled outside the +timed Python region. Compare total tokens divided by total update time, timing +drift, matched losses/gradient norms, and allocated/reserved VRAM. Profiles are +separate runs; their instrumented throughput is excluded from timing results. + +## Reproduction + +Keep production training paused and reserve both GPUs before running: + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO \ +NCCL_DEBUG_SUBSYS=INIT,GRAPH,TUNING,ENV \ +.server-venv/bin/torchrun --standalone --nproc-per-node=2 \ + ops/benchmark_training.py \ + --checkpoint work/run/latest.pt --data work/data/train.bin \ + --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \ + --no-activation-checkpointing --autocast-cache \ + --compile-cores --compile-mode default --loss-mode compiled-full \ + --bf16-residuals --gradient-mode compiled-autograd-overlap \ + --ddp-bucket-cap-mb 128 --optimizer-state resume --vram-limit-gib 88 \ + --warmup-updates 8 --measured-updates 40 --backward-layout paired \ + --output-dir work/benchmark-backward-layout-reproduction/paired-1 +``` + +Use `--backward-layout baseline` for the control and a fresh output directory for +every process. For dispatch and overlap profiles, use `--profile` with one +measured update after the same warmup. Existing comparison and communication +profile tools consume the saved JSON reports without opening model weights. + +The underlying kernel arithmetic and original isolated measurements are in +[BACKWARD_LAYOUT_EXPERIMENT.md](BACKWARD_LAYOUT_EXPERIMENT.md). + +## Training validation results + +One production-shaped core matched the compiled baseline exactly for forward +output, input gradients, and all nine core parameter gradients. The opaque +operator's schema-purity and fake-metadata checks passed on CUDA. + +The two-rank training fixture uses production attention/FFN dimensions, one +logical layer, three accumulation microbatches, two full updates, and a final +nine-token update with an entirely masked rank. It compares every parameter +gradient after the first, middle, and final synchronization backward, then +checks each parameter and Adam-state tensor after every optimizer update. + +An initial 512-vocabulary fixture with repeated input tokens passed numerical +comparison but failed byte-exact continuation: 33 embedding-weight elements +differed by at most `3.7253e-9`. Differences in optimizer state were also confined +to the embedding. Other model weights, RNG states, token counters, and execution +identity matched. Its report is retained as a diagnostic of the existing +embedding accumulation's nondeterminism, not counted as an exact-resume pass. + +The follow-up `--unique-inputs` fixture uses vocabulary 8,192 and a deterministic +permutation so each embedding row appears once per backward. It retains the +same attention geometry and precision without relaxing comparison thresholds. +Both ranks passed exact continuation for model, full optimizer, RNG, and cursor +state. All Adam parameters and moments also matched the baseline byte for byte +at every update. The largest gradient relative L2 difference was `1.22e-7`, +with maximum absolute difference `2.73e-12`, confined to the small +`memory_mix_logit` reduction. Both ranks agreed exactly on final model and Adam +state within each run. Every expected layout dispatch occurred. + +Reproduce the strict continuation fixture with a new output directory: + +```bash +CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/python -B \ + ops/validate_backward_layout_training.py --world-size 2 --unique-inputs \ + --output-dir work/validation/backward-layout-training-reproduction +``` + +The fixture initializes deterministic synthetic weights, tokens, and nonzero +FP32 Adam moments. This checks integration and continuation at production +attention geometry; it is not a byte-exact continuation claim for the complete +28-layer model on repeated-token corpus data. The reports are +`work/validation/backward-layout-training-20260912/` and +`work/validation/backward-layout-training-unique-20260912/`. + +A two-core follow-up covers the compiler's sharing of an identical zero constant +across layers. The matcher retains that strictly matched constant while another +region still uses it, then removes it after its final replacement. Shared +computed arithmetic still causes rejection. The installed compiler's actual +common-subexpression pass is exercised by a focused structural test. + +The final two-core, two-GPU fixture also passed exact model/optimizer/RNG/cursor +continuation. It required two dispatches per backward and observed every one. +All updated parameters and Adam states matched baseline byte for byte; maximum +gradient errors were `3.64e-12` absolute and `1.27e-7` relative L2. Add `--cores 2` +to the command above to reproduce it. Results are in +`work/validation/backward-layout-training-two-core-20260912/results.json`. +See [BACKWARD_LAYOUT_TRAINING_VALIDATION.md](BACKWARD_LAYOUT_TRAINING_VALIDATION.md) +for the full validation scope and source identities. + +The separate producer-scheduling fixture also passed the existing two-GPU +asynchronous-save/stop/resume test, including full model/optimizer/RNG equality +and a final masked rank. Its log is +`work/validation/backward-layout-training-producer-resume-20260912.log`. + +## Complete-model timing results + +Four fresh-process runs used the order control, paired, paired, control, each +with eight warmup and 40 measured updates. All started from step 40,081 and +token cursor 5,253,496,832 with the same retained checkpoint, data, optimizer, +and source identities. Profiling and evaluation/checkpoint pauses are +excluded; these numbers describe training-update throughput. + +| Run | Layout | Tokens/second | +| --- | --- | ---: | +| `control-1` | Baseline | 31,010.48 | +| `paired-2` | Paired | 31,197.36 | +| `paired-3` | Paired | 31,200.70 | +| `control-2` | Baseline | 30,992.67 | + +Aggregating total tokens over total measured time gives **31,001.57 to +31,199.03 tokens/second**, a **0.637%** gain. Mean update time falls from +4.22791 to 4.20116 seconds, saving **26.76 ms per update**. The two controls +differ by 0.0575%; the two candidates differ by 0.011%. Both candidates beat +both controls. This is a small, repeatable improvement on this hardware and +preset; the isolated region's 31% gain does not apply to whole-model training. + +Peak live allocator usage is unchanged at **65.7119 GiB per GPU**. Peak reserved +allocator memory falls from **66.9785 to 66.4785 GiB per GPU**. The 0.5 GiB +reduction is reservation, not a reduction in peak live tensors. Power limits +remain 300 W and measured-window temperatures and clock rates are comparable. + +Every measured candidate update dispatches exactly 224 paired operations per +rank: 28 logical layers times eight accumulation microbatches. The second +candidate run loads compiler caches and still records every runtime dispatch, +despite the graph matcher itself not being invoked on those cache hits. + +The first update's loss and gradient norm are exactly equal across all four +runs. Later trajectories are not byte-identical, including the repeated control. +Against the first control, the second control's maximum loss difference is +`1.28e-4` and relative gradient-norm difference is 0.219%. Candidate maxima are +`1.48e-4` / 0.179% and `1.35e-4` / 0.307%. These small scalar differences are +consistent with nondeterministic training reductions; they do not establish +byte equality of the complete model's gradients. The fixture checks above +provide the separate tensor-level evidence. + +Run artifacts are under `work/benchmark-backward-layout-integration-20260912/`. +The comparison and telemetry summary is +`work/validation/backward-layout-integration/abba-summary.json`. +The manually stopped `paired-1` compilation probe has no completed update and +is excluded from all timing results. + +## Profile verification + +Separate baseline and paired runs each use eight warmup updates and one profiled +update. Each candidate rank records 224 custom-operator calls and 448 actual +`_paired_kernel` launches: 224 rotary and 224 value/gate/U kernels. The original +rotary and projection-packing launch signatures are absent. Their combined +region time falls from **114.95 to 78.96 ms** on rank zero, and from **114.65 to +78.91 ms** on rank one, about 31% less CUDA time for this region. This saves +roughly 36 ms of kernel time; the repeated unprofiled whole-update improvement +is the smaller 26.76 ms reported above. + +The total CUDA kernel count stays at 17,477 per rank, and every copy/fill count +and byte total is unchanged. There are no added layout-materialization kernels. +The cached synchronized backward graph retains 45 identical generated Triton +bodies and the same 225 MM, 224 BMM, and one ADDMM calls; only the two recognized +layout bodies are removed. The replacement consumes native buffers directly. +These checks are recorded in +`work/validation/backward-layout-integration/profile-kernel-copy-comparison.json` +and `generated-sync-graph-comparison.json` in the same directory. + +The complete captured update retains all 59 collectives and +8,020,445,708 logical input bytes per rank, with identical collective signatures. +The following values are local-device timeline unions, not sums across GPUs: + +| Profile metric (ms/update) | Baseline rank 0 | Paired rank 0 | Baseline rank 1 | Paired rank 1 | +| --- | ---: | ---: | ---: | ---: | +| Noncommunication CUDA activity | 3,957.68 | 3,935.16 | 3,834.21 | 3,796.91 | +| Compute/communication overlap | 192.63 | 192.56 | 191.92 | 191.57 | +| Communication with no overlapping compute | 149.87 | 154.23 | 281.09 | 290.27 | + +Overlap is preserved. The exposed communication duration varies slightly upward +in this single profile pair; the measured layout gain is not a communication +speedup. Removing computation can also reduce work available to hide communication. +Instrumented profile durations are excluded from the repeated timing result. +The timeline accounting is in +`work/validation/backward-layout-integration/profile-communication.json`. + +## Separate producer scheduling experiment + +`--overlap-producer-schedule` is an opt-in communication experiment. Its +two separate runs use `--backward-layout baseline` and 128 MiB buckets, with +the same checkpoint, cursor, geometry, warmup, timing, and telemetry protocol +above. `producer-1` reached **31,133.27 tokens/second** (+0.425% against the +aggregate control); `producer-2` reached **31,090.54** (+0.287%). The repeat is +0.137% slower than the first screen, and its gain falls just below the approximate +0.3% threshold used to justify testing a combined preset. Peak allocated and +reserved VRAM match the control's 65.7119 and 66.9785 GiB per GPU. + +Both runs show a small positive average difference, but the evidence is weaker +than the layout's repeated gain. The producer option remains available for +experimentation and is **not selected** in the production launcher. No combined +layout/producer run was started; additive gains have not been established. +This is a conservative selection decision, not proof that the scheduler cannot +help. The run summaries remain under the same benchmark directory as the ABBA +measurements. + +The installed scheduler already moves ready collective launches earlier through +`simple_overlap`. This experiment additionally lets +`reorder_communication_preserving_peak_memory` move a collective with its +packing/casting producers, followed by the existing `sink_waits_iterative` pass. +Without runtime estimates, producer grouping stops at GEMM-producing nodes. +It therefore targets earlier bucket readiness rather than promising to reorder +all independent backward GEMMs. + +The installed `comms.py` docstring names a `sink_comms_and_waits` prerequisite +that has no callable implementation in this version. The installed `config.py` +instead recommends producer reordering and iterative wait sinking; ordinary +peak-memory scheduling runs before these passes. The initial experiment uses one +producer pass followed by one wait sink, retaining the existing final wait +scheduling behavior. + +Both passes explicitly prohibit changing collective launch order and allow +zero increase in their estimated peak memory. Producer groups cannot include +another collective. These settings override the producer pass's installed +unsafe-order and 20% memory defaults. The flag requires +`compiled-autograd-overlap` and initially rejects wait swapping, runtime +estimation, and a nonzero memory allowance. Its immutable execution setting is +recorded in benchmark metadata and checkpoint signatures; changing it on resume +requires `--reconfigure`. The timing and continuation checks above measure this +option separately from the paired layout. + +## Final checks and resume state + +The final CPU suite passed **356 tests**, with 24 CUDA tests skipped in that +CPU-only invocation and three existing Dynamo cache-wrapper warnings. Separate +CUDA operator checks, production-shaped one-/two-core gradient/optimizer and +exact-continuation fixtures, and the producer scheduling async-resume test are +documented above. The final CPU log is +`work/validation/backward-layout-integration/pytest-final-cpu.log`. + +All ten package-manifest entries validate. The launcher passes `bash -n`. +The retained full checkpoint at `work/run/latest.pt` passes the before/after +SHA256 check, retaining +`9745d93345dff7c389ac6fa65948dc78ec021c90f56b40e9caed8ddcd84e8c54`. +The verification is saved in `checkpoint-after.log` beside the CPU log. +It remains at step **40,081**, **5,253,496,832 tokens**, with recorded validation +loss **2.4545078724622726**. Production and preparation jobs remain stopped, +both GPUs are idle, and their power limits remain 300 W. Archived weights are +untouched. The selected launcher will use the paired layout when training is +next resumed; this experiment does not resume production. diff --git a/ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md b/ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md new file mode 100644 index 0000000000000000000000000000000000000000..434a4737a88d53f98dddccc39686cc2e17854f38 --- /dev/null +++ b/ops/BACKWARD_LAYOUT_TRAINING_VALIDATION.md @@ -0,0 +1,147 @@ +# Backward-layout training validation — September 12, 2026 + +The paired graph replacement was validated with two-GPU training and checkpoint +continuation fixtures with production attention and FFN dimensions. The fixtures +use one or two physical cores/logical layers, not the complete 28-layer model. +They verify complete parameter-gradient and optimizer behavior for those fixtures; +they are not throughput measurements or proof about every trained activation. + +`ops/validate_backward_layout_training.py` retains microbatch four, sequence +length 2,048, 32 heads, head dimension 64, chunk size 128, and FFN width 7,168. +It uses BF16 autocast/residuals, compiled cores and full loss, conservative +compiled-autograd communication overlap, 128 MiB DDP buckets, and FP32 master +parameters, gradients, all-reduces, and fused AdamW state. Weights and tokens are +deterministic synthetic fixtures. Adam starts with deterministic nonzero FP32 +moments, rather than fresh first-update moments or imported production state. + +Three accumulation microbatches exercise separate backward variants: first +`no_sync()` with absent gradients, later `no_sync()` accumulation, and final +accumulation with DDP synchronization. There are two full updates and a final +nine-token update, during which rank one has entirely masked targets. Each +candidate backward must execute the paired operation; an earlier dispatch cannot +hide fallback in a later graph. Expected totals per rank are seven dispatches +for continuous execution, three for the partial run, and four after resumption. + +The harness runs baseline, continuous candidate, partial candidate, and resumed +candidate in separate processes. It compares initial forward logits, every +parameter gradient after every backward, and every parameter/Adam state after +every update. It also requires byte-identical final model, optimizer, RNG, token +cursor, configuration, and execution/source identity across candidate stop/resume. +Both ranks' final model and optimizer states must agree exactly in each run. +Source hashes are checked between phases, and outputs require a fresh directory +under `work/validation`. No production checkpoint or dataset is opened or changed. + +The later `--cores` option selects a positive number of distinct physical cores +and the same number of logical layers. It scales both the per-backward dispatch +requirement and exact final dispatch count with logical depth. This covers +cross-core compilation that a single-core fixture cannot test. + +The numerical limits were declared before CUDA measurements. Gradient and Adam +moment errors must satisfy both L2 error at most 0.5% of reference L2 plus +`1e-8`, and maximum error at most 1% of reference maximum plus `1e-8`. Updated +parameters have a `2e-5` maximum absolute limit; losses have a `2e-4` absolute +limit. Adam step counters, initial forward logits, and complete candidate resume +state require byte equality. Every comparison separately records exact bytes, +finite values, and numerical error; a tolerance pass is not labeled byte equality. + +The initial fixture used a 512-token vocabulary with repeated input indices. +Its candidate gradient/optimizer comparisons passed, but strict continuation +failed. Differences were confined to the tied embedding: 33 parameter elements +with maximum difference `3.725290298461914e-9`, 50 first-moment elements with +maximum `9.516952559351921e-8`, and 28 second-moment elements with maximum +`4.3655745685100555e-11`. All other parameter/state tensors, RNG states, and +execution/cursor metadata matched exactly. Every run's two ranks agreed. This +is consistent with the existing relaxed embedding atomic accumulation for +repeated tokens. The original failure and numerical diagnostics remain recorded; +no comparison limit was relaxed to turn it into a pass. + +The successful `--unique-inputs` fixture uses an 8,192-token vocabulary and a +deterministically shuffled permutation of all input indices in every microbatch. +Each embedding row is therefore visited once per backward. Targets can still +repeat. This removes that repeated-index accumulation from the exact-continuation +fixture without changing the model's training implementation. The guarded pass +also gained a rejection for unsupported `disable_ftz=True` before this fresh run; +the measured default setting and kernel arithmetic were unchanged. + +The single-core run passed on both RTX PRO 6000 Blackwell Max-Q GPUs: + +| Check | Result | +| --- | --- | +| Initial forward logits | Byte-identical | +| Every required backward dispatch | Present; one per microbatch | +| Gradient comparisons across candidate/partial/resumed ranks | 28 complete parameter sets | +| Maximum gradient absolute / relative L2 error | `2.7284841053187847e-12` / `1.2185588970205807e-7` | +| Remaining gradient differences | Only the 32-element memory-mix parameter | +| Parameters and Adam moments after all 12 compared rank-updates | Byte-identical to baseline | +| Full candidate checkpoint continuation on both ranks | Byte-identical for every saved field | +| Model/optimizer agreement between ranks in every run | Byte-identical | +| Maximum allocated / reserved CUDA memory | 3.064 / 3.445 GiB | + +For that single-core run, the harness source hash was +`a2d24d27b81db2f24daac76fb23096a67116e86feed2b43621a866ec5b7d1656`; +and the guarded graph-pass hash was +`4029da453f54b470f8c091f3faf825913583d357d0285450fccf82082c4557c3`. +The JSON reports contain all other source hashes, per-tensor results, and runtime +dispatch/match counts. That run found four matching compiled backward +variants per rank with no missed candidates. The initial and final vocabulary +fixtures are different correctness experiments and must not be compared for speed. + +While the full-model probe was compiling, a CPU reproduction exposed a limitation +that a single-core fixture cannot detect: installed AOT common-subexpression +elimination shares the same zero constant between cores. The original matcher's +external-user guard rejected both regions in that reproduction. The disposable +full-model probe was stopped before its first update completed; it supplies no +timing result. The matcher was corrected to retain only that strictly +matched, immutable zero constant while another region still uses it; external +users of all other matched arithmetic remain rejected. The arithmetic kernels +were unchanged. A separate CPU test applies the installed CSE pass to two regions +and checks that both replacements succeed. + +The fresh two-core run used `--cores 2 --unique-inputs` and passed all the same +checks with unchanged numerical limits. It found eight matched regions per rank +across four backward variants, with no misses. Each backward dispatched twice; +the continuous/partial/resumed totals were exactly 14/6/8 per rank. Across 28 +complete gradient comparisons, the largest difference was +`3.637978807091713e-12` absolute and `1.2654258902165433e-7` relative L2. Every +parameter and Adam tensor after all 12 compared rank-updates remained +byte-identical to baseline. Complete candidate stop/resume state and model/Adam +agreement between ranks were byte-identical for every saved field. Maximum +allocated/reserved CUDA memory was 5.366/5.736 GiB. + +For the two-core run, the final harness source hash is +`5f2d161aa624b670b07205345c63177b3b0f46baa67eaef7e9d4d15482f42f6d` +and the graph-pass hash is +`174e97d06602471c258c70b7c0cef9efba7d14e7b78d71251ea012136cbc5ff9`. +These supersede the single-core hashes above; all older reports are preserved. +The CUDA kernel source stayed unchanged across both successful fixtures. + +The expanded CPU oracle suite passed 18 tests. Independently, the producer-scheduling +option passed the existing two-GPU asynchronous checkpoint/resumption test: +one passed, five deselected, in 71.06 seconds. That smaller fixture validates the +producer scheduler's continuation behavior; it does not exercise the paired +kernel or establish a combined full-model performance gain. + +Ignored evidence is retained at: + +- `work/validation/backward-layout-training-20260912/results.json`: repeated-index fixture, including strict-resume failure. +- `work/validation/backward-layout-training-20260912/resume-differences.json`: embedding-only continuation differences. +- `work/validation/backward-layout-training-unique-20260912/results.json`: successful single-core fixture and all source hashes. +- `work/validation/backward-layout-training-two-core-20260912/results.json`: successful two-core fixture after the shared-constant fix. +- `work/validation/backward-layout-training-producer-resume-20260912.log`: producer-scheduling continuation test. + +To reproduce, reserve both GPUs, keep production training paused, and use a new +output directory. Use `--unique-inputs` for the strict continuation fixture; +omitting it retains the repeated-index diagnostic, which can fail exact resume +because of the existing embedding accumulation nondeterminism. + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +CUDA_VISIBLE_DEVICES='' .server-venv/bin/python -B -m pytest -q test_backward_layout_training.py +CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/python -B \ + ops/validate_backward_layout_training.py --world-size 2 --unique-inputs --cores 2 \ + --output-dir work/validation/backward-layout-training-two-core-new +CUDA_VISIBLE_DEVICES=0,1 WARPSTATE_TEST_CUDA=1 \ + .server-venv/bin/python -B -m pytest -q test_training_optimization.py \ + -k producer-scheduled-overlap +``` diff --git a/ops/CHAT.md b/ops/CHAT.md new file mode 100644 index 0000000000000000000000000000000000000000..d1e3c787c8ca02bde01fa752eba739183d375109 --- /dev/null +++ b/ops/CHAT.md @@ -0,0 +1,134 @@ +# CPU chat with the latest completed checkpoint + +This is a text-generation probe of the pretrained **2.005B WarpState** model, +not a chat-finetuned assistant. The training corpus is English text plus EOS, +without role tokens. The default `Question: ... / Answer:` wrapper is only a +prompting convention. Use `/raw` to test ordinary text completion instead. +Fluency or one good answer does not by itself establish model quality. + +Run from `/workspace/WarpState`: + +```bash +bash rtx6000_40b/ops/chat-cpu.sh +``` + +The launcher uses CPU FP32 inference and 16 intra-op threads; it never +initializes CUDA or signals the training job. This allows both GPUs to keep +training. CPU responses are slower, and CPU/disk contention is still possible. +No training/model implementation, checkpoint writer, optimizer or data setting +is modified. No quantization, compilation or external service is introduced. + +## Snapshot safety + +At startup the tool opens the completed `work/run/latest.pt` once and pins that +inode using a read-only descriptor. The existing training writer publishes by +atomic rename, so a later save does not change the opened version. The tool +memory-maps that descriptor on CPU and exports only original model weights, +configuration, step/token boundary and provenance into a fresh directory under +`work/chat/`. Adam/RNG/dataset state are not exported: this is **inference-only**, +not a training-resume checkpoint. The training checkpoint is not overwritten, +hardlinked, chmodded, or otherwise mutated. + +An export uses about 7.47 GiB. Its new weight file is read-only, and +`snapshot.json` records its SHA256, tokenizer hash and source inode/boundary. +The hash is verified before strict model loading. Only load trusted local torch +checkpoints. A snapshot is frozen at its printed step, not refreshed silently +while chatting. To test a later checkpoint, quit and launch again without +`--snapshot`; this creates another independent export. Reuse an existing export +without consuming another export's storage: + +```bash +bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/work/chat/step-N-suffix/model.pt +``` + +`--snapshot` also accepts a full training checkpoint such as `work/run/latest.pt`. +When no `snapshot.json` is present, the tool validates the embedded training +format and creates an independent model-only export under `work/chat/` before +loading it. This creates a fresh export on each launch and prints its path: + +```bash +bash rtx6000_40b/ops/chat-cpu.sh --snapshot /workspace/WarpState/rtx6000_40b/work/run/latest.pt \ + --max-new-tokens 128 --repetition-penalty 1.2 --no-repeat-ngram-size 4 +``` + +`--checkpoint /path/to/latest.pt` remains the explicit way to capture training +weights. A chat export whose original `snapshot.json` is missing or invalid is +rejected; its integrity check is never bypassed or replaced with new metadata. +Restore the original metadata or create a fresh export from a training checkpoint. + +No generated snapshots, metadata or transcripts are committed to Git. + +## Controls and one-shot probes + +- `/new`: clear conversation context. +- `/raw`: clear context and switch to plain text completion. +- `/chat`: clear context and restore the Q/A wrapper. +- `/info`: show the exact snapshot boundary and metadata. +- `/help`, `/exit` (or Ctrl-D): help / quit. +- Ctrl-C during generation cancels that reply, not training. + +Each turn replays the retained prompt with one independent generation cache per +logical layer. A completed tile updates fast/slow memory only after producing +its own outputs. The total context budget defaults to 2048 tokens, including +the requested new tokens; an oversized prompt drops its oldest tokens. Long +histories therefore have substantial CPU prefill latency. + +```bash +# A short reproducible greedy probe; no interactive session. +bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/model.pt \ + --prompt 'What is Facebook?' --temperature 0 --max-new-tokens 32 + +# Sampling settings are explicit and do not affect training RNG. +bash rtx6000_40b/ops/chat-cpu.sh --snapshot /absolute/path/to/model.pt \ + --threads 16 --temperature 0.8 --top-k 40 --top-p 0.9 --seed 1337 \ + --max-new-tokens 128 +``` + +For a separate terminal, attach to the prepared session using +`tmux attach -t warpstate-chat`. Detach with Ctrl-b, then d. Chatting in that +session does not attach to or control the training console. + +## Repetition protection + +Chat and raw completion now default to `--repetition-penalty 1.2` and +`--no-repeat-ngram-size 4`. Already generated tokens receive a lower score, +and a token cannot complete a four-token sequence already emitted in the +same reply. These controls run before greedy selection, top-k and top-p; +EOS can still end a reply normally. The prompt and earlier turns are excluded +from the restrictions, so the model can reuse names and phrases from a question. +Protection resets for each reply. The active settings are printed at startup. + +This prevents exact token loops of that size and discourages shorter repeats; +it does not guarantee distinct meanings or turn a pretrained model into an +instruction-following assistant. Strict n-gram blocking can also restrict +intentional repetition in code, quotations or lists. For an unfiltered baseline, +pass `--repetition-penalty 1 --no-repeat-ngram-size 0`. Both options affect only +inference; no weights or training settings change. + +After updating a running chat, use `/exit` and relaunch it with the same +`--snapshot` path. This loads the new defaults and clears old repetitive +conversation context. Use `/new` to clear context later without reloading weights. + +## Verification + +The CPU test suite covers strict state loading, tokenizer identity, independent +depth caches, greedy/seeded sampling, repetition penalties for both logit signs, +phrase-loop blocking, per-reply reset, EOS, nonfinite logits, and snapshot +independence when `latest.pt` is atomically replaced during loading. It also +checks the CLI with `--snapshot latest.pt`, sealed snapshot reuse and rejection +of missing or invalid export metadata. It compares +incremental logits against full forward across three tile boundaries, with +both sequential and parallel memory. Agreement is tolerance-based: their +floating-point reduction orders are different, and CPU FP32 inference is not +claimed bitwise-equivalent to BF16 training. + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +CUDA_VISIBLE_DEVICES='' rtx6000_40b/.server-venv/bin/python -B -m pytest \ + rtx6000_40b/test_chat.py -q --basetemp=/workspace/WarpState/tmp/pytest-chat +``` + +The initial real-model smoke probe loaded an independent step-3250 snapshot +(425,984,000 training tokens) and produced finite greedy output on CPU. This is +a functionality check, not a new validation benchmark or a quality guarantee. diff --git a/ops/COMMUNICATION_EXPLORATION_20260912.md b/ops/COMMUNICATION_EXPLORATION_20260912.md new file mode 100644 index 0000000000000000000000000000000000000000..bfe9fe0f304aaddf1137f966262b9c0e59ac3230 --- /dev/null +++ b/ops/COMMUNICATION_EXPLORATION_20260912.md @@ -0,0 +1,254 @@ +# Communication scheduling exploration — September 12, 2026 + +These experiments use disposable copies of the step-40,081 full checkpoint. +Production training remains paused. The model, corpus cursor, 2,048-token +context, 131,072-token update, BF16 computation, FP32 gradient communication, +and full FP32 AdamW state are held fixed. The production resume preset retains +the previously selected conservative communication scheduler. + +## Explicit scheduler controls + +`run_40b.py` and `ops/benchmark_training.py` expose three opt-in controls for +`--gradient-mode compiled-autograd-overlap`: + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--overlap-swap-collectives` | Off | Move a wait and its dependent copies past later collective launches while preserving collective launch order. | +| `--overlap-runtime-estimates` | Off | Use measured GEMM estimates, aligned across distributed ranks, to guide wait placement. | +| `--overlap-memory-budget FRACTION` | `0.0` | Permit a fractional increase in the scheduler's estimated peak memory, bounded to 0–0.25. This does not increase the CUDA allocator cap. | + +The helper always disables unsafe collective reordering. Settings are immutable, +validated before compiler configuration, and recorded in execution metadata. +The runner records all three controls in checkpoint signatures and requires +explicit `--reconfigure` for changes. Legacy signatures normalize to the same +False/False/0 defaults, preserving existing resume behavior. + +Runtime estimation uses installed experimental PyTorch compiler APIs. It adds +GEMM benchmarking during compilation and distributed estimate alignment; this +is separate from the timed training updates. Configurations use fresh processes +because compiler/reducer settings are process-wide. Both ranks must compile the +same collective-bearing graphs. + +The installed FX cache explicitly includes `config_comms` in its key, and the +AOT cache uses the same payload. Actual cache-key checks distinguish these +options; they cannot silently reuse a differently configured scheduler merely +because the forward graph is unchanged. The runtime-estimate screen also +records an additional pre-update AllGather consistent with rank alignment. +`TORCH_LOGS=overlap` alone does not print the detailed swap decisions: the +installed pass additionally requires `reorder_sink_verbose_logging=True`. + +## Complete-model screens + +All rows use the same source versions and checkpoint. Changes are relative to +the fresh 128 MiB conservative control, not the earlier eager-gradient baseline. + +| Configuration / artifact directory | Tokens/s | Change | Reserved GiB/rank | +| --- | ---: | ---: | ---: | +| Conservative, 128 MiB (`control-128`) | 31,174.13 | — | 66.98 | +| Waits past launches, 128 MiB (`swap-128`) | 31,201.81 | +0.09% | 66.98 | +| Waits past launches, 64 MiB (`swap-64`) | 30,824.54 | -1.12% | 66.98 | +| Waits past launches, 256 MiB (`swap-256`) | 30,385.00 | -2.53% | 66.98 | +| Waits past launches + runtime estimates, 128 MiB (`swap-runtime-128`) | 31,117.39 | -0.18% | 66.46 / 66.96 | +| Same conservative control repeated (`control-repeat`) | 31,070.03 | -0.33% | 66.98 | + +The wait-only and runtime-estimate results are effectively tied with control; +neither establishes a throughput gain. The runtime-estimate choice reduces +allocator reservations by approximately 0.5 GiB on rank zero; rank one is +essentially unchanged. Peak allocated memory remains 65.71188 GiB on both +ranks, so this does not demonstrate a smaller model working set. +Smaller and larger buckets both lose throughput here. None of these results +justifies changing the production preset. + +Repeating the unchanged control moves throughput by -0.33%, larger than either +apparent wait-only or runtime-estimate effect. Its maximum loss difference is +0.00007796 and maximum relative gradient-norm difference is 0.0836%. Thus even +the unchanged full model does not give a bitwise scalar trajectory in these +screens. The other configurations' norm differences can be larger; a pair of +scalar traces is not proof of per-parameter equivalence or long-run quality. + +Control and wait-only runs both slow by about 1.14% from their first five to +last five measured updates. Telemetry shows similar temperatures and clocks +across the control, wait-only, and 64 MiB runs; their mean GPU temperatures are +approximately 52 C and 55 C by rank. One-second clock samples mix GEMM and +non-GEMM phases and are not measurements of sustained GEMM clock alone. + +`ops/compare_communication_benchmarks.py` validates checkpoint/data identities, +batch geometry, update cursors, recorded precision, and rank consistency before +comparing complete unprofiled windows. Its report preserves paired loss/norm +differences, timing variation, per-rank memory peaks, and source/input hashes. +These screens' largest loss difference is 0.00018311 and largest relative +gradient-norm difference is 0.366%; those are observed differences, not general +acceptance thresholds or evidence of improved model quality. + +Generated conservative and wait-only synchronization graphs have identical +Triton arithmetic bodies, GEMM/attention order, collective launch order, and +statement multisets; only wait/copy placement changes. Existing embedding +backward uses relaxed FP32 atomic additions to repeated-token rows. Small +cross-run differences are therefore consistent with existing nondeterministic +reductions amplified by subsequent BF16 rounding, although this observation +alone does not prove their cause. Both ranks agree on gradient norms in each +run. Same-mode repetition and explicit continuation tests are separate checks. + +The paired report is +`work/validation/communication-exploration/comparison-final.json`. It can be +rebuilt from the saved benchmark summaries and rank logs with: + +```bash +.server-venv/bin/python ops/compare_communication_benchmarks.py \ + --control work/benchmark-communication-20260912/control-128 \ + --candidate work/benchmark-communication-20260912/swap-128 \ + work/benchmark-communication-20260912/swap-64 \ + work/benchmark-communication-20260912/swap-256 \ + work/benchmark-communication-20260912/swap-runtime-128 \ + work/benchmark-communication-20260912/control-repeat \ + --output work/validation/communication-exploration/comparison-rebuilt.json +``` + +## Method + +### Fresh trace comparison + +Separate profiles use eight warmup updates and one instrumented update, with +identical checkpoint/corpus offsets. Their throughput is not included in the +unprofiled comparison above. + +| CUDA interval budget | Conservative rank 0 / rank 1 | Wait swap rank 0 / rank 1 | +| --- | ---: | ---: | +| NCCL overlapping other CUDA work | 191.805 / 191.069 ms | 193.185 / 191.983 ms | +| NCCL with no other CUDA work | 153.136 / 266.595 ms | 152.460 / 274.337 ms | + +Profiled update wall time is 4.17505 seconds for control and 4.18611 for swap. +Both contain 59 collective calls including scalar metrics, with identical CPU +operator/shape/type submission fingerprints. The structural wait movement does +not create a material increase in observed overlap on this model. + +Rank one's larger exposed interval includes waiting for its peer: its +noncommunication CUDA union is approximately 3.833 seconds, versus 3.946 on +rank zero in the control. Exposed intervals must not be added across ranks or +treated as wholly recoverable communication overhead. + +The report is `work/validation/communication-exploration/profile-comparison.json`. +Rebuild it with `ops/analyze_communication_profile.py`, the four +`profile-{control,swap}/profile-rank-{0,1}-trace.json` files under the benchmark +directory, and a fresh `--output` JSON path under `work/`. + +### Next scheduling work + +The installed compiler already enables `simple_overlap` after the iterative +wait pass. Enabling it again is not a new optimization. Its adjacent launch +moves stop at a gradient producer, an earlier collective, or estimated memory +growth. The next useful experiment would move the producer work with the +collective, making a bucket ready sooner rather than only delaying its wait. + +`reorder_communication_preserving_peak_memory` can group packing/casting +producers with collectives. A future screen must explicitly set +`reorder_iterative_unsafe_collectives_reorder=False` and +`reorder_iterative_peak_memory_budget=0.0`; the installed defaults are True and +0.2. Its non-estimated mode stops at GEMM producers; aligned runtime estimates +allow considering larger producer groups. Another option, `raise_comms`, +prioritizes the next collective's producer ancestors but has no peak-memory +guard. Neither pass was implemented or benchmarked by this exploration. + +The nonzero memory-allowance option is exercised by the continuation fixture; +nonzero allowances were not included in the complete-model throughput sweep. +Bucket readiness still depends on every member gradient, and the final +reductions must complete before clipping and AdamW. These are real limits on +how much overlap any scheduler can obtain. + +### Timing procedure + +The unprofiled comparisons use eight complete warmup updates followed by twenty +timed updates, with microbatch four per GPU and eight accumulation microbatches. +Each starts at the same checkpoint and token offset. The benchmark neither +modifies nor writes production checkpoints. Both GPUs retain their 300 W limits +and an 88 GiB PyTorch allocator cap. Matrix recipes, external weight caching, +and standalone activation replacements are off. + +NVIDIA-SMI samples clocks, power, temperature, and memory once per second outside +the timed Python regions. Initial diagnostic comparisons use identical NCCL +logging settings. A single sub-percent timing difference is insufficient to +select a default, particularly when sustained clock or thermal drift is similar +in size. Loss and gradient-norm sequences must be compared at identical update +indices, including warmups. + +Profiles are separate diagnostics. CUDA intervals are unioned per rank/device; +overlapping streams and annotation ranges are not summed as elapsed time. +Communication-only time is the union of NCCL intervals outside other CUDA +activity. It is an exposed-work budget, not a guarantee that every millisecond +can be hidden. Ranks' times must not be added together. + +`ops/analyze_communication_profile.py` produces reproducible overlap reports +from the benchmark's Chrome traces. Collective payload metadata describes +logical input bytes, not measured physical link traffic. Its CPU collective +sequence fingerprint covers operation names and shapes; it is supporting +evidence and does not establish tensor identity or numerical correctness. + +## Transport evidence + +The fresh control's NCCL initialization confirms direct `P2P/CUMEM` transport +in both directions and two collective channels. Its tuning messages identify +large all-reduces as RING/SIMPLE. Network/socket initialization messages are not +evidence that gradient traffic uses sockets; kernel symbol suffixes alone also +do not establish the selected protocol. The GPUs have a SYS path across CPU +NUMA nodes, so direct peer access still traverses the system interconnect. + +The benchmark records NCCL version, relevant environment overrides, and CPU +affinity. Raw logs and telemetry are under +`work/benchmark-communication-20260912/` and remain ignored. + +## Reproduction + +Reserve both GPUs with production training paused, then use a fresh output +directory for each process: + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO \ +NCCL_DEBUG_SUBSYS=INIT,GRAPH,TUNING,ENV \ +.server-venv/bin/torchrun --standalone --nproc-per-node=2 \ + ops/benchmark_training.py \ + --checkpoint work/run/latest.pt --data work/data/train.bin \ + --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \ + --no-activation-checkpointing --autocast-cache \ + --compile-cores --compile-mode default --loss-mode compiled-full \ + --bf16-residuals --gradient-mode compiled-autograd-overlap \ + --optimizer-state resume --vram-limit-gib 88 \ + --warmup-updates 8 --measured-updates 20 \ + --output-dir work/benchmark-communication-reproduction/control-128 +``` + +Append `--overlap-swap-collectives` for wait movement. Vary +`--ddp-bucket-cap-mb 64|128|256` in separate runs. The Python reducer groups +whole parameter gradients; a nominal bucket capacity does not split a larger +individual parameter. Runtime estimates and an estimated memory allowance +must be identified separately in results. + +The backward memory-traffic prototype is a separate experiment; see +[BACKWARD_LAYOUT_EXPERIMENT.md](BACKWARD_LAYOUT_EXPERIMENT.md). Isolated kernel +speedups cannot be added to complete-model throughput results. + +## Validation and retained state + +The final CPU suite passed **214 tests** with 17 CUDA checks skipped in that +CPU-only invocation. It covers option validation, legacy signatures, explicit +execution reconfiguration, gradient accumulation, rank-aligned estimates, +trace interval accounting, paired-run consistency, and backward-layout math. +The separate backward-layout CUDA comparisons are documented in its report. + +Both selected two-GPU continuation fixtures passed in 136.90 seconds: +wait movement with a 2% estimated memory allowance and 64 MiB buckets, and +wait movement with aligned runtime estimates and 128 MiB buckets. Each compares +continuous training with an asynchronous save/stop/resume, including a final +update with an entirely masked rank. Model tensors, complete FP32 optimizer +state, and both ranks' RNG state match bitwise within the same execution mode. +These small-model continuation checks are distinct from the full-model +cross-configuration numerical differences above. + +Validation logs and reports are under +`work/validation/communication-exploration/`, including `pytest-final-cpu.log`, +`pytest-resume.log`, `comparison-final.json`, and `profile-comparison.json`. +The retained step-40,081 checkpoint passes its before/after SHA256 check +(`checkpoint-before.sha256`, `checkpoint-after.log`). All package manifest +entries also verify. Production remains paused, and no new scheduler setting +or backward-layout adapter is selected in the production preset. diff --git a/ops/DATA_PREPARATION.md b/ops/DATA_PREPARATION.md new file mode 100644 index 0000000000000000000000000000000000000000..12cd3bfb292201e085e43bfed68ef22f1ba4615a --- /dev/null +++ b/ops/DATA_PREPARATION.md @@ -0,0 +1,78 @@ +# WarpState training data preparation + +The managed job prepares 40,000,000,000 training tokens and 1,048,576 +validation tokens from `HuggingFaceFW/fineweb-edu`, configuration +`sample-100BT`, using the package's unchanged SentencePiece tokenizer. +The upstream commit is pinned in the preparation state and final manifest. + +All job files, dependencies, caches, logs, and temporary files are below +`/workspace/WarpState`. This job uses CPU and network resources, with 16 +tokenizer threads and a lower CPU scheduling priority. It does not train a +model. The package's original Python files are unchanged. + +The launcher uses `ops/prepare_data.py` to avoid a reproduced PyArrow/Python +shutdown crash on this host. It waits for the original pipeline to return +(after token/state files are flushed and committed), flushes console output, +then exits without interpreter finalization. Exceptions are logged and keep +a nonzero exit code, so Supervisor can restart from committed progress. +Garbage collection alone did not eliminate the crash. This follows the +same process-exit approach documented in NVIDIA's +[Cosmos launcher](https://github.com/NVIDIA/cosmos-framework/blob/main/cosmos_framework/scripts/train.py). + +The launcher was checked on a real FineWeb-Edu stream with 1,000,000 train +tokens and 64 validation tokens. A SIGTERM stop committed its partial batch +and exited successfully. Resuming produced byte-identical files to an +uninterrupted run, and both completion manifests matched the computed +SHA256 checksums. Changed data settings were rejected with a nonzero exit +code. The results are in +`work/validation/live-resume-exitguard/validation.json`. + +## Paths + +- Python environment: `rtx6000_40b/.data-venv` +- Resolved Python dependencies: `rtx6000_40b/ops/data-requirements-resolved.txt` +- Training tokens: `rtx6000_40b/work/data/train.bin` +- Validation tokens: `rtx6000_40b/work/data/val.bin` +- Resumable state: `rtx6000_40b/work/data/prepare-state.pkl` +- Completion manifest and SHA256 checksums: `rtx6000_40b/work/data/manifest.json` +- Preparation log: `rtx6000_40b/work/logs/prepare.log` +- Hugging Face cache: `rtx6000_40b/work/hf-cache` +- Package download cache: `.cache/uv` +- Temporary files: `tmp` + +The final token files contain little-endian uint16 IDs and occupy +80,002,097,152 bytes together. An incomplete `train.bin` is not a completed +dataset. Completion requires `manifest.json` with `complete: true`; the job +computes both file checksums before writing that manifest. + +## Monitor and control + +Run from `/workspace/WarpState/rtx6000_40b`: + +```bash +ops/datactl.sh status +tail -f work/logs/prepare.log + +# Gracefully stop, committing the current preparation batch: +ops/datactl.sh stop warpstate-prepare + +# Resume from the saved iterator and committed token counts: +ops/datactl.sh start warpstate-prepare +``` + +Supervisor runs independently of the terminal, restarts the job after an +unexpected exit, and leaves it stopped after successful completion. Its +configuration, Unix socket, PID, and logs are local to this project. It does +not change the instance's system Supervisor configuration or expose a port. + +If this project's Supervisor itself is no longer running, start it with: + +```bash +TMPDIR=/workspace/WarpState/tmp PYTHONDONTWRITEBYTECODE=1 \ + /usr/local/bin/supervisord -c ops/supervisord.conf +``` + +The prepared data can later be used with the existing trainer's +`--stage train --work-dir /workspace/WarpState/rtx6000_40b/work` and its +default dataset, revision, tokenizer, and token-count arguments. Training +also requires a CUDA PyTorch environment and compatible initial weights. diff --git a/ops/DPO_CHAT.md b/ops/DPO_CHAT.md new file mode 100644 index 0000000000000000000000000000000000000000..79582ebf0eb3b251b3fa6acd20ee4bbed12694ea --- /dev/null +++ b/ops/DPO_CHAT.md @@ -0,0 +1,57 @@ +# Final DPO archive and chat + +DPO finished on 2026-09-15 at 01:05:06.882 UTC, with the guardian's final +verification complete at 01:05:42 UTC. It completed 2,334 updates over 74,661 +preference pairs. Training remains paused. + +The historical archive is `work/archives/step-00076294-dpo-final/` relative to +`rtx6000_40b`. It preserves the full FP32 model/Adam/RNG checkpoint and a +tensor-exact inference export, plus the tokenizer, evaluation, recipes, +reference scores, and provenance. Files are sealed read-only and checksummed. +Existing raw and SFT archives are preserved. + +From `/workspace/WarpState`, create or verify the archive and prepare the +independent chat copy: + +```bash +source rtx6000_40b/ops/training-env.sh +rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/archive_dpo.py +rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/prepare_dpo_chat.py +``` + +The chat copy is `rtx6000_40b/work/chat/dpo-final/`: separate physical files, +with hashes equal to the archive. The chat launcher reads this copy by default. +Neither chat nor preparation modifies archived weights. + +```bash +rtx6000_40b/ops/chat-dpo.sh +# Or attach to the already-open interface: +tmux attach -t warpstate-chat +``` + +Type a message at `You:`. Ctrl-C discards unfinished input or cancels the current +reply and returns to `You:`, preserving completed conversation turns. +`/reset` clears conversation history; `/quit` exits. +Detach tmux with Ctrl-B followed by D. The interface uses GPU 0, BF16 compute, +original FP32 master weights, a 2,048-token context, and greedy decoding. +Responses print when complete. Alternate CUDA devices can be selected with +`--device cuda:1`; `--temperature 0.7` enables sampling. + +Prompt serialization matches the segmented role headers, BOS and assistant EOS +used during posttraining. Cached inference follows the model's parallel chunk +memory; shape-dependent BF16 rounding can differ from full-sequence execution. + +Run reproducible sample prompts, choosing a new output filename each time: + +```bash +rtx6000_40b/ops/chat-dpo.sh --prompts-json rtx6000_40b/configs/posttrain_smoke_prompts.json --max-new-tokens 128 --output-json rtx6000_40b/work/validation/dpo-smoke.json +``` + +These few prompts are sanity checks, not a capability benchmark. DPO's final +held-out preference loss was 0.5790; 245 of 339 pairs had positive relative +reward margins. This 72.27% is not a generation win rate or answer accuracy. +Exponentiating DPO loss does not produce language-model perplexity. + +For a matched full SFT holdout evaluation, `ops/evaluate_sft_snapshot.py` +accepts `--snapshot`, `--device`, and a new `--output` path. Reports and generated +responses stay under ignored `work/`; source scripts and prompts are tracked. diff --git a/ops/DPO_HANDOFF.md b/ops/DPO_HANDOFF.md new file mode 100644 index 0000000000000000000000000000000000000000..64d6ce475488b299c51025b91d4fdc5d6554a868 --- /dev/null +++ b/ops/DPO_HANDOFF.md @@ -0,0 +1,144 @@ +# Archive completed SFT and continue into DPO + +The user authorized this sequence on 2026-09-14: + +**Finish SFT → archive its full state → independent initialization → DPO → pause.** + +The separate `warpstate-dpo` Supervisor job waits while the existing SFT job +continues unchanged. The original SFT plan still says pause because it is part +of that run's immutable resume signature; its guardian must complete and exit +before this newly authorized continuation can start. The active DPO plan is +[`../configs/dpo_10b.json`](../configs/dpo_10b.json). + +## Completion and archival gate + +The DPO guardian requires all of the following before any GPU work: + +- SFT Supervisor state `EXITED`, exit status 0, and its guardian's durable state + `complete`. +- Exclusive SFT guardian, base launcher and pipeline locks, then recheck. +- Exactly 10,312 SFT updates, 329,973 samples, 212,321,372 shifted input tokens, + and 143,695,039 supervised assistant tokens. +- A completed full SFT checkpoint with Adam and both ranks' RNG state, and its + matching final evaluation of all 1,603 held-out conversations. +- Unchanged parent SFT plan, tokenizer, prepared-data manifest and source pins. + +The archive is `rtx6000_40b/work/archives/step-00076294-sft-final/`. It contains: + +| File | Purpose | +| --- | --- | +| `checkpoint.pt` | Physical byte-for-byte copy of the full SFT checkpoint, including optimizer/RNG/cursor/signatures | +| `model.pt`, `snapshot.json` | Original FP32 model-only export, with every tensor compared exactly to the full checkpoint | +| `tokenizer.model` | Exact original SentencePiece tokenizer | +| `evaluation.json`, `completion.json`, `settings.json`, `metrics.jsonl` | Original final evaluation, completion, execution settings and complete SFT history | +| `sft-execution.json`, `sft-handoff.json`, `PROVENANCE.json` | Source/data identities and both raw-pretraining and SFT stage counters | +| `ARCHIVE.md`, `SHA256SUMS` | Retention instructions and integrity checks | + +All files are sealed read-only before the complete directory is atomically +published without replacement. Existing archives are validated and reused, or +rejected without alteration. Interrupted hidden staging directories remain for +diagnosis. The original SFT resume checkpoint and all raw historical archives +remain in place. + +DPO reads a physically independent copy of `model.pt`, `snapshot.json` and the +tokenizer under `work/posttrain/step-00076294-dpo/initialization/`. It does not +inherit SFT Adam state or write into either SFT archive or SFT run directory. +The full SFT archive is roughly 32 GB decimal; its counters describe SFT after +the raw step-76,294 model, not an additional 10B-token pretraining run. + +## Preference objective and execution + +This first DPO experiment uses: + +| Setting | Value | +| --- | --- | +| Data | 74,661 training pairs and 339 held-out pairs already prepared | +| Objective | Length-normalized DPO, beta 5 | +| Epochs | 1; 2,334 optimizer updates | +| Global batch | 32 preference pairs; 2 pairs per rank × 2 ranks × 8 accumulation microbatches | +| Context | 2,048 tokens per chosen/rejected branch; no truncation or cross-conversation packing | +| Learning rate | Peak `5e-7`, 10% warmup, linear decay to zero | +| Optimizer | Fresh FP32 AdamW, betas 0.9/0.95, zero weight decay, clipping at 1 | +| Precision | BF16 compute, original FP32 parameters/gradients/Adam; no FP8 | +| Evaluation/save | Every 100 DPO updates; full holdout at start and completion | + +For each branch, sum the log probabilities of **only the final assistant +response and its actual EOS token**. System/user/tool messages, earlier +assistant turns, role headers and padding are conditioning or masked. Divide +each policy-minus-reference log-probability sum by that branch's response-token +count, subtract rejected from chosen, multiply by beta, and apply negative +log-sigmoid. Average over actual preference pairs across ranks; short final +batches and empty rank slots do not change the normalization. + +This uses the length-normalized objective and conservative hyperparameters +described in the [Tülu 3 paper, equation 6 and table 20](https://arxiv.org/html/2411.15124v3). +It is a pilot for this 2B/10B base, not a reproduced or tuned Tülu result. +Ordinary summed-log-probability DPO is a separate supported objective and its +usual beta cannot be substituted for beta 5 here without changing the recipe. + +Before policy updates, score the immutable SFT model in BF16 on all selected +chosen/rejected branches. Each GPU saves resumable reference-score shards; the +final cache is bound to SFT weights, data, tokenizer, masks, execution settings, +library version and source hashes. Reusing or rebuilding it must preserve that +identity. The reference remains the completed SFT model after every policy +checkpoint/resume. Once scores are cached, release the reference model before +allocating policy optimizer state, avoiding a second resident full model. + +The output head uses checkpointed token chunks to avoid retaining full-sequence +vocabulary logits for both responses. A tiny two-GPU BF16 preflight checks the +actual DPO/reference/continuation path after SFT releases the GPUs. CPU setup +tests do not claim that this deferred GPU gate or production-size startup has +already run. Any failed gate stops automatic progression with the archive kept. + +## State, monitoring and controls + +Paths below are relative to `rtx6000_40b/work/posttrain/step-00076294-dpo/`. + +| Path | Purpose | +| --- | --- | +| `control/handoff.json`, `control/execution.json` | Guardian phase and frozen execution provenance | +| `initialization/` | Independent immutable SFT model copy | +| `preflight/` | Tiny GPU validation and its acceptance report | +| `reference-cache/` | Fixed SFT reference log probabilities and resumable scoring shards | +| `run/latest.pt` | Independent full DPO resume checkpoint, format `warpstate-dpo-v1` | +| `run/metrics.jsonl`, `run/evaluation.json`, `run/completion.json` | Training history, evaluation and durable completion | + +From `/workspace/WarpState`: + +```bash +rtx6000_40b/ops/datactl.sh status warpstate-posttrain warpstate-dpo +rtx6000_40b/ops/dpo-handoff.sh --check +tmux attach -t warpstate-dpo +tail -f rtx6000_40b/work/logs/dpo.log +``` + +To cancel automatic DPO while SFT continues, or gracefully stop active DPO: + +```bash +rtx6000_40b/ops/datactl.sh stop warpstate-dpo +``` + +To explicitly resume the unchanged guardian/reference preparation/DPO state: + +```bash +rtx6000_40b/ops/datactl.sh start warpstate-dpo +``` + +No restart of SFT is needed. The new guardian preserves the SFT guardian and +base-launch exclusion locks throughout DPO. Every worker holds a shared base +pipeline lock, so surviving workers block accidental base/SFT/DPO relaunches +even if their launcher dies. Stop files and lock files are never shared with +the live SFT stop request. The program has `autostart=false` and +`autorestart=false`; a whole-machine/Supervisor restart requires explicitly +restarting the appropriate jobs. A completed DPO run stays paused when its +guardian is started again. + +On normal completion, validate final counters, all 339 held-out pairs, fixed +reference-cache identity and full archive/init hashes before recording success. +No additional training stage starts automatically. Next compare the archived +SFT model with DPO on instruction following and tool use. DPO loss, reward +margin and preference accuracy describe this objective; exponentiating DPO +loss does not produce language-model perplexity. + +All generated data, scores, archives, checkpoints, logs and environments remain +ignored under `work/`. Only source/configuration/tests/documentation are tracked. diff --git a/ops/LOCAL_RESUME.md b/ops/LOCAL_RESUME.md new file mode 100644 index 0000000000000000000000000000000000000000..02567bc34a487f63f568547ff13fbafdebad2013 --- /dev/null +++ b/ops/LOCAL_RESUME.md @@ -0,0 +1,207 @@ +# Resume the B200 checkpoint on the RTX workstation + +This configuration continues the B200 training run on the local RTX PRO 6000 +workstation. +The source checkpoint was saved on the B200 at **step 680 / 89,128,960 +tokens**, with validation loss **4.39469** and SHA256: + +```text +bf689a10a030842699bfa3a25ff62e488cfa2d5b022b2b6e3bfc860359eb8b4f +``` + +After the verified import, `work/run/latest.pt` is the local resume checkpoint. +Later local saves replace it atomically; the checksum above identifies the +imported step-680 checkpoint. [B200_HANDOFF.md](B200_HANDOFF.md) and the B200 +performance documents record the remote run and its historical controls. +Use the workstation controls below on this host. +The B200 source history was imported through commit `752563e`. The generated +local migration record is `work/validation/local-resume/import.json`, relative +to `rtx6000_40b`; it records the verified checkpoint and continuation state. + +[`transfer_artifacts.py`](transfer_artifacts.py) accepts `--pull` to retrieve +a remote source into a local destination with parallel SSH connections. +Use `--expected-sha256` with the checkpoint hash above, keep the private key +on the local host, and use the existing verified host-key file. The download +uses resumable chunks and verifies the entire file before publishing its +final name. No second download of the unchanged 40B-token corpus is needed. + +The prepared 40,000,000,000 training tokens and 1,048,576 validation tokens +are unchanged. Their immutable upstream revision, tokenizer hash, split rule, +and preparation dependencies remain in +[`../configs/data_40b.json`](../configs/data_40b.json). The existing data +signature retains the original requested revision `main`, which was resolved +to the pinned upstream commit. Preserve that signature when resuming. + +## Local execution + +[`train-local-resume.sh`](train-local-resume.sh) requires an existing full +checkpoint and selects both physical workstation GPUs by UUID. Both GPUs +became available before the local resume. This launcher uses the measured +BF16 configuration and continues the existing run. Check both GPUs' +availability before every start: + +| Setting | Value | +| --- | --- | +| Sequence length | 2,048 | +| Microbatch per GPU / gradient accumulation | 4 / 8 | +| GPUs / global tokens per optimizer update | 2 / 131,072 | +| Parameters, gradients, full AdamW moments | FP32 on the GPU | +| Forward/backward computation / residual activations | BF16 autocast / BF16 | +| Core activation checkpointing | Disabled | +| Core compilation / autocast weight cache | Enabled, default mode / enabled | +| Vocabulary loss | Compiled full loss | +| Gradient execution / communication | Compiled autograd with delayed waits/copies; FP32 reduction, 128 MiB buckets | +| Allocator cap / extra startup headroom per GPU | 88 GiB / 3 GiB | +| Checkpoint writing | Synchronous immutable CPU snapshot; background serialization and atomic replacement | +| Save / evaluation interval | Every 100 / 100 updates | +| Validation display | Fresh results only, labeled with their evaluation step | + +The current preset includes gradient conversion/accumulation fusion and moves +collective waits with their dependent copies past independent backward GEMMs. +The repeated 20-update comparison measured **31,045 tokens/s** versus +**30,578 tokens/s** without those changes (**1.53% faster**), with **66.98 GiB +reserved per GPU**. Both GPU ranks remain data-parallel; communication overlaps +their local computation. See [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md) +for the profile, numerical comparisons and exact two-GPU checkpoint/resume tests. +Shape-specific BF16 matrix plans are available through `--matmul-plan`. The +experimental fused-epilogue/TMA recipe measured 31,156 versus 31,022 tokens/s +with the same overlap mode in one comparison (+0.43%); that additional gain is +not yet confirmed across repeated windows, so the launcher keeps its current +matrix implementation. See [MATMUL_TUNING.md](MATMUL_TUNING.md) for the recipe, +tile tuning, persistent kernels, precision checks, and full-model measurements. +The earlier loss/residual changes are documented in +[RTX_PROFILE_20260910.md](RTX_PROFILE_20260910.md), and asynchronous checkpoints +in [RTX_ASYNC_TRAINING.md](RTX_ASYNC_TRAINING.md). + +These rates exclude evaluation, checkpoint pauses and compilation. Compiled +autograd can spend approximately three minutes compiling its first full update; +even with cached kernels, each new process has graph-capture/setup costs. +The allocator cap excludes CUDA context/library allocations. Startup requires +91 GiB free on each GPU and does not reserve memory for other jobs. +Keep at least **53 GiB of free SSD space** beyond existing files for full +checkpoint writing and replacement. + +Asynchronous saves use about **22.4 GiB of additional host memory** for one +full model/optimizer snapshot. GPU-to-CPU staging still pauses both workers; +serialization, file synchronization and publication overlap later updates. +There is at most one outstanding snapshot, and a new save waits for the +previous write if necessary. A logged `Staged step` is not yet durable: use +the corresponding `Saved ... (step ..., ... background write ...)` message. +Normal completion and graceful stop wait for the final write. A save failure +is reported to both workers at the next coordinated check, save or close and +stops training; a failure during an update can allow that update to finish. + +Training step lines display `val ... (step ...)` only when that step has a +fresh validation result. Every step still records `val_loss`, `val_step` and +`val_tokens` in `metrics.jsonl`; these fields retain the latest measured +result between evaluations. Checkpoints retain the same validation metadata. +This adds no per-step evaluation work. +Startup still evaluates the resumed weights, and graceful stop/final completion +still evaluates and saves even when the step is not a multiple of 100. + +`--reconfigure` preserves the model, full optimizer, token cursor, objective, +and learning-rate schedule while recording execution changes in the checkpoint. +The original B200 import restored rank-zero random states and seeded the new +rank reproducibly. Subsequent two-GPU resumes restore both ranks' states. +BF16 residual activations retain FP32 master parameters, gradients and Adam +moments, but change activation rounding. A small validation change at the +same saved weights is not training progress. Disabling them requires +`--no-bf16-residuals` with an explicit reconfiguration. + +## Controls and current progress + +Run from `/workspace/WarpState`: + +```bash +rtx6000_40b/ops/datactl.sh status +nvidia-smi --query-gpu=index,memory.used,memory.free,utilization.gpu --format=csv +rtx6000_40b/ops/datactl.sh start warpstate-train +tail -n 10 rtx6000_40b/work/logs/train.log +tail -n 3 rtx6000_40b/work/run/metrics.jsonl +``` + +The status command and latest metrics identify the actual running state and +current cursor. This document records the import and launcher configuration; +it does not imply that a job is running. Preparation and training both have +automatic startup and restart disabled. After changing Supervisor +configuration, use `datactl.sh reread` and `datactl.sh update` before an +explicit start. + +Stop with `rtx6000_40b/ops/datactl.sh stop warpstate-train`. The launcher +requests a stop through its stop file, waits for the current update and full +checkpoint, and has a 900-second Supervisor stop allowance. Check the saved +step in the log before treating an in-memory update as durable. + +## Resuming after a deliberate pause + +Keep a deliberately paused run stopped until the user requests resumption. +As requested on 2026-09-14, the workstation Supervisor command includes +`--stop-at-step 76294`: the trainer finishes optimizer update 76,294, evaluates, +and publishes the full checkpoint before exiting. This boundary corresponds to +10,000,007,168 consumed tokens. No update 76,295 runs. Async checkpoint writing +is flushed before exit, and Supervisor has automatic restart disabled. + +`--stop-at-step` is an absolute limit, so an intervening restart cannot move +the boundary. `--max-steps` still limits additional updates in one invocation; +if both are supplied, the earlier boundary wins. Neither option changes the +40B-token learning-rate schedule or the checkpoint's training signature. +The active boundary is also recorded in `work/run/settings.json` under +`runtime.stop_at_step` and `runtime.end_step`. + +When the user requests training beyond this boundary, first remove or increase +`--stop-at-step` in the `warpstate-train` command in `ops/supervisord.conf`, then +run `ops/datactl.sh reread` and `ops/datactl.sh update warpstate-train` from the +package directory before starting the job. A restart retaining a boundary at +or below the loaded checkpoint step performs no further optimizer updates. + +The generated `work/validation/paused-run.json` records the last verified +pause: checkpoint path, SHA256, step, tokens consumed, validation loss, +optimizer/RNG checks, execution settings, source commit and next update. +`work/validation/paused-run.sha256` provides a directly checkable checksum. +Both records are excluded from Git. Subsequent training replaces `latest.pt`, +so these records identify the last deliberate pause rather than live progress. + +Before resuming a stopped run, inspect its saved state and available hardware: + +```bash +cd /workspace/WarpState +rtx6000_40b/ops/datactl.sh status +cat rtx6000_40b/work/validation/paused-run.json +sha256sum -c rtx6000_40b/work/validation/paused-run.sha256 +nvidia-smi --query-gpu=index,memory.used,memory.free,utilization.gpu --format=csv +``` + +Use the existing full checkpoint at `work/run/latest.pt`, which contains +model weights, full AdamW state and both ranks' RNG states. Preserve +`work/data/`, `work/run/`, the `.server-venv` environment, tokenizer and +existing launchers. No data preparation or checkpoint conversion is needed. +Both workstation GPUs must have at least 91 GiB free before launch. The +launcher retains BF16 compute, the 131,072-token global batch, the 40B-token +schedule, asynchronous saves, and 100-step save/evaluation intervals. + +When resumption is requested: + +```bash +cd /workspace/WarpState +rtx6000_40b/ops/datactl.sh start warpstate-train +tmux attach -t warpstate-train +``` + +The launcher clears the previous stop request under its process lock. If the +host has rebooted and Supervisor is absent, initialize it once with +`/usr/local/bin/supervisord -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf` +before using the controls. Its training and preparation jobs start paused. +If the tmux monitor was lost during reboot, recreate it with: + +```bash +tmux new-session -d -s warpstate-train -n training -c /workspace/WarpState \ + 'tail -n 30 -F /workspace/WarpState/rtx6000_40b/work/logs/train.log' +``` + +Keep the active token files, their manifest/preparation state, the latest full +checkpoint, active environment, and controls required to continue training. +Git retains source, pinned dependency specifications, tokenizer, and input +provenance for rebuilding. Obsolete checkpoints, experimental outputs, +download staging, caches, and disposable test artifacts can be removed once +their replacement or source recipe is verified; generated runtime files stay +out of Git. diff --git a/ops/LOSS_FORECAST_20260913.md b/ops/LOSS_FORECAST_20260913.md new file mode 100644 index 0000000000000000000000000000000000000000..c7122fa280fd60ceb3ef49efb693b10ea85d69f7 --- /dev/null +++ b/ops/LOSS_FORECAST_20260913.md @@ -0,0 +1,208 @@ +# WarpState validation-loss forecasting, 2026-09-13 + +**Neither proposed token budget currently guarantees validation loss below 2.0.** +The observed curve supports continued improvement, but the crossing estimate is +extremely sensitive to the assumed eventual loss floor. A power-law fit to the +1–7B-token history crosses near 1.07T tokens; excluding the first billion tokens +of that fit moves its extrapolated floor above 2.0. This is uncertainty about the +forecast, not evidence of a demonstrated architecture limit. + +This research did not change training, checkpoints, the dataset, or the schedule. +Generated snapshots, numerical results, and figures belong under +`work/validation/loss-forecast-20260913/` and remain excluded from Git. + +## Current measurements and interpretation of the budgets + +The frozen validation history ends at step 54,300: **7,117,209,600 tokens, +validation loss 2.42521**, with 555 evaluation records. Recent values fluctuate +around 2.41–2.43. The complete 6.75–7.00B bin averages 2.41219. Loss 2.0 corresponds +to perplexity `exp(2) = 7.389` on this tokenizer and evaluation distribution. + +Interpret `8000*256` and `8000*1024` as **total optimizer steps**, not additional +steps or individual tokens. The active configuration processes +`2 GPUs * 4 sequences * 2048 tokens * 8 accumulation microsteps = 131072` +tokens per optimizer update. + +| Budget | Optimizer steps | Total training tokens | Tokens per model parameter | Passes over the existing 40B-token corpus | Remaining wall time | +| --- | ---: | ---: | ---: | ---: | ---: | +| `8000*256` | 2,048,000 | 268,435,456,000 | 133.88 | 6.711 | 100.5 days | +| `8000*1024` | 8,192,000 | 1,073,741,824,000 | 535.50 | 26.844 | 410.3 days | + +The model has 2,005,111,424 parameters. Wall-time arithmetic uses the live metrics +at step 54,360 on 2026-09-13 11:41:14 UTC: the preceding 3,000 optimizer steps +averaged **4.35589249 seconds/update**, or **30,090.7 effective tokens/second**, +including periodic evaluation and checkpoint overhead. Durations assume the +current two-GPU workstation, unchanged batch size and throughput, and uninterrupted +operation. They estimate the time to consume a budget, not the time to attain a +particular loss. From zero, the budgets would take 103.3 and 413.0 days. + +The configured objective and prepared token file are still **40B tokens**. +The larger budgets require an explicit continuation schedule and data plan. +If the question meant additional steps, add the current consumed tokens and +about 2.74 days to the corresponding remaining-duration arithmetic. + +## What the primary research establishes + +Language-model learning curves often admit power-law approximations after an +initial transient. These describe diminishing returns: each additional billion +tokens generally produces a smaller gain. Published coefficients depend on the +model, tokenization, data, and experimental setup. +[Kaplan et al., Sections 4–5](https://arxiv.org/pdf/2001.08361). + +Chinchilla fits endpoint loss as +`L(N,D) = E + A/N^alpha + B/D^beta`. At fixed model size this has the form +`L(D) = F + B/D^beta`, where `F` includes the model-size contribution as well as +irreducible uncertainty. Its compute-optimal parameter/data allocation does not +define a stopping point for a fixed model. Appendix B also shows that the cosine +schedule duration must be calibrated to the intended training duration. +[Hoffmann et al., Section 3.3 and Appendix B](https://arxiv.org/pdf/2203.15556). + +Schedule-aware work separates continuing optimization from the extra improvement +caused by lowering the learning rate. Tissue et al. model validation loss using +both accumulated learning rate and an annealing-history term; their Appendix C.3 +uses shorter constant and cosine runs to forecast a 1.7B model trained on 1.4T +tokens. Such calibrated experiments support forecasting, but do not establish +that one early training trace can identify a 150-fold extension. +[Scaling Law with Learning Rate Annealing](https://arxiv.org/html/2408.11029). + +Constant-rate training followed by a terminal cooldown provides a way to compare +different training budgets while reusing checkpoints. A loss reached after +cooldown need not have the same subsequent slope as a loss reached at a high +learning rate. +[Hägele et al., Sections 2–3](https://arxiv.org/html/2405.18392v3). + +Long training of small models can remain useful. Sardana et al. study a 2.46B +model up to 500 tokens/parameter, close to the larger proposed budget here. Their +study also warns that fits from shorter training can overstate the benefits of +much longer training. Its reported final training loss and unique-data setup +cannot be converted into WarpState validation loss 2.0. +[Sardana et al., Sections 3–4 and Appendix C](https://arxiv.org/html/2401.00448). +Gadre et al. successfully forecast a 1.4B/900B-token model using calibrated +experiments, further supporting the feasibility of forecasting with the right +measurements. +[ICLR 2025 paper](https://proceedings.iclr.cc/paper_files/paper/2025/hash/a91869936a63d814971b6423990ecf6e-Abstract-Conference.html). + +Repeated tokens have diminishing value compared with fresh tokens. Muennighoff +et al. find small penalties through roughly four epochs in their experiments, +with progressively smaller returns from further repetition. That finding is not +a universal epoch cutoff. Repeating our corpus 6.7 or 26.8 times is a different +experiment from acquiring 268B or 1.07T comparable fresh tokens. +[Scaling Data-Constrained Language Models](https://arxiv.org/html/2305.16264v3). + +## Fits to WarpState: scenarios, not confidence intervals + +Use actual evaluations from `work/run/validation.jsonl`, rather than the latest +validation value repeated in every training-metric row. Verify strictly increasing, +unique token counts, start at 1B tokens to exclude early execution changes, and +average complete 0.25B-token bins. Each bin receives equal weight, at its actual +mean token position. The frozen snapshot contains 24 complete bins from 1–7B; +the partial final bin is excluded from fitting. + +Fit in loss space, using `t = tokens / 1e9`: + +- Logarithmic: `L(t) = a + b*ln(t)`. +- Power law with a floor: `L(t) = F + A*t^(-beta)`, with `A >= 0`, `F >= 0`, + and positive `beta`. +- Fixed-floor sensitivity: choose an illustrative `F` and fit `A` and `beta`. + These chosen floors are assumptions, not measured properties of WarpState. + +For the power-law scenario, a finite crossing exists only if `F < 2`, and its +location is `t = (A/(2-F))^(1/beta)`. A small uncertainty in `F` close to 2 thus +creates a very large uncertainty in the crossing time. + +| Fit | Loss at 268.435B | Loss at 1.073742T | Crossing below 2.0 | +| --- | ---: | ---: | --- | +| Logarithmic, starts at 3/4/5B and ends at 7B | 1.68–1.85 | 1.40–1.64 | 53–103B | +| Power law, free floor, 1–7B history | 2.052 | approximately 2.000 | approximately 1.07T | +| Power law, free floor, 2–7B history | 2.096 | 2.058 | No finite crossing in this fitted curve | +| Power law, **assumed floor 1.5**, 1–7B history | 1.905 | 1.799 | 103B | +| Power law, **assumed floor 1.8**, 1–7B history | 2.001 | 1.932 | 272B | +| Power law, **assumed floor 1.9**, 1–7B history | 2.043 | 1.989 | 762B | + +The 1–7B free-floor fit is approximately +`L(t) = 1.918124 + 0.975783*t^(-0.355255)`. +The 2–7B fit has floor 2.009366 and exponent 0.419261. Neither fitted floor is a +measured asymptotic limit. Shifting the start to 3, 4, or 5B yields floors around +2.19–2.33, showing how poorly the eventual floor is identified by this history. +The numerical 1.07T crossing should not be interpreted as a precise prediction +or as evidence that the larger budget was chosen correctly. + +The logarithmic fit eventually predicts unboundedly negative loss, so it is a +local trend approximation. It is included because it illustrates an optimistic +extrapolation, not because it is a suitable global asymptotic law. Small residuals +on fitted data also do not establish long-range predictive accuracy. + +Historical forward checks fit an earlier interval and predict the next complete +billion tokens. The table shows mean predicted minus observed loss; negative +values mean the forecast was too optimistic. + +| Fit interval → held-out interval | Logarithmic bias | Free-floor power-law bias | +| --- | ---: | ---: | +| 2–4B → 4–5B | -0.00197 | +0.00695 | +| 2–5B → 5–6B | -0.00604 | -0.00194 | +| 2–6B → 6–7B | -0.01665 | -0.01023 | + +Neither family wins every historical comparison. Both overpredicted the latest +interval's improvement, particularly the logarithmic fit. These checks use +0.25B-bin means, so their residuals should not be confused with errors on raw +individual validation points. + +## Why there is no dependable crossing date yet + +The observations extend to about 7.1B tokens, whereas the requested budgets are +roughly **38 and 151 times** that amount. Historical one-billion-token holdouts +can reject some overly optimistic local trends; they cannot validate such a +large extrapolation. Successive measurements are correlated observations of one +training run, not independent experimental replications. + +The run still uses a learning rate near `1.40e-4`, about 93% of its `1.50e-4` +peak, with a planned 40B-token cosine decay toward `1.50e-5`. The simple fits +above do not model that future annealing. Cooldown may improve loss beyond their +predictions; a changed continuation schedule or repeated data may produce a +different curve. There is no fitted magnitude for those effects in WarpState. + +Current evaluation samples eight fixed sequences of length 2,048: **16,384 +target tokens**, drawn at fixed spaced positions from the 1,048,576-token +held-out file. A single reported value below 2 would be weaker evidence than a +sustained crossing and confirmation on more held-out tokens. Absolute loss 2 +also cannot be compared directly with another paper's tokenizer or validation +distribution, and does not by itself determine downstream task performance. + +The useful next forecast experiment is a matched control and cooldown branch +from the same full checkpoint, retaining optimizer state, data order, batch +geometry, BF16 computation, and the same evaluation metric. Evaluate additional +held-out tokens separately, preserving the historical metric for comparison. +Combining those measurements with later 10–15B-token checkpoints would help +separate continued-learning gains from annealing gains and allow an actual +out-of-sample test of a schedule-aware model. This document recommends that +experiment; it does not launch it. + +For planning, treat sub-2.0 loss as an unresolved target. **268B tokens might be +enough, and 1.07T provides much more training, but the present evidence does not +justify promising either budget will cross the threshold.** + +## Reproduction + +The input snapshot SHA256 is +`13f7fa766a59b884d376515132e5f41fe07b771438e7e66ed294dce030e9f38e`. +The analysis program is `ops/loss_curve_forecast.py`. It requires NumPy; +Matplotlib is optional for the figure. The generated JSON records dependency +versions and hashes of both the snapshot and analysis source. The isolated +`work/analysis-venv` keeps analysis dependencies separate from training. +This analysis used Python 3.12.3, NumPy 2.5.3, and Matplotlib 3.11.2. + +From `/workspace/WarpState/rtx6000_40b`: + +```bash +source ops/training-env.sh +python3.12 -m venv work/analysis-venv # Only when rebuilding an absent environment. +work/analysis-venv/bin/python -m pip install 'numpy==2.5.3' 'matplotlib==3.11.2' +work/analysis-venv/bin/python -B ops/loss_curve_forecast.py \ + --input work/validation/loss-forecast-20260913/validation-snapshot.jsonl \ + --output work/validation/loss-forecast-20260913/forecast.json \ + --plot work/validation/loss-forecast-20260913/loss-scenarios.png +``` + +The same command with an `.svg` plot extension generates a vector figure. +Use a new output directory when analyzing a later frozen snapshot so that the +original forecast remains available for comparison with subsequent observations. diff --git a/ops/MATMUL_PROFILE_20260912.md b/ops/MATMUL_PROFILE_20260912.md new file mode 100644 index 0000000000000000000000000000000000000000..789083c8084c42c51fc8474af923ba69840650ac --- /dev/null +++ b/ops/MATMUL_PROFILE_20260912.md @@ -0,0 +1,316 @@ +# WarpState matrix multiplication profile, 2026-09-12 + +The largest dense BF16 products already sustain approximately **243–263 TFLOPS +per GPU** in the refreshed isolated test. Their observed clocks and 300 W board +power explain much of the gap from the boost-clock specification. The best next +experiments are selective algorithm/tile selection and small, measured GEMM +fusions. A global backend replacement has not demonstrated a useful improvement. + +This analysis uses the refreshed production-configuration profile and new +isolated GEMM timings. Production remains paused at step 40,081 with its full +resume state preserved. All step-8,000, step-16,000, and step-40,000 archives are +unchanged. No matrix experiment in this report writes production checkpoints. + +## Profile and attribution + +The configuration is the two RTX PRO 6000 Blackwell Max-Q GPUs, microbatch 4 per +rank, sequence length 2,048, eight accumulation microbatches, 131,072 global +tokens/update, compiled cores in default mode, compiled full loss, BF16 residual +activations, and the existing FP32 fused AdamW state. The unprofiled 20-update +baseline is **30,799 tokens/s during optimizer updates**; the separate profiled +update took 4.1781 seconds after two warmups. + +Each GPU kernel's `External id` is joined to its `aten::mm`, `aten::bmm`, or +`aten::addmm` CPU event. Dimensions, strides, and dtypes come from that event. +The square output projection's forward and input-gradient shapes are identical; +the RHS transpose stride distinguishes them. This avoids assigning the same +square GEMM to both directions. + +For `A[M,K] @ B[K,N]`, arithmetic is counted as `2*M*N*K` FLOPs, multiplied by +batch and call count. Achieved TFLOPS divide these FLOPs by matched GPU kernel +time, not CPU launch time or the entire training update. CPU operator totals can +also include associated fills/copies; the tables below use kernel durations. + +All **5,400 matrix kernels per rank** are accounted for: + +| Profile | Rank 0 matrix time | Rank 1 matrix time | FLOPs per rank/update | +| --- | ---: | ---: | ---: | +| Current baseline | 3,166.449 ms | 3,049.219 ms | 791.597 trillion | +| Compiled-autograd candidate | 3,122.129 ms | 3,005.404 ms | 791.597 trillion | + +Baseline matrix kernels represent 71.19% and 68.62% of summed GPU activity on +ranks 0 and 1. Streams overlap with communication and other work, so these +percentages are not partitions of elapsed update time. The matrix operations +themselves execute on the main compute stream in this trace. The compiled +autograd comparison changes surrounding execution and communication overlap; +its slightly shorter matrix times do not establish a faster full training run. + +## Which products dominate + +For a linear layer, forward computes `Y = X @ W.T`, the input derivative +computes `dX = dY @ W`, and the weight derivative computes `dW = dY.T @ X`. +Every core projection executes 224 times per direction: 28 cores times eight +microbatches. Backward therefore includes two dense products for each forward. + +| Projection family | Rank 0 forward | Rank 0 dX | Rank 0 dW | Total | Share of matrix time | +| --- | ---: | ---: | ---: | ---: | ---: | +| FFN up and gate, 2,048 → 14,336 | 405.939 ms | 433.643 ms | 427.218 ms | 1,266.800 ms | 40.01% | +| Q/K/V/gate/memory input, 2,048 → 10,240 | 284.852 ms | 312.126 ms | 305.255 ms | 902.232 ms | 28.49% | +| FFN down, 7,168 → 2,048 | 218.817 ms | 209.116 ms | 198.243 ms | 626.176 ms | 19.78% | +| Core output, 2,048 → 2,048 | 57.486 ms | 56.861 ms | 57.828 ms | 172.176 ms | 5.44% | +| Vocabulary head, 2,048 → 32,768 | 33.869 ms | 34.131 ms | 35.123 ms | 103.124 ms | 3.26% | +| State-memory batched products | — | — | — | 95.942 ms | 3.03% | + +FFN up/gate, input projection, and FFN down together consume **88.28% of matrix +time**. Dense linear products perform 788.350 trillion FLOPs per rank/update; +state-memory batched products account for the remaining 3.247 trillion. + +The vocabulary head now uses eight complete 8,192-token products per direction. +The September 10 profile used chunked head loss and repeated head forward +products during backward. Its larger forward-head count should not be reused +to estimate the current configuration. + +The dominant shapes and their rank-0 kernel performance are: + +| Product | M | N | K | Mean kernel time | Achieved TFLOPS | +| --- | ---: | ---: | ---: | ---: | ---: | +| FFN up/gate forward | 8,192 | 14,336 | 2,048 | 1.8122 ms | 265.4 | +| FFN up/gate dX | 8,192 | 2,048 | 14,336 | 1.9359 ms | 248.5 | +| FFN up/gate dW | 14,336 | 2,048 | 8,192 | 1.9072 ms | 252.2 | +| Input projection forward | 8,192 | 10,240 | 2,048 | 1.2717 ms | 270.2 | +| Input projection dX | 8,192 | 2,048 | 10,240 | 1.3934 ms | 246.6 | +| Input projection dW | 10,240 | 2,048 | 8,192 | 1.3627 ms | 252.1 | +| FFN down forward | 8,192 | 2,048 | 7,168 | 0.9769 ms | 246.2 | +| FFN down dX | 8,192 | 7,168 | 2,048 | 0.9336 ms | 257.6 | +| FFN down dW | 2,048 | 7,168 | 8,192 | 0.8850 ms | 271.8 | + +These are timings within the full-model profile. They can differ from isolated +tests because clocks, cache state, memory pressure, and communication differ. +The isolated tests below use longer warmup and repeated timing trials. + +## What executes inside the dense kernels + +Most dense kernels use symbols beginning +`cutlass_80_tensorop_bf16_s16816gemm_relu_bf16`. Forward commonly selects +`128x256_32x3_tn_align8`; dX uses `256x128_32x3_nn_align8`; dW uses +`128x256_32x3_nt_align8`. The head forward uses a `256x128` TN variant. + +These are tiled tensor-core kernels: tiles load operands, reuse them while +advancing through K in pipelined stages, accumulate partial products, then store +the output. The symbols encode tile/pipeline/operand variants. The embedded +`80` and `relu` text does not establish obsolete hardware execution or an actual +ReLU in this model; semantic attribution comes from the originating operators. + +Observed launches use 256 threads, approximately 218–232 registers per thread, +and 73,728 bytes of dynamic shared memory. Several have a grid Z dimension of +2 or 4. That is consistent with additional partitioned work, but launch metadata +alone does not prove the precise split-K/reduction implementation. A proposed +split-K experiment must first identify what the selected library algorithm +already does. + +Profiler occupancy estimates report zero for some kernels that demonstrably +execute. They are not valid measured occupancy. The previous hardware-counter +attempt was denied by the driver's `ERR_NVGPUCTRPERM`; this analysis makes no +new occupancy, tensor-pipe-utilization, or bandwidth-counter claim. + +The large dense shapes are multiples of tensor-core alignment and ordinary +tile sizes. Their transpose operands are views with unit stride in one +dimension, not evidence of a full physical transpose before every GEMM. +Blindly adding `.contiguous()` would create extra traffic; any alternative +layout must include its preparation cost in the benchmark. + +## Where parallel execution already happens + +Both GPUs already train concurrently using data parallelism. Each holds the +model and processes **65,536 of the 131,072 global tokens per update**: eight +8,192-token microbatches per rank. There is no intentional schedule that runs +GPU 0's training first and GPU 1's afterward. Gradient synchronization combines +their contributions to the same optimizer update. + +The original compiled-autograd candidate lost overlap between computation and +communication **within each GPU's schedule**. That is different from serializing +the two GPUs: both ranks still compute concurrently, but local waits block +independent backward GEMMs while collectives finish. Recovering that overlap +is the immediate parallelization target identified by the trace. + +The refreshed `nvidia-smi topo -m` reports the GPU-to-GPU path as `SYS`, meaning +PCIe plus an inter-NUMA CPU interconnect, with no NVLink entry. Reported CPU +affinities are `0-95,192-287` for GPU 0 and `96-191,288-383` for GPU 1. Topology +identifies the route; it does not measure achieved communication bandwidth. +No CPU/GPU affinity or power settings were changed for this analysis. + +Some dX and dW products are mathematically independent and could use separate +CUDA streams. However, the large GEMMs already consume much of each GPU's +compute and power capacity; concurrent launches share those resources rather +than doubling them. A focused stream experiment could test thin or final-tail +products, including their synchronization cost, before changing the full backward +schedule. Model or pipeline parallelism would add activation transfers and +potential pipeline bubbles over this topology, with additional implementation +complexity. It is not an automatic improvement over the existing two-GPU data +parallel execution. + +## Refreshed isolated performance and compute headroom + +The existing `benchmark_gemm.py` tested all 12 core shape/direction pairs on +GPU 0 using deterministic BF16 operands with production transpose strides. +Each case had at least two seconds of burn-in and five CUDA-event timing trials +of 30 products. The existing BF16 intermediate-reduction allowance was preserved; +no lower-precision model or gradient representation was introduced. + +NVIDIA specifies 438.9 dense BF16 TFLOPS with FP32 accumulation at a 2,280 MHz +boost clock, 300 W board power, and 1,792 GB/s memory bandwidth for this Max-Q +GPU. Source: Appendix A of the +[RTX Blackwell architecture whitepaper](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46). + +The final column below divides achieved performance by +`438.9 * observed_SM_clock_MHz / 2280`. It is an illustrative clock-scaled +comparison, not a hardware-counter utilization measurement. Clocks are telemetry +snapshots after each timed case, not time-integrated clock measurements. + +| Product | Median time | TFLOPS | Observed SM clock | Clock-scaled comparison | +| --- | ---: | ---: | ---: | ---: | +| FFN up/gate forward | 1.8473 ms | 260.4 | 1,492 MHz | 90.7% | +| FFN up/gate dX | 1.9148 ms | 251.2 | 1,402 MHz | 93.1% | +| FFN up/gate dW | 1.8925 ms | 254.2 | 1,425 MHz | 92.7% | +| Input projection forward | 1.3337 ms | 257.6 | 1,485 MHz | 90.1% | +| Input projection dX | 1.3958 ms | 246.2 | 1,417 MHz | 90.2% | +| Input projection dW | 1.3531 ms | 253.9 | 1,447 MHz | 91.2% | +| FFN down forward | 0.9879 ms | 243.5 | 1,440 MHz | 87.8% | +| FFN down dX | 0.9162 ms | 262.5 | 1,530 MHz | 89.1% | +| FFN down dW | 0.9399 ms | 255.9 | 1,485 MHz | 89.5% | +| Core output forward | 0.2656 ms | 258.8 | 1,612 MHz | 83.4% | +| Core output dX | 0.2625 ms | 261.8 | 1,620 MHz | 83.9% | +| Core output dW | 0.2578 ms | 266.5 | 1,725 MHz | 80.3% | + +Every timed case's final power sample was approximately 300 W. The dominant +families remain close to the earlier measured range and approximately 88–93% +of this clock-scaled estimate. The smaller output projection has more apparent +relative headroom, but represents only 5.44% of matrix time. Neither comparison +implies all remaining theoretical capacity can be recovered by software. + +## Compute versus operand traffic + +For one idealized BF16 read of A/B and one write of C, arithmetic intensity is +`2*M*N*K / (2*(M*K + K*N + M*N))`. It is about 1,470 FLOPs/byte for the FFN +up/gate products, 1,412 for the input projection, 1,334 for FFN down, and 910 +for the square output projection. Actual traffic includes reloads, cache effects, +and possible intermediate reductions; these are operand-footprint estimates. + +This supports a compute-efficiency focus for the large dense GEMMs. It does not +prove that every tile load is perfectly hidden. A persistent/TMA implementation +can improve operand delivery or scheduling even when the overall dense product +is predominantly compute-limited. + +State-memory products are different. Shapes with batch 128 and `M=16, N=4096, +K=16` have only about 8 FLOPs/byte under that estimate; the `128x64x64` and +`64x64x128` batched products have about 25.6. Observed rates are roughly +10–61 TFLOPS, with 16x16/32x32 WMMA kernel tiles. They merit smaller-shape or +producer/consumer fusion experiments, but their entire current kernel budget +is only 95.94 ms, approximately 2.3% of the profiled update. + +## Prior experiments and specific next tests + +The September 10 experiments already covered eager cuBLAS, cuBLASLt preference, +ATEN-only compilation, default Triton autotuning, and persistent TMA. See +[`RTX_PROFILE_20260910.md`](RTX_PROFILE_20260910.md). + +| FFN up/gate product | Prior cuBLAS | Prior default Triton | Prior persistent TMA | Current cuBLAS | +| --- | ---: | ---: | ---: | ---: | +| Forward | 1.828 ms | 1.981 ms | 1.798 ms | 1.847 ms | +| dX | 1.885 ms | 2.057 ms | 1.890 ms | 1.915 ms | +| dW | 1.871 ms | 1.869 ms | 1.810 ms | 1.892 ms | + +These are separate runs with different observed clocks, not a controlled +same-clock speedup claim. Default Triton usually lost to cuBLAS; persistent TMA +closed much of that gap and won some shapes modestly. cuBLASLt preference was +mixed, and its earlier up/gate dW result coincided with a higher clock. Repeating +a global backend toggle is therefore a weak next experiment. + +The following targeted experiments have a better basis: + +1. **Select algorithms per dominant shape/direction.** Retest persistent TMA + against ATEN/cuBLAS for up/gate forward and dW first, with alternating warmed + trials and clock samples. Keep cuBLAS for shapes where it wins. Measure an + entire layer forward/backward and then the full two-GPU update so template + fusion and DDP overlap are included. Preserve the BF16 dW rounding before + FP32 accumulation. +2. **Test a GEMM epilogue that performs gradient conversion and accumulation.** + The current conversion/addition budget is 129.076 ms/update. A fused kernel + must retain the BF16 GEMM result's rounding before its FP32 addition and keep + autograd/DDP ownership correct. Direct FP32-output `addmm` was already tested: + it changes rounding and was not consistently better. The present compiled + autograd candidate also demonstrates why local fusion needs full-update + communication validation. +3. **Explore a paired up/gate epilogue only if its GEMM remains competitive.** + The combined projection is already one GEMM, with up and gate columns 7,168 + apart. A tile/layout that places matching pairs together could compute SwiGLU + before writing its activated output. Backward still needs the projected up + and gate values, so their storage cannot simply disappear. The removable + standalone forward budget is about 41.16 ms/update; recomputing the projection + would cost about 405.94 ms and would defeat that purpose. Repacking must use + temporary views/copies and retain canonical archive/checkpoint layout. +4. **Explore the small state-memory GEMMs separately.** Their low intensity and + small dimensions make tiled producer/consumer fusion more plausible than + expecting dense-projection TFLOPS. Account for both gradient products and + layout preparation; a low TFLOPS number on its own does not establish waste. + +The installed compiler already has epilogue/prologue fusion enabled for suitable +templates. Its `kernel/mm.py` adds persistent TMA choices only through the +max-autotune template route, gated by `ENABLE_PERSISTENT_TMA_MATMUL`. Enabling +that variable alone does not replace every existing library GEMM. + +The distinction between GPU architectures matters: this workstation is SM120. +The installed `use_triton_blackwell_tma_template()` additionally requires +`is_datacenter_blackwell_arch()`, whose predicate is `100 <= arch < 110`. +The named datacenter-Blackwell warp-specialized template and the CUTEDSL route +therefore are not automatic options for this RTX GPU; generic persistent TMA +is the relevant already-tested path. The environment also lacks the `cutlass` +and `cutlass_library` packages and the configured CUTLASS source directory, so +CUTLASS/NVGEMM are not available simply by changing the backend flag. + +As a conditional scale check, retaining the measured 3.1664 seconds of matrix +time and deleting every other cost would give about **41.4K tokens/s**. This is +not a hardware limit or an achievable configuration. Reaching 50K with all +other costs zero would still require about 17.2% less matrix time. The isolated +results give no evidence for a software-only reduction of that scale across all +products at the present power limit. + +## Evidence and reproduction + +Generated evidence remains ignored in Git: + +- `work/validation/rtx-opt-20260912/matmul/matmul-attribution.json`: both ranks, + both profiles, shapes/strides/dtypes, call counts, exact kernel names and launch + metadata, FLOPs, operand intensities, and current clock-scaled comparisons. +- `work/benchmark-opt-20260912/profile-baseline/`: source traces, kernel TSVs, + operator tables, and category summaries. +- `work/benchmark-opt-20260912/profile-compiled-autograd/`: corresponding + candidate traces. Include its eight BF16 `aten::addmm` head-dW calls when + counting arithmetic; otherwise the head's weight-gradient FLOPs are omitted. +- `work/benchmark-gemm-20260912-current/results.json`: refreshed isolated trials + with telemetry for each product. +- `work/benchmark-gemm-20260910/`, `work/benchmark-gemm-cublaslt-20260910/`, and + `work/benchmark-gemm-tma-20260910/`: earlier backend comparisons. + +Installed compiler source evidence is under +`.server-venv/lib/python3.12/site-packages/torch/_inductor/`: `kernel/mm.py`, +`config.py`, `utils.py`, and `codegen/cuda/cuda_env.py`. These observations apply +to the installed `2.14.0+cu130` build and should be rechecked after an upgrade. + +To repeat the isolated sweep in an exclusive GPU window, with a new output +directory: + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +CUDA_VISIBLE_DEVICES=0 .server-venv/bin/python -B ops/benchmark_gemm.py \ + --families ffn_up_gate in_proj ffn_down out_proj \ + --operations fwd dx dw --rows 8192 --backends eager \ + --burnin-seconds 2 --repeats 30 --trials 5 \ + --output-dir work/benchmark-gemm-20260912-recheck +``` + +The full-model profiling command and checkpoint protections are documented in +[`OPTIMIZATION_20260912.md`](OPTIMIZATION_20260912.md). Compilation and benchmark +outputs must stay under the workspace, and a new experiment must not overwrite +the retained reports or archived weights. diff --git a/ops/MATMUL_TUNING.md b/ops/MATMUL_TUNING.md new file mode 100644 index 0000000000000000000000000000000000000000..153952214a032c1296a6997eae9edf2babef7ffa --- /dev/null +++ b/ops/MATMUL_TUNING.md @@ -0,0 +1,342 @@ +WarpState's matrix experiments select BF16 kernels by operation, exact matrix +dimensions, and operand strides. They are opt-in through `--matmul-plan` in +`run_40b.py` and `ops/benchmark_training.py`. The workstation preset selects +compiled autograd with communication overlap. Matrix kernels remain explicit +experiments; isolated GEMM improvements alone do not select them for production. +The complete two-GPU comparisons and validation are recorded below. + +The implementation is split across these source files: + +| File | Responsibility | +| --- | --- | +| [matmul_kernels.py](matmul_kernels.py) | BF16 ATen, tiled, persistent, and persistent TMA matrix launches; tile validation and shape/stride signatures. | +| [matmul_epilogue.py](matmul_epilogue.py) | Paired up/gate matrix multiplication with a SwiGLU epilogue; returns both packed projection and activation. | +| [matmul_execution.py](matmul_execution.py) | Strict plan loading, registered autograd, and selected core-linear/SwiGLU training adapters. | +| [tune_matmul.py](tune_matmul.py) | Disposable ordinary/fused FFN matrix tuning with warmed controls and generated candidate recipes. | +| [activation_execution.py](activation_execution.py) | Existing BF16 SwiGLU derivative used by the fused epilogue's backward. | + +The adapter is applied before core compilation and DDP construction. Original +FP32 parameters remain the autograd leaves. Their differentiable BF16 casts stay +inside the visible forward graph; the adapter does not retain external casts +across updates or write leaf gradients through hooks. Matrix operands, returned +matrix gradients, projections, and activations are BF16. Custom dot products +accumulate in FP32. Parameter-gradient accumulation, clipping, DDP reduction, +FP32 master parameters, and AdamW state remain under the existing training +framework. The token cursor, objective, optimizer schedule, and checkpoint keys +retain their existing meanings. CPU and evaluation model execution use the +original module paths. + +Selected kernel launches use opaque registered custom operations with fake +output metadata, surrounded by the adapter's registered autograd. This is +necessary because installed PyTorch 2.14 compiled autograd cannot retrace AOT's +functionalized Triton launch operator. Unselected ATen products remain visible +to the compiler. The CUDA suite exercises this second backward capture as well +as ordinary eager/AOT execution; a plain forward compilation test would miss it. + +Each JSON plan has format `warpstate-matmul-plan-v1`, an optional description, +and an `entries` array. For `A[M,K] @ B[K,N]`, `m`, `n`, and `k` describe that +operation; strides are in elements. They are essential: a transposed weight and +a contiguous matrix with the same dimensions can require different kernels. +The complete dispatch key is `(kind, m, n, k, a_stride, b_stride)`. Missing entries +use ATen. An empty plan selects no custom kernels. Duplicate JSON keys, duplicate +dispatch entries, unknown fields, invalid tiles, and non-BF16 entries are rejected. + +This is a **candidate example**, not a selected production recipe: + +```json +{ + "format": "warpstate-matmul-plan-v1", + "description": "Candidate paired FFN epilogue; requires complete-model validation.", + "entries": [ + { + "kind": "swiglu", + "name": "ffn_up_gate.fwd", + "m": 8192, + "n": 14336, + "k": 2048, + "a_stride": [2048, 1], + "b_stride": [1, 2048], + "dtype": "bfloat16", + "algorithm": "tiled", + "tile": { + "block_m": 128, + "block_n": 128, + "block_k": 32, + "num_warps": 8, + "num_stages": 3 + } + } + ] +} +``` + +`kind: "mm"` selects an ordinary matrix product. Forward and backward products +have independent entries. The FFN up/gate forward uses `(M,N,K) = +(8192,14336,2048)` with strides `[2048,1]` and `[1,2048]`. Its weight-gradient +product uses `(14336,2048,8192)` with strides `[1,14336]` and `[2048,1]`, matching +the original `dY.T @ X` orientation. A forward winner therefore does not select +that weight-gradient kernel automatically. + +`kind: "swiglu"` selects the fused forward. Here `N=2H` is the complete packed +projection width, while `block_n` counts columns in **each** half. Backward uses +the saved BF16 packed projection, the existing SwiGLU derivative, and independently +selected `mm` entries for `dX` and `dW`. + +| Algorithm | Execution | Supported plan kinds | +| --- | --- | --- | +| `aten` | Calls the installed PyTorch matrix implementation with its existing process settings. | `mm`, `swiglu` | +| `tiled` | Launches one program for each output tile, with grouped row tiles for weight reuse. | `mm`, `swiglu` | +| `persistent` | Launches at most one program per SM; each program strides over several output tiles using ordinary loads. | `mm`, `swiglu` | +| `persistent_tma` | Persistent scheduling with Triton tensor descriptors and hardware TMA operand loads. | `mm` | + +Persistent scheduling alone does not imply TMA or warp specialization. The TMA +implementation uses an opaque registered custom-op boundary because the installed +PyTorch 2.14 AOT path does not preserve the host descriptor metadata through a +plain `triton_op` launch. It does not secretly substitute another algorithm. +TMA operands need an inner stride of one and 16-byte-aligned rows and storage +offsets. The training selector falls back to ATen for an incompatible layout; +the explicit kernel launcher reports unsupported requests. The paired SwiGLU +kernel does not currently support TMA. + +Tiles specify `(block_m, block_n, block_k, num_warps, num_stages)`. Block sizes +must be powers of two from 16 through 256, with 4 or 8 warps and 1 through 4 +pipeline stages. A conservative SM120 operand-pipeline estimate rejects ordinary +tiles above `(BM+BN)*BK*2*stages = 96 KiB` and paired tiles above +`(BM+2*BN)*BK*2*stages = 96 KiB`. This is an estimate, not a guarantee about +compiler register/shared-memory allocation. The tuner records compilation failures +without promoting those candidates. + +The paired epilogue arranges adjacent logical columns as up/gate pairs, using +their physical packed offsets `j` and `j+H`. One wider tensor-core dot reuses the +input tile, then splits the accumulator in registers. Both FP32 projection halves +are rounded to BF16 **before** the activation. The kernel stores that rounded +packed projection for backward and also writes the BF16 activation. It saves a +subsequent packed-projection read and activation launch; it does not remove the +projection storage required by backward. + +The `eager_rounding` argument additionally rounds the SiLU result to BF16 before +multiplication when following eager execution. Compiled execution follows the +existing compiler's fused SiLU/product arithmetic, while still rounding the GEMM +projection first. No FP8, FP16, quantization, split-K atomics, or reduced-precision +gradient communication is introduced. Exact equality on measured aligned shapes +does not promise bitwise equality for every GEMM layout or compiler version. +For example, the tested ragged `K=193` cuBLAS reference used a different reduction +result: its packed relative-L2 difference was 0.269%, while the custom result +differed by 0.000394% from an FP32-dot reference. Tests separately check projection +accuracy and that activation consumes the saved rounded BF16 projection. + +On GPU 1 of the RTX PRO 6000 Blackwell Max-Q workstation, the final paired kernel +was compared with `torch.compile(fullgraph=True)` around the complete BF16 +`mm -> split -> SiLU -> multiply` forward, returning both packed and activation +outputs. Inputs had shape `[8192,2048]` and transposed weights `[2048,14336]`. +With seed 563, 400 warmup calls per variant, and 15 alternating-order batches of +40 CUDA-event-timed calls, the selected microbenchmark tile measured: + +| Forward implementation | Median milliseconds | +| --- | ---: | +| Compiled ATen matrix multiplication plus SwiGLU | 1.950789 | +| Paired single-dot epilogue, tiled `(128,128,32,8,3)` | 1.905561 | + +This is **2.37% greater primitive throughput**, or approximately 45 microseconds +less per call; paired batch gains ranged from 2.08% to 2.75%. Packed projections +and activations matched the compiled reference bitwise for two production-size +random input seeds. The final epilogue suite passed 33 tests, covering both +schedules, eager/compiled rounding, ragged dimensions, and the default/best tiles. +A subsequent CPU-only invocation passed 9 and skipped 24 CUDA fixtures. + +Applying the measured 45-microsecond difference to 28 cores and eight microbatches +gives about 10 milliseconds per update before interaction with backward, +communication, compiler scheduling, and allocator behavior. That arithmetic is +not a measured complete-training gain. The earlier two-dot implementation lost +to the control; it is superseded by the single-dot source. + +Generated evidence remains under +`work/validation/rtx-opt-20260912/epilogue/`: + +- `paired-swiglu-confirm.json`: final steady-state comparison and second input seed. +- `paired-swiglu-single-dot.json`: final single-dot implementation's tile/schedule screen. +- `paired-swiglu.json` and `paired-swiglu-more-tiles.json`: superseded two-dot screens. + +The ordinary GEMM screen tested three algorithms and five tiles for each FFN +up/gate forward and weight-gradient shape: 30 candidates total. Every candidate +matched the ATen BF16 output exactly. All forward candidates lost. The selected +weight-gradient candidate used persistent TMA with tile `(256,128,32,8,3)`: +median candidate time 1.811495 ms and median paired throughput gain 3.39%. +Its weakest of three paired trials gained 1.30%. The first trial's two controls +drifted from 1.589741 to 1.765839 ms, so this short screen alone is insufficient +to select a production configuration. The complete generated sweep and candidate +recipe are in `work/benchmark-matmul-tiles-20260912/`. + +To reproduce the ordinary FFN forward/weight-gradient tuning, reserve an idle +GPU first. From the repository root, the following command uses physical GPU 0 +and keeps outputs and compiler caches inside the workspace. The output directory +must be a new `work/benchmark-*` directory; existing directories are rejected. + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +CUDA_VISIBLE_DEVICES=0 .server-venv/bin/python -B ops/tune_matmul.py \ + --kind mm --operations fwd dw --algorithms tiled persistent persistent_tma \ + --rows 8192 --seed 1337 --warmup 10 --burnin-seconds 1 \ + --repeats 30 --trials 3 --minimum-gain 0.01 \ + --maximum-relative-rms 0.003 \ + --output-dir work/benchmark-matmul-tuning-reproduction +``` + +To recheck the measured fused tile with the same dimensions, seed, and operand +scale, use the fused mode on an idle GPU: + +```bash +CUDA_VISIBLE_DEVICES=1 .server-venv/bin/python -B ops/tune_matmul.py \ + --kind swiglu --operations fwd --algorithms tiled \ + --tile 128,128,32,8,3 --rows 8192 --seed 563 --input-scale 0.1 \ + --warmup 400 --burnin-seconds 1 --repeats 40 --trials 15 \ + --minimum-gain 0.01 --maximum-relative-rms 0.003 \ + --output-dir work/benchmark-matmul-swiglu-reproduction +``` + +This uses the tuner's bracketed control/candidate/control protocol. The historical +15-batch report used alternating pairs, so compare the resulting measurements +as independent checks rather than expecting identical timing samples. Fused +mode compiles the complete matrix-plus-SwiGLU reference and validates both packed +projection and activation separately. It supports only `--operations fwd` with +`tiled` or `persistent`. Omit `--tile` to use that mode's bounded tile list, or +repeat it to specify several candidates. `--input-scale` defaults to 1.0 for both +operands; the recorded epilogue comparisons used 0.1. + +`--list-cases` prints the requested dimensions and strides without using CUDA. +The tuner generates deterministic BF16 random operands from the supplied seed; +it does not open a model, checkpoint, or token database. In this disposable process +it disables BF16 reduced-precision reduction and TF32 for the ATen reference. +Every timed candidate has preceding and following warmed controls. Eligibility +requires finite results, the configured relative RMS tolerance, and at least +the configured gain in **every** trial. Compilation failures remain in +`results.json`. The best eligible entry for each tested operation is written to +`candidate-policy.json`; an empty recipe means no candidate passed selection. +Results include source hashes, software/device metadata, input signatures, +timing triples, per-output numerical error, and GPU telemetry. Generated reports and +candidate recipes remain ignored under `work/`; a recipe selected after complete +validation should be copied into tracked `configs/` with its provenance. + +Use the same checkpoint, input offsets, microbatch, loss/gradient modes, and +warmup/measurement counts to compare a candidate with a control. For example, +after reserving both workstation GPUs, this disposable command tests the generated +recipe against the retained full checkpoint: + +```bash +CUDA_VISIBLE_DEVICES=0,1 .server-venv/bin/torchrun --standalone --nproc-per-node=2 \ + ops/benchmark_training.py --checkpoint work/run/latest.pt \ + --data work/data/train.bin --micro-batch 4 --seq-len 2048 \ + --global-tokens-per-update 131072 --no-activation-checkpointing \ + --compile-cores --compile-mode default --loss-mode compiled-full \ + --bf16-residuals --gradient-mode compiled-autograd-overlap \ + --optimizer-state resume --warmup-updates 8 --measured-updates 20 \ + --matmul-plan work/benchmark-matmul-tuning-reproduction/candidate-policy.json \ + --output-dir work/benchmark-matmul-training-candidate +``` + +Run a matching control with `--matmul-plan` omitted and a different new output +directory. Neither benchmark advances the production checkpoint. Run numerical +backward/optimizer and checkpoint/resume tests before selecting a training recipe. +The production runner accepts the same `--matmul-plan PATH` option alongside its +existing resume arguments; it is not an instruction to start a new run or change +the established global token batch. External core-weight cast caching and the +`reduce-overhead` CUDA-graph mode are explicitly incompatible with this adapter. +Standalone `--activation-mode swiglu|rmsnorm|all` kernels require eager gradient +execution; their transparent backward launches have the same compiled-autograd +limitation described above. Both CLIs reject that unsupported combination. +Use `--activation-mode baseline` with compiled autograd; matrix-plan fused +epilogues still work through their separate opaque boundaries. +Combining a matrix plan and compiled autograd also requires activation +checkpointing off. The installed compiler cannot trace the saved-tensor unpack +hook when the runner wraps an already compiled core in an eager checkpoint. +Recomputation captured inside AOT passes the small kernel tests, but does not +establish support for that different runner nesting. + +Plan loading happens once, before execution. It computes a canonical recipe +SHA256 and implementation SHA256 values for `matmul_execution.py` and +`matmul_kernels.py`; a fused plan also includes `matmul_epilogue.py` and +`activation_execution.py`. The runner records this metadata in settings and the +checkpoint training signature, and the benchmark records it with its source +provenance. A changed recipe or implementation therefore changes the execution +signature. Legacy checkpoints normalize to no matrix plan. Resuming with an +execution change requires the existing explicit `--reconfigure` mechanism, +which still protects the data, context, global token batch, and optimizer schedule. +Reformatting JSON without changing its parsed canonical content leaves the recipe +identity unchanged; changing a description or an entry changes it. + +The complete-model comparisons resume the same step-40,081 checkpoint into +disposable GPU model/AdamW copies. Each uses eight warmup updates and 20 timed +updates, BF16 residuals, compiled full loss, the same 131,072-token global batch, +and compiled autograd with delayed communication waits. They exclude compilation, +evaluation and checkpoint pauses. The generated rank logs and summaries are +under `work/benchmark-opt-20260912/`. + +| Configuration | Timed tokens/s | Change from control | Reserved GiB/rank | +| --- | ---: | ---: | ---: | +| Communication-overlap control (`matmul-control`) | 31,022.34 | — | 66.98 | +| Persistent TMA dW only (`matmul-dw`) | 31,063.57 | +0.13% | 66.99 | +| Fused forward + TMA dW, initial boundary (`matmul-combined`) | 31,002.83 | -0.06% | 66.54 | +| Fused forward + TMA dW, shared weight buffer (`matmul-combined-native`) | 31,156.20 | +0.43% | 66.98 | + +These first comparisons are effectively tied. Over all 28 updates, the initial +combined candidate's maximum loss difference from the fresh control was +0.00008535, and the maximum relative gradient-norm difference was 0.124%. +Small BF16 reduction/fusion differences do not imply training progress. + +After the weight-layout correction, the candidate averaged 4.20693 seconds per +update versus 4.22508 for the control. Its maximum loss difference was 0.00010061 +and its maximum relative gradient-norm difference was below 0.114%; all observed losses +and norms were finite. This single comparison shows a small +0.43% gain, which +has not been confirmed across repeated timing windows. The matrix recipe remains +opt-in. The resume launcher retains the repeatedly verified communication-overlap +configuration from [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md), which +measured a 1.53% improvement over eager gradients in the repeated comparison. +These percentages come from separate experiments and must not be added into a +claimed combined speedup. The generated numerical/timing comparison is +`work/validation/rtx-opt-20260912/matmul/full-model-comparison.json`. + +Generated-code inspection verified the selected forward and dW routes, with +224 uses each per rank/update inferred from the compiled graphs and loop counts. +It also found an avoidable cost in the initial fused boundary: passing `W.T` +into an opaque operation made Inductor store two identical BF16 weight buffers, +while the control shared one between forward and backward. The final adapter +passes native-layout BF16 `W` and forms its transpose inside the launch. The +actual-size compiled core now shares one allocation and one cast with backward, +removing 58,720,256 extra bytes of stores per core call, or 13,153,337,344 bytes +per rank/update. This is generated-code evidence, not a measured bandwidth +counter or a guaranteed wall-time saving. The TMA dW route already reuses its +gradient buffer and required no corresponding change. + +The tracked +[`matmul_rtx6000_bf16_experimental.json`](../configs/matmul_rtx6000_bf16_experimental.json) +contains both exact candidate shapes and tiles. The final native-weight boundary +comparison uses this recipe directly. Static dispatch and buffer evidence is +recorded in `work/validation/rtx-opt-20260912/matmul/compiled-dispatch-evidence.json` +and `dw-dispatch-evidence.json`. Production remains paused; these experiments +leave the step-40,081 checkpoint and archived weights unchanged. + +The integration validation passed 208 CPU tests, with 83 CUDA fixtures skipped +and three existing compiler warnings. Separate reserved-GPU runs passed the +33-test epilogue suite and 35 adapter tests, including tiled, persistent, and +TMA forward/dX/dW, FP32 gradient accumulation and AdamW, ordinary AOT, and compiled +autograd. The adapter suite includes two alignment regressions: shifted inputs +use exact ATen fallback in eager mode and match a compiled ATen control when +compiled. Inductor can realign an input before the opaque boundary, making +TMA valid; the original caller's offset need not survive compilation. + +A two-GPU runner test combined the fused persistent epilogue, persistent TMA +weight gradients, and delayed communication waits. Continuous execution matched +asynchronous save/stop/resume exactly for model weights, full FP32 AdamW state, +both ranks' RNG state, losses, and token cursors. Its final partial update also +covered a rank with entirely masked targets. The final implementation passed in +73.63 seconds; its log is +`work/validation/rtx-opt-20260912/pytest-matmul-resume-final.log`. +The CPU and alignment regression logs are `pytest-matmul-final-cpu.log` and +`pytest-offset-guard.log` in the same validation directory. These checks prove +continuation consistency within the selected execution mode, not bitwise +equivalence to a different BF16 reduction/fusion strategy. +The final CLI compatibility/signature checks passed 103 tests with eight GPU +fixtures skipped (`pytest-matmul-cli-final.log`). The complete package manifest +was verified after updating the runner's SHA256/byte count. After the final +benchmark, the production checkpoint's SHA256 still matched the paused record. diff --git a/ops/OPTIMIZATION_20260912.md b/ops/OPTIMIZATION_20260912.md new file mode 100644 index 0000000000000000000000000000000000000000..d2e1fcce8eb5fa6b96ebeee90fc7db90e99f6d3d --- /dev/null +++ b/ops/OPTIMIZATION_20260912.md @@ -0,0 +1,349 @@ +# RTX training optimization, 2026-09-12 + +The refreshed baseline delivers **30,799 tokens/s during optimizer updates** on +the two RTX PRO 6000 Blackwell Max-Q GPUs. The experiments below address +conversion and gradient accumulation, DDP communication, SwiGLU/RMSNorm, and +repeated core weight casts. The communication-overlap candidate beats the baseline in two full-model +comparisons. The workstation resume preset selects this mode; isolated kernel +improvements are not evidence of the same gain in complete training. See +[MATMUL_PROFILE_20260912.md](MATMUL_PROFILE_20260912.md) for the matrix breakdown. + +## Protected training state + +Production training was gracefully paused at step **40,081**, after +**5,253,496,832 tokens**. The next update is step 40,082. Its full resume checkpoint +is `work/run/latest.pt`, containing FP32 model parameters, AdamW moments, optimizer +step 40,081, and both ranks' RNG state. All inspected model and optimizer tensors +were finite. The model has 2,005,111,424 parameters. + +The checkpoint is 24,061,689,997 bytes; its SHA256 at the pause was +`9745d93345dff7c389ac6fa65948dc78ec021c90f56b40e9caed8ddcd84e8c54`. +The generated verification record is +[`work/validation/rtx-opt-20260912/paused-checkpoint.json`](../work/validation/rtx-opt-20260912/paused-checkpoint.json). + +The step-8,000, step-16,000, and step-40,000 archives under `work/archives/` are +immutable inputs. These experiments do not replace, re-export, or edit them. +The benchmark reads the production checkpoint through a read-only CPU mapping, +loads independent CUDA model/optimizer tensors, and writes no checkpoints. +Benchmark updates do not advance the production token cursor or RNG state. + +Production remains paused while disposable experiments run. A later production +resume must use the retained full checkpoint and record any selected execution +changes through the runner's explicit reconfiguration mechanism. + +## Baseline and profile method + +The baseline resumes the full AdamW state from step 40,081 and uses deterministic +training-token offsets. It measures 20 updates after three warmup updates: + +| Setting | Baseline | +| --- | --- | +| PyTorch / CUDA | `2.14.0+cu130` / `13.0` | +| Devices | Two RTX PRO 6000 Blackwell Max-Q GPUs | +| Sequence length / per-rank microbatch | 2,048 / 4 | +| Accumulation / global tokens per update | 8 / 131,072 | +| Core execution | `torch.compile`, default mode | +| Loss / residual activations | Compiled full-vocabulary loss / BF16 | +| Activation checkpointing | Off | +| DDP | FP32 gradients, 128 MiB buckets, bucket views | +| Optimizer | Fused AdamW, original FP32 state | +| Measured throughput | 30,799.17 tokens/s | +| Peak allocated / reserved memory, rank 0 | 65.49 / 66.44 GiB | + +This throughput includes gradient zeroing, forward/backward, accumulation, +gradient clipping, and optimizer updates. It excludes periodic production +evaluation/checkpoint pauses and initial compilation. It must not be presented +as sustained wall-clock throughput over a long training run. + +The separate profile records one complete update after two warmups. That +instrumented update took **4.1781 seconds**. Use the unprofiled 20-update baseline +for speed comparisons; the shorter profiled run has a different warmup history +and instrumentation overhead. + +| GPU activity | Rank 0 summed milliseconds | Rank 1 summed milliseconds | +| --- | ---: | ---: | +| Matrix multiplication | 3,166.45 | 3,049.22 | +| Compiled elementwise operations and reductions | 512.12 | 515.05 | +| NCCL communication | 422.56 | 532.72 | +| Other tensor kernels | 206.16 | 206.27 | +| Local attention | 99.41 | 98.82 | +| Fused AdamW | 38.50 | 38.46 | +| Explicit copies and fills | 2.96 | 3.08 | + +Matrix multiplication accounts for 71.19% and 68.62% of summed GPU activity. +The totals exceed wall time because streams overlap. Rank 0's GPU activity union +is 4.1257 seconds; its summed durations are 4.4482 seconds. Name-based kernel +categories are attribution aids, not hardware utilization or bandwidth counters. +Operator and kernel tables describe the same work and must not be added together. + +The explicit copy category excludes memory traffic inside arithmetic kernels. +It therefore does not measure total VRAM bandwidth consumption. This profile +does not establish a hardware bandwidth ceiling. + +## Work outside matrix multiplication + +The following rank-0 costs are subsets of the preceding categories, measured +over all eight accumulation microbatches and 28 cores: + +| Candidate work | Summed milliseconds per update | +| --- | ---: | +| SwiGLU forward / backward | 41.156 / 87.166 | +| Core gradient BF16-to-FP32 conversion / FP32 addition | 39.749 / 89.327 | +| Repeated core weight casts | 54.006 | +| Normalization forward / backward | 12.087 / 50.377 | +| Two groups of backward attention/projection transforms | 82.969 / 32.730 | +| BF16 attention layout copy | 14.036 | +| DDP parameter/bucket scaling, 256 calls | Approximately 7.884 | + +The conversion-plus-accumulation budget is **129.076 ms/update**. Its attribution +uses CUDA external IDs mapped to CPU operator input dimensions to distinguish +parameter gradients from activation gradients. Weight casts are the three +compiled transpose/copy families with durations 16.138, 14.107, and 23.761 ms. + +NCCL overlaps substantial computation. Intersecting communication intervals with +the union of other GPU activity leaves approximately **100.05 ms on rank 0** and +**215.80 ms on rank 1** containing only communication. These exposed intervals +are not a guaranteed speedup: moving bucket boundaries can alter overlap and +contention elsewhere in the update. + +The current compiler already fuses many operations. A custom normalization +kernel may save local kernel time while preventing a larger surrounding fusion. +The candidate budgets therefore cannot be summed into a projected speedup. + +## Implemented execution options + +The runner and disposable benchmark expose the following options. Defaults +retain the prior execution, and the runner's resume signature includes the new +settings. Legacy checkpoints normalize to these defaults; a selected change +requires explicit resume reconfiguration while preserving the objective, global +token batch, training cursor, optimizer, and schedule. + +| Option | Default | Implementation and scope | +| --- | --- | --- | +| `--gradient-mode eager\|compiled-autograd\|compiled-autograd-overlap` | `eager` | Captures backward through leaf accumulation so Inductor can fuse BF16 gradient conversion with FP32 addition. | +| `--activation-mode baseline\|swiglu\|rmsnorm\|all` | `baseline` | Optional Triton activation/normalization forwards and registered autograd backwards. | +| `--cache-core-weight-casts` | Off | Reuses differentiable BF16 core linear-weight views for one accumulation update. | +| `--ddp-bucket-cap-mb` | `128` | Changes communication grouping while retaining FP32 reduction. | +| `--ddp-batched-grad-copy` | Off | Batches gradient copying/scaling in the C++ DDP reducer. | + +Compiled autograd uses the Python DDP reducer and cannot be combined with the +C++ reducer's batched gradient copy option. The installed PyTorch can mishandle +a saved CPU scalar in compiled CUDA backward; `scale_loss()` materializes that +same scaling coefficient on the loss device for this mode. Unsupported requested +DDP options fail explicitly. The helper also supports capacity lists for +controlled programmatic experiments; the current CLI exposes a uniform capacity. + +The weight cast cache creates fresh views at the start of each optimizer update +and clears them on normal or exceptional exit, before clipping/optimizer, +evaluation, or checkpoint operations. FP32 master identities and version counters +are checked. Its current scope requires one use of each core per forward, +activation checkpointing off, and unaliased trainable linear weights. It does not +cache the tied embedding/output weight. Extra BF16 views consume VRAM; a complete +model measurement must establish their memory cost and benefit. + +These options retain BF16 tensor-core computation and BF16 activations. FP32 +master parameters, parameter gradients, normalization reductions, DDP gradient +communication, and AdamW moments retain their original representation. No FP8, +FP16, quantized weights, or reduced-precision gradient communication is introduced. +Canonical checkpoint keys and the default CPU chat/evaluation paths are unchanged. + +## Numerical checks and isolated measurements + +The activation suite passed **22 tests**, including eager and compiled CUDA +forward/backward comparisons, BF16 residual integration, FP32 AdamW, canonical +state reload, and unchanged CPU/evaluation behavior. Its final log is +`work/validation/rtx-opt-20260912/activation-tests-final.log`. + +The activation implementation distinguishes eager rounding from current compiled +arithmetic. Inductor can eliminate intermediate BF16 cast pairs; the compiled +variant follows that behavior instead of forcing additional eager truncations. +The comparison does not promise bitwise equality for every reduction shape or +future compiler configuration. + +On GPU 1, five timing trials of 20 iterations after five warmup iterations used +the actual activation shapes, seed 951, BF16 random upstream gradients, and +compiled forward plus backward. Compilation used `fullgraph=True, dynamic=False`; +CUDA events timed each trial: + +| Primitive | Compiled baseline | Custom | Numerical comparison | +| --- | ---: | ---: | --- | +| SwiGLU, packed `8192 x 14336` | 0.61335 ms | 0.59965 ms | Outputs and input gradients bitwise equal in this fixture. | +| RMSNorm, `8192 x 2048` | 0.43256 ms | 0.39248 ms | Relative RMS error: output `1.64e-6`, input gradient `1.17e-5`, FP32 weight gradient `0.001604`. | + +These are approximately 2.23% and 9.27% primitive improvements, not complete +training gains. RMSNorm's large-shape baseline can retain a final BF16 rounding +at a compiler reduction boundary; the custom compiled reduction retains FP32. +That explains the approximately 0.16% weight-gradient discrepancy and requires +whole-model numerical screening before selection. + +A separate large-linear gradient fixture verifies that compiled autograd can +replace repeated conversion/addition work with fused kernels while preserving +the accumulated gradient exactly in that fixture. Its measured cycle was +44.668 ms eager versus 41.013 ms compiled autograd. The fixture is not a complete +training update. Small-model comparisons show some changed BF16 rounding in +embedding, depth modulation, and final normalization, so exact linear gradients +do not establish bitwise equality for the whole model. + +DDP tests exercise repeated eight-microbatch accumulation and optimizer updates, +including multiple buckets. They preserve FP32 reduction and optimizer state. +Full-model two-GPU tests remain the criterion for selecting an execution change. + +## Whole-model candidate results + +All screens start from the same step-40,081 full checkpoint and token cursor, +with three warmups and ten measured updates. The initial baseline measures 20 +updates. The benchmark holds the checkpoint's learning rate fixed for every +candidate; production continues its existing schedule. The measurements below +exclude compilation, evaluation and saving. + +| Candidate / artifact directory | Tokens/s | Peak reserved/GPU | Maximum absolute loss difference from baseline | +| --- | ---: | ---: | ---: | +| Baseline, 20 updates (`baseline`) | 30,799.17 | 66.44 GiB | 0 | +| Batched DDP copies, 64 MiB (`ddp-copy64`) | 30,896.39 | 67.42 GiB | 0 | +| Compiled autograd (`compiled-autograd`) | 30,498.20 | 66.45 GiB | 0.00009394 | +| Cached core casts (`cache`) | 29,956.94 | 66.44 GiB | 0.00008345 | +| Custom SwiGLU (`swiglu`) | 30,749.24 | 66.44 GiB | 0 | +| Custom RMSNorm (`rmsnorm`) | 30,737.22 | 66.14 GiB | 0.00012708 | +| Compiled autograd with wait sinking (`ca-overlap`) | **31,331.80** | 66.98 GiB | 0.00016069 | + +A second pair of complete, unprofiled **20-update** measurements confirms the +communication fix: `ca-overlap-confirm` reaches **31,045.42 tokens/s**, versus +**30,578.17 tokens/s** for the subsequent `baseline-confirm` (**1.53% faster**). +The initial ten-update screen was 1.73% above its baseline. These are modest, +repeatable gains, not a claim of 50K throughput or long-run model quality. + +Custom activation/normalization kernels and cached casts remain opt-in +experiments. No complete-model gain justifies selecting them in this preset. +The cache's BF16 views also stop DDP's compiler partitioner from counting those +inputs as `nn.Parameter`s. Generated layouts match baseline, but whole-core +backward regions can delay leaf-gradient readiness and change fusion boundaries. +Do not attribute that regression to a confirmed matrix-transpose problem. + +### Why fusion alone lost, and the communication fix + +Compiled autograd removes nearly all native gradient-add kernels. Rank 0's +other-kernel category falls from 206.17 to 51.36 ms while compiled elementwise +work rises from 512.12 to 585.32 ms, saving roughly 82 ms across those categories. +However, its default schedule puts each wait and gradient copy immediately after +the collective, before independent backward GEMMs. The 57 gradient collectives +carry the same 8,020,445,696 FP32 bytes. + +| Communication time/update | Baseline | Compiled autograd | With wait sinking | +| --- | ---: | ---: | ---: | +| Overlapping compute, rank 0 | 322.51 ms | 0 ms | **190.74 ms** | +| Overlapping compute, rank 1 | 316.91 ms | 0 ms | **190.58 ms** | +| Communication only, rank 0 | 100.05 ms | 285.17 ms | **166.64 ms** | +| Communication only, rank 1 | 215.80 ms | 407.35 ms | **267.61 ms** | + +`--gradient-mode compiled-autograd-overlap` enables Inductor's +`sink_waits_iterative` scheduler. It moves waits together with their dependent +copies past independent computation, retaining collective launch order and +requiring no increase in the scheduler's **estimated** peak memory. It does not +permit unsafe collective reordering or moves past subsequent collectives. +Actual allocator peaks remain measured separately. + +The two CA profiles contain identical kernel counts: 5,400 matmuls, 59 +communication events, 9,577 compiled pointwise/reduction kernels and 1,531 other +kernels per rank. Profiled update time changes from 4.25901 to 4.12683 seconds; +the gain is restored overlap, not less arithmetic. Compared with baseline, +52–67 ms more communication remains exposed. That is potential scheduling work, +not a guaranteed further saving. + +Cold first-update compilation took approximately 195–196 seconds in these +full-model CA screens. A cached second process still incurs graph capture and +compiler setup; restarts can spend minutes before regular updates appear. + +Whole-backward compilation can fuse a tied-head BF16 matrix product and gradient +addition into BF16 `addmm`, changing an intermediate rounding boundary. Original +FP32 master weights, gradients, AdamW state and communication remain FP32; GEMM +operands and outputs remain BF16. Short loss/gradient comparisons use explicit +numerical tolerances rather than claiming a bitwise baseline trajectory. + +The CLI rejects weight cast caching with either compiled-autograd mode, and +rejects either feature with `--compile-mode reduce-overhead`. Cache plus +activation checkpointing is rejected before loading; it requires +`--no-activation-checkpointing`. Batched DDP copying uses the C++ reducer and is +incompatible with both compiled-autograd modes' Python reducer. + +### Validation and selected preset + +The completed CPU suite passed **159 tests** (28 CUDA tests skipped in that +CPU-only invocation). Separate activation, gradient and cast-cache CUDA checks +passed. The final two-GPU runner suite passed **both** continuation cases in +136.79 seconds: combined custom activations/cached casts/batched DDP copies, and +compiled autograd with communication overlap. Each compares uninterrupted +training against an asynchronous stop/resume, including an entirely masked +final rank. Model tensors, full FP32 optimizer state and both ranks' RNG states +match **bitwise** between continuous and resumed execution of the same mode. +This is separate from the bounded numerical differences versus baseline. + +`train-local-resume.sh` selects `--gradient-mode compiled-autograd-overlap`. +Activation kernels, external weight-cast caching and batched DDP copies remain +at their baseline settings. BF16 compute, FP32 state, the global token batch, +context, data, optimizer schedule and save/evaluation intervals are preserved. +Production remains paused at step 40,081 until resumption is requested. + +Follow-up scheduling controls and paired measurements are documented in +[COMMUNICATION_EXPLORATION_20260912.md](COMMUNICATION_EXPLORATION_20260912.md). +The controls are opt-in; the preset above keeps its conservative defaults. +The separate [backward layout experiment](BACKWARD_LAYOUT_EXPERIMENT.md) +measures a fused rotary-gradient and projection-packing region without changing +production model execution. + +Test logs are `work/validation/rtx-opt-20260912/pytest-communication-final.log` +and `work/validation/rtx-opt-20260912/pytest-optimized-resume.log`. + +## Reproducing and locating results + +All paths above are relative to `rtx6000_40b/`. Source the workspace environment +before any test or benchmark. GPU experiments require training paused and an +exclusive device window. Run each candidate in a fresh process and a new output +directory because compiler/reducer settings are process-wide. + +The baseline can be reproduced with: + +```bash +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +.server-venv/bin/torchrun --standalone --nproc-per-node=2 \ + ops/benchmark_training.py \ + --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \ + --no-activation-checkpointing --autocast-cache \ + --compile-cores --compile-mode default --loss-mode compiled-full \ + --bf16-residuals --optimizer-state resume --vram-limit-gib 88 \ + --warmup-updates 3 --measured-updates 20 \ + --output-dir work/benchmark-opt-20260912/baseline-recheck +``` + +Use a new output directory and append the candidate flags for each comparison. +For the profile, use two warmup updates, one measured update, and `--profile`. +The original run artifacts are: + +- `work/benchmark-opt-20260912/baseline/summary.json`: configuration, per-rank + update measurements, source hashes, checkpoint identity, and memory peaks. +- `work/benchmark-opt-20260912/profile-baseline/`: both ranks' Chrome traces, + operator tables, kernel TSV files, and summary/category JSON. +- `work/validation/rtx-opt-20260912/activation/actual-shape-compiled-arithmetic.json`: + final primitive timings and random-gradient differences. The earlier + `actual-shape-kernels.json` tested eager rounding against compiled math and is + superseded for comparing the current implementation. +- `work/validation/rtx-opt-20260912/gradient/cuda-linear-proof.json`: isolated + accumulation timing, gradient equality, and kernel trace counts. +- `work/validation/rtx-opt-20260912/gradient/cuda-gradient-deltas.json`: initial + small-model per-parameter differences for compiled autograd. + +The source implementations are `activation_execution.py`, +`gradient_execution.py`, `weight_cast_execution.py`, and `distributed_execution.py` +in this directory. Their focused tests live at the package root. Generated +benchmarks, traces, validation outputs, data, and checkpoints remain ignored in +Git; this document records methods and findings for source tracking. + +The subsequent paired attention-backward integration and separate gradient +producer-scheduling experiment are documented in +[BACKWARD_LAYOUT_INTEGRATION.md](BACKWARD_LAYOUT_INTEGRATION.md), with gradient +and checkpoint-continuation checks in +[BACKWARD_LAYOUT_TRAINING_VALIDATION.md](BACKWARD_LAYOUT_TRAINING_VALIDATION.md). +The completed integration selects `--backward-layout paired` in +`train-local-resume.sh`: repeated update throughput improves from 31,001.57 to +31,199.03 tokens/second (+0.637%). The separate producer scheduler remains +disabled after a smaller, less consistent gain. Production remains paused. diff --git a/ops/OPTIMIZED_TRAINING.md b/ops/OPTIMIZED_TRAINING.md new file mode 100644 index 0000000000000000000000000000000000000000..0cccdb9c15bab725bdad18877262d1056b2c89cd --- /dev/null +++ b/ops/OPTIMIZED_TRAINING.md @@ -0,0 +1,228 @@ +# Training with both GPUs available + +**Currently paused at the user's request.** Training and benchmarks are +stopped so another session can resume VORTEX. The latest full checkpoint is +step 448 / 58,720,256 tokens. Start commands below are for a future +user-requested resume after checking GPU availability. + +The selected configuration measured **29,490 training tokens/second** on two +NVIDIA RTX PRO 6000 Blackwell Max-Q GPUs, with approximately **68.8 GiB reserved +and 25 GiB free per GPU**. The measurements below were taken on September 9, +2026, after VORTEX and the previous WarpState training process had stopped. + +See [PERFORMANCE.md](PERFORMANCE.md) for the subsequent bottleneck analysis, +GPU profiling method, and the compute requirements of a 100K-token/second +target. + +## Selected configuration + +[`train-fast.sh`](train-fast.sh) launches two workers through +[`launch_training.py`](launch_training.py). Each worker holds the complete +2,005,111,424-parameter model and AdamW state; DDP synchronizes gradients. + +| Setting | Value | +| --- | --- | +| GPUs / workers | 2, one worker per GPU | +| Sequence length | 2,048 tokens | +| Microbatch per GPU | 4 sequences | +| Gradient accumulation | 8 microbatches per GPU | +| Global tokens per optimizer update | **2 × 4 × 2,048 × 8 = 131,072** | +| Parameters, gradients, AdamW moments | FP32 on each GPU | +| Matrix computation | BF16 autocast, weight cache enabled | +| Compilation | Each core's training forward, full graph, fixed shapes | +| Core activation checkpointing | Disabled | +| Loss projection chunk | 1,024 tokens; loss projection recomputation remains enabled | +| DDP gradient buckets | 128 MiB target, gradients use bucket views | +| PyTorch allocator cap | 88 GiB per GPU | +| Additional free memory required at startup | 3 GiB per GPU | +| Validation | 8 batches of 1 sequence, initially and every 100 updates | +| Checkpoint | Every 50 updates, on graceful stop, and at completion | + +The global batch, sequence length, data cursor, optimizer state, and learning +rate schedule are preserved when moving from the previous shared-GPU run. +The launcher's `--reconfigure` allows execution changes while the runner +checks those invariants and records the transition in `execution_history`. +Compilation and changed batch grouping can change floating-point rounding; +this is not a promise of bitwise equivalence across configurations. + +The allocator cap excludes CUDA context and library allocations. The startup +headroom check does not reserve memory for another process. This configuration +assumes VORTEX remains paused and both GPUs are available. + +## Measured configuration grid + +All rows use sequence length 2,048, 131,072 global tokens per update, BF16 +autocast with its cache enabled, FP32 parameters and full fused AdamW state. +Memory is the largest steady reserved allocation on any participating GPU. +The single-GPU trials used a 90 GiB allocator cap; DDP trials used 88 GiB. + +| GPUs | Microbatch/GPU | Core execution | Core checkpointing | Loss chunk | Tokens/s | Reserved/GPU | +| ---: | ---: | --- | --- | ---: | ---: | ---: | +| 1 | 2 | Eager | Off | 1,024 | 12,180 | 51.72 GiB | +| 1 | 4 | Eager | Off | 1,024 | 12,252 | 68.83 GiB | +| 1 | 8 | Eager | Off | 1,024 | Out of memory | Exceeded 90 GiB cap | +| 1 | 8 | Eager | On | 1,024 | 9,249 | 45.13 GiB | +| 1 | 4 | Compiled cores | Off | 1,024 | 15,142 | 60.44 GiB | +| 1 | 8 | Compiled cores | Off | 1,024 | 15,014 | 86.08 GiB | +| 1 | 4 | Compiled cores | Off | 8,192 | 15,321 | 62.89 GiB | +| **2** | **4** | **Compiled cores** | **Off** | **1,024** | **29,490** | **68.76 GiB** | +| 2 | 2 | Compiled cores | Off | 1,024 | 28,140 | 55.33 GiB | + +Compiled cores improved single-GPU throughput by about 24% over eager +microbatch 4 and reduced reserved memory by about 8.4 GiB. Increasing the +compiled microbatch to 8 consumed another 25.6 GiB without improving speed. +Activation checkpointing saved memory but repeated enough computation to +make the tested eager microbatch-8 configuration slower. + +The 8,192-token loss chunk improved single-GPU throughput by only 1.2%, used +another 2.5 GiB, and produced slightly different gradient rounding. Retaining +1,024 keeps the existing loss computation closer without a material measured +speed penalty. Among the two-GPU trials, microbatch 4 was 4.8% faster than +microbatch 2 and still left substantial memory available. + +`nvidia-smi topo -m` reported a **SYS** connection between these GPUs: traffic +crosses PCIe and the CPU interconnect between NUMA nodes. There is no NVLink +connection between this pair. Despite that topology, the measured compiled +microbatch-4 DDP run achieved **1.95×** the corresponding single-GPU throughput. + +## Measurement method and limits + +[`benchmark_training.py`](benchmark_training.py) loaded the trusted local +checkpoint `work/run/before-optimization-step-237.pt`, an immutable hardlink +to the checkpoint at step 237 and 31,064,064 training tokens. It mmap-loaded +the checkpoint on the CPU, copied model parameters and every optimizer +tensor into separate CUDA storage, and optimized those disposable copies. +It read the prepared corpus in place and never saved a training checkpoint. + +Each case performed complete forward/backward, gradient clipping, and fused +AdamW updates over 131,072 global tokens. Timings include data batching and +device transfer. DDP timings include gradient communication and use the +slowest rank's elapsed time. Compilation and the first update were excluded +as warmup; DDP excluded two warmup updates to allow gradient bucket rebuilding. +Every successful result aggregates **three measured updates**. The full +14.94 GiB of FP32 Adam moments was present on each GPU, and all measured +losses and gradient norms were finite. + +These are short steady-compute samples, not long-run throughput guarantees. +They exclude checkpoint loading/saving, validation, process startup, and +initial compilation. At an unchanged 29,490 tokens/second, 40B tokens take +about 15.7 days of training compute; operational overhead adds time. + +Generated evidence remains ignored by Git: + +- `work/benchmark-grid-summary.json`: all nine cases and throughput ranges. +- `work/benchmark-*/summary.json`: exact arguments, source hashes, checkpoint + identity, per-update times, and first-update/steady memory peaks. +- `work/benchmark-*/rank-*.jsonl`: individual worker measurements. +- `work/benchmark-eager-mb8/failure.json`: the bounded allocation failure. + +The source hashes in each result identify the files present for that run. +Runner source evolved during this optimization task; the full-model benchmark +used its own update loop and the same batching/model computation throughout. + +## Live continuation and validation + +The production run resumed at step 237 with 31,064,064 tokens consumed and +completed twelve updates using the selected configuration. Steady updates +ran at approximately 29.0–29.7K tokens/second, with 67.75 GiB allocated, +68.64 GiB reserved, and 25.31 GiB physically free per GPU. The previous +shared settings measured 8.5K tokens/second after VORTEX stopped and +5.5–6K while VORTEX was training. + +A real `datactl.sh stop warpstate-train` completed the current update and +saved step 249 at 32,636,928 tokens. Both workers exited and released GPU +memory. Inspection of the 22.41 GiB checkpoint confirmed all 256 canonical +model tensors, full FP32 Adam moments and step counters, both rank random +states, the original initialization metadata, and the execution transition +at step 237. Restart restored validation loss 5.56880 from that checkpoint. +The original step-237 checkpoint remains at +`work/run/before-optimization-step-237.pt`. + +Verification also passed 64 CPU tests and four CUDA integration tests +(two original single-GPU cases and two distributed cases). These cover +exact resume within one configuration, one-to-two-GPU continuation, +the final partial update with an empty rank, and graceful/forced worker +cleanup. A full-model eager-versus-compiled comparison had a maximum +loss difference of 0.000012 and relative gradient-norm difference below +0.072% across three updates. Initial validation across the two execution +configurations was 5.63291 versus 5.63279, consistent with small rounding +differences. + +The generated live checkpoint verification is recorded in +`work/optimization-resume-validation.json`. + +## Production controls + +In optimized mode, the `warpstate-train` program in +[`supervisord.conf`](supervisord.conf) must point to `ops/train-fast.sh`. +Use the project-private Supervisor through `datactl.sh`: + +```bash +cd /workspace/WarpState +rtx6000_40b/ops/datactl.sh status +rtx6000_40b/ops/datactl.sh start warpstate-train +tail -n 10 rtx6000_40b/work/logs/train.log +tail -n 3 rtx6000_40b/work/run/metrics.jsonl +``` + +To stop and preserve progress: + +```bash +rtx6000_40b/ops/datactl.sh stop warpstate-train +``` + +The launcher receives the stop signal and creates +`work/control/training-stop-request`. Both workers finish the current update; +the runner saves the model, full optimizer state, progress, and rank random +states before exiting. The launcher allows up to 820 seconds for this, +followed by 45 seconds for termination if workers fail to exit. Supervisor's +stop timeout is 900 seconds. Use the launcher/Supervisor control rather than +signalling an individual DDP worker. Starting the same program resumes +`work/run/latest.pt`; do not delete the checkpoint to change execution options. + +Keep at least 53 GiB free for a full checkpoint and atomic replacement. +Training artifacts, caches, and runtime control files remain inside +`/workspace/WarpState` and are excluded from Git. + +## Reproducing benchmarks + +**Stop training first and wait until both GPUs are free.** A benchmark uses +disposable model state, but it still competes for GPU memory and computation. +VORTEX must also remain paused. From the repository root: + +```bash +rtx6000_40b/ops/datactl.sh stop warpstate-train +rtx6000_40b/ops/datactl.sh status +nvidia-smi +rtx6000_40b/ops/datactl.sh start warpstate-benchmark +tail -n 10 rtx6000_40b/work/logs/benchmark.log +``` + +[`benchmark-supervisord.conf`](benchmark-supervisord.conf) specifies the +winning two-GPU configuration and sets `autostart=false` and +`autorestart=false`. Its command, expressed with environment variables, is: + +```bash +cd /workspace/WarpState/rtx6000_40b +WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 \ +TORCHINDUCTOR_COMPILE_THREADS=8 ops/benchmark.sh \ + --checkpoint /workspace/WarpState/rtx6000_40b/work/run/before-optimization-step-237.pt \ + --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \ + --no-activation-checkpointing --compile-cores --autocast-cache \ + --loss-chunk-size 1024 --vram-limit-gib 88 \ + --warmup-updates 2 --measured-updates 3 +``` + +Use Supervisor for routine runs; the command above documents its effective +arguments. `benchmark.sh` sources `training-env.sh` and launches `torchrun`. +Each invocation automatically receives a fresh +`work/benchmark-` directory. An explicit `--output-dir` +must name a new directory under `work/benchmark-*`; existing rank log files +are not overwritten. If reproducing against another trusted checkpoint, +change the configured checkpoint path and record which checkpoint was used. + +For a different candidate, edit only the benchmark program's command or +environment, then apply it with `ops/datactl.sh reread` and +`ops/datactl.sh update warpstate-benchmark` from the package directory. +Starting that program runs the selected finite benchmark. Once it has exited, +check `nvidia-smi` and restart `warpstate-train` to resume production. diff --git a/ops/PERFORMANCE.md b/ops/PERFORMANCE.md new file mode 100644 index 0000000000000000000000000000000000000000..f6a5747c2a35e595699d5d8eebbd32e3c99089af --- /dev/null +++ b/ops/PERFORMANCE.md @@ -0,0 +1,236 @@ +# WarpState training bottlenecks + +For the current single-B200 measurements and selected preset, see +[B200_PERFORMANCE.md](B200_PERFORMANCE.md). This document records the original +workstation experiments. + +**Paused at the user's request:** production and benchmarks are stopped, +with the full production checkpoint saved at step 448 / 58,720,256 tokens. +Another session will resume VORTEX. Do not start the reproduction commands +until the user requests further WarpState work and the GPUs are available. + +Measurements and analysis from September 9, 2026, using the configuration in +[OPTIMIZED_TRAINING.md](OPTIMIZED_TRAINING.md): two RTX PRO 6000 Blackwell +Max-Q GPUs, 300 W each, compiled cores, BF16 matrix computation, FP32 AdamW, +microbatch 4 per GPU, accumulation 8, sequence length 2,048. + +## Sustained throughput + +Production updates 251–315 averaged 28,928 training tokens/second inside +the timed update loop. Their median update took 4.544 seconds for 131,072 +global tokens. The interval between completed updates averaged 26,711 +tokens/second including the validation/checkpoint pause after step 300. +That pause produced a 28.31-second gap between updates 300 and 301. + +The GPUs had about 68.64 GiB reserved and 25.31 GiB physically free each. +They were busy at the 300 W power limit, with power-cap limiting observed +and no thermal slowdown. Available VRAM does not imply available compute. +The driver reported a maximum configurable power limit of 325 W; that is +only 8.3% above the current power allowance, and does not imply an equal +throughput gain. Hardware settings were left unchanged. + +## GPU profile + +One full update was profiled after two warmup updates on disposable copies +of checkpoint step 357. The update took 4.452 seconds. Rank 0 showed: + +| GPU activity | Summed activity time | +| --- | ---: | +| Matrix multiplication | 3.215 seconds | +| Compiled pointwise functions and reductions | 0.561 seconds | +| NCCL communication | 0.439 seconds | +| Local attention | 0.100 seconds | +| Fused AdamW kernels | 0.039 seconds | +| Other kernels and copies | Approximately 0.382 seconds | + +These are actual CUDA kernel/copy activities, excluding nested GPU annotation +ranges. They sum to more than elapsed time because streams overlap. Across +the two ranks, matrix multiplication was approximately 65–68% of summed +activity time and attention approximately 2.1%. Operator shapes attribute +1.888 seconds on rank 0 to FFN matrix multiplication, 0.919 seconds to the +input projection, and 0.141 seconds to the full-vocabulary head's matrix +multiplications. Normalization, gating, state operations, casts, gradient +accumulation, and other memory traffic account for much of the remaining +work. GPU utilization near 100% does not mean peak tensor arithmetic +throughput. + +Communication overlaps computation, so its summed time is not the amount +recoverable by removing it. The previous two-GPU benchmark scaled 1.95 times +the equivalent single-GPU configuration. Neither more data-loader workers +nor a different attention implementation addresses the dominant matrix cost. + +The raw evidence is under `work/benchmark-profile-20260909`. Profiler +instrumentation was confined to the benchmark; production resumed from +the unchanged step-357 checkpoint afterward. + +## Compute required for 100K tokens/second + +The following counts use two FLOPs per multiply-add. Let `d=2048`, +`f=7168`, `L=28`, and `V=32768`. Each core has an input projection with +`5*d*d` weights, an output projection with `d*d` weights, and SwiGLU +matrices with `3*d*f` weights. Forward and backward together require +approximately six FLOPs per dense matrix weight per token. + +| Matrix computation | Billion FLOPs / training token | +| --- | ---: | +| FFNs: `6*L*3*d*f` | 7.398752 | +| Core input/output projections: `6*L*6*d*d` | 4.227858 | +| Tied output head with forward recomputation: `8*d*V` | 0.536871 | +| Parallel tensor-memory operations | 0.049545 | +| Local causal attention, including backward | Approximately 0.044–0.103 | +| **Total matrix computation** | **Approximately 12.26–12.32** | + +The attention range reflects causal triangular arithmetic versus tiled work +and backward recomputation. These counts exclude normalization, nonlinear +functions, casts, optimizer work, and communication. Core activation +checkpointing is disabled; the chunked loss projection still recomputes +its forward during backward. + +At 100,000 tokens/second, the matrix operations alone require approximately +1.23 PFLOPS. NVIDIA specifies 438.9 TFLOPS of dense BF16 with FP32 +accumulation per Max-Q GPU, or 877.8 TFLOPS for this pair. Thus the +advertised ideal ceiling is about 71K tokens/second before other work. +Actual clocks, kernel efficiency, and memory traffic reduce that ceiling. +This rules out 100K through ordinary BF16 configuration tuning for the +unchanged dense computation. NVIDIA's 3,511 AI TOPS figure refers to sparse +FP4 and does not describe this BF16 training. See the +[NVIDIA architecture whitepaper, Appendix A, pages 45–46](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1.0.pdf#page=45). + +## What to investigate next + +Completed BF16 experiments used disposable copies of the same step-448 +checkpoint, two warmup updates and three measured updates per case: + +| Configuration | Tokens/second | Peak reserved/GPU | +| --- | ---: | ---: | +| Existing compiled-core baseline | 29,396 | 68.64 GiB | +| Full loss without recomputation | 29,576 | 72.03 GiB | +| Compiled full loss | 30,315 | 68.99 GiB | +| Compiled full loss + cuBLASLt / 32 MiB workspace | 30,329 | 68.99 GiB | +| Compiled full loss + matrix autotuning | 30,403 | 68.99 GiB | +| Compiled full loss + BF16 residual activations, microbatch 4 | 31,111 | 66.42 GiB | + +The 50K target was not reached. None of these experimental configurations +was promoted to production. The microbatch-8 residual trial has no completed +summary and must not be treated as a measured result. All benchmark losses +and gradient norms in the completed cases were finite. The compiled-full +cases differed from baseline by up to 0.00078 loss and 2.47% gradient norm +after several updates; execution changes are not bitwise equivalent. + +The added loss and residual helpers passed CPU numerical/parameter-identity +checks, and the full-model benchmarks exercised their GPU execution. +The new compiled-full two-GPU checkpoint/resume regression and a production +continuation/validation trial remain outstanding. Complete those before +promoting a new execution configuration. The production launcher retains +its previously validated BF16 settings. No FP8 GPU test or training ran. +At pause handoff, the combined CPU suite passed 84 tests; five CUDA cases +were skipped to leave the GPUs available for VORTEX. + +The next requested target is 50K tokens/second, about 1.73 times the current +steady rate. It is below the advertised ideal BF16 ceiling, but attaining it +in BF16 would require unusually high sustained efficiency at this power cap. +As a rough illustration, halving the observed 3.215 seconds of matrix work +while leaving everything else fixed would reduce a 4.452-second update to +about 2.845 seconds, or 46K tokens/second. Overlap and power behavior make +this an illustration, not a measured prediction. Reaching 50K likely needs +both faster matrix computation and reductions in the remaining costs. + +The requested constraint is BF16 matrix computation with FP32 master +weights and optimizer state. The experiments therefore focus on selecting +faster BF16 matrix kernels, using the spare memory to remove loss +recomputation, and reducing activation-memory traffic. Lower-precision +matrix training is outside this task's scope. + +A fused linear cross-entropy implementation could reduce repeated output +projection and intermediate-memory traffic. The current 32,768-way head +is only 4.4% of the counted dense linear FLOPs, although its memory costs +can consume a larger fraction of execution time. An implementation such as +[Cut Cross Entropy](https://github.com/apple/ml-cross-entropy) needs a +numerical comparison and an end-to-end benchmark; its default gradient +filtering should not be silently introduced into this pretraining run. + +Less frequent full checkpoints can recover some wall-clock throughput. +The current interval is 50 updates, about four minutes of compute. Raising +that interval increases the amount of work lost after an abrupt failure. +Checkpoint frequency does not change tokens/second inside the update loop. + +Reaching 100K could instead use more aggregate GPU compute, or a smaller, +low-rank, or conditionally activated model. Those architecture changes +alter the experiment and require their own training/quality comparison. +VORTEX uses low-rank projections, conditional parameter pages, and a +hierarchical output head. Its total parameter count does not imply the +same computation per token as WarpState's 28 dense cores. + +## Planning for one B200 GPU + +The user specified one B200 GPU and BF16 training. NVIDIA lists 180 GB +HBM3e, approximately 7.7 TB/s memory bandwidth, and 2.25 PFLOPS dense +BF16 for one HGX B200 GPU. The larger 4.5 PFLOPS BF16 number uses +sparsity. These specifications differ from eight-GPU DGX totals. Sources: +[NVIDIA Blackwell datasheet](https://dam-cdn.nvd.orangelogic.com/AssetLink/y441155802qub41q118b2852i557jem5.pdf), +[NVIDIA performance reference](https://github.com/NVIDIA/exemplar-performance/blob/main/README.md#peak-theoretical-throughput). + +For this custom model, 60–90K BF16 training tokens/second is an engineering +planning estimate, not a measured range or guarantee. The 75K midpoint +roughly preserves the current fraction of advertised compute throughput; +bandwidth, larger useful microbatches, B200 kernel selection, and removal +of two-GPU gradient synchronization can change the result. The theoretical +matrix-only ceiling is approximately 183K tokens/second. Reaching 100K +would need roughly 55% of advertised dense BF16 compute before accounting +for non-matrix work, so it is a plausible tuning target on B200 that still +requires an actual benchmark. + +| Assumed update throughput | Training tokens/day | Compute time for 40B | +| ---: | ---: | ---: | +| 60K/s | 5.18 billion | 7.7 days | +| 75K/s | 6.48 billion | 6.2 days | +| 90K/s | 7.78 billion | 5.1 days | +| 100K/s | 8.64 billion | 4.6 days | + +Checkpointing and validation add elapsed time to these compute estimates. +The current 50-update checkpoint interval becomes relatively more +expensive as updates get faster. + +Storage is independent of GPU choice. In decimal GB, the prepared corpus +uses 80.002 GB, a complete model/Adam checkpoint uses 24.062 GB, and atomic +replacement needs another 24.062 GB temporarily. The current Python +environment is approximately 5.7 GB. Extra retained checkpoints each add +24.062 GB. A deployment with token data, one checkpoint, and its replacement +therefore needs roughly 134 GB before OS, cache, and reserve space. The +runner also checks for about 53 GiB free at startup for checkpoint writing. + +Allocate **250 GB as a practical minimum**, or **500 GB NVMe SSD for +comfortable checkpoint retention and working space**. This assumes copying +the already prepared tokens, not retaining raw source downloads. A 1 TB +SSD is not required for this 40B-token run. SSD capacity does not cap the +lifetime number of training tokens: further data can be prepared or +streamed after old data is released. + +## Reproducing the GPU profile + +Stop production gracefully and wait for both GPUs to be free before running +this finite benchmark. It reads the latest checkpoint into disposable CUDA +copies and writes no training checkpoint. VORTEX must remain paused. + +```bash +cd /workspace/WarpState/rtx6000_40b +ops/datactl.sh stop warpstate-train +nvidia-smi +WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 \ +TORCHINDUCTOR_COMPILE_THREADS=8 ops/benchmark.sh \ + --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt \ + --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \ + --no-activation-checkpointing --compile-cores --autocast-cache \ + --loss-chunk-size 1024 --vram-limit-gib 88 \ + --warmup-updates 2 --measured-updates 1 --profile +ops/datactl.sh start warpstate-train +``` + +Only the first measured update is profiled. Each rank exports a Chrome +trace, an operator table grouped by shape, a kernel table, and a JSON +summary under its fresh `work/benchmark-*` directory. Profiling collects +CPU and CUDA activity with shapes, without Python stacks or memory +tracking. Use unprofiled runs to measure throughput: recording adds +overhead. CUDA kernels on different streams can overlap, so summed kernel +time is not the same as elapsed time. Operator and kernel tables describe +the same work and must not be added together. diff --git a/ops/POINTWISE_HEADROOM_20260913.md b/ops/POINTWISE_HEADROOM_20260913.md new file mode 100644 index 0000000000000000000000000000000000000000..775f9faed0403a821c8dcda6b2123baa856f3928 --- /dev/null +++ b/ops/POINTWISE_HEADROOM_20260913.md @@ -0,0 +1,195 @@ +# Remaining pointwise work and GEMM saturation — September 13, 2026 + +This analysis uses the saved September 12 full-model profile of the selected +paired backward layout. It does not launch a new GPU profile, pause production, +or change the training recipe. At the initial status check, production was +running past step 53,854, at approximately 31.2K tokens/second during updates. + +## Timing scope + +The profile is one update after eight warmups, with two RTX PRO 6000 Blackwell +Max-Q GPUs, microbatch four per rank, sequence 2,048, eight accumulation +microbatches, and 131,072 global tokens. Model computation uses BF16; master +parameters, accumulated gradients, communication, and AdamW state use FP32. +The communication preset uses conservative wait sinking and 128 MiB buckets. + +The profiled update lasts 4.16463 seconds. Repeated unprofiled measurements of +this preset averaged 4.20116 seconds, or 31,199.03 tokens/second. Profiles include +instrumentation; neither number includes periodic validation/checkpoint pauses. +Kernel durations from separate streams can overlap and must not be summed into +elapsed time without interval accounting. Individual operator totals and their +kernel totals describe the same work. + +## Remaining pointwise and reduction work + +The name-based compiled-pointwise/reduction category contains 475.795 ms on +rank zero and 479.331 ms on rank one. This includes arithmetic, conversions, +copies, and reductions fused into generated kernels. It excludes the selected +custom paired layout, whose two kernels take approximately 78.96 ms per rank. +Consequently, neither the small explicit memcpy category nor the 476 ms category +alone describes all memory traffic or all work outside matrix multiplication. + +The largest individually attributed groups in that category are: + +| Work | Rank 0 ms/update | Rank 1 ms/update | Current boundary | +| --- | ---: | ---: | --- | +| SwiGLU backward | 85.70 | 85.96 | Packed activation-gradient output | +| Parameter gradient conversion/accumulation | 69.55 | 70.29 | Includes already fused FP32 accumulation | +| Core normalization/residual backward | 68.42 | 68.10 | Several operations share each kernel | +| Attention/memory forward transforms | 66.84 | 69.35 | Rotary, gating, and layout work | +| Core FP32-to-BF16 weight casts | 49.59 | 49.54 | Four weight families, eight microbatches | +| SwiGLU forward | 49.20 | 49.13 | Already fused SiLU and multiplication | +| Core normalization forward | 11.20 | 11.18 | Includes fused surrounding work | + +These are disjoint groups within the compiled category, not the full category. +Attention/memory backward transforms, residuals, bucket operations, loss, and small +parameter reductions account for the remainder. Attribution groups profiler +operators by complete input shapes and checked generated-code bodies. The same +generic kernel number can represent different operations in different graphs; +names alone do not establish the mapping. The generated report is +`work/validation/pointwise-headroom-20260913/semantic-families.json` and retains +the explicit per-operator mapping, input shapes, source paths, and provenance. + +Cutting rank zero's 475.795 ms budget by 25% would remove about 119 ms. If every +saved millisecond reduced elapsed time and other execution stayed unchanged, +4.20116 seconds would become 4.08221 seconds: approximately 2.9% more throughput. +This is conditional arithmetic, not a measured candidate or a promised gain. + +Prior experiments already tested standalone SwiGLU and RMSNorm replacements. +SwiGLU forward/backward improved from 0.61335 to 0.59965 ms in its isolated +production-shaped fixture; RMSNorm improved from 0.43256 to 0.39248 ms. Neither +standalone mode improved the full-model screen. External weight-cast caching +also regressed overall and changed compiler/DDP partition boundaries. Their +switches are not supported with the selected compiled-autograd configuration. +See [OPTIMIZATION_20260912.md](OPTIMIZATION_20260912.md). + +The earlier fused SwiGLU-forward/TMA-weight-gradient recipe produced one +0.43% +full-model comparison after correcting duplicated weight storage. It lacks a +repeated gain and has not been combined with the selected paired layout. +The isolated epilogue's 45-microsecond gain predicts only about 10 ms per update +before interactions. See [MATMUL_TUNING.md](MATMUL_TUNING.md). + +### Best focused next experiment + +Start with the **85.7 ms SwiGLU backward region**, using a replacement inside +the compiled backward graph rather than the previously screened standalone +activation mode. Its generated kernel traverses a packed `8192 x 14336` output +and selects between the two derivative halves. The source has five masked +loads across those branches. A paired-output implementation can load the +up value, gate value, and upstream derivative once for each logical pair, then +write both derivative halves. This targets repeated operand access and branch +work while retaining native packed storage and the existing GEMMs. +The existing `activation_execution.py` already contains a paired backward +kernel. First compare that arithmetic and layout directly against this actual +compiled region; a new primitive implementation is not the starting requirement. +The integration route would preserve the surrounding compiled-autograd graph. + +Arithmetic must follow the compiled reference. The up derivative uses +`g / (1 + libdevice.exp(-g))`, while the gate derivative uses `tl.sigmoid(g)`. +Sharing one sigmoid value by algebraic rewriting is not automatically equivalent +to those two implementations. Preserve their evaluation and BF16 output rounding +unless a separately declared numerical comparison justifies a change. + +The relevant body is `triton_poi_fused_5` near line 436 of +`.cache/torchinductor/cw/ccwrp762vinzoemcnamsrl2wt54aoanpxjxxqobh7e7oxxutputc.py`. +Its synchronized variant is classified separately by matching operand shapes. +The source suggests an opportunity; it does not prove an instruction-count, +physical-bandwidth, or elapsed-time saving. A 25% reduction of this region +would save about 21.4 ms per update, approximately 0.5% more throughput if fully +realized. No new candidate kernel was timed or selected in this analysis. + +The next candidate is tuning the **68.4 ms normalization/residual backward +group as a whole**. The existing kernels already fuse FP32 row reductions, +parameter-gradient partials, residual additions, and BF16 output stores. +Tune tiling and intermediate-buffer traffic while preserving those combinations; +the entire group is not removable normalization overhead. +One specific subregion accounts for approximately 24.5 ms and loops twice over +2,048 channels, rereading inputs for the normalization input derivative and +residual output. A persistent row kernel could retain loaded intermediates, +but increased register pressure could offset the saved reads. This is a narrower +test than replacing every normalization operation. + +Gradient conversion/accumulation is already fused after many matrix products. +Further integration into a GEMM epilogue must preserve the BF16 product's +rounding before FP32 accumulation and keep gradient readiness/communication +ordering. Hoisting weight casts across accumulation microbatches could reduce +the 49.6 ms cast budget, but must solve the prior autograd/DDP-boundary regression. +These are more involved follow-ups than the isolated SwiGLU backward replacement. + +Use the same sequence as the selected layout integration: exact-shape primitive +comparison, guarded graph matching and runtime dispatch evidence, full gradient/ +optimizer/resume fixtures, then repeated complete-model timing. An isolated +kernel gain alone is insufficient to change the training preset. + +## Where matrix time is spent + +Every one of the 5,400 matrix kernels per rank is joined by its CUDA External ID +to the originating `aten::mm`, `aten::bmm`, or `aten::addmm` event. Exact operand +shapes and strides identify families and forward/input-gradient/weight-gradient +directions. This includes all eight vocabulary-head weight-gradient ADDMM calls. +The calculation counts 791.597 trillion matrix FLOPs per rank per update. +All matrix kernels use the same compute stream in these traces. + +| Matrix family | Rank 0 ms | Rank 1 ms | Rank 0 share of matrix time | +| --- | ---: | ---: | ---: | +| FFN up/gate | 1,281.23 | 1,225.96 | 40.05% | +| Attention/memory input projection | 907.66 | 862.34 | 28.38% | +| FFN down | 632.02 | 602.74 | 19.76% | +| Core output projection | 174.37 | 171.46 | 5.45% | +| Vocabulary head | 103.37 | 98.64 | 3.23% | +| Small state-memory batched products | 100.09 | 96.61 | 3.13% | +| Total | 3,198.75 | 3,057.76 | 100% | + +The first three families account for **88.19% / 88.01% of matrix time** and +**67.74% / 64.62% of the profiled update** on ranks zero/one. All matrix kernels +occupy 76.81% / 73.42% of that update. These are duration shares, not measures +of compute saturation. The generated attribution is +`work/validation/pointwise-headroom-20260913/paired-matmul-attribution.json`. +Its source profiles and reviewed shape/stride mapping are hashed in the report. + +## What the saturation evidence establishes + +The dominant dense families sustain approximately 252–256 TFLOPS on rank zero +and 264–268 TFLOPS on rank one in this stored full-model profile. Earlier warmed +isolated tests measured approximately 243–263 TFLOPS for their individual shapes. +Relative to an illustrative peak adjusted for their observed clocks, those +isolated results reached approximately **88–93%**. That estimate scales the +published 438.9 dense BF16 TFLOPS by observed MHz / 2,280 MHz. The specification +uses boost clocks; the denominator is not a measured sustained hardware limit. +[NVIDIA architecture specifications](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46). + +Those isolated clock samples are snapshots, not time-integrated clocks for the +new profile. The results support a compute-efficiency focus for large dense +GEMMs, while small state-memory products have much lower arithmetic intensity. +They do **not** mean that 88% of matrix time is fully saturated, or that each +dense kernel spends 88–93% of its lifetime at full utilization. Operand loads, +Tensor Core instructions, and output work overlap within a pipelined GEMM; +kernel duration alone cannot partition that execution into saturated and +unsaturated intervals. + +The exact requested fraction remains unmeasured. A prior Nsight Compute attempt +failed with `ERR_NVGPUCTRPERM`; the read-only driver check still reports +`RmProfilingAdminOnly=1`. No counter-access policy was changed for this analysis. +The session's effective `CAP_SYS_ADMIN` and `CAP_PERFMON` capabilities are both +absent. The read-only policy/capability check does not collect GPU counters. +The prior failure is saved in +`work/validation/rtx-profile-20260910/ncu-metrics.txt`. +NVIDIA documents this as restricted access to GPU performance counters. +[Counter permission reference](https://developer.nvidia.com/nvidia-development-tools-solutions-err_nvgpuctrperm-permission-issue-performance-counters). + +After counter access is available, a bounded measurement should use disposable, +warmed copies of the dominant GEMM shapes, with the same BF16 operands, strides, +power limits, and relevant cache conditions. Collect Tensor-pipeline throughput, +SM activity, memory throughput, and scheduler statistics. Prefer elapsed-cycle +metrics when assessing whole-device utilization: active-cycle metrics exclude +idle cycles. Query the installed tool's supported metrics rather than assuming +a metric name from another architecture. + +To answer a literal fraction-of-time question, collect supported PM-sampling +timelines and declare a near-saturation threshold, for example 90% of the +appropriate Tensor-pipeline peak. Correlate above-threshold sample intervals +with the GEMM execution intervals, and report sampling resolution and coverage. +Per-kernel average utilization alone cannot determine that time fraction. +Nsight Compute documents both active/elapsed normalization and time-resolved PM +sampling, including examples of operand-delivery stalls and underfilled tails. +[Profiling guide](https://docs.nvidia.com/nsight-compute/ProfilingGuide/#pm-sampling). diff --git a/ops/POSTTRAIN.md b/ops/POSTTRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..204a939ac5e298942712d226de410a2dd453761d --- /dev/null +++ b/ops/POSTTRAIN.md @@ -0,0 +1,165 @@ +# Step 76,294 archive and supervised fine-tuning + +The SFT stage runs from +[`../configs/posttrain_10b.json`](../configs/posttrain_10b.json). The user later +authorized **archiving the full completed SFT state, then automatically running +DPO**. A separate `warpstate-dpo` Supervisor job implements that continuation; +see [DPO_HANDOFF.md](DPO_HANDOFF.md). The active SFT plan/source bytes stay fixed +to preserve their checkpoint/resume signatures. Its original guardian still +finishes and exits cleanly before the DPO guardian can take over. + +`prepare_posttrain_data.py` prepares conversations and preference pairs. The +separate `posttrain.py` runner implements assistant-only supervised training. + +## Automatic sequence + +1. `warpstate-posttrain-data` prepares the pinned dataset recipe on CPU now. +2. `warpstate-posttrain` waits for `warpstate-train` to exit successfully. It + never changes the base run or resumes an earlier stopped base checkpoint. +3. Under the base launch/pipeline locks, require the full checkpoint to contain + step **76,294**, **10,000,007,168 tokens**, and final validation at that same + boundary. A checkpoint before the boundary remains waiting; an overshoot, + stale validation, or mismatched tokenizer fails without starting SFT. +4. Export the original FP32 model using the same `chat.capture_latest` method as + the 8K, 16K and 40K archives. Compare every exported tensor exactly against the + stopped checkpoint; copy the exact tokenizer; write provenance/checksums; + seal files read-only; publish the complete archive atomically without + replacing an existing directory. +5. Make a physical, independent copy of the archive as SFT initialization. The + full pretraining checkpoint, corpus, historical archives and chat selection + are retained. SFT starts with a fresh AdamW optimizer on its first launch. +6. Verify prepared SFT data, run a tiny two-GPU BF16 training/resume preflight, + then start the full SFT model. Failed preflight or data checks stop the + handoff and retain the raw archive for diagnosis. +7. Save final SFT state and evaluation results, verify the raw archive and + initialization remain unchanged, then exit. Neither program automatically + restarts training after completion or failure. The separately armed DPO job + then archives this full SFT state and continues according to DPO_HANDOFF.md. + +The boundary triggers archival, checksum verification, compilation/preflight +and initialization before the first SFT update; allow several minutes for this +handoff. GPU preflight is deliberately deferred until base training releases +the GPUs. Its result is not claimed by the CPU-only setup tests. + +## Paths + +All paths below are relative to `/workspace/WarpState/rtx6000_40b`. + +| Purpose | Path | +| --- | --- | +| Untouched full base resume checkpoint | `work/run/latest.pt` | +| Raw historical model archive | `work/archives/step-00076294/` | +| Independent SFT initialization | `work/posttrain/step-00076294/initialization/` | +| Prepared SFT/DPO data and final manifest | `work/posttrain/data/` | +| SFT checkpoints, settings and metrics | `work/posttrain/step-00076294/run/` | +| Final evaluation / completion | `run/evaluation.json`, `run/completion.json` under the SFT directory | +| Handoff state and frozen execution provenance | `work/posttrain/step-00076294/control/` | +| GPU preflight artifacts | `work/posttrain/step-00076294/preflight/` | +| Handoff and training log | `work/logs/posttrain.log` | +| CPU preparation log | `work/logs/posttrain-data.log` | + +Archives contain model weights only, without Adam or training RNG state. SFT's +`latest.pt` is a separate full `warpstate-posttrain-v1` checkpoint with optimizer, +per-rank RNG, deterministic sample cursor, source/data identities and base +provenance. It is not interchangeable with the base runner's checkpoint. + +## SFT recipe and validation + +This first run uses one epoch, two GPUs, sequence length 2,048, microbatch 4 per +rank and four accumulation microbatches: **32 conversations per update**. There +is no cross-conversation packing. All parameters train with BF16 compute, +original FP32 parameters/gradients and fresh FP32 AdamW state; no FP8 is used. +The learning rate peaks at `5e-6`, with 3% warmup and cosine decay to zero, +zero weight decay and gradient clipping at 1. Evaluation/checkpoints occur +every 100 SFT updates and at the end. Initial and final validation use the full +SFT holdout; intermediate checks use the same fixed 128-example subset. +Settings and effective epoch length are +recorded in the SFT run's `settings.json` once data is loaded. + +The learning rate is a conservative experimental starting point, not a +validated optimum for this 2B model. This is not a reproduction of Tülu 3's +SFT schedule: its published learning rate was calibrated using summed token +loss, and its models/context/batch sizes differ. See the +[Tülu 3 paper, section 4.3.2](https://arxiv.org/html/2411.15124v3). + +Role headers use ordinary tokens from the existing SentencePiece vocabulary. +Only assistant content and the actual EOS ID contribute to the loss. Prompt, +role-header and padding labels are masked; DDP gradients are normalized by the +global count of supervised tokens, including uneven final batches. The shared +renderer is `ops/posttrain_data.py`; its hash is pinned in the data manifest. + +SFT metrics report actual nonpadding input tokens and supervised assistant +tokens separately. SFT validation loss/perplexity uses the assistant-token +objective on the prepared holdout; it is **not directly comparable** with the +base model's all-token pretraining validation loss. Evaluation reports include +the exact evaluated sample/token counts, dataset and base-weight identities. +Behavioral instruction-following and tool-use evaluation is the next manual +decision after the full authorized SFT/DPO sequence pauses; a lower SFT loss +alone does not establish those capabilities. The final SFT archive preserves +the exact model and its evaluation for comparison with DPO. + +## Monitor, pause and resume + +From the repository root: + +```bash +rtx6000_40b/ops/datactl.sh status warpstate-train warpstate-posttrain-data warpstate-posttrain +rtx6000_40b/ops/posttrain-handoff.sh --check +tmux attach -t warpstate-posttrain +tail -f rtx6000_40b/work/logs/posttrain.log +``` + +The tmux session displays the durable Supervisor log. Detaching tmux does not +stop the handoff or training. `RUNNING` on the posttraining program can mean +waiting for the base boundary; inspect `--check` or its phase log. + +To pause the handoff or gracefully checkpoint an active SFT run: + +```bash +rtx6000_40b/ops/datactl.sh stop warpstate-posttrain +``` + +To explicitly re-enable the unchanged handoff or resume an interrupted SFT run: + +```bash +rtx6000_40b/ops/datactl.sh start warpstate-posttrain +``` + +The saved plan/source identity must still match. A completed run stays paused +even if the program is started again. Do not remove its completion state to +repeat SFT. Use a separately reviewed plan and output directory for a new +experiment. Do not clear lock files or launch the base trainer manually while +SFT workers are alive: every SFT rank holds a shared base pipeline lock, and +the base/handoff require its exclusive lock. The launcher also retains its +exclusion locks across child launch and unexpected watcher death. + +SFT Supervisor includes are in `ops/posttrain-supervisord.conf`; both programs +have `autostart=false` and `autorestart=false`. Setup explicitly starts the +authorized jobs. A whole-machine/Supervisor restart requires an explicit +restart of the appropriate jobs; there is no hidden reboot training launch. + +To cancel automatic continuation into DPO while allowing SFT to finish, stop +the separate waiting job with `rtx6000_40b/ops/datactl.sh stop warpstate-dpo`. +Stopping SFT itself leaves the DPO guardian waiting for verified completion. + +## Rebuild the prepared inputs + +The recipe in `configs/posttrain_data.json` pins immutable dataset revisions, +the tokenizer identity, sample counts/seed, split rule, renderer and required +preparation packages. Preparation records downloaded file hashes, schemas, +source/licensing metadata, row/token counts, rejections and output hashes. +Prompt groups use the same deterministic partition across SFT and DPO; exact +duplicate conversations are excluded. Overlength examples and DPO branches +are rejected rather than truncated. Gated xLAM data is recorded as skipped +without automatically accepting terms or requiring interactive input. + +To rebuild the isolated CPU environment, source `ops/training-env.sh`, create +`work/posttrain-data-venv` with Python 3.12, then install +`ops/requirements-posttrain-data.lock` into that environment. Do not change the +active training environment. `ops/prepare-posttrain-data.sh` runs the recipe; +do not regenerate/replace data while SFT is running. The completed manifest is +published only after every selected output has been written and hashed. + +Source/configuration/tests/dependency metadata belong in Git. Downloaded data, +token caches, generated manifests/reports, checkpoints, archives, environments +and logs under `work/` remain ignored. diff --git a/ops/RESTART_HANDOFF.md b/ops/RESTART_HANDOFF.md new file mode 100644 index 0000000000000000000000000000000000000000..ac710b948d15debb5b8264fc54e73abe0830b444 --- /dev/null +++ b/ops/RESTART_HANDOFF.md @@ -0,0 +1,97 @@ +# RTX workstation restart handoff + +Prepared on 2026-09-15. All paths below are relative to `/workspace/WarpState`. +Training, posttraining, and benchmarks have finished; leave them stopped. +Use the existing environments and prepared data. BF16 compute remains required. + +## Saved model state + +| Stage | Final step | Full resumable checkpoint | Immutable model archive | +| --- | ---: | --- | --- | +| Raw pretraining | 76,294 | `rtx6000_40b/work/run/latest.pt` | `rtx6000_40b/work/archives/step-00076294/` | +| SFT | 10,312 | `rtx6000_40b/work/archives/step-00076294-sft-final/checkpoint.pt` | `rtx6000_40b/work/archives/step-00076294-sft-final/` | +| DPO | 2,334 | `rtx6000_40b/work/archives/step-00076294-dpo-final/checkpoint.pt` | `rtx6000_40b/work/archives/step-00076294-dpo-final/` | + +Raw training reached 10,000,007,168 tokens with validation loss 2.3496886492. +SFT and DPO checkpoints both record `completed=true`. Their validation losses +are 1.1157395900 and 0.5790014537 respectively, for different objectives. +The earlier step 8,000, 16,000, and 40,000 archives are also retained. +Do not modify archived files. Chat uses an independent copy in +`rtx6000_40b/work/chat/dpo-final/`. + +With user approval, five byte-identical working weight files were removed +after comparison with their retained archives, reclaiming 72,184,948,679 bytes. +These are the SFT and DPO working `run/latest.pt` and `initialization/model.pt` +files, plus `work/chat/step-8000-s536yem8/model.pt` (under `rtx6000_40b/`). +All archives and the independent current chat copy remain intact. + +Before any future posttraining continuation or rerun of completion checks, +restore independent working files from the following sources. Paths in this +table are relative to `rtx6000_40b/work/`: + +| Archived source | Working destination | +| --- | --- | +| `archives/step-00076294/model.pt` | `posttrain/step-00076294/initialization/model.pt` | +| `archives/step-00076294-sft-final/checkpoint.pt` | `posttrain/step-00076294/run/latest.pt` | +| `archives/step-00076294-sft-final/model.pt` | `posttrain/step-00076294-dpo/initialization/model.pt` | +| `archives/step-00076294-dpo-final/checkpoint.pt` | `posttrain/step-00076294-dpo/run/latest.pt` | + +Verify the archive's `SHA256SUMS`, copy only to an absent destination using +`cp --reflink=auto --no-clobber`, and compare the copy with its source before +use. Never train against an archive path or a hardlink to archived weights. +Restoring these files does not authorize starting training. + +Prepared pretraining tokens remain in `rtx6000_40b/work/data/`; posttraining +data remain in `rtx6000_40b/work/posttrain/data/`. Benchmark results are in +`rtx6000_40b/work/validation/nord-dpo/` and +`rtx6000_40b/work/validation/nord-stage-comparison/COMPARISON.md`. + +## After reboot + +Start in `/workspace/WarpState` and read `AGENTS.md` and this handoff. +Check `nvidia-smi` and `git status --short --branch`. +To restore post-DPO chat in its own tmux session: + +```bash +cd /workspace/WarpState +tmux has-session -t warpstate-chat 2>/dev/null || tmux new-session -d -s warpstate-chat -n dpo-chat -c /workspace/WarpState 'rtx6000_40b/ops/chat-dpo.sh' +tmux attach -t warpstate-chat +``` + +Ctrl-C cancels typing or generation and returns to the next input. `/reset` +clears conversation history, `/quit` exits, and Ctrl-B then D detaches tmux. +Tmux sessions and in-memory chat history do not survive reboot; captured +terminal history is available in the restart record below. + +All workspace Supervisor jobs have `autostart=false` and `autorestart=false`. +If job management is needed, check `rtx6000_40b/ops/datactl.sh status` first. +Initialize the workspace Supervisor only when it is absent: + +```bash +/usr/local/bin/supervisord -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf +``` + +This starts no training. The raw launcher still stops at step 76,294, and +posttraining has completed. Further training requires a new user request and +an explicitly chosen continuation configuration; do not simply restart jobs. +Do not use the B200 launcher on this RTX workstation. + +Jupyter currently uses HTTPS port 8080 with the existing authentication. +After reboot, check the platform-managed Jupyter service before launching +another server. Runtime files and logs from the last standalone launch are +under `rtx6000_40b/work/control/jupyter/`; treat them as private because they +can contain authentication data. Never commit them or expose their contents. +The raw model download path on the Jupyter host is +`/files/workspace/WarpState/rtx6000_40b/work/archives/step-00076294/model.pt?download=1`. +For post-DPO weights, substitute `step-00076294-dpo-final`. + +## Restart record + +`rtx6000_40b/work/restart/LATEST` identifies the private local restart record. +It contains available tmux history, checkpoint metadata, file sizes, archive +checksum results, job state, and a Git bundle of committed sources. +The earlier checkpoint inventory is a pre-cleanup snapshot; the current paths +are listed above. `rtx6000_40b/work/restart/duplicate-weight-deletion.json` +records the approved removals and their retained counterparts. +Artifacts and this bundle are on the same machine, not an off-machine backup. +Keep datasets, weights, runtime files, and this generated record out of Git. diff --git a/ops/RTX_ASYNC_TRAINING.md b/ops/RTX_ASYNC_TRAINING.md new file mode 100644 index 0000000000000000000000000000000000000000..282fce78449609157357b79ec398f8aad35439f2 --- /dev/null +++ b/ops/RTX_ASYNC_TRAINING.md @@ -0,0 +1,114 @@ +# RTX BF16 execution and overlapping checkpoints + +The local launcher selects the fastest complete-update experiment from +[RTX_PROFILE_20260910.md](RTX_PROFILE_20260910.md): compiled cores in default +mode, compiled full vocabulary loss, BF16 residual activations, microbatch 4 +per GPU, and accumulation 8 across two RTX PRO 6000 GPUs. The measured update +rate was 30,707 tokens/second versus 29,081 for the previous configuration, +a 5.59% increase. These measurements exclude checkpoint and evaluation pauses. +The larger microbatch experiments and standalone GEMM/gradient prototypes +are not enabled in the production launcher. + +The model, tokenizer, token corpus, 131,072-token global batch, context length, +optimizer and learning-rate schedule are preserved. Forward/backward compute +uses BF16. Master parameters, gradients and Adam moments remain FP32 on the +GPU. BF16 residual execution changes activation rounding; the first validation +at an unchanged checkpoint measures that numerical difference, not learning. + +## Checkpoint lifecycle + +`--async-checkpoint` works with one GPU or DDP. Every save captures random +states from every rank at a completed optimizer update. Only rank zero +constructs the checkpoint and owns the background writer. + +1. Finish the previous write, if it is still pending. +2. Copy all checkpoint tensors and mutable metadata into an immutable CPU + snapshot. Both ranks wait; later updates cannot mutate this snapshot. +3. Submit serialization to one CPU thread and let both GPUs resume training. +4. Write `latest.pt.tmp`, synchronize the file, atomically replace `latest.pt`, + and synchronize its directory. The previous published checkpoint stays + available until replacement. + +One full snapshot costs approximately 22.4 GiB of host RAM for this model and +optimizer. There is no additional full checkpoint clone on either GPU, and +at most one snapshot/write is outstanding. The local launcher now saves and +evaluates every 100 updates. The initial promotion used a 50-update save +interval; increasing it to 100 reduces time spent staging checkpoints. + +Step lines display validation loss only when that step has a fresh result, +together with its evaluation step. Metrics and checkpoints still record +`val_step` and `val_tokens` alongside `val_loss`, retaining the latest measured +values between evaluations. This avoids repeated validation values in the +terminal and adds no GPU work. Startup evaluation and final evaluation and +saving on a graceful stop remain enabled. + +The main training threads coordinate initialization, construction, staging, +completed background errors and final close. The writer thread performs no +CUDA or distributed operations. Every update first checks the prior writer's +status on both ranks. A failure observed there is raised on both ranks before +the next update; a failure occurring during an update is observed at the next +check or save. Normal completion and graceful stop join the final write. +Exceptional exit joins only the local writer, avoiding extra collectives +when another worker may already have failed. + +`Staged step ...` identifies a frozen snapshot, not durable progress. A later +`Saved ... (step ..., tokens=...; background write ...s)` identifies completed +publication. GPU-to-CPU staging and any wait for a previous save remain +synchronous; only disk serialization and publication overlap training. + +## Validation and reproduction + +The promotion checks passed 112 CPU tests and two CUDA continuation tests +(one GPU and two GPUs). The full-model canary started from the verified +production checkpoint at step 5,618 / 736,362,496 tokens. Its initial +validation remained 2.97925 to five decimal places. After startup, the first +three measured updates ran at 31,637, 31,482 and 31,450 tokens/second with +66.57 GiB reserved per GPU. This short canary validates operation; the +20-update comparison above remains the controlled performance experiment. + +The canary staged step 5,620 in 11.41 seconds, then completed updates 5,621 +and 5,622 before that checkpoint's 10.62-second background write finished. +It published the final step-5,622 full checkpoint before process exit. +This confirms real overlap and also shows that the CPU snapshot remains a +material pause. The initial production save interval was 50 updates; the +canary used two updates to exercise overlap and queue backpressure quickly. + +Reloading that full checkpoint completed two further updates and published +step 5,624 / 737,148,928 tokens. These six canary updates belong only to the +disposable run; production resumes its original step-5,618 checkpoint. The +canary's logs, exact launcher arguments and integrity report remain in +`work/validation/async-promotion/`. + +The distributed writer tests inject failures during state construction, CPU +snapshot allocation, previous-save waits, background checks, flush and close. +Two CPU/Gloo processes must both report each failure without hanging. Tests +also verify immutable snapshots, rank-zero-only writing, file and directory +synchronization, and cleanup without collectives on exceptional exit. + +The CUDA execution regression compares continuous synchronous training with +an asynchronous split/resume run using compiled full loss and BF16 residuals. +Model tensors, Adam state, RNG state, validation loss and progress must match +exactly. It exercises periodic saves, final flush and a partially filled last +update where one rank has no valid tokens. This verifies exact continuation +for the test model and selected execution configuration; it does not establish +bitwise equivalence to the previous FP32-residual configuration. + +Run from `rtx6000_40b` with both GPUs available: + +```bash +source ops/training-env.sh +mkdir -p work/validation/async-promotion +.server-venv/bin/python -m pytest -q \ + test_checkpoint_writer.py test_distributed_checkpoint_writer.py \ + --basetemp="$PWD/work/validation/async-promotion/pytest-writer" +.server-venv/bin/python -m pytest -q test_training_execution.py \ + -k 'two_gpu_bf16_residual_async_full_state_resume_and_empty_final_rank or max-autotune-no-cudagraphs-async' \ + --basetemp="$PWD/work/validation/async-promotion/pytest-gpu" +``` + +Full-model validation uses a separate work directory with read-only access to +the prepared data and an initial hardlink to the paused checkpoint. Saving +atomically replaces only the disposable checkpoint path. Source checkpoint +integrity, canary logs and generated verification reports are retained under +`work/validation/async-promotion/`, outside Git. Production controls and status +commands are in [LOCAL_RESUME.md](LOCAL_RESUME.md). diff --git a/ops/RTX_PROFILE_20260910.md b/ops/RTX_PROFILE_20260910.md new file mode 100644 index 0000000000000000000000000000000000000000..d6377113354572c37157ea74d6351ce8e06c4d92 --- /dev/null +++ b/ops/RTX_PROFILE_20260910.md @@ -0,0 +1,453 @@ +# RTX training profile, 2026-09-10 + +This report records the experiments before production promotion. The selected +BF16 configuration and subsequent checkpoint-overlap implementation are +documented in [RTX_ASYNC_TRAINING.md](RTX_ASYNC_TRAINING.md); use +[LOCAL_RESUME.md](LOCAL_RESUME.md) for the current launcher and status controls. + +The production run was gracefully paused to measure training bottlenecks and +compare disposable execution experiments. Production remains paused while the +experiments run. Benchmark updates do not advance the saved training run. + +## Preserved training state + +The completed checkpoint was verified at 18:12:57 UTC: + +| Field | Value | +| --- | --- | +| Checkpoint | `work/run/latest.pt` | +| Step | 5,618 | +| Tokens consumed | 736,362,496 | +| Validation loss at pause | 2.979250341653824 | +| Bytes | 24,061,700,237 (22.409 GiB) | +| Saved optimizer entries | 256 | +| Saved RNG ranks | 2 | +| SHA256 | `0f2e2a47f13b9d91d0049a0e99109c289399fe593c244bab046f120c1f7ea835` | + +The benchmark loads this full checkpoint, including optimizer state, into +disposable processes. It does not write production checkpoints. Its configuration +and final summary record the checkpoint file identity before and after the run. +The verification record is generated at +`work/validation/rtx-profile-20260910/paused-checkpoint.json`. + +## Configuration and measurement method + +The hardware is two NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition +GPUs, each with approximately 96 GB VRAM. The 2,005,111,424-parameter model uses +BF16 autocast with FP32 master parameters, gradients, and full fused AdamW state. +No FP8 or reduced-precision optimizer state is used. + +The production configuration uses sequence length 2,048, microbatch 4 per GPU, +and eight accumulation microsteps. The global batch is therefore 131,072 tokens +per optimizer update. It compiles the model cores in `default` mode, uses the +checkpointed vocabulary loss with 1,024-token chunks, enables the autocast cache, +and disables activation checkpointing for the model blocks. Production saves +every 50 updates and evaluates every 100 updates. + +The unprofiled baseline restores the checkpoint's optimizer state and runs two +warmup updates followed by 20 measured updates. Its aggregate throughput was +**29,081.0 tokens/s**, with a median of **28,996.8 tokens/s**. Peak allocated VRAM +was **67.747 GiB** and reserved VRAM **68.643 GiB** per GPU, with a minimum of +**25.311 GiB** free. The short baseline excludes evaluation and checkpoint writes +and has a transient speed change as the GPUs warm up. Sustained production +measurements below provide the wall throughput reference. + +Profiler runs identify operators and timing relationships. Their instrumented +update durations must not be presented as unprofiled training throughput. +Summed CUDA operator durations may overlap; they do not automatically partition +wall time. Candidate speed comparisons must use unprofiled measurements with +the same global token batch and full optimizer state. + +Baseline artifacts are in `work/benchmark-rtx-20260910/baseline/`. Telemetry is +in `work/validation/rtx-profile-20260910/baseline-gpu-telemetry.json`. + +## Production overhead + +The last 200 completed production intervals end at step 5,618 and start just +after step 5,418. This interval stays within the restart segment and excludes +the shutdown gap. It includes four periodic checkpoints and two evaluations; +the final pause-time evaluation and save occur after its last update timestamp. + +| Measurement | Result | +| --- | ---: | +| Tokens processed | 26,214,400 | +| Sum of measured optimizer-update time | 909.490 s | +| Elapsed time between metric timestamps | 1,000.906 s | +| Throughput within updates | **28,823.2 tokens/s** | +| Throughput including recurring overhead | **26,190.7 tokens/s** | +| Wall time outside updates | **9.133%** | +| Median ordinary inter-update overhead | **2.10 ms** | + +For each consecutive metric pair, outside-update time is the difference between +record timestamps minus the later record's update duration. Metrics are emitted +before periodic evaluation and saving, so the gap following a save step captures +that save's stall. + +| After step | Operations before next update | Outside-update time | +| --- | --- | ---: | +| 5,450 | Checkpoint | 22.730 s | +| 5,500 | Validation and checkpoint | 23.026 s | +| 5,550 | Checkpoint | 22.505 s | +| 5,600 | Validation and checkpoint | 22.743 s | + +Checkpoint-only gaps average 22.618 s. The difference between grouped means +suggests approximately 0.27 s for validation in this window, but validation was +not separately timed and checkpoint duration also varies. The wider restart +segment gives approximately 22.49 s per checkpoint and a 0.35 s grouped +validation difference. Both windows show that synchronous checkpoint saving +dominates the approximately 9% operational overhead. + +These gaps combine GPU-to-host staging, serialization, filesystem writes, +durability synchronization, and waiting. Dividing checkpoint bytes by the pause +gives approximately 1.06 GB/s for the combined path; this is not an isolated SSD +bandwidth measurement. Raw interval calculations are recorded in +`work/validation/rtx-profile-20260910/production-overhead.json`. + +### Checkpoint cadence model + +Holding compute time and evaluation cadence fixed, amortizing approximately +22.6 s per checkpoint gives the following estimates. These are predictions, +not measured candidate runs; production cadence has not been changed. + +| Save every | Estimated wall throughput | Approximate gain | Maximum replay after a crash | +| --- | ---: | ---: | --- | +| 50 updates, current | 26.2K tokens/s | Reference | 6.55M tokens, 4.2 min | +| 100 updates | 27.4K tokens/s | 4.7% | 13.11M tokens, 8.0 min | +| 200 updates | 28.1K tokens/s | 7.3% | 26.21M tokens, 15.5 min | +| 500 updates | 28.5K tokens/s | 8.9% | 65.54M tokens, 38.4 min | + +Eliminating all measured outside-update time would raise this production window +from 26.19K to 28.82K tokens/s: an approximately **10.1% upper bound** for +operational improvements alone. Substantially higher throughput requires faster +optimizer updates as well. + +`run_40b.py` currently saves model and optimizer GPU tensors synchronously on +rank 0 and then waits at a distributed barrier. Its `--async-checkpoint` option +explicitly rejects a world size other than one. The existing +`ops/checkpoint_writer.py` stages an immutable CPU snapshot before overlapping +serialization and durable publication with training. Adapting this path to DDP +requires coordinated error handling and validation; the flag cannot simply be +enabled for the current two-GPU run. CPU snapshot staging would still block, +and approximately one full checkpoint of additional host memory is required. + +## GPU profile + +One complete update, after two warmup updates, took 4.4745 seconds with profiler +instrumentation. Actual CUDA activity occupied approximately 98.8% of that +interval. This is the union of kernel/copy intervals, not tensor-core utilization. + +| GPU activity | Rank 0 summed time | Rank 1 summed time | +| --- | ---: | ---: | +| Matrix multiplication | 3.231 s | 3.122 s | +| Compiled pointwise functions and reductions | 0.561 s | 0.563 s | +| NCCL communication | 0.448 s | 0.545 s | +| Other kernels | 0.378 s | 0.378 s | +| Local attention | 0.100 s | 0.099 s | +| Fused AdamW | 0.0385 s | 0.0384 s | +| CUDA copies and fills | 0.0033 s | 0.0031 s | + +Matrices represent 65.7–67.9% of summed GPU activity. Streams overlap, so this +table must not be added up as a partition of elapsed time. Rank 0 communication +overlaps other activity for 337 ms; its communication-only intervals total +111 ms. Rank 1 has 215 ms of communication-only activity. Those intervals are +not guaranteed recoverable by removing communication. + +The eight data batches require only 5.65/5.12 ms of host time for construction, +pinning and transfer submission. Increasing loader workers does not address a +measured bottleneck. Gradient clipping's 280/380 ms host range mostly waits for +previously queued GPU work during the nonfinite check. Actual clipping-related +CUDA work is approximately 16 ms. Optimizer and attention replacements likewise +have small direct budgets compared with dense projections. + +Raw traces and operator/kernel tables are under +`work/benchmark-rtx-20260910/profile/`. The generated interpretation is +`work/validation/rtx-profile-20260910/gpu-attribution.json`. + +## What the matrix multiplication contains + +For a linear projection with input X and weight W, training computes three +matrix products: `Y = X @ W.T`, `dX = dY @ W`, and `dW = dY.T @ X`. Each product +costs approximately `2*M*N*K` FLOPs. Thus backward contributes approximately +twice the dense arithmetic of forward, before other backward operations. + +Each rank processes 65,536 tokens per update as eight 8,192-token microbatches. +Each core projection therefore executes 224 times: 28 cores times eight +microbatches. The following rank-0 times sum all calls within the profiled update. + +| Projection | Forward input/output width | Forward | Input gradient | Weight gradient | +| --- | --- | ---: | ---: | ---: | +| FFN up and gate | 2,048 → 14,336 | 407.4 ms | 437.1 ms | 431.5 ms | +| FFN down | 7,168 → 2,048 | 222.2 ms | 210.5 ms | 195.5 ms | +| Q/K/V/gate/memory input | 2,048 → 10,240 | 291.0 ms | 314.9 ms | 309.1 ms | +| Core output | 2,048 → 2,048 | 57.8 ms | 57.4 ms | 57.5 ms | +| Vocabulary head | 2,048 → 32,768 | 71.2 ms | 39.8 ms | 31.2 ms | + +Head forward includes recomputation: 64 original and 64 recomputed projections +of 1,024 tokens, with 64 calls for each derivative. Core activation checkpointing +is disabled; head checkpointing still repeats the head's forward projection. +State-memory batched matrix operations add another 96.5 ms on rank 0. + +The dense linear products alone account for 797.146 trillion FLOPs per rank +per update, or 12.1635 billion FLOPs per training token. Their summed execution +time is 3.1342 s, approximately 254.3 TFLOPS during those operations. FFNs alone +take 1.905 s, and the five-way input projection takes 0.916 s. These are the +primary matrix optimization targets. + +The large forward GEMMs have ideal one-pass BF16 operand arithmetic intensities +of roughly 910–1,470 FLOPs/byte. This is an operand-footprint calculation, not +measured device-memory traffic: tile reloads, caches, casts and accumulation +change actual traffic. It supports investigating compute efficiency before +assuming that the large GEMMs are starved for external memory bandwidth. + +Shapes, strides, dtype, counts, FLOPs and matching kernel symbols are recorded +in `work/validation/rtx-profile-20260910/matmul-detail.json`. The dominant kernel +names begin `cutlass_80_tensorop_bf16`, with 128×256 or 256×128 tiles, K32 and +three pipeline stages. The symbol identifies an implementation family; it does +not establish that the GPU is executing in an obsolete architecture mode. + +## Compute ceiling at the measured power limit + +NVIDIA specifies **438.9 TFLOPS of dense BF16 with FP32 accumulation per Max-Q +GPU**, based on a **2,280 MHz boost clock**, with **300 W board power** and +**1,792 GB/s memory bandwidth**. The larger sparsity figure is not applicable +to this dense model. See Appendix A of the +[NVIDIA RTX Blackwell architecture whitepaper](https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/quadro-product-literature/pdf/NVIDIA-RTX-Blackwell-PRO-GPU-Architecture-v1_1.pdf#page=46). + +During loaded baseline samples, median SM clocks were approximately 1,537 and +1,571 MHz. The software power cap was active; thermal throttling was not. +Scaling the advertised rate by those clocks gives roughly 296–302 TFLOPS per +GPU as an illustrative instantaneous ceiling, substantially below boost-rated +peak. This calculation assumes arithmetic capacity scales linearly with SM +clock; it is not a hardware-counter utilization measurement. + +The isolated GEMM tests below sustain approximately 244–263 TFLOPS with +observed clocks of 1,417–1,537 MHz and approximately 300 W. Relative to the +clock-scaled specification, these correspond to approximately **88–94%**. +Telemetry is sampled after each timed case and clocks vary within a case, so +this percentage is approximate. Nevertheless, the evidence indicates that the +large library GEMMs already use much of the compute available at this power +limit. Whole-update efficiency is lower because normalization, activations, +gradient handling, memory operations and communication still consume time. + +Nsight Compute 2025.1.1 is installed, but its hardware-counter query was rejected +with `ERR_NVGPUCTRPERM` by the host driver. No driver permissions or power settings +were changed. Profiler occupancy fields reporting zero for active kernels were +not treated as measured occupancy. The conclusion above uses CUDA timings, +shape-derived FLOPs and clock observations, with those limits made explicit. + +## Isolated GEMM algorithm experiments + +[`benchmark_gemm.py`](benchmark_gemm.py) generates deterministic BF16 operands +with the projection's actual transpose strides. It preserves the production +BF16 reduction flag, excludes compilation, warms each case for at least two +seconds, then measures five CUDA-event trials of 30 products. The inputs are +reused and no autograd, communication, optimizer or training checkpoint is +involved. These are kernel comparisons, not training throughput measurements. + +| Product | cuBLAS | cuBLASLt preference | Default Triton-only autotune | +| --- | ---: | ---: | ---: | +| FFN up/gate forward | 1.828 ms | 1.838 ms | 1.981 ms | +| FFN up/gate dX | 1.885 ms | 1.885 ms | 2.057 ms | +| FFN up/gate dW | 1.871 ms | 1.777 ms | 1.869 ms | +| Input projection forward | 1.326 ms | 1.314 ms | 1.441 ms | +| Input projection dX | 1.389 ms | 1.376 ms | 1.474 ms | +| Input projection dW | 1.351 ms | 1.336 ms | 1.350 ms | +| FFN down forward | 0.984 ms | 0.965 ms | 1.076 ms | +| FFN down dX | 0.914 ms | 0.902 ms | 0.993 ms | +| FFN down dW | 0.935 ms | 0.955 ms | 0.993 ms | + +ATEN-only compilation was essentially equal to eager cuBLAS. Default +Triton-only selection was usually slower; swapping backends globally is not an +established improvement. cuBLASLt gave mixed shape-specific results. Its 5.3% +up/gate dW gain coincided with a higher observed clock, so it must not be +attributed solely to a better algorithm. Some Triton input-gradient cases +differed from the BF16 cuBLAS reference by about 0.30% relative RMS; finite +outputs do not establish numerical equivalence. + +Enabling `ENABLE_PERSISTENT_TMA_MATMUL=1` added persistent TMA candidates and +selected them for the three up/gate products. Measured times were 1.798 ms +forward, 1.890 ms dX and 1.810 ms dW: approximately 268, 255 and 266 TFLOPS. +This closes the default Triton gap and gives small shape-specific improvements +over cuBLAS, rather than a large untapped acceleration. Autotuner trial times +were faster than the subsequent warmed measurements; the latter are reported. + +Evidence is under `work/benchmark-gemm-20260910/`, +`work/benchmark-gemm-cublaslt-20260910/`, and +`work/benchmark-gemm-tma-20260910/`. + +## Fusion opportunities + +The two FFN input projections already share one GEMM. Q/K/V/gate/memory likewise +share one projection. The compiled core already fuses residual addition, +RMSNorm and BF16 conversion, as well as SwiGLU split/SiLU/multiply and several +attention transforms. Generated wrappers still use `extern_kernels.mm` between +these fused regions. + +The FFN computes `(X @ W_up * silu(X @ W_gate)) @ W_down`. The nonlinearity +prevents collapsing both projection stages into a single ordinary linear map. +A custom fused implementation still performs both matrix stages. + +| Remaining target | Rank-0 measured kernel-time budget | Practical constraint | +| --- | ---: | --- | +| Weight-gradient conversion and FP32 accumulation | 130.5 ms | Preserve BF16 rounding and DDP gradient hooks | +| SwiGLU backward | 88.0 ms | Paired-element derivative can reuse loads and sigmoid | +| Repeated core weight casts | 56.4 ms | Cached BF16 copies need optimizer-step invalidation and correct autograd | +| SwiGLU forward | 41.1 ms | Up/gate columns are 7,168 apart; backward still needs saved values | +| Forward norms | 29.6 ms | Row reductions complicate fusion into tiled GEMMs | +| Forward Q/K/V/U transforms | 43.9 ms | Multiple attention and memory consumers need compatible layouts | + +These are budgets, not promised savings, and are not additive end-to-end +speedups. A slower custom GEMM can erase the benefit of its fused epilogue. +The complete SwiGLU forward/backward pointwise cost is only 129 ms, about 2.9% +of the profiled update. Recomputing the up/gate projection to avoid saving its +activations would add approximately 407 ms, exceeding its forward fusion budget. + +[`benchmark_gradient_accumulation.py`](benchmark_gradient_accumulation.py) +compares existing BF16 dW followed by FP32 accumulation, fused conversion/add, +and direct FP32-output `addmm`. The direct variant changes the original +BF16-round-before-add semantics. Neither helper implements production autograd +or DDP integration, and neither has been enabled in the training run. + +The final accumulation test uses eight distinct deterministic input pairs. It +creates the first gradient and then performs seven additions, matching the +production accumulation order. Each case uses two seconds of burn-in and five +timing trials, each containing ten eight-gradient cycles. + +| Weight-gradient family | Separate BF16 GEMM/cast/add | Fused cast/add | Direct FP32 GEMM accumulation | +| --- | ---: | ---: | ---: | +| FFN up/gate | 16.928 ms | 15.850 ms | 15.438 ms | +| Input projection | 11.804 ms | 11.308 ms | 11.620 ms | +| FFN down | 8.141 ms | 7.802 ms | 7.873 ms | + +Fused conversion/add improves these complete gradient cycles by **4.3–6.8%**, +with zero output difference on all three tested fixtures. Scaling the measured +cycle differences over 28 blocks suggests approximately 54 ms per update, +around **1.2%** of profiled wall time. This is a hypothesis for integration, +not a measured whole-model improvement; autograd, DDP hooks and overlap remain +to be tested. Direct FP32 accumulation is not consistently faster than the +rounding-preserving fusion and differs by approximately 0.166–0.203% relative +RMS from the baseline on these synthetic gradients. + +Final evidence is in `work/benchmark-gradient-distinct-20260910/`. The earlier +`work/benchmark-gradient-20260910/` experiment used eight identical inputs and +a zero-reset-plus-eight-add cycle; it is retained as exploratory evidence and +is not used for the table or integration estimate above. + +## Complete training-update experiments + +Each successful case uses the same checkpoint and token cursor, two warmup +updates and 20 measured updates, full FP32 optimizer state and BF16 computation. +Global batch and context are unchanged. The table excludes validation and saves. + +| Execution | Tokens/s | Gain over baseline | Peak reserved/GPU | +| --- | ---: | ---: | ---: | +| Production baseline, microbatch 4 | 29,081 | Reference | 68.64 GiB | +| Compiled full vocabulary loss, microbatch 4 | 30,050 | 3.3% | 68.99 GiB | +| Compiled full loss + BF16 residuals, microbatch 4 | 30,707 | 5.6% | 66.42 GiB | +| Chunked loss + BF16 residuals, microbatch 8 | 30,394 | 4.5% | 89.49 GiB | +| Compiled full loss + BF16 residuals, microbatch 8 | Out of memory | — | Exceeded 91 GiB cap | +| Baseline repeated after the other experiments | 29,073 | -0.03% | 68.64 GiB | + +The repeated baseline also measured 20 updates and returned within 0.03% of +the initial aggregate throughput, supporting the small measured candidate gains. + +The full-loss microbatch-8 trial failed while allocating its 1 GiB logits +buffer, before completing a warmup update. Retaining chunked loss made that +microbatch fit, but it was slower than microbatch-4 compiled-full with BF16 +residuals. Available VRAM did not translate into a larger useful speed gain. + +Against baseline's corresponding 20 updates, maximum absolute loss differences +were 0.000210 for compiled-full, 0.000235 for compiled-full plus BF16 residuals, +and 0.000146 for microbatch-8 chunked residuals. Maximum relative gradient-norm +differences were 0.236%, 0.384% and 0.362%, respectively. All completed updates +had finite losses and norms. These short comparisons do not establish long-run +quality or distributed checkpoint/resume equivalence for an experimental path. + +The immediate measured compute improvement is approximately 3–6%. A further +implementation effort should prioritize gradient conversion/accumulation, +selective GEMM tuning, and checkpoint staging/overlap. Preserving the current +save frequency is especially useful after the recent abrupt shutdown. Production +remains paused at step 5,618; no experiment has been promoted to its launcher. + +## Reproduction and paused-state controls + +These commands target the two-GPU workstation. The dedicated B200 host instead +uses `ops/b200ctl.sh` and its own handoff instructions. + +Start at the repository root and inspect the production job. If it is running, +request a graceful stop and wait for `STOPPED` and the final checkpoint save +before starting a benchmark. A stopped job may make the status command return a +nonzero shell exit status; inspect its output. + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +rtx6000_40b/ops/datactl.sh status warpstate-train +# Only when the production job is running: +rtx6000_40b/ops/datactl.sh stop warpstate-train +rtx6000_40b/ops/datactl.sh status warpstate-train +tail -n 12 rtx6000_40b/work/logs/train.log +``` + +Use a fresh output directory for every disposable run. The benchmark restores +full optimizer state, preserves the global batch, and writes no checkpoint: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +rtx_profile_run=$(date -u +%Y%m%dT%H%M%S) +WARPSTATE_BENCH_DEVICES=0,1 WARPSTATE_BENCH_PROCESSES=2 TORCHINDUCTOR_COMPILE_THREADS=8 \ + rtx6000_40b/ops/benchmark.sh \ + --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt \ + --data /workspace/WarpState/rtx6000_40b/work/data/train.bin \ + --output-dir "/workspace/WarpState/rtx6000_40b/work/benchmark-rtx-${rtx_profile_run}/baseline" \ + --micro-batch 4 --seq-len 2048 --global-tokens-per-update 131072 \ + --compile-cores --compile-mode default \ + --no-activation-checkpointing --autocast-cache \ + --loss-mode checkpointed --loss-chunk-size 1024 \ + --optimizer-state resume --warmup-updates 2 --measured-updates 20 \ + --vram-limit-gib 88 --threads 4 --ddp-bucket-cap-mb 128 +``` + +For the operator trace, use another fresh output directory, add `--profile`, and +`--diagnose-stalls`, and use `--measured-updates 1`. Only the first measured update is traced. Inspect +the output summary and both ranks' profile artifacts, then compare throughput +using separate unprofiled runs. Leave production paused after the experiments. + +Candidate overrides, each with a new output directory, are `--loss-mode +compiled-full`; then additionally `--bf16-residuals`; and, for the larger batch, +`--micro-batch 8 --vram-limit-gib 91`. The successful larger-batch case retains +`--loss-mode checkpointed`. Do not run these GPU experiments concurrently. + +The isolated matrix and gradient comparisons use no training data or checkpoint: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +export CUDA_VISIBLE_DEVICES=0 +export TORCHINDUCTOR_COMPILE_THREADS=8 +rtx6000_40b/.server-venv/bin/python rtx6000_40b/ops/benchmark_gemm.py \ + --backends eager aten triton --warmup 10 --burnin-seconds 2 --repeats 30 --trials 5 +rtx6000_40b/.server-venv/bin/python rtx6000_40b/ops/benchmark_gemm.py \ + --backends eager --blas-backend cublaslt +ENABLE_PERSISTENT_TMA_MATMUL=1 rtx6000_40b/.server-venv/bin/python \ + rtx6000_40b/ops/benchmark_gemm.py --families ffn_up_gate --backends triton +rtx6000_40b/.server-venv/bin/python \ + rtx6000_40b/ops/benchmark_gradient_accumulation.py +``` + +Each invocation defaults to a fresh timestamped output directory and refuses +to overwrite an existing directory. `--list-shapes` validates and lists the +selected layouts on CPU. GPU comparisons save source hashes, arguments, +precision settings, numerical errors and timing samples. The two new helpers +passed CPU syntax, layout/FLOP and accumulation-control checks, and actual BF16 +GPU comparisons; they do not alter the production model or launcher. + +When the user requests production resume, use the existing verified checkpoint +and `rtx6000_40b/ops/datactl.sh start warpstate-train`. The tmux session named +`warpstate-train` is the monitoring session; Supervisor controls the training +process. Never use a from-scratch launcher to resume this run. + +Raw profiles, benchmark summaries, telemetry, and verification reports remain +ignored under `work/`. This source document records the recipe and interpretation +without adding generated artifacts or checkpoints to Git. diff --git a/ops/TRAINING.md b/ops/TRAINING.md new file mode 100644 index 0000000000000000000000000000000000000000..93280c3ecbf07f244df40885ebeda5cc3af10187 --- /dev/null +++ b/ops/TRAINING.md @@ -0,0 +1,148 @@ +# Training environment and shared GPU configuration + +This document records the original two-GPU workstation configuration. Its +Supervisor and launchers are not the controls for the single-B200 host. +Current B200 settings, checkpoint and pause/resume controls are in +[B200_HANDOFF.md](B200_HANDOFF.md). For workstation history, see +[OPTIMIZED_TRAINING.md](OPTIMIZED_TRAINING.md); the shared GPU settings below +preceded that workstation preset. + +The completed corpus in `work/data` contains exactly 40,000,000,000 training +tokens and 1,048,576 validation tokens. Its generated manifest records the +SHA256 checksums. The model has 2,005,111,424 parameters; 40B is the token +budget, not the parameter count. + +No initial WarpState checkpoint was supplied. The shared-GPU launcher uses +`--from-scratch` with seed 1337 for its first launch. Once `work/run/latest.pt` +exists, subsequent launches restore its model, full AdamW state, token cursor, +learning-rate schedule, and random states. `--from-scratch` never overwrites +an existing run. To initialize another run from trusted weights, invoke +`run_40b.py --weights ...` with a separate work directory instead. + +## Shared GPU settings + +`train-shared.sh` selects physical GPU 1 by its UUID. VORTEX uses both GPUs, +so concurrent training shares GPU compute and can slow both jobs. +On another machine, set `WARPSTATE_CUDA_DEVICE` to the intended GPU UUID or +index before launching. + +| Setting | Value | +| --- | --- | +| Sequence length | 2,048 tokens | +| Microbatch | 1 sequence | +| Gradient accumulation | 64 microbatches | +| Tokens per optimizer update | 131,072 | +| Parameters, gradients, AdamW moments | FP32 on the GPU | +| Forward/backward matrix operations | BF16 autocast | +| Activation checkpointing | Enabled | +| Autocast weight cache | Disabled to reduce retained weight copies | +| Loss projection chunks | 256 tokens | +| PyTorch allocator limit | 33.5 GiB | +| Required extra free memory at startup | 3 GiB | +| Validation | 8 batches initially and every 100 updates | +| Checkpoint | Every 50 updates, on a graceful stop, and at completion | + +The September 9, 2026 live trial completed two full updates, saved a +22.41 GiB checkpoint, and resumed through step 5. Peak memory was 31.54 GiB +allocated and 32.02 GiB reserved, leaving about 4.7 GiB physically free. +Observed training throughput was approximately 5,500–6,000 tokens/second. +At that unchanged rate, 40B tokens would take roughly 77–84 days before +checkpoint/evaluation overhead; this is an early measurement, not an ETA +that accounts for future GPU availability. VORTEX continued advancing, +with overlapping updates around 68–69K tokens/second versus roughly 100K +before the trial. + +The fixed FP32 parameters, gradients, and two AdamW moments alone occupy +about 29.88 GiB. Activations and allocator overhead must also fit the cap. +The launcher refuses to start if the cap plus startup headroom exceeds +available memory. This is a check, not a reservation for VORTEX. + +PyTorch's [allocator memory limit](https://docs.pytorch.org/docs/2.14/generated/torch.cuda.memory.set_per_process_memory_fraction.html) +excludes CUDA context and library allocations. Compare both the allocated +and reserved peaks in `work/run/metrics.jsonl` with physical memory in +`nvidia-smi`. A different VORTEX workload may need different headroom. + +Each full checkpoint is approximately 22.4 GiB. Keep at least 53 GiB free +for checkpoint writing and atomic replacement. Checkpoints, metrics, data, +and environments stay inside this project and are ignored by Git. + +## Environment + +The CUDA training environment is separate from the CPU data-preparation +environment and from VORTEX's environment. Recreate it with Python 3.12: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +python3 -m venv rtx6000_40b/.server-venv +uv pip install --python rtx6000_40b/.server-venv/bin/python \ + -r rtx6000_40b/ops/training-requirements-resolved.txt \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ + --index-strategy unsafe-best-match +uv pip check --python rtx6000_40b/.server-venv/bin/python +``` + +The lock pins PyTorch 2.14.0+cu130 and its installed dependencies. Always +source `training-env.sh` when testing or provisioning so caches and temporary +files remain under `/workspace/WarpState`. The launcher sets these paths +itself and uses `--no-bootstrap`; the minimal GPU environment does not +contain the separate data-preparation dependencies. + +## Operation + +The existing project-private Supervisor manages both preparation and training. +Its training command is now `train-fast.sh`, which needs both GPUs available. +`datactl.sh` is its control wrapper, including for the training job: + +```bash +cd /workspace/WarpState +rtx6000_40b/ops/datactl.sh status +rtx6000_40b/ops/datactl.sh start warpstate-train +tail -n 10 rtx6000_40b/work/logs/train.log +tail -n 3 rtx6000_40b/work/run/metrics.jsonl +nvidia-smi +``` + +To stop, use `rtx6000_40b/ops/datactl.sh stop warpstate-train`. The runner +finishes its current optimizer update and saves before exiting. Supervisor +allows up to 15 minutes for that. Starting the same job resumes its latest +checkpoint. `--reconfigure` permits changes to execution settings while keeping +the sequence length, global tokens per update, data, and optimizer schedule +unchanged. Changes to accumulation or GPU count can change floating-point +rounding; the model and full optimizer state are retained. + +Training is started explicitly and does not automatically restart after a +failure, avoiding repeated allocation attempts on a shared GPU. Inspect the +log and available memory before restarting a failed job. A container restart +also requires starting the project Supervisor and the training job again. + +For a bounded trial of the current configuration, run +`ops/train-fast.sh --max-steps 2` from the package directory while the +supervised training job is stopped. This saves after two +additional updates while preserving the full 40B schedule. Only one runner +can hold `work/pipeline.lock` at a time. + +To return to the original shared GPU configuration, stop the supervised job +and run `ops/train-shared.sh --reconfigure` with enough free memory on GPU 1. +For persistent supervision, update the program command to that launcher plus +`--reconfigure` and use `stopsignal=TERM` before `datactl.sh reread` and +`datactl.sh update warpstate-train`. The shared launcher runs the worker +directly; the optimized launcher relays stop requests to distributed workers. +Do not run the benchmark or another launcher alongside the production job. + +## Checks + +The tests exercise interrupted data preparation, exact final partial updates, +the memory-headroom refusal, and exact CUDA checkpoint/resume behavior for +both imported weights and fresh initialization. The CUDA tests use a tiny +model and a 0.5 GiB allocator cap: + +```bash +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +CUDA_VISIBLE_DEVICES=1 rtx6000_40b/.server-venv/bin/python \ + -m pytest rtx6000_40b/test_server.py -q +``` + +Use `CUDA_VISIBLE_DEVICES=''` to skip CUDA tests when there is insufficient +free memory. Test artifacts use the project-local `TMPDIR`. diff --git a/ops/activation_execution.py b/ops/activation_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..3770aec54606202039ac5988dae9dcfd8a229f75 --- /dev/null +++ b/ops/activation_execution.py @@ -0,0 +1,253 @@ +"""Opt-in BF16 SwiGLU and RMSNorm execution without replacing parameters. + +Eager execution retains the model's BF16 rounding boundaries. Compiled execution +uses the current Inductor arithmetic, eliding intermediate BF16 casts while +retaining BF16 kernel inputs/outputs. Normalization statistics and master +parameter gradients remain FP32. FP32 arithmetic reductions can differ +slightly from Inductor's reduction order; this is an execution experiment, not a +claim of bitwise equivalence. CPU, evaluation and non-BF16 paths stay original. + +Apply before compiling cores. Checkpoints and inference archives are unchanged. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl +from triton.language.extra.cuda import libdevice +from torch.library import triton_op, wrap_triton + + +@triton.jit +def _swiglu_forward_kernel(X, Y, N: tl.constexpr, H: tl.constexpr, + BLOCK: tl.constexpr, ROUND: tl.constexpr): + i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = i < N + packed = (i // H) * (2 * H) + i % H + up = tl.load(X + packed, mask, 0).to(tl.float32) + gate = tl.load(X + packed + H, mask, 0).to(tl.float32) + silu = gate / (1.0 + libdevice.exp(-gate)) + if ROUND: + silu = silu.to(tl.bfloat16).to(tl.float32) + tl.store(Y + i, silu * up, mask) + + +@triton.jit +def _swiglu_backward_kernel(X, DY, DX, N: tl.constexpr, H: tl.constexpr, + BLOCK: tl.constexpr, ROUND: tl.constexpr): + i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = i < N + packed = (i // H) * (2 * H) + i % H + up = tl.load(X + packed, mask, 0).to(tl.float32) + gate = tl.load(X + packed + H, mask, 0).to(tl.float32) + dy = tl.load(DY + i, mask, 0).to(tl.float32) + sigmoid = 1.0 / (1.0 + tl.exp(-gate)) + silu = gate / (1.0 + libdevice.exp(-gate)) + if ROUND: + silu = silu.to(tl.bfloat16).to(tl.float32) + # aten's multiply backward rounds before aten's silu backward executes. + dsilu = dy * up + if ROUND: + dsilu = dsilu.to(tl.bfloat16).to(tl.float32) + dgate = (dsilu * sigmoid) * (1.0 + gate * (1.0 - sigmoid)) + tl.store(DX + packed, dy * silu, mask) + tl.store(DX + packed + H, dgate, mask) + + +@triton_op("warpstate::bf16_swiglu", mutates_args={}) +def bf16_swiglu(packed: torch.Tensor, eager_rounding: bool = True) -> torch.Tensor: + """SwiGLU of a contiguous BF16 CUDA [*, 2*hidden] projection. + + Set ``eager_rounding=False`` to match the current compiled core arithmetic. + """ + hidden = packed.shape[-1] // 2 + output = torch.empty((*packed.shape[:-1], hidden), device=packed.device, + dtype=packed.dtype) + wrap_triton(_swiglu_forward_kernel)[(triton.cdiv(output.numel(), 1024),)]( + packed, output, output.numel(), hidden, 1024, eager_rounding, + enable_fp_fusion=not eager_rounding) + return output + + +@triton_op("warpstate::bf16_swiglu_backward", mutates_args={}) +def _swiglu_backward(packed: torch.Tensor, grad: torch.Tensor, + eager_rounding: bool) -> torch.Tensor: + grad = grad.contiguous() + output = torch.empty_like(packed) + wrap_triton(_swiglu_backward_kernel)[(triton.cdiv(grad.numel(), 1024),)]( + packed, grad, output, grad.numel(), packed.shape[-1] // 2, 1024, eager_rounding, + enable_fp_fusion=not eager_rounding) + return output + + +def _swiglu_setup(ctx, inputs, output): + ctx.save_for_backward(inputs[0]) + ctx.eager_rounding = inputs[1] + + +def _swiglu_autograd(ctx, grad): + return _swiglu_backward(ctx.saved_tensors[0], grad, ctx.eager_rounding), None + + +bf16_swiglu.register_autograd(_swiglu_autograd, setup_context=_swiglu_setup) + + +@triton.jit +def _rms_forward_kernel(X, W, Y, R, D: tl.constexpr, EPS: tl.constexpr, + BLOCK: tl.constexpr, ROUND: tl.constexpr): + row = tl.program_id(0) + col = tl.arange(0, BLOCK) + x = tl.load(X + row * D + col, col < D, 0).to(tl.float32) + w = tl.load(W + col, col < D, 0) + if ROUND: + w = w.to(tl.bfloat16).to(tl.float32) + r = tl.rsqrt(tl.sum(x * x, 0) / D + EPS) + z = x * r + if ROUND: + z = z.to(tl.bfloat16).to(tl.float32) + tl.store(Y + row * D + col, z * w, col < D) + tl.store(R + row, r) + + +@triton.jit +def _rms_backward_kernel(X, W, R, DY, DX, DW_PART, + M: tl.constexpr, D: tl.constexpr, + ROWS: tl.constexpr, BLOCK: tl.constexpr, ROUND: tl.constexpr): + group = tl.program_id(0) + rows = group * ROWS + tl.arange(0, ROWS) + cols = tl.arange(0, BLOCK) + mask = (rows[:, None] < M) & (cols[None, :] < D) + x = tl.load(X + rows[:, None] * D + cols[None, :], mask, 0).to(tl.float32) + dy = tl.load(DY + rows[:, None] * D + cols[None, :], mask, 0).to(tl.float32) + w = tl.load(W + cols, cols < D, 0) + if ROUND: + w = w.to(tl.bfloat16).to(tl.float32) + r = tl.load(R + rows, rows < M, 0) + z = x * r[:, None] + dz = dy * w[None, :] + if ROUND: + z = z.to(tl.bfloat16).to(tl.float32) + dz = dz.to(tl.bfloat16).to(tl.float32) + dr = tl.sum(dz * x, 1) + # Keep the same explicit derivative chain as rsqrt(mean(x**2) + eps). + dsquare = (-0.5 * dr * (r * r * r)) / D + dx = dz * r[:, None] + dsquare[:, None] * (2.0 * x) + dw = dy * z + if ROUND: + dw = dw.to(tl.bfloat16).to(tl.float32) + partial = tl.sum(dw, 0) + tl.store(DX + rows[:, None] * D + cols[None, :], dx, mask) + tl.store(DW_PART + group * D + cols, partial, cols < D) + + +@triton.jit +def _rms_weight_reduce_kernel(PART, DW, D: tl.constexpr, GROUPS: tl.constexpr, + BLOCK_G: tl.constexpr, BLOCK_D: tl.constexpr, ROUND: tl.constexpr): + cols = tl.program_id(0) * BLOCK_D + tl.arange(0, BLOCK_D) + groups = tl.arange(0, BLOCK_G) + partial = tl.load(PART + groups[:, None] * D + cols[None, :], + (groups[:, None] < GROUPS) & (cols[None, :] < D), 0) + # Eager weight.to(BF16) makes sum-to-size return BF16 before its gradient is + # cast back to FP32. The compiled path retains the FP32 reduction; Inductor's + # own partitioning can retain a final BF16 cast at some large shapes, so this + # comparison is numerically bounded rather than universally bitwise equal. + total = tl.sum(partial, 0) + if ROUND: + total = total.to(tl.bfloat16).to(tl.float32) + tl.store(DW + cols, total, cols < D) + + +@triton_op("warpstate::bf16_rmsnorm", mutates_args={}) +def bf16_rmsnorm(x: torch.Tensor, weight: torch.Tensor, + eps: float, eager_rounding: bool = True) -> tuple[torch.Tensor, torch.Tensor]: + """BF16 CUDA RMSNorm with FP32 statistics; also return saved row rsqrts.""" + width = x.shape[-1] + rows = x.numel() // width + output = torch.empty_like(x) + rstd = torch.empty((rows,), device=x.device, dtype=torch.float32) + block = triton.next_power_of_2(width) + wrap_triton(_rms_forward_kernel)[(rows,)]( + x, weight, output, rstd, width, eps, block, eager_rounding, + num_warps=4 if block <= 4096 else 8, enable_fp_fusion=not eager_rounding) + return output, rstd + + +@triton_op("warpstate::bf16_rmsnorm_backward", mutates_args={}) +def _rms_backward(x: torch.Tensor, weight: torch.Tensor, rstd: torch.Tensor, + grad: torch.Tensor, eager_rounding: bool) -> tuple[torch.Tensor, torch.Tensor]: + grad = grad.contiguous() + width = x.shape[-1] + rows = x.numel() // width + group_rows = 4 + groups = triton.cdiv(rows, group_rows) + output = torch.empty_like(x) + partial = torch.empty((groups, width), device=x.device, dtype=torch.float32) + weight_grad = torch.empty_like(weight) + block = triton.next_power_of_2(width) + wrap_triton(_rms_backward_kernel)[(groups,)]( + x, weight, rstd, grad, output, partial, rows, width, group_rows, block, eager_rounding, + num_warps=4 if block <= 2048 else 8, enable_fp_fusion=not eager_rounding) + wrap_triton(_rms_weight_reduce_kernel)[(triton.cdiv(width, 8),)]( + partial, weight_grad, width, groups, triton.next_power_of_2(groups), 8, eager_rounding, + num_warps=8, enable_fp_fusion=False) + return output, weight_grad + + +def _rms_setup(ctx, inputs, output): + ctx.save_for_backward(inputs[0], inputs[1], output[1]) + ctx.mark_non_differentiable(output[1]) + ctx.eager_rounding = inputs[3] + + +def _rms_autograd(ctx, grad, _grad_rstd): + x, weight, rstd = ctx.saved_tensors + dx, dw = _rms_backward(x, weight, rstd, grad, ctx.eager_rounding) + return dx, dw, None, None + + +bf16_rmsnorm.register_autograd(_rms_autograd, setup_context=_rms_setup) + + +def apply_activation_execution(model: torch.nn.Module, mode: str = "baseline") -> None: + """Patch training forwards only, retaining canonical FP32 parameter objects. + + Modes select fused ``swiglu``, ``rmsnorm``, or ``all``; ``baseline`` is a no-op. + Inputs outside the supported contiguous BF16 CUDA shapes use eager model + code. Evaluation retains the original path, avoiding validation graph churn. + Eager invocation preserves eager casts; core graph capture selects the + current Inductor arithmetic with elided intermediate casts. + """ + from warpstate.model import RMSNorm, SwiGLU + + if mode not in {"baseline", "swiglu", "rmsnorm", "all"}: + raise ValueError(f"Unknown activation execution mode: {mode}") + if getattr(model, "_activation_execution_mode", "baseline") != "baseline": + raise ValueError("Activation execution has already been applied") + if mode == "baseline": + return + if any(p.dtype != torch.float32 for p in model.parameters()): + raise ValueError("Activation execution requires FP32 master parameters") + + for module in model.modules(): + original = module.forward + if isinstance(module, SwiGLU) and mode in {"swiglu", "all"}: + def swiglu_forward(x, _module=module, _original=original): + if (_module.training and torch.is_grad_enabled() and x.is_cuda + and torch.is_autocast_enabled("cuda") + and torch.get_autocast_dtype("cuda") == torch.bfloat16): + packed = _module.up_gate(x) + return _module.down(bf16_swiglu(packed.contiguous(), + eager_rounding=not torch.compiler.is_compiling())) + return _original(x) + module.forward = swiglu_forward + elif isinstance(module, RMSNorm) and mode in {"rmsnorm", "all"}: + def rms_forward(x, _module=module, _original=original): + if (_module.training and torch.is_grad_enabled() and x.is_cuda + and x.dtype == torch.bfloat16 and x.is_contiguous() + and 0 < x.shape[-1] <= 8192 and x.numel() > 0): + return bf16_rmsnorm(x, _module.weight, _module.eps, + eager_rounding=not torch.compiler.is_compiling())[0] + return _original(x) + module.forward = rms_forward + model._activation_execution_mode = mode diff --git a/ops/analyze_communication_profile.py b/ops/analyze_communication_profile.py new file mode 100644 index 0000000000000000000000000000000000000000..b012e8beb675811d89901f8cef28b3fd71e77a66 --- /dev/null +++ b/ops/analyze_communication_profile.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Summarize compute/communication overlap from saved Chrome traces, without CUDA. + +Each rank and device is reported separately. All time values use microseconds. +Noncommunication activity includes kernels, copies, and fills. Logical NCCL +input payload bytes are distinct from bytes physically transported over a link. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +import hashlib +import json +import math +from pathlib import Path +import re + + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +ACTIVITY_CATEGORIES = {"kernel", "gpu_memcpy", "gpu_memset"} +COLLECTIVE_OPERATORS = { + "c10d::allreduce_", "c10d::allreduce_coalesced_", "c10d::allgather_", + "c10d::_allgather_base_", "c10d::allgather_into_tensor_coalesced_", + "c10d::reduce_scatter_", "c10d::_reduce_scatter_base_", + "c10d::reduce_scatter_tensor_coalesced_", "c10d::broadcast_", "c10d::reduce_", + "c10d::alltoall_", "c10d::alltoall_base_", "c10d::gather_", "c10d::scatter_", + "c10d::barrier", +} +DTYPE_BYTES = {"float": 4, "float32": 4, "double": 8, "float64": 8, + "half": 2, "float16": 2, "bfloat16": 2, "long": 8, + "int64": 8, "int": 4, "int32": 4, "short": 2, + "int16": 2, "char": 1, "int8": 1, "byte": 1, "uint8": 1, + "bool": 1} + + +def merge_intervals(intervals): + """Return disjoint intervals, counting overlapping streams only once.""" + merged = [] + for begin, end in sorted(intervals): + if not math.isfinite(begin) or not math.isfinite(end) or end < begin: + raise ValueError("Activity intervals must have finite, ordered endpoints") + if begin == end: + continue + if merged and begin <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(end, merged[-1][1])) + else: + merged.append((begin, end)) + return merged + + +def interval_duration(intervals): + return sum(end - begin for begin, end in intervals) + + +def intersection_duration(left, right): + """Measure intersection of two disjoint, ordered interval lists.""" + total = 0.0 + i = j = 0 + while i < len(left) and j < len(right): + total += max(0.0, min(left[i][1], right[j][1]) - max(left[i][0], right[j][0])) + if left[i][1] <= right[j][1]: + i += 1 + else: + j += 1 + return total + + +def collective_metadata(kernels): + """Deduplicate channel kernels using process-group name and sequence number. + + Unknown/inconsistent metadata yields null totals instead of guessed bytes or + a kernel count mislabeled as a collective count. Scalar metric collectives + are included; the returned sequence permits separating them from gradients. + """ + calls = {} + missing_identity = 0 + for event in sorted(kernels, key=lambda item: item["ts"]): + args = event.get("args", {}) + sequence_id = args.get("Seq") + if (args.get("Process Group Name") is None or not args.get("Collective name") + or type(sequence_id) not in (int, str) or not str(sequence_id).isdigit()): + missing_identity += 1 + continue + key = (str(args["Process Group Name"]), str(args["Seq"])) + count = args.get("In msg nelems") + width = DTYPE_BYTES.get(str(args.get("dtype", "")).lower()) + byte_count = count * width if type(count) is int and count >= 0 and width else None + row = {"process_group": key[0], "sequence": key[1], + "collective": args["Collective name"], "dtype": args.get("dtype"), + "input_elements": count, "input_payload_bytes": byte_count} + previous = calls.get(key) + if previous is None: + calls[key] = {**row, "kernel_count": 1, "metadata_consistent": True} + else: + previous["kernel_count"] += 1 + if any(previous[name] != value for name, value in row.items()): + previous["metadata_consistent"] = False + previous["input_payload_bytes"] = None + sequence = list(calls.values()) + known_bytes = sum(row["input_payload_bytes"] or 0 for row in sequence) + complete = not missing_identity and all(row["metadata_consistent"] and + row["input_payload_bytes"] is not None + for row in sequence) + return {"nccl_kernel_count": len(kernels), + "collective_call_count": len(sequence) if not missing_identity else None, + "known_collective_call_count": len(sequence), + "kernels_missing_collective_identity": missing_identity, + "logical_input_payload_bytes": known_bytes if complete else None, + "known_logical_input_payload_bytes": known_bytes, + "collectives": sequence} + + +def cpu_collective_fingerprint(events): + """Structural CPU submission order, never double-counting NCCL annotations. + + Shapes/types do not identify tensor contents. This can detect many bucket + order changes but cannot prove equal tensor order for equal-shaped buckets. + """ + selected = sorted((event for event in events if event.get("ph") == "X" + and event.get("cat") == "cpu_op" + and event.get("name") in COLLECTIVE_OPERATORS), + key=lambda event: event["ts"]) + sequence = [{"operator": event["name"], + "input_dims": event.get("args", {}).get("Input Dims"), + "input_types": event.get("args", {}).get("Input type")} + for event in selected] + payload = json.dumps(sequence, sort_keys=True, separators=(",", ":")) + return {"call_count": len(sequence), "sequence": sequence, + "sha256": hashlib.sha256(payload.encode()).hexdigest() if sequence else None, + "all_input_shapes_recorded": all(row["input_dims"] is not None for row in sequence)} + + +def summarize_trace(trace): + events = trace.get("traceEvents", []) + devices = {} + excluded = 0 + for event in events: + if event.get("ph") != "X": + continue + category = event.get("cat") + if category == "gpu_user_annotation": + excluded += 1 + if category not in ACTIVITY_CATEGORIES: + continue + begin, duration = event.get("ts"), event.get("dur") + if (not isinstance(begin, (int, float)) or not isinstance(duration, (int, float)) + or not math.isfinite(begin) or not math.isfinite(duration) or duration < 0): + raise ValueError("CUDA activities require finite timestamps and nonnegative durations") + device = str(event.get("args", {}).get("device", event.get("pid", "unknown"))) + group = devices.setdefault(device, {"communication": [], "compute": [], "kernels": []}) + communication = category == "kernel" and "nccl" in event.get("name", "").lower() + group["communication" if communication else "compute"].append((begin, begin + duration)) + if communication: + group["kernels"].append(event) + summaries = [] + for device, group in sorted(devices.items()): + compute = merge_intervals(group["compute"]) + communication = merge_intervals(group["communication"]) + active = merge_intervals([*compute, *communication]) + overlap = intersection_duration(compute, communication) + compute_us = interval_duration(compute) + communication_us = interval_duration(communication) + span = active[-1][1] - active[0][0] if active else 0.0 + active_us = interval_duration(active) + summaries.append({"device": device, "activity_span_us": span, + "active_union_us": active_us, "idle_within_activity_span_us": span - active_us, + "noncommunication_union_us": compute_us, + "communication_union_us": communication_us, + "compute_only_us": max(0.0, compute_us - overlap), + "compute_communication_overlap_us": overlap, + "communication_only_us": max(0.0, communication_us - overlap), + "communication_overlap_fraction": overlap / communication_us if communication_us else None, + "summed_noncommunication_us": interval_duration(group["compute"]), + "summed_communication_us": interval_duration(group["communication"]), + "nccl_kernel_names": dict(Counter(event["name"] for event in group["kernels"])), + **collective_metadata(group["kernels"])}) + return {"rank": trace.get("distributedInfo", {}).get("rank"), + "distributed_info": trace.get("distributedInfo"), "devices": summaries, + "excluded_gpu_annotation_ranges": excluded, + "cpu_collective_fingerprint": cpu_collective_fingerprint(events)} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("traces", nargs="+", type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + args.traces = [path.expanduser().resolve() for path in args.traces] + args.output = args.output.expanduser().resolve() + if any(not path.is_relative_to(WORKSPACE) for path in args.traces): + parser.error("Trace inputs must stay inside the WarpState workspace") + if not args.output.is_relative_to(PACKAGE / "work") or args.output.suffix != ".json": + parser.error("Output must be a JSON file under rtx6000_40b/work") + if args.output.exists(): + parser.error("Output already exists; select a new report path") + report = {"format": "warpstate-communication-profile-v1", + "analyzer_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "ranks": [], "limitations": [ + "Per-rank/device budgets; never sum ranks or devices into a wall-time estimate.", + "Compute includes all non-NCCL kernels, memory copies, and fills.", + "GPU annotation ranges are excluded; real CUDA activities alone define intervals.", + "Communication-only time is an exposed interval, not a guaranteed removable delay.", + "Payload bytes describe logical collective inputs, not physical transport traffic.", + "Scalar benchmark metric collectives remain included in counts and bytes.", + "CPU fingerprints compare operator/shape/type submission sequences, not tensor identities.", + "Activity spans exclude untraced startup/tail time; profiling adds instrumentation overhead."]} + for path in args.traces: + payload = path.read_bytes() + summary = summarize_trace(json.loads(payload)) + if summary["rank"] is None: + match = re.search(r"profile-rank-(\d+)", path.name) + if match: + summary["rank"] = int(match.group(1)) + report["ranks"].append({"trace": str(path), "trace_sha256": hashlib.sha256(payload).hexdigest(), + **summary}) + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_name(args.output.name + ".partial") + with temporary.open("x") as output: + output.write(json.dumps(report, indent=2, allow_nan=False) + "\n") + temporary.replace(args.output) + print(json.dumps({"output": str(args.output), "traces": len(report["ranks"])})) + + +if __name__ == "__main__": + main() diff --git a/ops/archive_dpo.py b/ops/archive_dpo.py new file mode 100644 index 0000000000000000000000000000000000000000..d5432e4ed20e1589d7d64e97c9e7ac52af5630e3 --- /dev/null +++ b/ops/archive_dpo.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Seal a completed DPO run as independent, read-only full-state/model archives. + +Run from /workspace/WarpState after the DPO guardian exits successfully. The +default destination is work/archives/step-00076294-dpo-final. Existing archives +are verified and reused; they are never replaced or modified. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import math +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +sys.path.insert(0, str(PACKAGE)) + +import numpy as np +import torch + +from dpo import DEFAULTS, identity_digest +from ops.dpo_handoff import load_plan, read_json, seal, verify_files, verify_dpo_completion +from ops.launch_training import workspace_path +from ops.posttrain_handoff import (atomic_json, clean_exit, copy_regular, exclusive_lock, + process_status, publish_directory, sha256, tensors_equal, utc_now) + +FILES = ('checkpoint.pt', 'model.pt', 'snapshot.json', 'tokenizer.model', 'plan.json', + 'settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl', + 'dpo-execution.json', 'dpo-handoff.json', 'gpu-preflight.json', + 'data-manifest.json', 'reference-manifest.json', 'reference-train.npy', + 'reference-val.npy', 'PROVENANCE.json', 'ARCHIVE.md') + + +def validate_state(checkpoint, evaluation, completion, settings_report, data, reference, plan): + """Validate an archive using its own preserved records, without a live run.""" + settings = DEFAULTS | plan['training'] + processes = plan['handoff']['processes'] + train, val = (data['outputs'][name] for name in ('dpo_train.jsonl', 'dpo_val.jsonl')) + total_steps = math.ceil(train['rows'] / (settings['micro_batch'] * settings['grad_accum'] * processes)) * settings['epochs'] + if settings['max_steps'] is not None: + raise ValueError('Archive requires a completed whole DPO epoch, without a plan step cap') + expected = dict(step=total_steps, total_steps=total_steps, + sample_cursor=train['rows'] * settings['epochs'], + tokens_seen=(train['tokens'] - 2 * train['rows']) * settings['epochs'], + response_tokens_seen=train['supervised_tokens'] * settings['epochs'], val_step=total_steps) + if (checkpoint.get('format') != 'warpstate-dpo-v1' or checkpoint.get('completed') is not True or + any(checkpoint.get(key) != value for key, value in expected.items()) or + not math.isfinite(checkpoint.get('val_loss', float('nan')))): + raise ValueError('DPO checkpoint is not the fully completed, finally evaluated state') + signature = checkpoint['signature'] + if (signature.get('plan') != plan or signature.get('settings') != settings or + signature.get('world_size') != processes or + signature.get('tokenizer_sha256') != plan['sft']['tokenizer_sha256'] or + signature.get('data_sha256') != plan['handoff']['data_manifest_sha256'] or + settings_report.get('signature') != signature or + settings_report.get('total_steps') != total_steps or + settings_report.get('train_rows') != train['rows'] or + settings_report.get('val_rows') != val['rows']): + raise ValueError('DPO checkpoint/settings signatures or dataset counts differ') + if (data.get('format') != 'warpstate-posttrain-data-v1' or data.get('complete') is not True or + data.get('tokenizer_sha256') != signature['tokenizer_sha256'] or + data.get('max_length') != settings['sequence_length']): + raise ValueError('DPO data provenance differs from its signature') + origin = checkpoint['origin'] + if (origin.get('stage') != 'sft' or origin.get('step') != plan['sft']['step'] or + origin.get('weights_sha256') != signature['base_sha256'] or + origin.get('tokens_seen') != plan['sft']['tokens_seen'] or + origin.get('origin', {}).get('step') != plan['sft']['base_step'] or + origin.get('origin', {}).get('tokens_seen') != plan['sft']['base_tokens_seen']): + raise ValueError('DPO SFT/raw ancestry differs from the authorized plan') + if (not checkpoint.get('optimizer', {}).get('state') or + len(checkpoint.get('rng_by_rank', [])) != processes or + any('torch' not in value or 'cuda' not in value for value in checkpoint['rng_by_rank'])): + raise ValueError('DPO optimizer or per-rank RNG state is missing') + for value in checkpoint['optimizer']['state'].values(): + if float(value['step']) != total_steps: + raise ValueError('DPO Adam step differs from the final checkpoint') + for name in ('exp_avg', 'exp_avg_sq'): + if value[name].dtype != torch.float32: + raise ValueError('DPO Adam state must retain its original FP32 tensors') + if not checkpoint.get('model'): + raise ValueError('DPO model tensors are missing') + for name, tensor in checkpoint['model'].items(): + if tensor.is_floating_point() and tensor.dtype != torch.float32: + raise ValueError(f'DPO model tensor is not original FP32: {name}') + counters = ('step', 'total_steps', 'sample_cursor', 'tokens_seen', 'response_tokens_seen', 'val_loss') + if (completion.get('format') != 'warpstate-dpo-completion-v1' or completion.get('stage') != 'dpo' or + completion.get('completed') is not True or completion.get('paused_for_evaluation') is not True or + any(completion.get(key) != checkpoint[key] for key in counters)): + raise ValueError('DPO completion report differs from its checkpoint') + if (evaluation.get('stage') != 'dpo' or evaluation.get('step') != total_steps or + evaluation.get('full_holdout') is not True or evaluation.get('evaluated_samples') != val['rows'] or + evaluation.get('available_validation_samples') != val['rows'] or + evaluation.get('objective') != settings['objective'] or + any(evaluation.get(key) != checkpoint['validation'].get(key) for key in + ('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll', 'evaluated_samples')) or + any(evaluation.get(key) != signature[key] for key in + ('base_sha256', 'data_sha256', 'tokenizer_sha256', 'reference_sha256')) or + not all(math.isfinite(evaluation.get(key, float('nan'))) for key in + ('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'))): + raise ValueError('DPO final full-holdout evaluation differs from its checkpoint') + if not 0 <= evaluation['reward_accuracy'] <= 1 or evaluation['chosen_nll'] < 0: + raise ValueError('DPO evaluation contains impossible accuracy/NLL') + identity = reference.get('identity', {}) + if (reference.get('format') != 'warpstate-dpo-reference-v1' or reference.get('complete') is not True or + reference.get('identity_sha256') != identity_digest(identity) or + identity.get('weights_sha256') != signature['base_sha256'] or + identity.get('data_sha256') != signature['data_sha256'] or + identity.get('tokenizer_sha256') != signature['tokenizer_sha256'] or + identity.get('sources') != signature['sources'] or + identity.get('renderer_sha256') != data.get('renderer_sha256') or + identity.get('mask_format') != signature['mask'] or + identity.get('device_type') != signature['device_type'] or + identity.get('world_size') != processes or + identity.get('torch_version') != signature['torch_version'] or + any(value != settings.get(key) for key, value in identity.get('settings', {}).items()) or + completion.get('reference_sha256') != signature['reference_sha256']): + raise ValueError('DPO fixed-reference identity differs from its final checkpoint') + return expected + + +def verify_dpo_archive(folder, *, expected_checkpoint_hash=None): + folder = Path(folder) + entries = verify_files(folder, FILES) + if stat.S_IMODE(folder.stat().st_mode) & 0o222 or {path.name for path in folder.iterdir()} != set(FILES) | {'SHA256SUMS'}: + raise ValueError('DPO archive directory is not sealed or contains unexpected files') + if expected_checkpoint_hash and entries['checkpoint.pt'] != expected_checkpoint_hash: + raise ValueError('Archived DPO state differs from the source checkpoint') + checkpoint = torch.load(folder / 'checkpoint.pt', map_location='cpu', weights_only=False, mmap=True) + plan, data = read_json(folder / 'plan.json'), read_json(folder / 'data-manifest.json') + reference = read_json(folder / 'reference-manifest.json') + validate_state(checkpoint, read_json(folder / 'evaluation.json'), read_json(folder / 'completion.json'), + read_json(folder / 'settings.json'), data, reference, plan) + if (entries['data-manifest.json'] != checkpoint['signature']['data_sha256'] or + entries['reference-manifest.json'] != checkpoint['signature']['reference_sha256'] or + entries['tokenizer.model'] != checkpoint['signature']['tokenizer_sha256']): + raise ValueError('Archived data/reference/tokenizer hashes differ from checkpoint signatures') + for split in ('train', 'val'): + entry = reference['files'][split + '.npy'] + name = f'reference-{split}.npy' + rows = data['outputs'][f'dpo_{split}.jsonl']['rows'] + values = np.load(folder / name, allow_pickle=False, mmap_mode='r') + if (entries[name] != entry['sha256'] or entry['rows'] != rows or + values.shape != (rows, 2) or values.dtype != np.float64 or + not np.isfinite(values).all() or not (values <= 0).all()): + raise ValueError('Archived fixed-reference score array differs') + metadata = read_json(folder / 'snapshot.json') + if (metadata.get('format') != 'warpstate-chat-v1' or metadata.get('inference_only') is not True or + metadata.get('stage') != 'dpo' or metadata.get('config') != checkpoint['config'] or + any(metadata.get(key) != checkpoint[key] for key in + ('step', 'tokens_seen', 'response_tokens_seen', 'val_loss', 'origin')) or + metadata.get('tokenizer_sha256') != entries['tokenizer.model'] or + metadata.get('snapshot_sha256') != entries['model.pt'] or + metadata.get('snapshot_bytes') != (folder / 'model.pt').stat().st_size or + metadata.get('source', {}).get('sha256') != entries['checkpoint.pt']): + raise ValueError('DPO model export metadata differs from the full archive') + exported = torch.load(folder / 'model.pt', map_location='cpu', weights_only=False, mmap=True) + if {key: value for key, value in exported.items() if key != 'model'} != { + key: value for key, value in metadata.items() if key not in ('snapshot_sha256', 'snapshot_bytes')}: + raise ValueError('DPO exported model metadata differs from snapshot.json') + tensors_equal(checkpoint['model'], exported['model']) + return metadata, entries + + +def archive_dpo(work, tokenizer, destination, plan, revision): + """Caller owns all training exclusion locks; no source files are changed.""" + work, tokenizer, destination = map(Path, (work, tokenizer, destination)) + run, reference_dir = work / 'run', work / 'reference-cache' + source = run / 'latest.pt' + if source.is_symlink() or not source.is_file(): + raise ValueError('DPO checkpoint must be a regular completed file') + before = source.stat() + checkpoint = torch.load(source, map_location='cpu', weights_only=False, mmap=True) + data_path = workspace_path(WORKSPACE / plan['handoff']['data_dir']) / 'manifest.json' + validate_state(checkpoint, read_json(run / 'evaluation.json'), read_json(run / 'completion.json'), + read_json(run / 'settings.json'), read_json(data_path), read_json(reference_dir / 'manifest.json'), plan) + full_hash = sha256(source) + if destination.exists() or destination.is_symlink(): + return verify_dpo_archive(destination, expected_checkpoint_hash=full_hash) + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix='.dpo-final-', dir=destination.parent)) + copy_regular(source, staging / 'checkpoint.pt') + if sha256(staging / 'checkpoint.pt') != full_hash: + raise ValueError('DPO full-state archive copy differs from its source') + metadata = dict(format='warpstate-chat-v1', inference_only=True, stage='dpo', + config=checkpoint['config'], step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'], + response_tokens_seen=checkpoint['response_tokens_seen'], val_loss=checkpoint['val_loss'], + tokenizer_sha256=checkpoint['signature']['tokenizer_sha256'], origin=checkpoint['origin'], + captured_at_utc=utc_now(), source=dict(path=str(source), device=before.st_dev, + inode=before.st_ino, size=before.st_size, mtime_ns=before.st_mtime_ns, sha256=full_hash)) + with (staging / 'model.pt').open('xb') as stream: + torch.save(dict(metadata, model=checkpoint['model']), stream) + stream.flush() + os.fsync(stream.fileno()) + metadata.update(snapshot_sha256=sha256(staging / 'model.pt'), + snapshot_bytes=(staging / 'model.pt').stat().st_size) + atomic_json(staging / 'snapshot.json', metadata) + atomic_json(staging / 'plan.json', plan) + copies = {name: run / name for name in ('settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl')} + copies.update({'tokenizer.model': tokenizer, 'dpo-execution.json': work / 'control/execution.json', + 'dpo-handoff.json': work / 'control/handoff.json', 'gpu-preflight.json': work / 'preflight/report.json', + 'data-manifest.json': data_path, 'reference-manifest.json': reference_dir / 'manifest.json', + 'reference-train.npy': reference_dir / 'train.npy', 'reference-val.npy': reference_dir / 'val.npy'}) + for name, path in copies.items(): + copy_regular(path, staging / name) + atomic_json(staging / 'PROVENANCE.json', dict(format='warpstate-dpo-archive-v1', + repository_revision=revision, archive_command_sha256=sha256(Path(__file__)), + source_checkpoint_sha256=full_hash, model_sha256=metadata['snapshot_sha256'], + dpo_step=checkpoint['step'], dpo_input_tokens=checkpoint['tokens_seen'], + dpo_response_tokens=checkpoint['response_tokens_seen'], dpo_pairs=checkpoint['sample_cursor'], + sft_step=plan['sft']['step'], sft_model_sha256=checkpoint['signature']['base_sha256'], + raw_base_step=plan['sft']['base_step'], raw_base_tokens=plan['sft']['base_tokens_seen'], + signature=checkpoint['signature'])) + note = f'''# KEEP: completed DPO after raw step {plan['sft']['base_step']} + +User-requested historical archive created {utc_now()}. Preserve during routine +cleanup, further training and chat refreshes. Delete only on explicit request. + +- DPO update {checkpoint['step']:,}; {checkpoint['sample_cursor']:,} preference pairs. +- DPO input tokens {checkpoint['tokens_seen']:,}; response targets {checkpoint['response_tokens_seen']:,}. +- Parent SFT update {plan['sft']['step']:,}; raw base update {plan['sft']['base_step']:,}. +- Final full-holdout DPO objective loss: {checkpoint['val_loss']:.12f}. +- `checkpoint.pt`: byte-identical full checkpoint, including FP32 model and Adam, + per-rank RNG, sample cursor, model configuration and immutable signatures. +- `model.pt`: tensor-exact FP32 inference export with the original tokenizer. +- Plan, data/reference manifests and reference scores, original settings, + evaluation, completion, metrics, preflight and execution provenance are included. +- Stage counters are distinct. DPO objective loss is not language-model perplexity. +- All files are independent physical copies, read-only, without hardlinks/reflinks. +- Existing raw/SFT archives and the original DPO run remain unchanged. +- Repository revision at archival time: `{revision}`. + +Verify with `sha256sum -c SHA256SUMS`, or run `archive_dpo.py --verify-only` +with this directory as `--destination`. This is an on-device archive. +''' + with (staging / 'ARCHIVE.md').open('x') as stream: + stream.write(note) + stream.flush() + os.fsync(stream.fileno()) + after = source.stat() + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns): + raise ValueError('DPO checkpoint changed during archival') + seal(staging, FILES) + result = verify_dpo_archive(staging, expected_checkpoint_hash=full_hash) + publish_directory(staging, destination) + return result + + +def run(config, destination=None, *, program='warpstate-dpo', verify_only=False): + config = workspace_path(config) + plan, paths = load_plan(config) + work, base = paths['work_dir'], paths['base_work_dir'] + destination = workspace_path(destination or (base / 'archives' / f"step-{plan['sft']['base_step']:08d}-dpo-final")) + if (destination.parent != base / 'archives' or + destination == paths['archive_dir']): + raise ValueError('DPO archive must use a distinct direct child of work/archives') + if verify_only: + metadata, checksums = verify_dpo_archive(destination) + else: + with contextlib.ExitStack() as locks: + for path in (work / 'control/archive.lock', work / 'control/handoff.lock', + paths['sft_work_dir'] / 'control/handoff.lock', + base / 'control/training-launcher.lock', base / 'pipeline.lock'): + locks.enter_context(exclusive_lock(path)) + if not clean_exit(process_status(paths['supervisor_socket'], program)): + raise ValueError('DPO guardian must have exited successfully before archival') + if read_json(work / 'control/handoff.json').get('phase') != 'complete': + raise ValueError('DPO guardian has not completed its final integrity checks') + model_hash = sha256(work / 'initialization/model.pt') + if model_hash != sha256(paths['archive_dir'] / 'model.pt'): + raise ValueError('Original SFT archive and DPO initialization differ') + verify_dpo_completion(work / 'run', plan, model_hash) + revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=WORKSPACE, text=True).strip() + metadata, checksums = archive_dpo(work, paths['tokenizer'], destination, plan, revision) + print(json.dumps(dict(archive=str(destination), step=metadata['step'], stage='dpo', + checkpoint_sha256=checksums['checkpoint.pt'], model_sha256=checksums['model.pt'], + tokens_seen=metadata['tokens_seen'], response_tokens_seen=metadata['response_tokens_seen']), indent=2)) + return metadata, checksums + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', type=Path, default=PACKAGE / 'configs/dpo_10b.json') + parser.add_argument('--destination', type=Path) + parser.add_argument('--program', default='warpstate-dpo') + parser.add_argument('--verify-only', action='store_true') + args = parser.parse_args() + torch.set_num_threads(8) + run(args.config, args.destination, program=args.program, verify_only=args.verify_only) + + +if __name__ == '__main__': + main() diff --git a/ops/b200-requirements.txt b/ops/b200-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a4e93eb4ab88072c4c672503e3a9d49cf1886080 --- /dev/null +++ b/ops/b200-requirements.txt @@ -0,0 +1,2 @@ +-r training-requirements-resolved.txt +supervisor==4.3.0 diff --git a/ops/b200-supervisord.conf b/ops/b200-supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..d377faf04e7d7eb8c804ee46b4bf1070f2d17d6f --- /dev/null +++ b/ops/b200-supervisord.conf @@ -0,0 +1,35 @@ +[unix_http_server] +file=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisor.sock +chmod=0700 + +[supervisord] +logfile=/workspace/WarpState/rtx6000_40b/work/logs/b200-supervisor.log +logfile_maxbytes=10MB +logfile_backups=3 +pidfile=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisord.pid +childlogdir=/workspace/WarpState/rtx6000_40b/work/logs +directory=/workspace/WarpState/rtx6000_40b +umask=077 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///workspace/WarpState/rtx6000_40b/work/control/b200-supervisor.sock +history_file=/workspace/WarpState/rtx6000_40b/work/control/b200-supervisorctl-history + +[program:warpstate-train] +command=/workspace/WarpState/rtx6000_40b/ops/train-b200.sh +directory=/workspace/WarpState/rtx6000_40b +autostart=false +autorestart=false +startsecs=5 +startretries=0 +stopsignal=USR1 +stopwaitsecs=900 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/b200-train.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=5 +redirect_stderr=true diff --git a/ops/b200ctl.sh b/ops/b200ctl.sh new file mode 100644 index 0000000000000000000000000000000000000000..97db2bc635cca765ebbf3c6c94a066852d7ff3c7 --- /dev/null +++ b/ops/b200ctl.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +exec /workspace/WarpState/rtx6000_40b/.server-venv/bin/supervisorctl \ + -c /workspace/WarpState/rtx6000_40b/ops/b200-supervisord.conf "$@" diff --git a/ops/backward_layout.py b/ops/backward_layout.py new file mode 100644 index 0000000000000000000000000000000000000000..4c2be75e2c84976c7063f87dd08d88797c2be8eb --- /dev/null +++ b/ops/backward_layout.py @@ -0,0 +1,283 @@ +"""Isolated BF16 attention-backward layout experiment, not a training adapter. + +The reference boundary is the September 12 compiled backward's rotary kernel +followed by Q/K/V/gate/U packing. Both BF16 intermediate roundings are retained. +No reductions, GEMMs, parameter gradients or optimizer state are changed here. +All saved tensors use [batch,chunks,heads,chunk_size,head_dim] contiguous layout, +except transposed K-write gradients and token-major output gradients. +""" + +from __future__ import annotations + +from typing import NamedTuple + +import torch +import triton +import triton.language as tl + + +PROFILE_SHAPE = (4, 16, 32, 128, 64) +ALGORITHMS = ("profile", "paired", "single") + + +class LayoutInputs(NamedTuple): + q_memory: torch.Tensor + q_local: torch.Tensor + sin: torch.Tensor + cos: torch.Tensor + k_write: torch.Tensor + tanh_k: torch.Tensor + k_local: torch.Tensor + v_local: torch.Tensor + grad_y: torch.Tensor + memory: torch.Tensor + local: torch.Tensor + gate: torch.Tensor + u_write: torch.Tensor + tanh_u: torch.Tensor + + +def validate_inputs(inputs: LayoutInputs, *, cuda=False): + if not isinstance(inputs, LayoutInputs): + raise TypeError("Expected named LayoutInputs") + shape = tuple(inputs.q_memory.shape) + if len(shape) != 5 or min(shape) < 1 or shape[-1] % 2: + raise ValueError("Expected positive [B,N,H,C,D] with an even head dimension") + b, n, h, c, d = shape + device = inputs.q_memory.device + if cuda and device.type != "cuda": + raise ValueError("Triton layout kernels require CUDA") + for name, tensor in zip(inputs._fields, inputs, strict=True): + wanted = shape + dtype = torch.bfloat16 + if name in ("sin", "cos"): + wanted, dtype = (c, d), torch.float32 + elif name == "k_write": + wanted = (b, n, h, d, c) + elif name == "grad_y": + wanted = (b, n * c, h * d) + if tuple(tensor.shape) != wanted or tensor.dtype != dtype: + raise ValueError(f"{name} must have shape {wanted} and dtype {dtype}") + if tensor.device != device or not tensor.is_contiguous(): + raise ValueError(f"{name} must be contiguous on {device}") + return shape + + +def make_inputs(shape=PROFILE_SHAPE, *, device="cpu", seed=817) -> LayoutInputs: + """Deterministic synthetic saved activations/gradients, no model/data reads.""" + b, n, h, c, d = shape + generator = torch.Generator(device=device).manual_seed(seed) + + def random(dims=shape, scale=0.1): + return torch.randn(dims, device=device, dtype=torch.bfloat16, + generator=generator) * scale + + from warpstate.model import rope_cos_sin + cos, sin = rope_cos_sin(c, d, torch.device(device), torch.float32) + return LayoutInputs(random(), random(), sin, cos, random((b, n, h, d, c)), + random(scale=1).float().tanh().bfloat16(), random(), random(), + random((b, n * c, h * d)), random(), random(), + random(scale=1).float().sigmoid().bfloat16(), random(), + random(scale=1).float().tanh().bfloat16()) + + +def reference_layout(inputs: LayoutInputs) -> torch.Tensor: + """Independent tensor expression for both stages, including BF16 boundaries. + + FP32 operation contraction may differ between eager PyTorch and Triton; + use the tracked profile kernels as the exact GPU rounding oracle. + """ + b, n, h, c, d = validate_inputs(inputs) + q = inputs.q_memory.float() + inputs.q_local.float() + k = (inputs.k_write.transpose(-2, -1).float() + * (1 - inputs.tanh_k.float().square()) + inputs.k_local.float()) + + def inverse_rotate(value): + return torch.stack((value[..., 1::2], -value[..., ::2]), dim=-1).flatten(-2) + + q_rounded = (inverse_rotate(q * inputs.sin) + q * inputs.cos).bfloat16() + k_rotated_rounded = inverse_rotate(k * inputs.sin).bfloat16() + k_packed = (k_rotated_rounded.float() + k * inputs.cos).bfloat16() + dy = inputs.grad_y.view(b, n, c, h, d).permute(0, 1, 3, 2, 4).float() + gate = inputs.gate.float() + dg = ((-(dy * (inputs.memory.float() * d**-0.5)) + + dy * inputs.local.float()) * (gate * (1 - gate))).bfloat16() + du = (inputs.u_write.float() * (1 - inputs.tanh_u.float().square())).bfloat16() + + def token_major(value): + return value.permute(0, 1, 3, 2, 4).reshape(b, n * c, h * d) + + return torch.cat([token_major(value) for value in + (q_rounded, k_packed, inputs.v_local, dg, du)], dim=-1) + + +@triton.jit +def _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT, + ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr, + D: tl.constexpr, BLOCK: tl.constexpr, program): + pairs = program * BLOCK + tl.arange(0, BLOCK) + even = pairs * 2 + odd = even + 1 + mask = even < ELEMENTS + token = (even // D) % C + channel = even % D + group = even // (C * D) + rope = token * D + channel + s0 = tl.load(SIN + rope, mask, 0) + s1 = tl.load(SIN + rope + 1, mask, 0) + c0 = tl.load(COS + rope, mask, 0) + c1 = tl.load(COS + rope + 1, mask, 0) + q0 = (tl.load(QM + even, mask, 0).to(tl.float32) + + tl.load(QL + even, mask, 0).to(tl.float32)) + q1 = (tl.load(QM + odd, mask, 0).to(tl.float32) + + tl.load(QL + odd, mask, 0).to(tl.float32)) + # The profiled interleave sums each active rotary lane with an inactive +0. + # Retain that IEEE signed-zero boundary as well as its BF16 store below. + q_even_rot = q1 * s1 + 0.0 + q_odd_rot = (-q0 * s0) + 0.0 + q_even = (q_even_rot + q0 * c0).to(tl.bfloat16) + q_odd = (q_odd_rot + q1 * c1).to(tl.bfloat16) + kw_offset = group * C * D + channel * C + token + kw0 = tl.load(KW + kw_offset, mask, 0).to(tl.float32) + kw1 = tl.load(KW + kw_offset + C, mask, 0).to(tl.float32) + tk0 = tl.load(TK + even, mask, 0).to(tl.float32) + tk1 = tl.load(TK + odd, mask, 0).to(tl.float32) + k0 = kw0 * (1.0 - tk0 * tk0) + tl.load(KL + even, mask, 0).to(tl.float32) + k1 = kw1 * (1.0 - tk1 * tk1) + tl.load(KL + odd, mask, 0).to(tl.float32) + # Stage 17 stores these separately, before adding the cosine term in 18. + # Retain the truncation even though the fused implementation has no scratch. + k_even_rot = (k1 * s1 + 0.0).to(tl.bfloat16).to(tl.float32) + k_odd_rot = ((-k0 * s0) + 0.0).to(tl.bfloat16).to(tl.float32) + k_even = k_even_rot + k0 * c0 + k_odd = k_odd_rot + k1 * c1 + out = ((group // H) * C + token) * (5 * H * D) + (group % H) * D + channel + tl.store(OUT + out, q_even, mask) + tl.store(OUT + out + 1, q_odd, mask) + tl.store(OUT + out + H * D, k_even, mask) + tl.store(OUT + out + H * D + 1, k_odd, mask) + + +@triton.jit +def _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT, + ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr, + D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr, program): + index = program * BLOCK + tl.arange(0, BLOCK) + mask = index < ELEMENTS + token = (index // D) % C + channel = index % D + group = index // (C * D) + token_row = (group // H) * C + token + feature = (group % H) * D + channel + dy = tl.load(DY + token_row * H * D + feature, mask, 0).to(tl.float32) + memory = tl.load(MEMORY + index, mask, 0).to(tl.float32) + local = tl.load(LOCAL + index, mask, 0).to(tl.float32) + gate = tl.load(GATE + index, mask, 0).to(tl.float32) + # Keep the original chain rather than algebraically factoring dy. + dgate = (-(dy * (memory * SCALE)) + dy * local) * (gate * (1.0 - gate)) + uw = tl.load(UW + index, mask, 0).to(tl.float32) + tu = tl.load(TU + index, mask, 0).to(tl.float32) + du = uw * (1.0 - tu * tu) + out = token_row * 5 * H * D + feature + tl.store(OUT + out + 2 * H * D, tl.load(VL + index, mask, 0), mask) + tl.store(OUT + out + 3 * H * D, dgate, mask) + tl.store(OUT + out + 4 * H * D, du, mask) + + +@triton.jit +def _paired_kernel(QM, QL, SIN, COS, KW, TK, KL, VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT, + ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr, + D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr, + ROTARY: tl.constexpr): + if ROTARY: + _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT, + ELEMENTS, H, C, D, BLOCK, tl.program_id(0)) + else: + _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT, + ELEMENTS, H, C, D, SCALE, BLOCK, tl.program_id(0)) + + +@triton.jit +def _single_kernel(QM, QL, SIN, COS, KW, TK, KL, VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT, + ELEMENTS: tl.constexpr, H: tl.constexpr, C: tl.constexpr, + D: tl.constexpr, SCALE: tl.constexpr, BLOCK: tl.constexpr): + rotary_programs = tl.cdiv(ELEMENTS // 2, BLOCK) + program = tl.program_id(0) + if program < rotary_programs: + _rotary_store(QM, QL, SIN, COS, KW, TK, KL, OUT, + ELEMENTS, H, C, D, BLOCK, program) + else: + _other_store(VL, DY, MEMORY, LOCAL, GATE, UW, TU, OUT, + ELEMENTS, H, C, D, SCALE, BLOCK, program - rotary_programs) + + +def prepare_layout(inputs: LayoutInputs, algorithm="paired", *, block=512, warps=4, + output=None, scratch=None): + """Validate once and return an allocation-free launch callable. + + Callers must keep the inputs/output/scratch metadata and storage stable. + No GPU kernel executes until the returned callable is invoked. + """ + b, n, h, c, d = validate_inputs(inputs, cuda=True) + if algorithm not in ALGORITHMS: + raise ValueError(f"Unknown backward layout algorithm: {algorithm}") + if block not in (128, 256, 512, 1024) or warps not in (4, 8): + raise ValueError("Use block 128/256/512/1024 with 4/8 warps") + shape = (b, n * c, 5 * h * d) + if output is None: + output = torch.empty(shape, dtype=torch.bfloat16, device=inputs.q_memory.device) + if (tuple(output.shape) != shape or output.dtype != torch.bfloat16 + or output.device != inputs.q_memory.device or not output.is_contiguous()): + raise ValueError("Output must be a contiguous BF16 packed gradient on the input device") + if any(output.untyped_storage().data_ptr() == value.untyped_storage().data_ptr() + for value in inputs): + raise ValueError("Output may not alias a saved input") + if algorithm == "profile": + if (b, n, h, c, d) != PROFILE_SHAPE: + raise ValueError("The exact profiled baseline supports PROFILE_SHAPE only") + from ops.backward_layout_baseline import _profile_pack, _profile_rotary + if scratch is None: + scratch = (torch.empty_like(inputs.q_memory), torch.empty_like(inputs.q_memory)) + if (len(scratch) != 2 or any(value.shape != inputs.q_memory.shape + or value.dtype != torch.bfloat16 or value.device != output.device + or not value.is_contiguous() for value in scratch)): + raise ValueError("The profile baseline needs two contiguous BF16 scratch tensors") + q, kr = scratch + storages = [value.untyped_storage().data_ptr() for value in (*inputs, output)] + if (q.untyped_storage().data_ptr() in storages + or kr.untyped_storage().data_ptr() in storages + or q.untyped_storage().data_ptr() == kr.untyped_storage().data_ptr()): + raise ValueError("Scratch tensors may not alias inputs, output, or one another") + + def operation(): + _profile_rotary[(1, 16384)]( + inputs.q_memory, inputs.q_local, inputs.sin, inputs.cos, + inputs.k_write, inputs.tanh_k, inputs.k_local, q, kr, + 262144, 64, YBLOCK=16, XBLOCK=64, num_warps=4, enable_fp_fusion=True) + _profile_pack[(160, 128)]( + q, kr, inputs.k_write, inputs.tanh_k, inputs.k_local, inputs.cos, + inputs.v_local, inputs.grad_y, inputs.memory, inputs.local, inputs.gate, + inputs.u_write, inputs.tanh_u, output, 8192, 10240, + YBLOCK=64, XBLOCK=64, num_warps=8, enable_fp_fusion=True) + return output + else: + elements = inputs.q_memory.numel() + args = (*inputs, output, elements, h, c, d, d**-0.5, block) + + def operation(): + if algorithm == "paired": + _paired_kernel[(triton.cdiv(elements // 2, block),)]( + *args, True, num_warps=warps, enable_fp_fusion=True) + _paired_kernel[(triton.cdiv(elements, block),)]( + *args, False, num_warps=warps, enable_fp_fusion=True) + else: + _single_kernel[(triton.cdiv(elements // 2, block) + triton.cdiv(elements, block),)]( + *args, num_warps=warps, enable_fp_fusion=True) + return output + return operation + + +def launch_layout(inputs: LayoutInputs, algorithm="paired", *, block=512, warps=4, + output=None, scratch=None) -> torch.Tensor: + """One checked launch; timed loops should use prepare_layout once instead.""" + return prepare_layout(inputs, algorithm, block=block, warps=warps, + output=output, scratch=scratch)() diff --git a/ops/backward_layout_baseline.py b/ops/backward_layout_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..0ec4d49307d33ec518617fde237afe50f599f0e4 --- /dev/null +++ b/ops/backward_layout_baseline.py @@ -0,0 +1,201 @@ +"""Tracked reference kernels from the September 12 CA-overlap backward profile. + +The bodies below reproduce the exact profiled arithmetic/indexing at shape +B=4, chunks=16, heads=32, chunk_size=128, head_dim=64. They are kept as source, +not imported from generated compiler caches. Names 17/18 in that trace consumed +31.907/77.576 ms per rank-0 update. Their measured launch choices were +XBLOCK=64/YBLOCK=16/4 warps and XBLOCK=64/YBLOCK=64/8 warps respectively. + +Provenance: ops/model sources and profile-ca-overlap trace map these bodies to +cache hashes cq665jkyfy6uac7wrzvwa5aigwids6rzt4n7vzip6xvufai4ajjw and +cbr6gfx3vjhnzjbykfch3pg5peunfivhgdrfvl5hnzolhvfjfgdy. The isolated launcher +preallocates both scratch tensors; production reused a dead buffer for one of +them. Compare kernel time and traffic, not an inferred peak-allocation saving. +""" + +import triton +import triton.language as tl +from torch._inductor.runtime import triton_helpers + + +@triton.jit +def _profile_rotary(in_ptr0, in_ptr1, in_ptr2, in_ptr3, in_ptr4, in_ptr5, in_ptr6, out_ptr0, out_ptr1, ynumel, xnumel, YBLOCK : tl.constexpr, XBLOCK : tl.constexpr): + ynumel = 262144 + xnumel = 64 + yoffset = (tl.program_id(1) + tl.program_id(2) * tl.num_programs(1)) * YBLOCK + yindex = yoffset + tl.arange(0, YBLOCK)[:, None] + ymask = yindex < ynumel + xoffset = tl.program_id(0) * XBLOCK + xindex = xoffset + tl.arange(0, XBLOCK)[None, :] + xmask = xindex < xnumel + x2 = xindex + y3 = yindex + y0 = (yindex % 128) + y1 = yindex // 128 + tmp33 = tl.load(in_ptr0 + (x2 + 64*y3), xmask & ymask).to(tl.float32) + tmp34 = tl.load(in_ptr1 + (x2 + 64*y3), xmask & ymask).to(tl.float32) + tmp36 = tl.load(in_ptr3 + (x2 + 64*y0), xmask & ymask, eviction_policy='evict_last') + tmp0 = (x2).to(tl.int64) + tmp1 = (tmp0).to(tl.int64) + tmp2 = tl.full([1, 1], 1, tl.int64) + tmp3 = tmp1 >= tmp2 + tmp4 = ((((-1) + x2) % 2)).to(tl.int64) + tmp5 = (tmp4).to(tl.int64) + tmp6 = tl.full([1, 1], 0, tl.int64) + tmp7 = tmp5 == tmp6 + tmp8 = tmp3 & tmp7 + tmp9 = tl.load(in_ptr0 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp10 = tl.load(in_ptr1 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp11 = tmp9 + tmp10 + tmp12 = tl.load(in_ptr2 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y0), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0) + tmp13 = tmp12.to(tl.float32) + tmp14 = tmp11 * tmp13 + tmp15 = -tmp14 + tmp16 = tl.full(tmp15.shape, 0.0, tmp15.dtype) + tmp17 = tl.where(tmp8, tmp15, tmp16) + tmp18 = tl.full([1, 1], 0.0, tl.float32) + tmp19 = tl.where(tmp8, tmp17, tmp18) + tmp20 = ((x2 % 2)).to(tl.int64) + tmp21 = (tmp20).to(tl.int64) + tmp22 = tmp21 == tmp6 + tmp23 = tl.load(in_ptr0 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp24 = tl.load(in_ptr1 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp25 = tmp23 + tmp24 + tmp26 = tl.load(in_ptr2 + (1 + 2*(x2 // 2) + 64*y0), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0) + tmp27 = tmp26.to(tl.float32) + tmp28 = tmp25 * tmp27 + tmp29 = tl.full(tmp28.shape, 0.0, tmp28.dtype) + tmp30 = tl.where(tmp22, tmp28, tmp29) + tmp31 = tl.where(tmp22, tmp30, tmp18) + tmp32 = tmp19 + tmp31 + tmp35 = tmp33 + tmp34 + tmp37 = tmp36.to(tl.float32) + tmp38 = tmp35 * tmp37 + tmp39 = tmp32 + tmp38 + tmp40 = tl.load(in_ptr4 + (y0 + 256*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 8192*y1), tmp8 & xmask & ymask, other=0.0).to(tl.float32) + tmp41 = tmp40.to(tl.float32) + tmp42 = tl.load(in_ptr5 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp43 = tmp42.to(tl.float32) + tmp44 = tmp43 * tmp43 + tmp45 = tl.full([1, 1], 1.0, tl.float32) + tmp46 = tmp45 - tmp44 + tmp47 = tmp41 * tmp46 + tmp48 = tmp47.to(tl.float32) + tmp49 = tl.load(in_ptr6 + (2*(triton_helpers.div_floor_integer((-1) + x2, 2)) + 64*y3), tmp8 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp50 = tmp48 + tmp49 + tmp51 = tmp50 * tmp13 + tmp52 = -tmp51 + tmp53 = tl.full(tmp52.shape, 0.0, tmp52.dtype) + tmp54 = tl.where(tmp8, tmp52, tmp53) + tmp55 = tl.where(tmp8, tmp54, tmp18) + tmp56 = tl.load(in_ptr4 + (128 + y0 + 256*(x2 // 2) + 8192*y1), tmp22 & xmask & ymask, other=0.0).to(tl.float32) + tmp57 = tmp56.to(tl.float32) + tmp58 = tl.load(in_ptr5 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp59 = tmp58.to(tl.float32) + tmp60 = tmp59 * tmp59 + tmp61 = tl.full([1, 1], 1.0, tl.float32) + tmp62 = tmp61 - tmp60 + tmp63 = tmp57 * tmp62 + tmp64 = tmp63.to(tl.float32) + tmp65 = tl.load(in_ptr6 + (1 + 2*(x2 // 2) + 64*y3), tmp22 & xmask & ymask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp66 = tmp64 + tmp65 + tmp67 = tmp66 * tmp27 + tmp68 = tl.full(tmp67.shape, 0.0, tmp67.dtype) + tmp69 = tl.where(tmp22, tmp67, tmp68) + tmp70 = tl.where(tmp22, tmp69, tmp18) + tmp71 = tmp55 + tmp70 + tl.store(out_ptr0 + (x2 + 64*y3), tmp39, xmask & ymask) + tl.store(out_ptr1 + (x2 + 64*y3), tmp71, xmask & ymask) + + +@triton.jit +def _profile_pack(in_ptr0, in_ptr1, in_ptr2, in_ptr3, in_ptr4, in_ptr5, in_ptr6, in_ptr7, in_ptr8, in_ptr9, in_ptr10, in_ptr11, in_ptr12, out_ptr0, ynumel, xnumel, YBLOCK : tl.constexpr, XBLOCK : tl.constexpr): + ynumel = 8192 + xnumel = 10240 + yoffset = tl.program_id(1) * YBLOCK + yindex = yoffset + tl.arange(0, YBLOCK)[:, None] + ymask = tl.full([YBLOCK], True, tl.int1)[:, None] + xoffset = tl.program_id(0) * XBLOCK + xindex = xoffset + tl.arange(0, XBLOCK)[None, :] + xmask = xindex < xnumel + x2 = xindex + y0 = (yindex % 2048) + y1 = yindex // 2048 + y3 = yindex + tmp0 = (x2).to(tl.int32) + tmp1 = tl.full([1, 1], 0, tl.int64) + tmp2 = tmp0 >= tmp1 + tmp3 = (x2).to(tl.int64) + tmp4 = (tmp3).to(tl.int64) + tmp5 = tl.full([1, 1], 2048, tl.int64) + tmp6 = tmp4 < tmp5 + tmp7 = tl.load(in_ptr0 + (64*((y0 % 128)) + 8192*((((x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + (((x2) % 64))), tmp6 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp8 = tmp4 >= tmp5 + tmp9 = tl.full([1, 1], 4096, tl.int64) + tmp10 = tmp4 < tmp9 + tmp11 = tmp8 & tmp10 + tmp12 = tl.load(in_ptr1 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp13 = tl.load(in_ptr2 + (128*((((-2048) + x2) % 2048)) + 262144*(y0 // 128) + 4194304*y1 + ((y0 % 128))), tmp11 & xmask, other=0.0).to(tl.float32) + tmp14 = tmp13.to(tl.float32) + tmp15 = tl.load(in_ptr3 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp16 = tmp15.to(tl.float32) + tmp17 = tmp16 * tmp16 + tmp18 = tl.full([1, 1], 1.0, tl.float32) + tmp19 = tmp18 - tmp17 + tmp20 = tmp14 * tmp19 + tmp21 = tmp20.to(tl.float32) + tmp22 = tl.load(in_ptr4 + (64*((y0 % 128)) + 8192*(((((-2048) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp23 = tmp21 + tmp22 + tmp24 = tl.load(in_ptr5 + (64*((y0 % 128)) + ((((-2048) + x2) % 64))), tmp11 & xmask, eviction_policy='evict_last', other=0.0) + tmp25 = tmp24.to(tl.float32) + tmp26 = tmp23 * tmp25 + tmp27 = tmp12 + tmp26 + tmp28 = tl.full(tmp27.shape, 0.0, tmp27.dtype) + tmp29 = tl.where(tmp11, tmp27, tmp28) + tmp30 = tmp4 >= tmp9 + tmp31 = tl.full([1, 1], 6144, tl.int64) + tmp32 = tmp4 < tmp31 + tmp33 = tmp30 & tmp32 + tmp34 = tl.load(in_ptr6 + (64*((y0 % 128)) + 8192*(((((-4096) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-4096) + x2) % 64))), tmp33 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp35 = tmp4 >= tmp31 + tmp36 = tl.full([1, 1], 8192, tl.int64) + tmp37 = tmp4 < tmp36 + tmp38 = tmp35 & tmp37 + tmp39 = tl.load(in_ptr7 + (2048*y3 + ((((-6144) + x2) % 2048))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp40 = tl.load(in_ptr8 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp41 = tl.full([1, 1], 0.125, tl.float32) + tmp42 = tmp40 * tmp41 + tmp43 = tmp39 * tmp42 + tmp44 = -tmp43 + tmp45 = tl.load(in_ptr9 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp46 = tmp39 * tmp45 + tmp47 = tmp44 + tmp46 + tmp48 = tmp47.to(tl.float32) + tmp49 = tl.load(in_ptr10 + (64*((y0 % 128)) + 8192*(((((-6144) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-6144) + x2) % 64))), tmp38 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp50 = tmp49.to(tl.float32) + tmp51 = tl.full([1, 1], 1.0, tl.float32) + tmp52 = tmp51 - tmp50 + tmp53 = tmp50 * tmp52 + tmp54 = tmp48 * tmp53 + tmp55 = tmp54.to(tl.float32) + tmp56 = tl.full(tmp55.shape, 0.0, tmp55.dtype) + tmp57 = tl.where(tmp38, tmp55, tmp56) + tmp58 = tmp0 >= tmp36 + tmp59 = tl.full([1, 1], 10240, tl.int64) + tmp60 = tmp0 < tmp59 + tmp61 = tl.load(in_ptr11 + (64*((y0 % 128)) + 8192*(((((-8192) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-8192) + x2) % 64))), tmp58 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp62 = tmp61.to(tl.float32) + tmp63 = tl.load(in_ptr12 + (64*((y0 % 128)) + 8192*(((((-8192) + x2) // 64) % 32)) + 262144*(y0 // 128) + 4194304*y1 + ((((-8192) + x2) % 64))), tmp58 & xmask, eviction_policy='evict_last', other=0.0).to(tl.float32) + tmp64 = tmp63.to(tl.float32) + tmp65 = tmp64 * tmp64 + tmp66 = tl.full([1, 1], 1.0, tl.float32) + tmp67 = tmp66 - tmp65 + tmp68 = tmp62 * tmp67 + tmp69 = tmp68.to(tl.float32) + tmp70 = tl.full(tmp69.shape, 0.0, tmp69.dtype) + tmp71 = tl.where(tmp58, tmp69, tmp70) + tmp72 = tl.where(tmp38, tmp57, tmp71) + tmp73 = tl.where(tmp33, tmp34, tmp72) + tmp74 = tl.where(tmp11, tmp29, tmp73) + tmp75 = tl.where(tmp6, tmp7, tmp74) + tl.store(out_ptr0 + (x2 + 10240*y3), tmp75, xmask) diff --git a/ops/backward_layout_benchmark.py b/ops/backward_layout_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..3cfeeb3a23d6cc555f5c98e677448b5a66d3d37a --- /dev/null +++ b/ops/backward_layout_benchmark.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Reproduce isolated BF16 rotary/packed-gradient traffic measurements on one GPU.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import sys +import time + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +sys.path.insert(0, str(PACKAGE)) + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--algorithms", nargs="+", choices=["paired", "single"], default=["paired", "single"]) + parser.add_argument("--blocks", nargs="+", type=int, choices=[128, 256, 512, 1024], default=[256, 512, 1024]) + parser.add_argument("--warps", nargs="+", type=int, choices=[4, 8], default=[4, 8]) + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--repeats", type=int, default=40) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--burnin-seconds", type=float, default=0.5) + parser.add_argument("--seed", type=int, default=817) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + for name in ("trials", "repeats", "warmup"): + if getattr(args, name) < 1: + parser.error(name + " must be positive") + if not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0: + parser.error("burnin-seconds must be finite and nonnegative") + args.output_dir = args.output_dir.expanduser().resolve() + work = PACKAGE / "work" + if (not args.output_dir.is_relative_to(work) or args.output_dir == work + or not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")): + parser.error("output-dir must be below rtx6000_40b/work/benchmark-*") + for name in ("TMPDIR", "XDG_CACHE_HOME", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"): + if not os.environ.get(name) or not Path(os.environ[name]).resolve().is_relative_to(WORKSPACE): + parser.error("Source ops/training-env.sh; " + name + " must stay inside the workspace") + return args + + +def output_errors(actual, expected): + """Check each projection independently so a bad derivative cannot hide.""" + import torch + result = {} + for name, value, wanted in zip(("q", "k", "v", "gate", "u"), + actual.chunk(5, -1), expected.chunk(5, -1), strict=True): + delta = value.float() - wanted.float() + finite = bool(torch.isfinite(value).all()) + result[name] = {"finite": finite, + "unequal": int((value != wanted).sum()), + "bitwise_unequal": int((value.view(torch.int16) != wanted.view(torch.int16)).sum()), + "relative_rms": (float(delta.norm() / wanted.float().norm().clamp_min(1e-30)) + if finite else None), + "max_abs": float(delta.abs().max()) if finite else None} + return result + + +def main(argv=None): + args = arguments(argv) + import torch + from ops.backward_layout import PROFILE_SHAPE, make_inputs, prepare_layout, reference_layout + if not torch.cuda.is_available() or torch.cuda.device_count() != 1: + raise RuntimeError("Reserve exactly one GPU with CUDA_VISIBLE_DEVICES") + torch.cuda.set_device(0) + inputs = make_inputs(PROFILE_SHAPE, device="cuda", seed=args.seed) + baseline_output = torch.empty((4, 2048, 10240), device="cuda", dtype=torch.bfloat16) + candidate_output = torch.empty_like(baseline_output) + scratch = tuple(torch.empty_like(inputs.q_memory) for _ in range(2)) + baseline = prepare_layout(inputs, "profile", output=baseline_output, scratch=scratch) + baseline() + independent = reference_layout(inputs) + baseline_errors = output_errors(baseline_output, independent) + del independent + if any(not v["finite"] or v["relative_rms"] > 0.002 for v in baseline_errors.values()): + raise RuntimeError("Tracked profile arithmetic disagrees with independent tensor reference") + args.output_dir.mkdir(parents=True, exist_ok=False) + sources = ["ops/backward_layout.py", "ops/backward_layout_baseline.py", + "ops/backward_layout_benchmark.py", "warpstate/model.py"] + report = {"format": "warpstate-backward-layout-v1", "started_utc": dt.datetime.now(dt.UTC).isoformat(), + "arguments": {k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()}, + "shape_b_n_h_c_d": PROFILE_SHAPE, "dtype": "bfloat16", "rope_dtype": "float32", + "torch": torch.__version__, "cuda": torch.version.cuda, + "device": torch.cuda.get_device_name(0), "cuda_visible_devices": os.environ["CUDA_VISIBLE_DEVICES"], + "source_sha256": {name: hashlib.sha256((PACKAGE / name).read_bytes()).hexdigest() for name in sources}, + "baseline_reference_errors": baseline_errors, + "static_removed_scratch_store_and_read_bytes_per_call": inputs.q_memory.numel() * 2 * 4, + "limitations": ["Isolated backward region, not a measured training speedup.", + "Preallocated outputs and scratch; CUDA event times include launch scheduling.", + "Synthetic saved inputs are reused; no model/checkpoint/token files are read.", + "Static removed scratch traffic is not a measured bandwidth counter.", + "Production reuses one dead buffer; this is not a claim of two saved allocations.", + "Integration must preserve separate local/memory gradients and rounding joins."], "results": []} + + def save(): + path = args.output_dir / "results.json" + temporary = path.with_suffix(".json.partial") + temporary.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + temporary.replace(path) + + def elapsed_ms(operation): + begin, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + begin.record() + for _ in range(args.repeats): + operation() + end.record() + end.synchronize() + return begin.elapsed_time(end) / args.repeats + + for algorithm in args.algorithms: + for block in args.blocks: + for warps in args.warps: + row = {"algorithm": algorithm, "block": block, "warps": warps} + try: + candidate = prepare_layout(inputs, algorithm, block=block, warps=warps, + output=candidate_output) + candidate() + row["errors"] = output_errors(candidate_output, baseline_output) + if any(not v["finite"] for v in row["errors"].values()): + raise RuntimeError("Nonfinite candidate output") + row["bitwise_equal"] = all(v["bitwise_unequal"] == 0 for v in row["errors"].values()) + for _ in range(args.warmup): + baseline() + candidate() + torch.cuda.synchronize() + begin = time.monotonic() + while time.monotonic() - begin < args.burnin_seconds: + for _ in range(10): + baseline() + candidate() + torch.cuda.synchronize() + trials = [] + for _ in range(args.trials): + trials.append({"baseline_before_ms": elapsed_ms(baseline), + "candidate_ms": elapsed_ms(candidate), + "baseline_after_ms": elapsed_ms(baseline)}) + gains = [((t["baseline_before_ms"] + t["baseline_after_ms"]) / 2 + / t["candidate_ms"] - 1) * 100 for t in trials] + row.update(status="ok", trials=trials, + median_candidate_ms=statistics.median(t["candidate_ms"] for t in trials), + median_baseline_ms=statistics.median((t["baseline_before_ms"] + + t["baseline_after_ms"]) / 2 for t in trials), + median_gain_percent=statistics.median(gains), minimum_gain_percent=min(gains)) + except Exception as error: + row.update(status="error", error=type(error).__name__ + ": " + str(error)[:4000]) + report["results"].append(row) + save() + print(json.dumps(row), flush=True) + report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat() + save() + if not any(row["status"] == "ok" for row in report["results"]): + raise RuntimeError("No candidate completed") + + +if __name__ == "__main__": + main() diff --git a/ops/backward_layout_execution.py b/ops/backward_layout_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..f81c4d0747404513c4264aad8a743134b76bc038 --- /dev/null +++ b/ops/backward_layout_execution.py @@ -0,0 +1,355 @@ +"""Opt-in replacement of the measured, normalized attention backward subgraph. + +The boundary is the existing compiled BF16 kernel arithmetic, not eager BF16 +ATen arithmetic: Inductor normally retains FP32 intermediates within a fused +kernel. Forward/autograd ownership, leaf accumulation and communication remain +unchanged. Unsupported graphs are retained; callers must require actual runtime +dispatches in opt-in validation. Configure once before compiling in a fresh process. +""" + +from __future__ import annotations + +from collections import Counter +from functools import lru_cache +import hashlib +import json +import operator +from pathlib import Path + +import torch +from torch._inductor.custom_graph_pass import CustomGraphPass +from torch.fx import Node +from torch.library import custom_op + + +BACKWARD_LAYOUT_MODES = ("baseline", "inspect", "paired") +_SHAPE = (4, 16, 32, 128, 64) +_NAMES = ("q_memory", "q_local", "sin", "cos", "k_write", "tanh_k", "k_local", + "v_local", "grad_y", "memory", "local", "gate", "u_write", "tanh_u") +_NATIVE_SHAPES = ( + (2048, 128, 64), (64, 32, 128, 64), (128, 64), (128, 64), + (2048, 64, 128), _SHAPE, (64, 32, 128, 64), (64, 32, 128, 64), + (8192, 2048), (2048, 128, 64), (64, 32, 128, 64), _SHAPE, + (2048, 128, 64), _SHAPE, +) +_COUNTS = Counter() +_MISSES = Counter() +_ACTIVE_MODE = "baseline" +_ACTIVE_PASS = None + + +def _aten_boundary(qm, ql, sin, cos, kw, tk, kl, vl, dy, memory, local, gate, uw, tu): + """Exact normalized ATen dataflow of the real production backward boundary. + + This recipe is traced on fake tensors once to construct a structural pattern. + Explicit operations retain every original cast, view, projection position, + and zero-filled scatter; arbitrary algebraic equivalents are not accepted. + It is never called to execute training or to provide a numeric GPU oracle. + """ + a = torch.ops.aten + cvt = torch.ops.prims.convert_element_type.default + reshape = a.reshape.default + head = lambda x: reshape(x, list(_SHAPE)) + sin = reshape(cvt(sin, torch.bfloat16), [1, 1, 1, 128, 64]) + cos = reshape(cvt(cos, torch.bfloat16), [1, 1, 1, 128, 64]) + zero = a.full.default(list(_SHAPE), 0, dtype=torch.bfloat16, + layout=torch.strided, device=qm.device, pin_memory=False) + + def rotary(value): + lanes = reshape(a.mul.Tensor(value, sin), [4, 16, 32, 128, 32, 2]) + odd = a.slice_scatter.default(zero, a.neg.default(a.select.int(lanes, -1, 0)), + 4, 1, 9223372036854775807, 2) + even = a.slice_scatter.default(zero, a.select.int(lanes, -1, 1), + 4, 0, 9223372036854775807, 2) + return a.add.Tensor(a.add.Tensor(odd, even), a.mul.Tensor(value, cos)) + + def tanh_backward(gradient, saved): + gradient = cvt(gradient, torch.float32) + saved = cvt(saved, torch.float32) + return cvt(a.mul.Tensor(gradient, a.sub.Tensor(1, a.mul.Tensor(saved, saved))), + torch.bfloat16) + + q = rotary(a.add.Tensor(head(qm), head(ql))) + kw = a.permute.default(reshape(kw, [4, 16, 32, 64, 128]), [0, 1, 2, 4, 3]) + k = rotary(a.add.Tensor(tanh_backward(kw, tk), head(kl))) + v = head(vl) + dy = a.permute.default(reshape(reshape(dy, [4, 2048, 2048]), + [4, 16, 128, 32, 64]), [0, 1, 3, 2, 4]) + dg = a.add.Tensor(a.neg.default(a.mul.Tensor(dy, a.div.Tensor(head(memory), 8.0))), + a.mul.Tensor(dy, head(local))) + gf = cvt(gate, torch.float32) + dg = cvt(a.mul.Tensor(cvt(dg, torch.float32), + a.mul.Tensor(gf, a.sub.Tensor(1, gf))), torch.bfloat16) + u = tanh_backward(head(uw), tu) + + def pack(value, clone=True): + value = a.permute.default(value, [0, 1, 3, 2, 4]) + if clone: + value = a.clone.default(value, memory_format=torch.contiguous_format) + return reshape(reshape(value, [4, 2048, 32, 64]), [4, 2048, 2048]) + + return a.cat.default([pack(q), pack(k), pack(v), pack(dg, False), pack(u)], 2) + + +@lru_cache(maxsize=1) +def _pattern(): + from torch._subclasses.fake_tensor import FakeTensorMode + from torch.fx.experimental.proxy_tensor import make_fx + + with FakeTensorMode(): + inputs = [torch.empty(shape, dtype=torch.float32 if name in ("sin", "cos") + else torch.bfloat16, device="cuda:0") + for name, shape in zip(_NAMES, _NATIVE_SHAPES, strict=True)] + gm = make_fx(_aten_boundary, tracing_mode="fake")(*inputs) + # make_fx normalizes explicit reshape to view; this hook runs after + # Inductor's view-to-reshape normalization. + for node in gm.graph.nodes: + if node.target == torch.ops.aten.view.default: + node.target = torch.ops.aten.reshape.default + return gm + + +def _metadata(value): + if not isinstance(value, torch.Tensor): + return None + # Symbolic/unsupported dimensions are rejected, not specialized implicitly. + if any(type(x) is not int for x in (*value.shape, *value.stride())): + return None + return (tuple(value.shape), tuple(value.stride()), value.dtype, value.device.type) + + +class _Mismatch(Exception): + pass + + +def _match(candidate, pattern): + binding, reverse = {}, {} + + def walk(actual, expected): + if isinstance(expected, Node): + if not isinstance(actual, Node): + raise _Mismatch("node kind") + if expected in binding: + if binding[expected] is not actual: + raise _Mismatch("shared dataflow") + return + if actual in reverse: + raise _Mismatch("distinct dataflow") + actual_meta, expected_meta = _metadata(actual.meta.get("val")), _metadata(expected.meta.get("val")) + if actual_meta is None or actual_meta != expected_meta: + raise _Mismatch("tensor metadata") + if expected.op != "placeholder": + if actual.op != expected.op or actual.target != expected.target: + raise _Mismatch("operator") + walk(actual.args, expected.args) + walk(actual.kwargs, expected.kwargs) + binding[expected] = actual + reverse[actual] = expected + elif isinstance(expected, (tuple, list)): + if not isinstance(actual, (tuple, list)) or len(actual) != len(expected): + raise _Mismatch("argument list") + for av, ev in zip(actual, expected, strict=True): + walk(av, ev) + elif isinstance(expected, dict): + if not isinstance(actual, dict) or set(actual) != set(expected): + raise _Mismatch("keyword arguments") + for key in expected: + walk(actual[key], expected[key]) + elif isinstance(expected, torch.device): + if not isinstance(actual, torch.device) or actual.type != expected.type: + raise _Mismatch("device") + elif type(actual) is not type(expected) or actual != expected: + raise _Mismatch("literal") + + output = next(node for node in pattern.graph.nodes if node.op == "output").args[0] + walk(candidate, output) + inputs = tuple(binding[node] for node in pattern.graph.nodes if node.op == "placeholder") + interior = {actual for expected, actual in binding.items() if expected.op != "placeholder"} + retained = set() + + def retain_view(node): + if node not in interior or node in retained: + return + # AOT CSE can share the exact zero-filled scatter base across cores. + # Its complete literal recipe was matched above; preserve it while + # another region still uses it, without retaining computed arithmetic. + if node.target == torch.ops.aten.full.default and not node.all_input_nodes: + retained.add(node) + return + if node.target not in (torch.ops.aten.reshape.default, torch.ops.aten.permute.default): + raise _Mismatch("external intermediate consumer") + retained.add(node) + for parent in node.all_input_nodes: + retain_view(parent) + + for node in interior - {candidate}: + if any(user not in interior for user in node.users): + # The raw output-gradient's head view also feeds the independent + # attention/memory VJPs. Retain that metadata-only chain in place; + # the opaque op takes its original contiguous MM result directly. + retain_view(node) + if len({value.meta["val"].device for value in inputs}) != 1: + raise _Mismatch("mixed input devices") + _validate_producers(inputs) + return inputs, interior - retained + + +def _validate_producers(inputs): + """The local branches must belong to one causal dropout-zero Flash VJP.""" + qm, ql, _sin, _cos, kw, _tk, kl, vl, dy, memory, local, _gate, uw, _tu = inputs + flash = None + for node, index in ((ql, 0), (kl, 1), (vl, 2)): + if (node.op != "call_function" or node.target is not operator.getitem + or len(node.args) != 2 or node.args[1] != index): + raise _Mismatch("Flash gradient ordering") + if flash is None: + flash = node.args[0] + elif node.args[0] is not flash: + raise _Mismatch("separate Flash VJPs") + if (not isinstance(flash, Node) + or flash.target != torch.ops.aten._scaled_dot_product_flash_attention_backward.default + or len(flash.args) != 14 or flash.args[8:12] != (128, 128, 0.0, True) + or dict(flash.kwargs) != {"scale": 0.125} or flash.args[4] is not local): + raise _Mismatch("Flash attention settings") + for node in (qm, kw, uw): + if node.op != "call_function" or node.target != torch.ops.aten.bmm.default: + raise _Mismatch("native backward GEMM producer") + if dy.op != "call_function" or dy.target != torch.ops.aten.mm.default: + raise _Mismatch("native output gradient producer") + + +@custom_op("warpstate::paired_backward_layout", mutates_args=()) +def _paired_backward_layout(q_memory: torch.Tensor, q_local: torch.Tensor, + sin: torch.Tensor, cos: torch.Tensor, k_write: torch.Tensor, + tanh_k: torch.Tensor, k_local: torch.Tensor, v_local: torch.Tensor, + grad_y: torch.Tensor, memory: torch.Tensor, local: torch.Tensor, + gate: torch.Tensor, u_write: torch.Tensor, + tanh_u: torch.Tensor) -> torch.Tensor: + """Opaque pure launch using native input storage, with no layout copies. + + Fake/compile guards already require native shapes and strides. The entrypoint + is internal to this strict rewrite; shape views and fresh output allocation + are the only tensor operations. Runtime evidence remains valid on cache hits. + """ + from ops.backward_layout import _paired_kernel + + args = (q_memory, q_local, sin, cos, k_write, tanh_k, k_local, v_local, + grad_y, memory, local, gate, u_write, tanh_u) + _validate_native_inputs(args) + output = torch.empty((4, 2048, 10240), dtype=torch.bfloat16, device=q_memory.device) + # All contiguous native shapes have exactly the raw layouts consumed by the + # prototype kernel; passing native pointers needs no view or contiguous call. + launch = (*args, output, 16777216, 32, 128, 64, 0.125, 256) + _paired_kernel[(32768,)](*launch, True, num_warps=8, enable_fp_fusion=True) + _paired_kernel[(65536,)](*launch, False, num_warps=8, enable_fp_fusion=True) + _COUNTS["runtime_calls"] += 1 + return output + + +@_paired_backward_layout.register_fake +def _paired_backward_layout_fake(*inputs): + _validate_native_inputs(inputs) + return inputs[0].new_empty((4, 2048, 10240)) + + +def _validate_native_inputs(inputs): + for name, value, shape in zip(_NAMES, inputs, _NATIVE_SHAPES, strict=True): + dtype = torch.float32 if name in ("sin", "cos") else torch.bfloat16 + if (tuple(value.shape) != shape or value.dtype != dtype or not value.is_contiguous() + or value.device != inputs[0].device or value.device.type != "cuda"): + raise ValueError(f"Unsupported paired backward input {name}") + + +class BackwardLayoutPass(CustomGraphPass): + def __init__(self, mode, identity): + self.mode, self.identity = mode, identity + + def uuid(self): + return self.identity + + def __call__(self, graph): + _COUNTS["graphs"] += 1 + candidates = [node for node in graph.nodes if node.op == "call_function" + and node.target == torch.ops.aten.cat.default + and isinstance(node.args[0], (tuple, list)) and len(node.args[0]) == 5] + for candidate in candidates: + _COUNTS["candidates"] += 1 + try: + inputs, interior = _match(candidate, _pattern()) + except _Mismatch as error: + _MISSES[str(error)] += 1 + continue + _COUNTS["matches"] += 1 + if self.mode == "inspect": + continue + with graph.inserting_before(candidate): + replacement = graph.call_function(torch.ops.warpstate.paired_backward_layout.default, inputs) + replacement.meta = dict(candidate.meta) + candidate.replace_all_uses_with(replacement) + for node in reversed(list(graph.nodes)): + if node in interior: + graph.erase_node(node) + _COUNTS["replacements"] += 1 + + +def _identity(mode): + directory = Path(__file__).resolve().parent + sources = {"ops/" + name: hashlib.sha256((directory / name).read_bytes()).hexdigest() + for name in ("backward_layout_execution.py", "backward_layout.py")} + metadata = {"format": "warpstate-backward-layout-v1", "mode": mode, + "source_sha256": sources, "torch_version": torch.__version__, + "shape": list(_SHAPE), "algorithm": "paired", "block": 256, "warps": 8} + metadata["identity"] = hashlib.sha256(json.dumps(metadata, sort_keys=True).encode()).hexdigest() + return metadata + + +def validate_backward_layout_config(cfg): + """Reject unsupported model recipes before allocation or resume mutation.""" + required = {"d_model": 2048, "n_heads": 32, "chunk_size": 128, + "parallel_memory": True, "dropout": 0.0} + for name, value in required.items(): + if getattr(cfg, name, None) != value: + raise ValueError(f"Paired backward layout requires {name}={value}") + + +def configure_backward_layout(mode="baseline"): + """Configure once before compilation, returning checkpoint/benchmark identity. + + A pre-existing custom pass is rejected, never replaced or silently composed; + composition could change the arithmetic boundary. Use a fresh process when + changing modes, matching Compiled Autograd's process-wide configuration. + """ + global _ACTIVE_MODE, _ACTIVE_PASS + if mode not in BACKWARD_LAYOUT_MODES: + raise ValueError(f"Unknown backward layout mode: {mode}") + from torch._inductor import config + if _ACTIVE_PASS is not None: + raise RuntimeError("Backward layout is already configured; use a fresh process") + if mode == "baseline": + return None + if config.post_grad_custom_post_pass is not None: + raise RuntimeError("A post-gradient custom pass already exists; refusing to replace it") + if (config.emulate_precision_casts or not config.triton.codegen_upcast_to_fp32 + or config.eager_numerics.disable_ftz): + raise ValueError("Paired backward layout requires the profiled FP32 pointwise fusion settings") + metadata = _identity(mode) + _ACTIVE_MODE = mode + _ACTIVE_PASS = BackwardLayoutPass(mode, metadata["identity"]) + config.post_grad_custom_post_pass = _ACTIVE_PASS + return metadata + + +def backward_layout_stats(): + """Process-local compilation and runtime evidence; no device synchronization.""" + return {"mode": _ACTIVE_MODE, **{name: _COUNTS[name] for name in + ("graphs", "candidates", "matches", "replacements", "runtime_calls")}, + "miss_reasons": dict(_MISSES)} + + +def require_backward_layout_dispatch(min_calls=1): + if isinstance(min_calls, bool) or not isinstance(min_calls, int) or min_calls < 1: + raise ValueError("min_calls must be a positive integer") + if _ACTIVE_MODE != "paired" or _COUNTS["runtime_calls"] < min_calls: + raise RuntimeError(f"Paired backward layout expected at least {min_calls} runtime calls; " + f"observed {backward_layout_stats()}") + return backward_layout_stats() diff --git a/ops/benchmark-supervisord.conf b/ops/benchmark-supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..a7b5c5bfb64b52236b0d0383894b35320731c3d1 --- /dev/null +++ b/ops/benchmark-supervisord.conf @@ -0,0 +1,16 @@ +[program:warpstate-benchmark] +command=/workspace/WarpState/rtx6000_40b/ops/benchmark.sh --checkpoint /workspace/WarpState/rtx6000_40b/work/run/latest.pt --micro-batch 4 --no-activation-checkpointing --compile-cores --autocast-cache --loss-chunk-size 1024 --vram-limit-gib 88 --warmup-updates 2 --measured-updates 3 +directory=/workspace/WarpState/rtx6000_40b +autostart=false +autorestart=false +startsecs=5 +startretries=0 +stopsignal=TERM +stopwaitsecs=30 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/benchmark.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=3 +redirect_stderr=true +environment=TORCHINDUCTOR_COMPILE_THREADS="8",WARPSTATE_BENCH_DEVICES="0,1",WARPSTATE_BENCH_PROCESSES="2" diff --git a/ops/benchmark.sh b/ops/benchmark.sh new file mode 100644 index 0000000000000000000000000000000000000000..98e2c1d893e88110f0606c5509e0c5cf188c342d --- /dev/null +++ b/ops/benchmark.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +ulimit -c 0 +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +cd /workspace/WarpState/rtx6000_40b +export CUDA_VISIBLE_DEVICES="${WARPSTATE_BENCH_DEVICES:-0}" +if [[ "${WARPSTATE_BENCH_PROCESSES:-1}" == 1 ]]; then + exec .server-venv/bin/python -u ops/benchmark_training.py "$@" +else + exec .server-venv/bin/torchrun --standalone \ + --nproc-per-node="$WARPSTATE_BENCH_PROCESSES" \ + --log-dir=work/logs/torchrun-benchmark ops/benchmark_training.py "$@" +fi diff --git a/ops/benchmark_gemm.py b/ops/benchmark_gemm.py new file mode 100644 index 0000000000000000000000000000000000000000..75709913a43d32e6737a8ec88ca847319f4340e9 --- /dev/null +++ b/ops/benchmark_gemm.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Time isolated BF16 GEMMs with the dominant WarpState projection layouts. + +Source ops/training-env.sh first. Only deterministic random fixtures are used; +this program never opens a checkpoint or token dataset. Compilation/autotuning +are excluded from CUDA-event timings. These timings exclude communication, +optimizer updates and most Python overhead, so are not training throughput. +""" +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import subprocess +import time + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +GIB = 1024**3 +FAMILIES = {"ffn_up_gate": (2048, 14336), "in_proj": (2048, 10240), + "ffn_down": (7168, 2048), "out_proj": (2048, 2048)} +BACKENDS = {"eager": None, "aten": "ATEN", "triton": "TRITON", "mixed": "ATEN,TRITON"} + + +def projection_cases(families, rows, operations): + """Return M,N,K and storage layouts for Y=X@W.T, dX=dY@W, dW=dY.T@X.""" + cases = [] + for family in families: + input_dim, output_dim = FAMILIES[family] + specs = { + "fwd": (rows, output_dim, input_dim, False, True), + "dx": (rows, input_dim, output_dim, False, False), + "dw": (output_dim, input_dim, rows, True, False), + } + for operation in operations: + m, n, k, at, bt = specs[operation] + cases.append({"name": family + "." + operation, "family": family, + "operation": operation, "m": m, "n": n, "k": k, + "a_transposed": at, "b_transposed": bt, + "a_shape": [m, k], "b_shape": [k, n], + "a_stride": [1, m] if at else [k, 1], + "b_stride": [1, k] if bt else [n, 1], + "flops": 2 * m * n * k}) + return cases + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--families", nargs="+", choices=FAMILIES, + default=["ffn_up_gate", "in_proj", "ffn_down"]) + parser.add_argument("--operations", nargs="+", choices=["fwd", "dx", "dw"], + default=["fwd", "dx", "dw"]) + parser.add_argument("--rows", type=int, default=8192, + help="Micro-batch times sequence length; production uses 8192") + parser.add_argument("--backends", nargs="+", choices=BACKENDS, + default=["eager", "aten", "triton"]) + parser.add_argument("--warmup", type=int, default=10) + parser.add_argument("--repeats", type=int, default=30) + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--burnin-seconds", type=float, default=2.0, + help="Sustain GEMM load before timing to reduce cold boost effects") + parser.add_argument("--seed", type=int, default=1337) + parser.add_argument("--device", type=int, default=0, + help="CUDA device index after CUDA_VISIBLE_DEVICES filtering") + parser.add_argument("--blas-backend", choices=["default", "cublas", "cublaslt"], + default="default") + parser.add_argument("--allow-reduced-precision-reduction", action=argparse.BooleanOptionalAction, + default=True, + help="Allow BF16 intermediate reductions, matching the current production setting") + parser.add_argument("--list-shapes", action="store_true", + help="Print selected shapes/strides and exit without importing torch") + parser.add_argument("--output-dir", type=Path, + default=PACKAGE / "work" / ("benchmark-gemm-" + + dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%S"))) + args = parser.parse_args(argv) + for key in ("rows", "warmup", "repeats", "trials"): + if getattr(args, key) < 1: + parser.error("--" + key + " must be positive") + if args.device < 0: + parser.error("--device must be nonnegative") + if not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0: + parser.error("--burnin-seconds must be finite and nonnegative") + for key in ("families", "operations", "backends"): + if len(getattr(args, key)) != len(set(getattr(args, key))): + parser.error("--" + key + " must not contain duplicates") + args.output_dir = args.output_dir.expanduser().resolve() + work = PACKAGE / "work" + if (not args.output_dir.is_relative_to(work) or + not args.output_dir.relative_to(work).parts or + not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")): + parser.error("--output-dir must be below rtx6000_40b/work/benchmark-*") + if not args.list_shapes: + for key in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", + "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"): + if not os.environ.get(key) or not Path(os.environ[key]).resolve().is_relative_to(WORKSPACE): + parser.error("Source ops/training-env.sh first; " + key + " must stay in the workspace") + return args + + +def gpu_telemetry(): + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=index,uuid,name,power.draw,power.limit,clocks.sm," + "clocks.mem,temperature.gpu,utilization.gpu,memory.used", "--format=csv,noheader"], + capture_output=True, text=True, timeout=10, check=False) + return {"csv": result.stdout.strip(), "returncode": result.returncode} + except (OSError, subprocess.TimeoutExpired) as error: + return {"error": str(error)} + + +def main(): + args = arguments() + cases = projection_cases(args.families, args.rows, args.operations) + if args.list_shapes: + print(json.dumps(cases, indent=2)) + return + # Reject an existing directory to preserve previous measurements. + args.output_dir.mkdir(parents=True, exist_ok=False) + import torch + from torch._inductor import config + + if not torch.cuda.is_available(): + raise SystemExit("CUDA is required for timings; --list-shapes is CPU-only") + torch.cuda.set_device(args.device) + device = torch.device("cuda", args.device) + if not torch.cuda.is_bf16_supported(): + raise SystemExit("This GPU does not support BF16") + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = args.allow_reduced_precision_reduction + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cuda.preferred_blas_library(args.blas_backend) + properties = torch.cuda.get_device_properties(device) + report = { + "started_utc": dt.datetime.now(dt.UTC).isoformat(), + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "arguments": {key: str(value) if isinstance(value, Path) else value + for key, value in vars(args).items()}, + "torch_version": torch.__version__, "cuda_version": torch.version.cuda, + "dtype": "bfloat16", "output_dtype": "bfloat16", + "allow_bf16_reduced_precision_reduction": args.allow_reduced_precision_reduction, + "compile_mode": "max-autotune-no-cudagraphs", + "autotune_fallback_to_aten": False, + "device": {"name": properties.name, "capability": list(torch.cuda.get_device_capability(device)), + "multiprocessors": properties.multi_processor_count, + "total_memory_gib": properties.total_memory / GIB}, + "environment": {key: os.environ.get(key) for key in + ("CUDA_VISIBLE_DEVICES", "CUBLAS_WORKSPACE_CONFIG", "OMP_NUM_THREADS", + "TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR", "PYTORCH_ALLOC_CONF", + "TORCHINDUCTOR_COMPILE_THREADS", "ENABLE_PERSISTENT_TMA_MATMUL", + "TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_BACKENDS", + "TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_SEARCH_SPACE", "CUTLASS_EPILOGUE_FUSION")}, + "telemetry_before": gpu_telemetry(), "results": [], + "notes": ["Shapes represent isolated linear algebra, not an end-to-end backward graph.", + "Each repeat allocates a BF16 output, as torch.mm normally does.", + "Timings reuse inputs and omit compilation/autotuning; compare to full training separately.", + "Error is against eager cuBLAS BF16 output, not an exact FP32 reference.", + "Telemetry is a snapshot, not a synchronized power integral."], + } + + def save(): + target = args.output_dir / "results.json" + temporary = args.output_dir / "results.json.partial" + temporary.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + temporary.replace(target) + + save() + with torch.inference_mode(): + for case_index, case in enumerate(cases): + generator = torch.Generator(device=device).manual_seed(args.seed + case_index) + + def fixture(rows, columns, transposed): + shape = (columns, rows) if transposed else (rows, columns) + tensor = torch.randn(shape, device=device, dtype=torch.bfloat16, generator=generator) + return tensor.T if transposed else tensor + + a = fixture(case["m"], case["k"], case["a_transposed"]) + b = fixture(case["k"], case["n"], case["b_transposed"]) + assert list(a.stride()) == case["a_stride"] + assert list(b.stride()) == case["b_stride"] + reference = torch.mm(a, b) + torch.cuda.synchronize(device) + for backend in args.backends: + row = {**case, "backend": backend, "status": "running", + "max_autotune_gemm_backends": BACKENDS[backend], + "telemetry_before": gpu_telemetry()} + print(json.dumps({"case": case["name"], "backend": backend, "status": "starting"}), flush=True) + try: + torch._dynamo.reset() + settings = ({"max_autotune_gemm_backends": BACKENDS[backend], + "autotune_fallback_to_aten": False} + if BACKENDS[backend] else {}) + with config.patch(settings) if settings else contextlib.nullcontext(): + operation = torch.mm if backend == "eager" else torch.compile( + lambda x, y: torch.mm(x, y), fullgraph=True, dynamic=False, + mode="max-autotune-no-cudagraphs") + compile_start = time.perf_counter() + output = operation(a, b) + torch.cuda.synchronize(device) + row["first_call_seconds"] = time.perf_counter() - compile_start + difference = output.float().sub_(reference.float()) + reference_rms = reference.float().square_().mean().sqrt().item() + row["max_abs_error"] = difference.abs().max().item() + row["rms_error"] = difference.square_().mean().sqrt().item() + row["relative_rms_error"] = row["rms_error"] / max(reference_rms, 1e-30) + if not all(math.isfinite(row[key]) for key in + ("max_abs_error", "rms_error", "relative_rms_error")): + raise RuntimeError("Nonfinite output error") + del difference, output + for _ in range(args.warmup): + output = operation(a, b) + del output + torch.cuda.synchronize(device) + burnin_start = time.perf_counter() + burnin_iterations = 0 + while time.perf_counter() - burnin_start < args.burnin_seconds: + for _ in range(10): + output = operation(a, b) + del output + torch.cuda.synchronize(device) + burnin_iterations += 10 + row["burnin_elapsed_seconds"] = time.perf_counter() - burnin_start + row["burnin_iterations"] = burnin_iterations + torch.cuda.reset_peak_memory_stats(device) + row["baseline_allocated_gib"] = torch.cuda.memory_allocated(device) / GIB + trial_ms = [] + for _ in range(args.trials): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(args.repeats): + output = operation(a, b) + end.record() + end.synchronize() + trial_ms.append(start.elapsed_time(end) / args.repeats) + del output + row["trial_ms_per_gemm"] = trial_ms + row["median_ms"] = statistics.median(trial_ms) + row["min_ms"] = min(trial_ms) + row["tflops"] = case["flops"] / (row["median_ms"] * 1e9) + row["peak_allocated_gib"] = torch.cuda.max_memory_allocated(device) / GIB + row["peak_reserved_gib"] = torch.cuda.max_memory_reserved(device) / GIB + row["telemetry_after"] = gpu_telemetry() + row["status"] = "ok" + except Exception as error: + row["status"] = "error" + row["error"] = type(error).__name__ + ": " + str(error)[:4000] + report["results"].append(row) + save() + print(json.dumps({key: row[key] for key in + ("name", "backend", "status", "median_ms", "tflops", + "max_abs_error", "rms_error", "error") if key in row}), flush=True) + del a, b, reference + torch.cuda.empty_cache() + report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat() + report["telemetry_after"] = gpu_telemetry() + save() + print("Results: " + str(args.output_dir / "results.json"), flush=True) + if any(row["status"] != "ok" for row in report["results"]): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/ops/benchmark_gradient_accumulation.py b/ops/benchmark_gradient_accumulation.py new file mode 100644 index 0000000000000000000000000000000000000000..0dc12504f669a6726069dda8ff36cc417a9d86b3 --- /dev/null +++ b/ops/benchmark_gradient_accumulation.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Disposable BF16 weight-gradient GEMM/FP32 accumulation comparisons. + +Source ops/training-env.sh first. Uses deterministic random matrices only; no +checkpoint or dataset access. None of these experiments is promoted to training. +Direct FP32-output addmm changes rounding versus BF16 GEMM then FP32 accumulation. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import time + +from benchmark_gemm import FAMILIES, GIB, PACKAGE, WORKSPACE, gpu_telemetry, projection_cases + +VARIANTS = ("baseline", "fused_cast_add", "direct_fp32_addmm") + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--families", nargs="+", choices=FAMILIES, + default=["ffn_up_gate", "in_proj", "ffn_down"]) + parser.add_argument("--variants", nargs="+", choices=VARIANTS, default=list(VARIANTS)) + parser.add_argument("--rows", type=int, default=8192) + parser.add_argument("--accumulation-steps", type=int, default=8) + parser.add_argument("--warmup", type=int, default=2, help="Complete accumulation cycles") + parser.add_argument("--repeats", type=int, default=10, help="Cycles per timed trial") + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--burnin-seconds", type=float, default=2.0) + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--seed", type=int, default=1337) + parser.add_argument("--blas-backend", choices=["default", "cublas", "cublaslt"], default="default") + parser.add_argument("--allow-reduced-precision-reduction", action=argparse.BooleanOptionalAction, + default=True) + parser.add_argument("--list-shapes", action="store_true", help="CPU-only validation and shape listing") + parser.add_argument("--output-dir", type=Path, default=PACKAGE / "work" / + ("benchmark-gradient-" + dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%S"))) + args = parser.parse_args(argv) + for key in ("rows", "accumulation_steps", "warmup", "repeats", "trials"): + if getattr(args, key) < 1: + parser.error(key + " must be positive") + if args.device < 0 or not math.isfinite(args.burnin_seconds) or args.burnin_seconds < 0: + parser.error("device must be nonnegative; burnin-seconds must be finite and nonnegative") + for key in ("families", "variants"): + if len(getattr(args, key)) != len(set(getattr(args, key))): + parser.error(key + " must not contain duplicates") + args.output_dir = args.output_dir.expanduser().resolve() + work = PACKAGE / "work" + if (not args.output_dir.is_relative_to(work) or not args.output_dir.relative_to(work).parts or + not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")): + parser.error("output-dir must be below rtx6000_40b/work/benchmark-*") + if not args.list_shapes: + for key in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"): + if not os.environ.get(key) or not Path(os.environ[key]).resolve().is_relative_to(WORKSPACE): + parser.error("Source ops/training-env.sh first; " + key + " must stay in the workspace") + return args + + +def accumulation_cycle(pairs, create, add): + """Adopt the first newly allocated gradient, then add subsequent gradients.""" + if not pairs: + raise ValueError("At least one gradient fixture is required") + accumulator = create(*pairs[0]) + for a, b in pairs[1:]: + add(accumulator, a, b) + return accumulator + + +def main(): + args = arguments() + cases = projection_cases(args.families, args.rows, ["dw"]) + if args.list_shapes: + print(json.dumps(cases, indent=2)) + return + args.output_dir.mkdir(parents=True, exist_ok=False) + import torch + + if not torch.cuda.is_available(): + raise SystemExit("CUDA is required for timings") + torch.cuda.set_device(args.device) + device = torch.device("cuda", args.device) + if not torch.cuda.is_bf16_supported(): + raise SystemExit("This GPU does not support BF16") + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = args.allow_reduced_precision_reduction + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cuda.preferred_blas_library(args.blas_backend) + report = {"started_utc": dt.datetime.now(dt.UTC).isoformat(), + "arguments": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "helper_source_sha256": hashlib.sha256(Path(__file__).with_name("benchmark_gemm.py").read_bytes()).hexdigest(), + "torch_version": torch.__version__, "cuda_version": torch.version.cuda, + "device": torch.cuda.get_device_name(device), "input_dtype": "bfloat16", + "accumulator_dtype": "float32", "results": [], + "notes": ["Eight deterministic, distinct BF16 gradient input pairs are used by default; every variant uses the same pairs.", + "Each cycle adopts its first newly allocated FP32 gradient, then performs seven in-place additions by default; there is no zero reset.", + "Timings include first-gradient creation and subsequent accumulation; each cycle output is released before the next cycle.", + "Effective TFLOPS counts only 2MNK GEMM FLOPs over this complete cycle time.", + "Direct FP32 addmm skips BF16 output rounding and is not numerically equivalent to baseline.", + "Experiments do not implement autograd/DDP integration and are not promoted to training."]} + + def save(): + partial = args.output_dir / "results.json.partial" + partial.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + partial.replace(args.output_dir / "results.json") + + def baseline(acc, a, b): + acc.add_(torch.mm(a, b).float()) + + def baseline_create(a, b): + return torch.mm(a, b).float() + + save() + with torch.inference_mode(): + for index, case in enumerate(cases): + generator = torch.Generator(device=device).manual_seed(args.seed + index) + pairs = [(torch.randn((case["k"], case["m"]), device=device, dtype=torch.bfloat16, generator=generator).T, + torch.randn(case["b_shape"], device=device, dtype=torch.bfloat16, generator=generator)) + for _ in range(args.accumulation_steps)] + assert all(list(a.stride()) == case["a_stride"] and list(b.stride()) == case["b_stride"] for a, b in pairs) + reference = accumulation_cycle(pairs, baseline_create, baseline) + for variant in args.variants: + row = {**case, "variant": variant, "status": "running", "telemetry_before": gpu_telemetry(), + "gemm_output_dtype": "float32" if variant == "direct_fp32_addmm" else "bfloat16", + "preserves_baseline_bf16_rounding": variant != "direct_fp32_addmm"} + print(json.dumps({"case": case["name"], "variant": variant, "status": "starting"}), flush=True) + try: + create = baseline_create + if variant == "baseline": + operation = baseline + elif variant == "fused_cast_add": + torch._dynamo.reset() + cast_add = torch.compile(lambda dest, grad: dest.add_(grad.float()), + fullgraph=True, dynamic=False) + operation = lambda dest, x, y: cast_add(dest, torch.mm(x, y)) + else: + create = lambda x, y: torch.mm(x, y, out_dtype=torch.float32) + operation = lambda dest, x, y: torch.addmm( + dest, x, y, beta=1, out_dtype=torch.float32, out=dest) + + def cycle(): + return accumulation_cycle(pairs, create, operation) + + first_start = time.perf_counter() + output = cycle() + torch.cuda.synchronize(device) + row["first_cycle_seconds"] = time.perf_counter() - first_start + difference = output.sub(reference) + row["max_abs_error"] = difference.abs().max().item() + row["rms_error"] = difference.square_().mean().sqrt().item() + row["relative_rms_error"] = row["rms_error"] / max(reference.square().mean().sqrt().item(), 1e-30) + if not all(math.isfinite(row[key]) for key in ("max_abs_error", "rms_error", "relative_rms_error")): + raise RuntimeError("Nonfinite output difference") + del difference, output + for _ in range(args.warmup): + output = cycle() + del output + torch.cuda.synchronize(device) + burnin_start = time.perf_counter() + burnin_cycles = 0 + while time.perf_counter() - burnin_start < args.burnin_seconds: + output = cycle() + del output + torch.cuda.synchronize(device) + burnin_cycles += 1 + row["burnin_elapsed_seconds"] = time.perf_counter() - burnin_start + row["burnin_cycles"] = burnin_cycles + torch.cuda.reset_peak_memory_stats(device) + row["baseline_allocated_gib"] = torch.cuda.memory_allocated(device) / GIB + times = [] + for _ in range(args.trials): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(args.repeats): + output = cycle() + del output + end.record() + end.synchronize() + times.append(start.elapsed_time(end) / args.repeats) + row["trial_ms_per_cycle"] = times + row["median_ms_per_cycle"] = statistics.median(times) + row["median_ms_per_accumulation"] = row["median_ms_per_cycle"] / args.accumulation_steps + row["effective_tflops"] = case["flops"] * args.accumulation_steps / (row["median_ms_per_cycle"] * 1e9) + row["peak_allocated_gib"] = torch.cuda.max_memory_allocated(device) / GIB + row["peak_reserved_gib"] = torch.cuda.max_memory_reserved(device) / GIB + row["status"] = "ok" + except Exception as error: + row["status"] = "error" + row["error"] = type(error).__name__ + ": " + str(error)[:4000] + row["telemetry_after"] = gpu_telemetry() + report["results"].append(row) + save() + print(json.dumps({key: row[key] for key in ("name", "variant", "status", "median_ms_per_cycle", + "effective_tflops", "max_abs_error", "relative_rms_error", "error") if key in row}), flush=True) + del pairs, reference + torch.cuda.empty_cache() + report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat() + save() + print("Results: " + str(args.output_dir / "results.json"), flush=True) + if any(row["status"] != "ok" for row in report["results"]): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/ops/benchmark_training.py b/ops/benchmark_training.py new file mode 100644 index 0000000000000000000000000000000000000000..63e2f350c3f4492edc68d1486c524f21af5978ae --- /dev/null +++ b/ops/benchmark_training.py @@ -0,0 +1,603 @@ +#!/usr/bin/env python3 +"""Measure disposable full-model AdamW updates without changing training state. + +Source ops/training-env.sh before launching. For two GPUs, use torchrun +--standalone --nproc-per-node=2 with the same arguments. Each configuration must +use a new output directory under work/benchmark-*. No checkpoints are written. +""" +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import sys +import time + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +GIB = 1024**3 +sys.path.insert(0, str(PACKAGE)) + + +def arguments(): + parser = argparse.ArgumentParser(description=__doc__) + run_id = os.environ.get('TORCHELASTIC_RUN_ID') or dt.datetime.now(dt.UTC).strftime('%Y%m%dT%H%M%S') + parser.add_argument('--checkpoint', type=Path, default=PACKAGE / 'work/run/latest.pt', + help='Trusted local checkpoint; mmap-loaded read-only and copied to CUDA') + parser.add_argument('--data', type=Path, default=PACKAGE / 'work/data/train.bin') + parser.add_argument('--output-dir', type=Path, + default=PACKAGE / 'work' / ('benchmark-' + run_id)) + parser.add_argument('--micro-batch', type=int, required=True) + parser.add_argument('--seq-len', type=int, default=2048) + parser.add_argument('--global-tokens-per-update', type=int, default=131072) + parser.add_argument('--activation-checkpointing', action=argparse.BooleanOptionalAction, default=False) + parser.add_argument('--autocast-cache', action=argparse.BooleanOptionalAction, default=True) + parser.add_argument('--loss-chunk-size', type=int, default=1024) + parser.add_argument('--loss-mode', choices=['checkpointed', 'full', 'compiled-full'], default='checkpointed', + help='Disposable full-vocabulary loss execution experiment; model/objective stay the same') + parser.add_argument('--bf16-residuals', action='store_true', + help='Experimental BF16 residual stream with unchanged FP32 master parameters') + parser.add_argument('--compile-cores', action=argparse.BooleanOptionalAction, default=False) + parser.add_argument('--compile-mode', choices=['default', 'reduce-overhead', 'max-autotune-no-cudagraphs'], + default='default') + parser.add_argument('--blas-backend', choices=['default', 'cublas', 'cublaslt'], default=None, + help='Explicit PyTorch BLAS preference for a controlled comparison; ' + 'when omitted, retain the environment/library preference') + parser.add_argument('--fused-rms-norm', action=argparse.BooleanOptionalAction, default=False) + parser.add_argument('--activation-mode', choices=['baseline', 'swiglu', 'rmsnorm', 'all'], + default='baseline', help='Opt-in activation/normalization kernels') + parser.add_argument('--matmul-plan', type=Path) + parser.add_argument('--gradient-mode', choices=['eager', 'compiled-autograd', 'compiled-autograd-overlap'], default='eager', + help='Capture backward and gradient accumulation with compiled autograd') + parser.add_argument('--backward-layout', choices=['baseline', 'inspect', 'paired'], default='baseline', + help='Opt-in production-shape packed attention backward graph experiment') + parser.add_argument('--overlap-swap-collectives', action='store_true', + help='Allow wait/copy groups past later collective launches, preserving launch order') + parser.add_argument('--overlap-runtime-estimates', action='store_true', + help='Use measured, rank-aligned GEMM estimates for communication scheduling') + parser.add_argument('--overlap-producer-schedule', action='store_true', + help='Move ready gradient producers with collectives under zero estimated memory growth') + parser.add_argument('--overlap-memory-budget', type=float, default=0.0, + help='Allowed fractional increase in estimated scheduler peak memory (0 to 0.25)') + parser.add_argument('--cache-core-weight-casts', action=argparse.BooleanOptionalAction, default=False) + parser.add_argument('--ddp-batched-grad-copy', action=argparse.BooleanOptionalAction, default=False, + help='Batch gradient-to-bucket copies and scaling in the C++ DDP reducer') + parser.add_argument('--warmup-updates', type=int, default=1) + parser.add_argument('--measured-updates', type=int, default=2) + parser.add_argument('--profile', action='store_true', + help='Trace only the first measured full update after warmup; timings include profiler overhead') + parser.add_argument('--diagnose-stalls', action='store_true', + help='Record host stage timings without extra CUDA synchronization; dump Python stacks ' + 'every 10 seconds during measured updates') + parser.add_argument('--cudagraph-diagnostics', action='store_true', + help='Count successful nonempty CUDA graph replays using installed PyTorch internals; ' + 'adds host instrumentation only, for disposable comparisons') + parser.add_argument('--optimizer-state', choices=['resume', 'fresh'], default='resume') + parser.add_argument('--vram-limit-gib', type=float) + parser.add_argument('--threads', type=int, default=4) + parser.add_argument('--seed', type=int, default=1337) + parser.add_argument('--ddp-bucket-cap-mb', type=int, default=128) + args = parser.parse_args() + for name in ('micro_batch', 'seq_len', 'global_tokens_per_update', 'loss_chunk_size', + 'measured_updates', 'threads', 'ddp_bucket_cap_mb'): + if getattr(args, name) < 1: + parser.error(name + ' must be positive') + if args.warmup_updates < 0: + parser.error('--warmup-updates must be nonnegative') + if args.vram_limit_gib is not None and (not math.isfinite(args.vram_limit_gib) or args.vram_limit_gib <= 0): + parser.error('--vram-limit-gib must be finite and positive') + if args.gradient_mode != 'eager' and args.ddp_batched_grad_copy: + parser.error('Batched gradient copy requires the C++ reducer; test it separately from compiled autograd') + if args.backward_layout != 'baseline': + if (args.gradient_mode != 'compiled-autograd-overlap' or not args.compile_cores + or args.activation_checkpointing or not args.bf16_residuals + or args.micro_batch != 4 or args.seq_len != 2048 + or args.activation_mode != 'baseline' or args.matmul_plan is not None + or args.compile_mode != 'default' or args.fused_rms_norm): + parser.error('--backward-layout requires compiled-autograd-overlap, --compile-cores, ' + '--no-activation-checkpointing, --bf16-residuals, microbatch 4, sequence 2048, ' + 'default compile mode, baseline activations/norm and no matmul plan') + if not math.isfinite(args.overlap_memory_budget) or not 0 <= args.overlap_memory_budget <= 0.25: + parser.error('--overlap-memory-budget must be finite and between 0 and 0.25') + if args.gradient_mode != 'compiled-autograd-overlap' and ( + args.overlap_swap_collectives or args.overlap_runtime_estimates or args.overlap_memory_budget + or args.overlap_producer_schedule): + parser.error('Overlap scheduling options require --gradient-mode compiled-autograd-overlap') + if args.overlap_producer_schedule and ( + args.overlap_swap_collectives or args.overlap_runtime_estimates or args.overlap_memory_budget): + parser.error('--overlap-producer-schedule must be tested separately from other overlap scheduling options') + if args.gradient_mode != 'eager' and args.activation_mode != 'baseline': + parser.error('Standalone activation modes require eager gradients; use baseline activations ' + 'for compiled autograd, including matmul-plan epilogues') + if args.activation_mode != 'baseline' and args.fused_rms_norm: + parser.error('Choose an activation mode or the ATen fused RMSNorm experiment') + if args.cache_core_weight_casts and args.activation_checkpointing: + parser.error('--cache-core-weight-casts requires --no-activation-checkpointing') + if args.cache_core_weight_casts and args.gradient_mode != 'eager': + parser.error('Core weight cast caching and compiled autograd must be tested separately') + if args.compile_mode == 'reduce-overhead' and (args.cache_core_weight_casts or args.gradient_mode != 'eager'): + parser.error('Weight cast caching and compiled autograd require a compile mode without CUDA graphs') + if args.matmul_plan is not None: + args.matmul_plan = args.matmul_plan.expanduser().resolve() + if not args.matmul_plan.is_relative_to(WORKSPACE): + parser.error('--matmul-plan must stay inside the WarpState workspace') + if args.cache_core_weight_casts or args.compile_mode == 'reduce-overhead': + parser.error('--matmul-plan cannot use external core cast caching or CUDA graphs') + if args.gradient_mode != 'eager' and args.activation_checkpointing: + parser.error('--matmul-plan with compiled autograd requires --no-activation-checkpointing') + for name in ('checkpoint', 'data', 'output_dir'): + path = getattr(args, name).expanduser().resolve() + if not path.is_relative_to(WORKSPACE): + parser.error(name + ' must be inside ' + str(WORKSPACE)) + setattr(args, name, path) + relative_output = args.output_dir.relative_to(PACKAGE / 'work') if args.output_dir.is_relative_to(PACKAGE / 'work') else None + if relative_output is None or not relative_output.parts or not relative_output.parts[0].startswith('benchmark-'): + parser.error('--output-dir must be below rtx6000_40b/work/benchmark-*') + for name in ('TMPDIR', 'XDG_CACHE_HOME', 'CUDA_CACHE_PATH', 'TRITON_CACHE_DIR', 'TORCHINDUCTOR_CACHE_DIR'): + if not os.environ.get(name) or not Path(os.environ[name]).resolve().is_relative_to(WORKSPACE): + parser.error('Source ops/training-env.sh first; ' + name + ' must stay inside the workspace') + return args + + +def file_identity(path): + stat = path.stat() + return {'path': str(path), 'size': stat.st_size, 'inode': stat.st_ino, 'mtime_ns': stat.st_mtime_ns} + + +def source_hash(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def copy_optimizer_state(value, device): + """Detach every tensor from the checkpoint mapping, including Adam step counters.""" + import torch + if isinstance(value, torch.Tensor): + return value.to(device=device, copy=True) + if isinstance(value, dict): + return {key: copy_optimizer_state(item, device) for key, item in value.items()} + if isinstance(value, list): + return [copy_optimizer_state(item, device) for item in value] + if isinstance(value, tuple): + return tuple(copy_optimizer_state(item, device) for item in value) + return value + + +def timed_stage_markers(totals, annotate): + """Measure host wall time, including implicit waits, without synchronizing CUDA.""" + @contextlib.contextmanager + def mark(name): + with annotate(name): + begin = time.perf_counter() + try: + yield + finally: + key = name.removeprefix('warpstate::') + totals[key] = totals.get(key, 0.0) + time.perf_counter() - begin + return mark + + +@contextlib.contextmanager +def stalled_update_tracebacks(enabled): + """Keep the diagnostic watchdog scoped to one measured benchmark update.""" + if not enabled: + yield + return + import faulthandler + faulthandler.dump_traceback_later(10, repeat=True) + try: + yield + finally: + faulthandler.cancel_dump_traceback_later() + + +def kernel_category(name): + """Coarse name-based labels; preserve raw kernels for manual attribution.""" + name = name.lower() + if 'nccl' in name: + return 'communication' + if 'memcpy' in name or 'memset' in name: + return 'copies_and_fills' + if 'adam' in name: + return 'optimizer' + if any(part in name for part in ('flash', 'fmha', 'efficient_attention', 'scaled_dot_product')): + return 'attention' + if any(part in name for part in ('gemm', 'xmma', 'matmul', 'bmm', 'persistent_tma', 'swiglu_mm')): + return 'matrix_multiply' + if 'triton_' in name: + return 'compiled_pointwise_or_reduction' + return 'other' + + +def cuda_activity_summary(trace_events): + """Count real CUDA activities; GPU annotation ranges contain the same work.""" + kernels = {} + intervals = [] + excluded_annotations = 0 + for event in trace_events: + if event.get('ph') != 'X': + continue + if event.get('cat') == 'gpu_user_annotation': + excluded_annotations += 1 + if event.get('cat') not in ('kernel', 'gpu_memcpy', 'gpu_memset'): + continue + name = event['name'] + device_index = event.get('args', {}).get('device', event.get('pid')) + key = (name, device_index) + row = kernels.setdefault(key, {'name': name, 'device_index': device_index, + 'category': kernel_category(name), 'count': 0, 'self_cuda_us': 0.0}) + row['count'] += 1 + row['self_cuda_us'] += event['dur'] + intervals.append((event['ts'], event['ts'] + event['dur'])) + kernels = sorted(kernels.values(), key=lambda row: row['self_cuda_us'], reverse=True) + total_kernel_us = sum(row['self_cuda_us'] for row in kernels) + categories = {} + for row in kernels: + category = categories.setdefault(row['category'], {'self_cuda_us': 0.0, 'count': 0}) + category['self_cuda_us'] += row['self_cuda_us'] + category['count'] += row['count'] + for category in categories.values(): + category['fraction_of_summed_cuda_time'] = category['self_cuda_us'] / total_kernel_us if total_kernel_us else 0.0 + # A union avoids counting overlapping streams twice when estimating CUDA occupancy. + active_us = 0.0 + previous_end = None + for begin, end in sorted(intervals): + active_us += max(0.0, end - max(begin, previous_end if previous_end is not None else begin)) + previous_end = max(previous_end if previous_end is not None else end, end) + return {'kernels': kernels, 'kernel_name_count': len(kernels), 'categories': categories, + 'summed_cuda_event_us': total_kernel_us, 'cuda_event_union_us': active_us, + 'cuda_activity_filter': ['kernel', 'gpu_memcpy', 'gpu_memset'], + 'excluded_gpu_annotation_ranges': excluded_annotations} + + +def profile_operator_table(operators): + lines = ['self_cuda_ms\ttotal_cuda_ms\tself_cpu_ms\tcount\toperator\tinput_shapes'] + for row in operators[:100]: + lines.append(f"{row['self_cuda_us']/1000:.3f}\t{row['total_cuda_us']/1000:.3f}\t" + f"{row['self_cpu_us']/1000:.3f}\t{row['count']}\t{row['name']}\t" + + json.dumps(row['input_shapes'])) + return '\n'.join(lines) + '\n' + + +def export_profile(profiler, output_dir, rank, update, elapsed): + """Export both operator attribution and CUDA activities without mixing totals.""" + import torch + prefix = output_dir / f'profile-rank-{rank}' + trace_path = prefix.with_name(prefix.name + '-trace.json') + profiler.export_chrome_trace(str(trace_path)) + averages = profiler.key_averages(group_by_input_shape=True) + operators = [] + for event in averages: + if event.device_type != torch.autograd.DeviceType.CPU: + continue + operators.append({'name': event.key, 'input_shapes': event.input_shapes, 'count': event.count, + 'self_cuda_us': event.self_device_time_total, + 'total_cuda_us': event.device_time_total, + 'self_cpu_us': event.self_cpu_time_total, 'total_cpu_us': event.cpu_time_total}) + operators.sort(key=lambda row: row['self_cuda_us'], reverse=True) + activity = cuda_activity_summary(json.loads(trace_path.read_text()).get('traceEvents', [])) + summary = {'rank': rank, 'update': update, 'profiled_update_seconds': elapsed, + 'trace': str(trace_path), 'operator_count': len(operators), 'operators': operators, + 'aggregation_source_sha256': source_hash(Path(__file__)), **activity, + 'limitations': ['Profiling includes instrumentation overhead; use unprofiled runs for throughput.', + 'CUDA event times can overlap across streams; summed category time is not wall time.', + 'Operator and kernel CUDA times describe the same work; do not add them.', + 'Kernel categories are name-based heuristics; raw names and operator shapes are retained.']} + summary_path = prefix.with_name(prefix.name + '-summary.json') + summary_path.write_text(json.dumps(summary, indent=2) + '\n') + table_path = prefix.with_name(prefix.name + '-operators.txt') + table_path.write_text(profile_operator_table(operators)) + kernel_lines = ['self_cuda_ms\tcount\tcategory\tkernel'] + kernel_lines.extend(f"{row['self_cuda_us']/1000:.3f}\t{row['count']}\t{row['category']}\t{row['name']}" + for row in activity['kernels']) + prefix.with_name(prefix.name + '-kernels.tsv').write_text('\n'.join(kernel_lines) + '\n') + return {'summary': str(summary_path), 'trace': str(trace_path), 'operators': str(table_path), + **{key: value for key, value in activity.items() if key != 'kernels'}} + + +def benchmark(args, cudagraph_diagnostics=None): + import numpy as np + import torch + import torch.distributed as dist + from torch.nn.parallel import DistributedDataParallel + from run_40b import batches + from warpstate import WarpStateConfig, WarpStateLM + + rank = int(os.environ.get('RANK', 0)) + world_size = int(os.environ.get('WORLD_SIZE', 1)) + local_rank = int(os.environ.get('LOCAL_RANK', 0)) + batch_tokens = args.micro_batch * args.seq_len + divisor = batch_tokens * world_size + if args.global_tokens_per_update % divisor: + raise ValueError('Global tokens/update must be divisible by micro_batch * seq_len * world_size') + accumulation = args.global_tokens_per_update // divisor + matmul_plan = None + if args.matmul_plan is not None: + from ops.matmul_execution import load_matmul_plan + matmul_plan = load_matmul_plan(args.matmul_plan) + if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): + raise RuntimeError('BF16 CUDA hardware is required') + torch.cuda.set_device(local_rank) + if args.blas_backend is not None: + torch.backends.cuda.preferred_blas_library(args.blas_backend) + device = torch.device('cuda', local_rank) + free, total = torch.cuda.mem_get_info(device) + if args.vram_limit_gib is not None: + limit = int(args.vram_limit_gib * GIB) + if limit > total or limit > free: + raise RuntimeError(f'Allocator limit exceeds available CUDA memory: free={free/GIB:.2f} GiB') + torch.cuda.set_per_process_memory_fraction(limit / total, device) + torch.set_num_threads(args.threads) + torch.set_float32_matmul_precision('high') + torch.backends.cuda.matmul.allow_tf32 = True + torch.manual_seed(args.seed) + np.random.seed(args.seed) + if world_size > 1: + dist.init_process_group('nccl', device_id=device) + + args.output_dir.mkdir(parents=True, exist_ok=True) + record_file = (args.output_dir / f'rank-{rank}.jsonl').open('x') + + def emit(record): + record['rank'] = rank + record['timestamp'] = time.time() + line = json.dumps(record, sort_keys=True) + record_file.write(line + '\n') + record_file.flush() + print(line, flush=True) + + checkpoint_identity = file_identity(args.checkpoint) + checkpoint = torch.load(args.checkpoint, map_location='cpu', mmap=True, weights_only=False) + cfg = WarpStateConfig(**checkpoint['config']) + cfg.validate() + if args.backward_layout != 'baseline': + from ops.backward_layout_execution import validate_backward_layout_config + validate_backward_layout_config(cfg) + if args.seq_len % cfg.chunk_size: + raise ValueError('Sequence length must be divisible by the model chunk size') + with torch.device('meta'): + raw_model = WarpStateLM(cfg) + raw_model.load_state_dict(checkpoint['model'], assign=True, strict=True) + # CPU -> CUDA conversion allocates separate parameter storage before training. + raw_model.to(device=device, dtype=torch.float32).requires_grad_(True) + if any(parameter.device.type != 'cuda' for parameter in raw_model.parameters()): + raise RuntimeError('Every optimized parameter must be detached from the CPU mapping') + cfg.gradient_checkpointing = args.activation_checkpointing + raw_model.loss_chunk_size = args.loss_chunk_size + for module in raw_model.modules(): + if hasattr(module, 'use_fused'): + module.use_fused = args.fused_rms_norm + count = raw_model.num_parameters() + if not 2_000_000_000 <= count <= 2_020_000_000: + raise ValueError('This benchmark requires the complete approximately 2B model') + optimizer = torch.optim.AdamW(raw_model.parameters(), lr=1.5e-4, betas=(0.9, 0.95), + eps=1e-8, weight_decay=0.1, fused=True) + if args.optimizer_state == 'resume': + state = copy_optimizer_state(checkpoint['optimizer'], device) + optimizer.load_state_dict(state) + del state + start_tokens = checkpoint.get('tokens_seen', 0) + start_step = checkpoint.get('step', 0) + del checkpoint + execution_experiments = {'loss_mode': args.loss_mode, 'bf16_residuals': args.bf16_residuals} + extra_sources = [] + if matmul_plan is not None: + extra_sources.extend(name for name, _ in matmul_plan.source_hashes) + execution_experiments['matmul_execution'] = matmul_plan.metadata if matmul_plan else None + execution_experiments['backward_layout_execution'] = None + if args.backward_layout != 'baseline': + from ops.backward_layout_execution import configure_backward_layout + layout_metadata = configure_backward_layout(args.backward_layout) + execution_experiments['backward_layout_execution'] = layout_metadata + extra_sources.extend(layout_metadata['source_sha256']) + from ops.gradient_execution import OverlapSettings, make_backward, scale_loss + overlap_settings = (OverlapSettings(swap_with_collectives=args.overlap_swap_collectives, + use_runtime_estimations=args.overlap_runtime_estimates, + producer_schedule=args.overlap_producer_schedule, + peak_memory_budget=args.overlap_memory_budget) + if args.gradient_mode == 'compiled-autograd-overlap' else None) + backward = make_backward(args.gradient_mode, distributed=world_size > 1, overlap_settings=overlap_settings) + extra_sources.append('ops/gradient_execution.py') + if args.activation_mode != 'baseline': + from ops.activation_execution import apply_activation_execution + apply_activation_execution(raw_model, args.activation_mode) + extra_sources.append('ops/activation_execution.py') + if matmul_plan is not None: + from ops.matmul_execution import apply_matmul_execution + apply_matmul_execution(raw_model, matmul_plan) + execution_experiments.update(gradient_mode=args.gradient_mode, + communication_overlap=overlap_settings.metadata if overlap_settings else None, + activation_mode=args.activation_mode, + cache_core_weight_casts=args.cache_core_weight_casts) + if cudagraph_diagnostics is not None: + extra_sources.append('ops/cudagraph_diagnostics.py') + if args.compile_cores: + # Compiling bound forwards preserves the original module/state-dict names. + for core in raw_model.cores: + core.forward = torch.compile(core.forward, fullgraph=True, dynamic=False, mode=args.compile_mode) + if args.loss_mode != 'checkpointed': + from ops.loss_experiment import apply_loss_execution + apply_loss_execution(raw_model, args.loss_mode) + extra_sources.append('ops/loss_experiment.py') + if args.bf16_residuals: + from ops.residual_experiment import apply_bf16_residuals + apply_bf16_residuals(raw_model) + extra_sources.append('ops/residual_experiment.py') + weight_cast_cache = None + if args.cache_core_weight_casts: + from ops.weight_cast_execution import apply_weight_cast_cache + weight_cast_cache = apply_weight_cast_cache(raw_model) + extra_sources.append('ops/weight_cast_execution.py') + model = raw_model + if world_size > 1: + from ops.distributed_execution import ddp_kwargs, ddp_runtime_settings + ddp_settings = ddp_runtime_settings(bucket_cap_mb=args.ddp_bucket_cap_mb, + batched_grad_copy=args.ddp_batched_grad_copy) + model = DistributedDataParallel(raw_model, device_ids=[local_rank], output_device=local_rank, + find_unused_parameters=False, **ddp_kwargs(ddp_settings)) + extra_sources.append('ops/distributed_execution.py') + execution_experiments.update(ddp_settings) + model.train() + # Graph-owned .grad outputs cannot safely accumulate across replay calls. + # Allocate persistent buffers outside capture and zero them in place. + graph_grad_buffers = args.compile_cores and args.compile_mode == 'reduce-overhead' + if graph_grad_buffers: + for parameter in raw_model.parameters(): + if parameter.requires_grad: + parameter.grad = torch.zeros_like(parameter) + train_data = np.memmap(args.data, dtype=' 1: + dist.barrier() + torch.cuda.synchronize(device) + torch.cuda.reset_peak_memory_stats(device) + profile_this_update = args.profile and update == args.warmup_updates + profiler = (torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA], + record_shapes=True, with_stack=False, profile_memory=False) + if profile_this_update else None) + mark = torch.profiler.record_function if profile_this_update else lambda name: contextlib.nullcontext() + stage_wall_seconds = {} + if args.diagnose_stalls: + mark = timed_stage_markers(stage_wall_seconds, mark) + with (stalled_update_tracebacks(args.diagnose_stalls and update >= args.warmup_updates), + profiler if profiler is not None else contextlib.nullcontext()): + begin = time.perf_counter() + with mark('warpstate::zero_gradients'): + optimizer.zero_grad(set_to_none=not graph_grad_buffers) + loss_sum = torch.zeros((), device=device) + with weight_cast_cache.update() if weight_cast_cache else contextlib.nullcontext(): + for micro in range(accumulation): + cursor = start_tokens + update * args.global_tokens_per_update + (micro * world_size + rank) * batch_tokens + with mark('warpstate::batch_and_transfer'): + x, y = batches(train_data, cursor, args.micro_batch, args.seq_len, batch_tokens, device) + synchronize = micro == accumulation - 1 or world_size == 1 + sync_context = contextlib.nullcontext() if synchronize else model.no_sync() + with sync_context: + with mark('warpstate::forward'): + with torch.autocast('cuda', dtype=torch.bfloat16, cache_enabled=args.autocast_cache): + _, loss = model(x, y, return_logits=False) + with mark('warpstate::backward'): + scaled = scale_loss(loss, 1.0 / accumulation, args.gradient_mode) + backward(scaled) + loss_sum += scaled.detach() + with mark('warpstate::clip_gradients'): + norm = torch.nn.utils.clip_grad_norm_(raw_model.parameters(), 1.0, error_if_nonfinite=True) + with mark('warpstate::adamw_step'): + optimizer.step() + if world_size > 1: + with mark('warpstate::loss_allreduce'): + dist.all_reduce(loss_sum, op=dist.ReduceOp.AVG) + with mark('warpstate::cuda_sync'): + torch.cuda.synchronize(device) + elapsed = time.perf_counter() - begin + if world_size > 1: + elapsed_tensor = torch.tensor(elapsed, device=device, dtype=torch.float64) + dist.all_reduce(elapsed_tensor, op=dist.ReduceOp.MAX) + elapsed = float(elapsed_tensor) + if args.backward_layout == 'paired': + require_backward_layout_dispatch(min_calls=layout_calls_before + cfg.logical_depth * accumulation) + actual_calls = backward_layout_stats()['runtime_calls'] - layout_calls_before + if update >= args.warmup_updates and actual_calls != cfg.logical_depth * accumulation: + raise RuntimeError('Measured backward layout dispatch count differs from logical depth times accumulation') + loss_value, norm_value = float(loss_sum), float(norm) + if not math.isfinite(loss_value) or not math.isfinite(norm_value): + raise RuntimeError('Non-finite loss or gradient norm') + optimizer_bytes = sum(value.numel() * value.element_size() + for state in optimizer.state.values() for value in state.values() + if isinstance(value, torch.Tensor)) + if optimizer_bytes < count * 8: + raise RuntimeError('The full FP32 Adam first and second moments were not materialized') + record = {'event': 'update', 'update': update + 1, + 'phase': 'warmup' if update < args.warmup_updates else 'measured', + 'profiled': profile_this_update, + 'global_tokens': args.global_tokens_per_update, 'seconds': elapsed, + 'tokens_per_second': args.global_tokens_per_update / elapsed, + 'loss': loss_value, 'grad_norm': norm_value, 'optimizer_state_gib': optimizer_bytes/GIB, + 'peak_allocated_gib': torch.cuda.max_memory_allocated(device)/GIB, + 'peak_reserved_gib': torch.cuda.max_memory_reserved(device)/GIB, + 'device_free_gib': torch.cuda.mem_get_info(device)[0]/GIB} + if args.diagnose_stalls: + record['stage_wall_seconds'] = stage_wall_seconds + if args.backward_layout != 'baseline': + from ops.backward_layout_execution import backward_layout_stats + record['backward_layout'] = backward_layout_stats() + if args.backward_layout == 'paired': + record['backward_layout_calls'] = record['backward_layout']['runtime_calls'] - layout_calls_before + if cudagraph_diagnostics is not None: + record['cudagraph_diagnostics'] = cudagraph_diagnostics.snapshot() + emit(record) + updates.append(record) + if profiler is not None: + emit({'event': 'profile', 'update': update + 1, + **export_profile(profiler, args.output_dir, rank, update + 1, elapsed)}) + measured = [record for record in updates if record['phase'] == 'measured'] + summary = {'event': 'summary', 'configuration': metadata, 'updates': updates, + 'measured_tokens_per_second': sum(row['global_tokens'] for row in measured) / sum(row['seconds'] for row in measured), + 'median_tokens_per_second': statistics.median(row['tokens_per_second'] for row in measured), + 'first_update_peak_allocated_gib': updates[0]['peak_allocated_gib'], + 'first_update_peak_reserved_gib': updates[0]['peak_reserved_gib'], + 'steady_peak_allocated_gib': max(row['peak_allocated_gib'] for row in measured), + 'steady_peak_reserved_gib': max(row['peak_reserved_gib'] for row in measured), + 'checkpoint_path_identity_after': file_identity(args.checkpoint)} + if world_size > 1: + rank_summaries = [None] * world_size + dist.all_gather_object(rank_summaries, summary) + if rank == 0: + summary['ranks'] = rank_summaries + dist.barrier() + dist.destroy_process_group() + if rank == 0: + (args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n') + emit({key: value for key, value in summary.items() if key not in ('configuration', 'updates', 'ranks')}) + record_file.close() + + +if __name__ == '__main__': + args = arguments() + if args.cudagraph_diagnostics: + from ops.cudagraph_diagnostics import collect_cudagraph_diagnostics + with collect_cudagraph_diagnostics() as diagnostics: + benchmark(args, diagnostics) + else: + benchmark(args) diff --git a/ops/chat-cpu.sh b/ops/chat-cpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..2f7a4d02011b0cb89acd46423e7c24a63e49c8f9 --- /dev/null +++ b/ops/chat-cpu.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /workspace/WarpState/rtx6000_40b +source ops/training-env.sh +# CPU-only inference keeps both training GPUs untouched. +export CUDA_VISIBLE_DEVICES="" +export OMP_NUM_THREADS=16 +export MKL_NUM_THREADS=16 +exec .server-venv/bin/python -B -u chat.py "$@" diff --git a/ops/chat-dpo.sh b/ops/chat-dpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..dc967eb12de1dc27e1221bc3509e34cdf5c0125b --- /dev/null +++ b/ops/chat-dpo.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +exec rtx6000_40b/.server-venv/bin/python -B rtx6000_40b/ops/posttrain_chat.py "$@" diff --git a/ops/checkpoint_writer.py b/ops/checkpoint_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..3487c8d8db8e085b316d2511428f34dd57829dcd --- /dev/null +++ b/ops/checkpoint_writer.py @@ -0,0 +1,260 @@ +"""Stage immutable CPU checkpoints, then serialize one at a time in a thread. + +GPU-to-CPU copies finish synchronously before training can update live tensors. +Only serialization and durable publication overlap subsequent training. The +snapshot needs host memory for model parameters and full optimizer state; no +additional full checkpoint is allocated on the GPU. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import copy +import os +from pathlib import Path +import threading +import time + +import torch +import torch.distributed as dist + + +def snapshot_to_cpu(state): + """Copy every tensor and mutable metadata without copying tensors on CUDA. + + Prepopulate deepcopy's memo with detached CPU tensors. This also retains + repeated tensor references, OrderedDict metadata, and NumPy/Python RNG + objects while preventing any alias to subsequently mutated live state. + """ + memo = {} + visited = set() + + def stage(value): + identity = id(value) + if identity in visited: + return + visited.add(identity) + if isinstance(value, torch.Tensor): + memo[identity] = value.detach().to(device='cpu', copy=True, non_blocking=False) + elif isinstance(value, dict): + for key, item in value.items(): + stage(key) + stage(item) + elif isinstance(value, (list, tuple)): + for item in value: + stage(item) + + stage(state) + return copy.deepcopy(state, memo) + + +def atomic_torch_save(path, state): + """Publish the existing single-file torch.save format only after fsync.""" + path = Path(path) + temporary = path.with_suffix(path.suffix + '.tmp') + try: + with temporary.open('wb') as handle: + torch.save(state, handle) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + +class AsyncCheckpointWriter: + """One outstanding immutable snapshot; errors surface on check/flush/close. + + submit() blocks on the previous save before allocating the next snapshot, + bounding both queued writes and additional host checkpoint memory to one. + Use as a context manager, or always close in a finally block: an exception + in training must still join an already staged, valid checkpoint write. + """ + + def __init__(self, path): + self.path = Path(path) + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='checkpoint-writer') + self._future = None + self._error = None + self._closed = False + self._lock = threading.RLock() + + def _raise_error(self): + if self._error is not None: + raise self._error + + def _finish(self): + self._raise_error() + if self._future is not None: + try: + self._future.result() + except BaseException as error: + self._error = error + raise + finally: + self._future = None + + def _write(self, snapshot): + begin = time.perf_counter() + atomic_torch_save(self.path, snapshot) + progress = '' + if isinstance(snapshot, dict) and 'step' in snapshot and 'tokens_seen' in snapshot: + progress = f'step {snapshot["step"]}, tokens={snapshot["tokens_seen"]:,}; ' + print(f'Saved {self.path} ({progress}background write {time.perf_counter()-begin:.2f}s)', flush=True) + + def submit(self, state): + """Wait for the prior write, stage a frozen CPU copy, return staging seconds.""" + with self._lock: + self._raise_error() + if self._closed: + raise RuntimeError('Checkpoint writer is closed') + self._finish() + begin = time.perf_counter() + snapshot = snapshot_to_cpu(state) + staging_seconds = time.perf_counter() - begin + self._future = self._executor.submit(self._write, snapshot) + return staging_seconds + + def check(self): + """Propagate a completed background failure without waiting on a live save.""" + with self._lock: + self._raise_error() + if self._future is not None and self._future.done(): + self._finish() + + def flush(self): + """Join the outstanding save and surface any previous failure.""" + with self._lock: + self._finish() + + def close(self): + with self._lock: + if self._closed: + self._raise_error() + return + try: + self._finish() + finally: + self._closed = True + self._executor.shutdown(wait=True) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_value is None: + self.close() + else: + _close_after_error(self, exc_value) + return False + + +def _close_after_error(writer, original_error): + """Join locally without replacing the exception that stopped training.""" + try: + writer.close() + except BaseException as error: + if error is not original_error: + original_error.add_note( + f'Checkpoint cleanup also failed: {type(error).__name__}: {error}' + ) + + +class DistributedCheckpointWriter: + """Coordinate one rank-zero writer from the training thread on every rank. + + All ranks must call check/submit/flush/close in the same order. A status + broadcast after each rank-zero operation keeps peers from entering the next + training collective if staging, waiting, or serialization failed. submit() + returns only after the snapshot is immutable; the subsequent CPU-only file + write overlaps training. No background thread uses CUDA or distributed APIs. + + Normal context exit joins the writer collectively. Exceptional exit only + joins the local writer: a training failure may be rank-local, so another + collective could hang instead of allowing the launcher to stop its peers. + """ + + def __init__(self, path, *, rank=0, device=None, distributed=False): + self.rank = rank + self.distributed = distributed + self.device = torch.device('cpu') if device is None else torch.device(device) + self._writer = None + self._status = ( + torch.empty(1, dtype=torch.int32, device=self.device) + if distributed else None + ) + + def initialize(): + self._writer = AsyncCheckpointWriter(path) + + self._coordinate('initialization', initialize) + + def _coordinate(self, operation, action): + result = None + error = None + if self.rank == 0: + try: + result = action() + except BaseException as caught: + error = caught + if self.distributed: + if self.rank == 0: + self._status.fill_(int(error is not None)) + dist.broadcast(self._status, src=0) + if self._status.item(): + details = [ + f'{type(error).__name__}: {error}' if self.rank == 0 else None + ] + dist.broadcast_object_list(details, src=0, device=self.device) + if error is None: + raise RuntimeError( + f'Rank 0 checkpoint {operation} failed: {details[0]}' + ) + if error is not None: + raise error + return result + + def check(self): + """Surface a completed write failure to all ranks before training.""" + return self._coordinate('check', lambda: self._writer.check()) + + def submit(self, state=None, *, state_factory=None): + """Freeze rank zero's state; peers wait for staging or a shared error. + + Pass a state_factory to build the checkpoint dictionary inside the + coordinated operation. Only rank zero calls it, after the previous + write finishes, so construction errors also reach every rank. + """ + def stage(): + if state_factory is not None: + if state is not None: + raise ValueError('Pass checkpoint state or state_factory, not both') + self._writer.flush() + return self._writer.submit(state_factory()) + return self._writer.submit(state) + + return self._coordinate('submit', stage) + + def flush(self): + """Wait for durable publication and report any failure to every rank.""" + return self._coordinate('flush', lambda: self._writer.flush()) + + def close(self): + """Join the writer and report its final status on every rank.""" + return self._coordinate('close', lambda: self._writer.close()) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_value is None: + self.close() + elif self._writer is not None: + _close_after_error(self._writer, exc_value) + return False diff --git a/ops/compare_communication_benchmarks.py b/ops/compare_communication_benchmarks.py new file mode 100644 index 0000000000000000000000000000000000000000..64a20fd80d1b19b4a237fcddb3eab5f247bf6af8 --- /dev/null +++ b/ops/compare_communication_benchmarks.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Compare explicitly paired, unprofiled training benchmarks using saved JSON only. + +This reads no checkpoints, tokens, or devices. It checks recorded identities and +batch geometry, reports differences, and does not declare a statistical winner. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +import statistics + + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +REQUIRED_CONFIG = ("checkpoint", "checkpoint_step", "checkpoint_tokens", "data", "world_size", + "grad_accum", "parameters", "parameter_dtype", "autocast_dtype", "optimizer", + "torch", "cuda", "gpu", "update_timing_includes_gradient_zeroing") +REQUIRED_ARGS = ("global_tokens_per_update", "seq_len", "micro_batch", "warmup_updates", + "measured_updates", "seed", "optimizer_state", "bf16_residuals") +PRECISION_CONTROLS = ("precision", "precision_flags", "float32_matmul_precision", "allow_tf32", + "allow_bf16_reduced_precision_reduction", "allow_fp16_reduced_precision_reduction", + "optimizer_hyperparameters") + + +def finite_number(value, name, *, positive=False): + if (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or (positive and value <= 0)): + raise ValueError(f"{name} must be a finite {'positive ' if positive else ''}number") + return value + + +def require_fields(mapping, keys, label): + missing = [key for key in keys if key not in mapping] + if missing: + raise ValueError(f"{label} lacks required fields: {', '.join(missing)}") + + +def differences(left, right): + return {key: {"control": left.get(key), "candidate": right.get(key), + "control_recorded": key in left, "candidate_recorded": key in right} + for key in sorted(set(left) | set(right)) + if key not in left or key not in right or left[key] != right[key]} + + +def validate_run(summary): + config = summary["configuration"] + args = config["args"] + require_fields(config, REQUIRED_CONFIG, "configuration") + require_fields(args, REQUIRED_ARGS, "arguments") + if args.get("profile") or config.get("checkpoint_writes"): + raise ValueError("Comparisons require unprofiled benchmarks without checkpoint writes") + for identity in ("checkpoint", "data"): + require_fields(config[identity], ("path", "size", "inode", "mtime_ns"), identity) + for key, value in (("world_size", config["world_size"]), ("grad_accum", config["grad_accum"]), + *[(key, args[key]) for key in ("global_tokens_per_update", "seq_len", "micro_batch", + "measured_updates")]): + if type(value) is not int or value < 1: + raise ValueError(f"{key} must be a positive integer") + if type(args["warmup_updates"]) is not int or args["warmup_updates"] < 0: + raise ValueError("warmup_updates must be a nonnegative integer") + for name in ("checkpoint_step", "checkpoint_tokens"): + if type(config[name]) is not int or config[name] < 0: + raise ValueError(f"{name} must be a nonnegative integer") + if args["global_tokens_per_update"] != (config["world_size"] * config["grad_accum"] + * args["seq_len"] * args["micro_batch"]): + raise ValueError("Global batch does not match world size, accumulation, microbatch, and sequence") + ranks = summary.get("ranks") or [summary] + if len(ranks) != config["world_size"]: + raise ValueError("Missing per-rank benchmark records") + by_rank = {} + for rank_summary in ranks: + rc = rank_summary["configuration"] + rank = rc.get("rank") + if type(rank) is not int or rank < 0 or rank >= config["world_size"] or rank in by_rank: + raise ValueError("Rank identities must be complete and unique") + for key in REQUIRED_CONFIG: + if key not in rc or rc[key] != config[key]: + raise ValueError(f"Rank {rank} configuration mismatch: {key}") + for key in REQUIRED_ARGS: + if rc["args"].get(key) != args[key]: + raise ValueError(f"Rank {rank} argument mismatch: {key}") + if rc["args"] != args: + changed = ", ".join(differences(args, rc["args"])) + raise ValueError(f"Rank {rank} execution argument mismatch: {changed}") + for key in PRECISION_CONTROLS: + if (key in rc) != (key in config) or rc.get(key) != config.get(key): + raise ValueError(f"Rank {rank} precision/optimizer control mismatch: {key}") + if rc["args"].get("profile"): + raise ValueError("Profiled rank configuration cannot be compared") + if rank_summary.get("checkpoint_path_identity_after") != config["checkpoint"]: + raise ValueError("Checkpoint identity changed during a benchmark or was not verified") + updates = rank_summary["updates"] + if len(updates) != args["warmup_updates"] + args["measured_updates"]: + raise ValueError("Recorded update count does not match warmup/measured counts") + for index, update in enumerate(updates): + phase = "warmup" if index < args["warmup_updates"] else "measured" + if update.get("update") != index + 1 or update.get("phase") != phase: + raise ValueError("Update indices/phases must describe one continuous benchmark") + if update.get("rank") != rank or update.get("global_tokens") != args["global_tokens_per_update"]: + raise ValueError("Update rank/global batch mismatch") + if update.get("profiled") is not False: + raise ValueError("Every update must explicitly be unprofiled") + finite_number(update["seconds"], "update seconds", positive=True) + for name in ("loss", "grad_norm", "peak_allocated_gib", "peak_reserved_gib"): + finite_number(update[name], name) + expected_tps = update["global_tokens"] / update["seconds"] + if not math.isclose(update.get("tokens_per_second", 0), expected_tps, rel_tol=1e-9): + raise ValueError("Recorded throughput disagrees with tokens/update and elapsed time") + cursor = config["checkpoint_tokens"] + index * args["global_tokens_per_update"] + if "token_cursor" in update and update["token_cursor"] != cursor: + raise ValueError("Recorded token cursor disagrees with checkpoint/batch progression") + by_rank[rank] = updates + if summary["updates"] != by_rank[config.get("rank", 0)]: + raise ValueError("Top-level and per-rank update records disagree") + return config, by_rank + + +def load_run(directory): + directory = directory.expanduser().resolve() + if not directory.is_relative_to(WORKSPACE): + raise ValueError("Benchmark directories must stay inside the WarpState workspace") + inputs = [] + + def read_input(path): + payload = path.read_bytes() + inputs.append({"path": str(path), "sha256": hashlib.sha256(payload).hexdigest()}) + return payload + + payload = read_input(directory / "summary.json") + summary = json.loads(payload) + config = summary["configuration"] + # Older summaries may omit embedded ranks. Reconstruct from the canonical + # per-rank JSONL files, without opening model, data, or environment files. + if not summary.get("ranks") and config["world_size"] > 1: + ranks = [] + for rank in range(config["world_size"]): + records = [json.loads(line) for line in read_input(directory / f"rank-{rank}.jsonl").splitlines()] + configurations = [row for row in records if row.get("event") == "configuration"] + if len(configurations) != 1: + raise ValueError("Expected one configuration per rank JSONL") + ranks.append({"configuration": configurations[0], + "updates": [row for row in records if row.get("event") == "update"], + "checkpoint_path_identity_after": summary.get("checkpoint_path_identity_after")}) + summary["ranks"] = ranks + validate_run(summary) + return {"directory": str(directory), "summary_sha256": hashlib.sha256(payload).hexdigest(), + "input_files": inputs, "summary": summary} + + +def timing_statistics(seconds, tokens_per_update): + early, late = statistics.mean(seconds[:5]), statistics.mean(seconds[-5:]) + return {"updates": len(seconds), "mean_seconds": statistics.mean(seconds), + "median_seconds": statistics.median(seconds), + "std_seconds": statistics.pstdev(seconds), "std_definition": "population", + "tokens_per_second": len(seconds) * tokens_per_update / sum(seconds), + "first_five_mean_seconds": early, "last_five_mean_seconds": late, + "first_to_last_seconds_drift_fraction": late / early - 1, + "drift_windows_overlap": len(seconds) < 10} + + +def compare_summaries(control, candidate): + cc, cr = validate_run(control) + dc, dr = validate_run(candidate) + for key in REQUIRED_CONFIG: + if cc[key] != dc[key]: + raise ValueError(f"Incompatible configuration: {key}") + for key in REQUIRED_ARGS: + if cc["args"][key] != dc["args"][key]: + raise ValueError(f"Incompatible arguments: {key}") + for key in PRECISION_CONTROLS: + if (key in cc) != (key in dc) or cc.get(key) != dc.get(key): + raise ValueError(f"Incompatible or incompletely recorded precision/optimizer control: {key}") + start = cc["args"]["warmup_updates"] + total = cc["args"]["global_tokens_per_update"] + micro_tokens = cc["args"]["seq_len"] * cc["args"]["micro_batch"] + paired = [] + for index in range(start, len(cr[0])): + control_seconds = max(updates[index]["seconds"] for updates in cr.values()) + candidate_seconds = max(updates[index]["seconds"] for updates in dr.values()) + cursor = cc["checkpoint_tokens"] + index * total + per_rank = [] + for rank in sorted(cr): + before, after = cr[rank][index], dr[rank][index] + per_rank.append({"rank": rank, + "microbatch_cursors": [cursor + (micro * cc["world_size"] + rank) * micro_tokens + for micro in range(cc["grad_accum"])], + "control_loss": before["loss"], "candidate_loss": after["loss"], + "loss_difference": after["loss"] - before["loss"], + "control_grad_norm": before["grad_norm"], "candidate_grad_norm": after["grad_norm"], + "grad_norm_difference": after["grad_norm"] - before["grad_norm"], + "grad_norm_relative_difference": (abs(after["grad_norm"] - before["grad_norm"]) + / max(abs(before["grad_norm"]), 1e-30))}) + paired.append({"update": index + 1, "token_cursor": cursor, + "control_seconds": control_seconds, "candidate_seconds": candidate_seconds, + "throughput_ratio": control_seconds / candidate_seconds, "ranks": per_rank}) + control_timing = timing_statistics([row["control_seconds"] for row in paired], total) + candidate_timing = timing_statistics([row["candidate_seconds"] for row in paired], total) + memory = [] + for rank in sorted(cr): + row = {"rank": rank} + for label, ranks in (("control", cr), ("candidate", dr)): + row[label] = {phase: {name: max(update[name] for update in updates) + for name in ("peak_allocated_gib", "peak_reserved_gib")} + for phase, updates in (("all_updates", ranks[rank]), ("measured_updates", ranks[rank][start:]))} + memory.append(row) + ignore = {"timestamp", "rank", "args", "source_sha256", "initial_device_free_gib"} + control_rank_configs = {row["configuration"]["rank"]: row["configuration"] + for row in (control.get("ranks") or [control])} + candidate_rank_configs = {row["configuration"]["rank"]: row["configuration"] + for row in (candidate.get("ranks") or [candidate])} + return {"control_timing": control_timing, "candidate_timing": candidate_timing, + "throughput_ratio": candidate_timing["tokens_per_second"] / control_timing["tokens_per_second"], + "maximum_absolute_loss_difference": max(abs(rank["loss_difference"]) for row in paired for rank in row["ranks"]), + "maximum_relative_grad_norm_difference": max(rank["grad_norm_relative_difference"] for row in paired for rank in row["ranks"]), + "paired_updates": paired, "rank_peak_vram": memory, + "argument_differences": differences(cc["args"], dc["args"]), + "source_differences": differences(cc.get("source_sha256", {}), dc.get("source_sha256", {})), + "configuration_differences": differences({key: value for key, value in cc.items() if key not in ignore}, + {key: value for key, value in dc.items() if key not in ignore}), + "rank_configuration_differences": [{"rank": rank, + "source_differences": differences(control_rank_configs[rank].get("source_sha256", {}), + candidate_rank_configs[rank].get("source_sha256", {})), + "configuration_differences": differences( + {key: value for key, value in control_rank_configs[rank].items() if key not in ignore}, + {key: value for key, value in candidate_rank_configs[rank].items() if key not in ignore})} + for rank in sorted(cr)], + "unrecorded_precision_optimizer_controls": [key for key in PRECISION_CONTROLS + if key not in ("precision", "precision_flags") and key not in cc + and not any(key in (cc.get(container) or {}) for container in ("precision", "precision_flags"))]} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--control", required=True, type=Path) + parser.add_argument("--candidate", required=True, nargs="+", action="append", type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + output = args.output.expanduser().resolve() + if not output.is_relative_to(PACKAGE / "work") or output.suffix != ".json" or output.exists(): + parser.error("Select a new JSON output file under rtx6000_40b/work") + control = load_run(args.control) + comparisons = [] + for directory in (path for group in args.candidate for path in group): + candidate = load_run(directory) + comparisons.append({"candidate_directory": candidate["directory"], + "candidate_summary_sha256": candidate["summary_sha256"], + "candidate_input_files": candidate["input_files"], + **compare_summaries(control["summary"], candidate["summary"])}) + report = {"format": "warpstate-communication-comparison-v1", + "comparator_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "control_directory": control["directory"], "control_summary_sha256": control["summary_sha256"], + "control_input_files": control["input_files"], + "comparisons": comparisons, "limitations": [ + "Identities are recorded path/stat metadata, not independent checkpoint/data content hashes.", + "Token cursors are derived from recorded checkpoint tokens and batch geometry.", + "Global elapsed time uses the slowest reported rank; ranks are never summed.", + "Sequential update pairs are not randomized trials or independent confidence estimates.", + "Source and execution differences are disclosed; changed sources can alter unrecorded behavior.", + "No threshold declares a winner; inspect drift, numerical differences, and peak VRAM."]} + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(output.name + ".partial") + with temporary.open("x") as stream: + stream.write(json.dumps(report, indent=2, allow_nan=False) + "\n") + temporary.replace(output) + print(json.dumps({"output": str(output), "comparisons": len(comparisons)})) + + +if __name__ == "__main__": + main() diff --git a/ops/cudagraph_diagnostics.py b/ops/cudagraph_diagnostics.py new file mode 100644 index 0000000000000000000000000000000000000000..31acb0e52877456ea9c6e5b85ae932e725a18fb1 --- /dev/null +++ b/ops/cudagraph_diagnostics.py @@ -0,0 +1,88 @@ +"""Optional CUDA graph replay evidence for disposable benchmarks. + +Uses private PyTorch 2.14 APIs, imported only when enabled. Counts describe host +capture/replay calls, not GPU durations or kernel coverage. No CUDA work or +synchronization is added. Use only one context at a time in a benchmark process. +""" + +from __future__ import annotations + +from collections import Counter +from contextlib import contextmanager +from dataclasses import dataclass +from importlib import import_module +from typing import Callable, Iterator + + +@dataclass +class CudagraphDiagnostics: + _replays: Counter[int] + _read_counters: Callable[[], dict[str, int]] + _initial_counters: dict[str, int] + + def snapshot(self) -> dict: + """Return cumulative counts since context entry; subtract for an interval.""" + current = self._read_counters() + return { + **{key: value - self._initial_counters[key] for key, value in current.items()}, + "replay_calls": sum(self._replays.values()), + "replay_calls_by_graph_id": { + str(key): value for key, value in sorted(self._replays.items()) + }, + } + + +@contextmanager +def collect_cudagraph_diagnostics() -> Iterator[CudagraphDiagnostics]: + """Count successful, non-kernel-free tree replays and restore the wrapper. + + ``capture_calls`` counts entries in PyTorch's timed recording metrics. + ``cudagraph_skips`` and ``recorded_non_static_inputs`` are compiler counter + deltas. These alone do not establish replay. ``replay_calls`` increments + after a real ``CUDAGraphNode.run_graph`` returns successfully; its initial + constructor replay is outside this method and is deliberately excluded. + Existing benchmark synchronization establishes completion of queued work. + """ + try: + trees = import_module("torch._inductor.cudagraph_trees") + utils = import_module("torch._dynamo.utils") + node_type = trees.CUDAGraphNode + original = node_type.run_graph + if not callable(original): + raise AttributeError("CUDAGraphNode.run_graph is not callable") + # Validate availability before changing any process-wide method. + utils.compilation_time_metrics.get("CUDAGraphNode.record", []) + utils.counters.get("inductor", {}) + except (ImportError, AttributeError) as error: + raise RuntimeError( + "CUDA graph diagnostics require the private PyTorch 2.14 " + "CUDAGraphNode and compiler counter APIs" + ) from error + + def read_counters() -> dict[str, int]: + inductor = utils.counters.get("inductor", {}) + return { + "capture_calls": len(utils.compilation_time_metrics.get("CUDAGraphNode.record", [])), + "cudagraph_skips": int(inductor.get("cudagraph_skips", 0)), + "recorded_non_static_inputs": int(inductor.get("cudagraph_recorded_non_static_inputs", 0)), + } + + replays: Counter[int] = Counter() + diagnostics = CudagraphDiagnostics(replays, read_counters, read_counters()) + + def counted_run_graph(node): + try: + kernel_free = node.wrapped_function.kernel_free_cudagraph + graph_id = node.id.id + except AttributeError as error: + raise RuntimeError("CUDA graph diagnostics encountered an unsupported node API") from error + result = original(node) + if not kernel_free: + replays[graph_id] += 1 + return result + + node_type.run_graph = counted_run_graph + try: + yield diagnostics + finally: + node_type.run_graph = original diff --git a/ops/data-requirements-resolved.txt b/ops/data-requirements-resolved.txt new file mode 100644 index 0000000000000000000000000000000000000000..066c5914aba259a8498e24e77f58ef1e89c123a9 --- /dev/null +++ b/ops/data-requirements-resolved.txt @@ -0,0 +1,37 @@ +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.15.1 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +datasets==5.0.1 +dill==0.4.1 +filelock==3.32.6 +frozenlist==1.8.0 +fsspec==2026.6.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.30.0 +idna==3.19 +multidict==6.7.1 +multiprocess==0.70.19 +numpy==2.5.3 +packaging==26.3 +pandas==3.0.5 +pip==24.0 +propcache==0.5.2 +pyarrow==25.0.1 +python-dateutil==2.9.0.post0 +pyyaml==6.0.3 +requests==2.34.2 +sentencepiece==0.2.2 +six==1.17.0 +tqdm==4.70.0 +typing-extensions==4.16.0 +urllib3==2.7.0 +xxhash==4.0.1 +yarl==1.24.5 diff --git a/ops/data-resume-validation.json b/ops/data-resume-validation.json new file mode 100644 index 0000000000000000000000000000000000000000..171efbe05fbd5549dba5389b73a7dd557d61fbc2 --- /dev/null +++ b/ops/data-resume-validation.json @@ -0,0 +1,33 @@ +{ + "checked_at": 1788943224.2171412, + "source_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "datasets_version": "5.0.1", + "graceful_stop": true, + "checks": { + "train": { + "committed_tokens": 174099785, + "file_bytes": 348199570, + "checked_prefix_bytes": 8388608, + "prefix_sha256": "fae5a1378eb16a878f83f8784974923cb00e47c490ab94aeb34487d7ce737d14", + "min_token": 2, + "max_token": 32767, + "eos_tokens": 159064 + }, + "val": { + "committed_tokens": 191498, + "file_bytes": 382996, + "checked_prefix_bytes": 382996, + "prefix_sha256": "e08fc41708fce35d5cab7cd90c471771292c4b3aac2495939631ebb9bca9d1a7", + "min_token": 2, + "max_token": 32766, + "eos_tokens": 168 + } + }, + "resume_verified": true, + "resumed_train_tokens": 239010719, + "resumed_val_tokens": 319888, + "resume_checked_at": 1788943296.4178915, + "shutdown_issue": "Mitigated by ops/prepare_data.py, which explicitly exits after the original pipeline commits and returns.", + "launcher_live_test": "work/validation/live-resume-exitguard/validation.json", + "launcher_live_test_passed": true +} diff --git a/ops/datactl.sh b/ops/datactl.sh new file mode 100644 index 0000000000000000000000000000000000000000..111fc04a0fed2b55a7e6c33c615319a112e61530 --- /dev/null +++ b/ops/datactl.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +export TMPDIR=/workspace/WarpState/tmp +export PYTHONDONTWRITEBYTECODE=1 +exec /usr/local/bin/supervisorctl \ + -c /workspace/WarpState/rtx6000_40b/ops/supervisord.conf "$@" diff --git a/ops/distributed_execution.py b/ops/distributed_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..dab066bb7aa09b33ade9cdfe527034148a996c61 --- /dev/null +++ b/ops/distributed_execution.py @@ -0,0 +1,79 @@ +"""Validated, opt-in DDP execution settings without changing gradient precision.""" + +import inspect +import math +from numbers import Real + + +def _capacity(value, name): + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{name} must be a positive finite number of MiB") + if not math.isfinite(value) or value <= 0 or value * 1024 * 1024 < 1: + raise ValueError(f"{name} must be finite and at least one byte in MiB") + return float(value) + + +def ddp_runtime_settings(*, bucket_cap_mb=128, batched_grad_copy=False, + bucket_cap_mb_list=None): + """Return serializable settings; a capacity list overrides the uniform cap. + + These options tune the existing FP32 gradient reducer. They do not install a + communication hook, change reduction precision, or alter accumulation. The + default keeps the historical 128 MiB buckets and individual gradient copies. + """ + capacity = _capacity(bucket_cap_mb, "bucket_cap_mb") + if not isinstance(batched_grad_copy, bool): + raise ValueError("batched_grad_copy must be a boolean") + capacities = None + if bucket_cap_mb_list is not None: + if not isinstance(bucket_cap_mb_list, (list, tuple)) or not bucket_cap_mb_list: + raise ValueError("bucket_cap_mb_list must be a nonempty list of capacities") + capacities = [_capacity(value, "bucket_cap_mb_list entry") + for value in bucket_cap_mb_list] + capacity = max(capacities) + return { + "ddp_bucket_cap_mb": capacity, + "ddp_batched_grad_copy": batched_grad_copy, + "ddp_bucket_cap_mb_list": capacities, + } + + +def ddp_kwargs(settings): + """Validate stored settings and build kwargs for the installed DDP version. + + Unsupported experimental options fail explicitly instead of silently running + a different benchmark. Unselected new arguments are omitted so historical + defaults still work with PyTorch versions that predate those arguments. + """ + from torch.nn.parallel import DistributedDataParallel + + resolved = ddp_runtime_settings( + bucket_cap_mb=settings.get("ddp_bucket_cap_mb", 128), + batched_grad_copy=settings.get("ddp_batched_grad_copy", False), + bucket_cap_mb_list=settings.get("ddp_bucket_cap_mb_list"), + ) + kwargs = { + "broadcast_buffers": False, + "gradient_as_bucket_view": True, + "bucket_cap_mb": resolved["ddp_bucket_cap_mb"], + } + requested = {} + if resolved["ddp_batched_grad_copy"]: + requested["batched_grad_copy"] = True + if resolved["ddp_bucket_cap_mb_list"] is not None: + requested["bucket_cap_mb_list"] = resolved["ddp_bucket_cap_mb_list"] + parameters = inspect.signature(DistributedDataParallel).parameters + for name, value in requested.items(): + if name not in parameters: + raise RuntimeError(f"Installed PyTorch DDP does not support requested {name}") + kwargs[name] = value + if requested: + import torch._dynamo + + if torch._dynamo.utils.get_optimize_ddp_mode() == "python_reducer": + # Compiled autograd uses per-parameter Python hooks and compiler + # communication fusion instead of the C++ reducer's copy/bucket + # implementation. Do not report inactive flags as measured options. + raise ValueError("batched_grad_copy and bucket_cap_mb_list require " + "the C++ DDP reducer; disable them for python_reducer") + return kwargs diff --git a/ops/dpo-handoff.sh b/ops/dpo-handoff.sh new file mode 100644 index 0000000000000000000000000000000000000000..e1bb363e0945aa9c82bdab7cdabded74d565d7e3 --- /dev/null +++ b/ops/dpo-handoff.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +exec rtx6000_40b/.server-venv/bin/python -B \ + rtx6000_40b/ops/dpo_handoff.py --config rtx6000_40b/configs/dpo_10b.json "$@" diff --git a/ops/dpo-supervisord.conf b/ops/dpo-supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..bd86cfe6584a5462ec43ba69935f0418a6f16c14 --- /dev/null +++ b/ops/dpo-supervisord.conf @@ -0,0 +1,17 @@ +# User-authorized follow-on job; archive completed SFT before starting DPO. +[program:warpstate-dpo] +command=/workspace/WarpState/rtx6000_40b/ops/dpo-handoff.sh +directory=/workspace/WarpState +autostart=false +autorestart=false +startsecs=5 +startretries=0 +exitcodes=0 +stopsignal=USR1 +stopwaitsecs=900 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/dpo.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=5 +redirect_stderr=true diff --git a/ops/dpo_data.py b/ops/dpo_data.py new file mode 100644 index 0000000000000000000000000000000000000000..2996246930e7a79c1a92ec5fb40709efa3980ee9 --- /dev/null +++ b/ops/dpo_data.py @@ -0,0 +1,88 @@ +"""Preference branches using the frozen SFT renderer, with response-only masks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ops.posttrain_data import encode_messages, renderer_sha256 + +MASK_FORMAT = 'warpstate-final-assistant-response-v1' + + +def encode_pair(row, tokenizer, max_length): + prompt = row.get('prompt') + if not isinstance(prompt, list) or not prompt or prompt[-1].get('role') not in ('user', 'tool'): + raise ValueError('DPO requires a nonempty shared prompt ending user/tool') + result = {} + for name in ('chosen', 'rejected'): + response = row.get(name) + if not isinstance(response, list) or len(response) != 1 or response[0].get('role') != 'assistant': + raise ValueError('DPO requires exactly one final assistant response per branch') + encoded = encode_messages(prompt + response, tokenizer, max_length=max_length) + tail = encode_messages(response, tokenizer) + bos = int(0 <= tokenizer.bos_id() < tokenizer.vocab_size()) + prefix_length = len(encoded['input_ids']) - len(tail['input_ids']) + bos + if encoded['input_ids'][prefix_length:] != tail['input_ids'][bos:]: + raise ValueError('Response segmentation differs from the frozen SFT renderer') + encoded['labels'][:prefix_length] = [-100] * prefix_length + if not any(value != -100 for value in encoded['labels'][1:]): + raise ValueError('Preference branch has no response targets') + result[name] = encoded + if result['chosen']['input_ids'] == result['rejected']['input_ids']: + raise ValueError('Chosen and rejected responses must differ') + return result + + +def load_data(data_dir, tokenizer, tokenizer_hash, sequence_length): + from posttrain import digest + directory = Path(data_dir) + manifest_path = directory / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + if manifest.get('format') != 'warpstate-posttrain-data-v1' or manifest.get('complete') is not True: + raise ValueError('Posttraining data is not complete') + if manifest.get('tokenizer_sha256') != tokenizer_hash: + raise ValueError('Prepared data tokenizer hash differs') + if manifest.get('renderer_sha256') != renderer_sha256(): + raise ValueError('Prepared data renderer hash differs') + if manifest.get('max_length') != sequence_length: + raise ValueError('Prepared data max_length differs from sequence_length') + result = [] + for filename in ('dpo_train.jsonl', 'dpo_val.jsonl'): + path = directory / filename + info = manifest.get('outputs', {}).get(filename) + if not info or not path.is_file() or digest(path) != info.get('sha256'): + raise ValueError(f'Missing or changed prepared file: {filename}') + records = [] + token_count = target_count = 0 + with path.open() as stream: + for number, line in enumerate(stream, 1): + row = json.loads(line) + encoded = encode_pair(row, tokenizer, sequence_length) + for branch in ('chosen', 'rejected'): + tokens = len(encoded[branch]['input_ids']) + targets = sum(value != -100 for value in encoded[branch]['labels'][1:]) + if (row.get('tokens', {}).get(branch) != tokens or + row.get('supervised_tokens', {}).get(branch) != targets): + raise ValueError(f'{filename}:{number}: token metadata differs') + token_count += tokens + target_count += targets + records.append(encoded) + if number % 10000 == 0: + print(json.dumps(dict(event='data_validation', file=filename, rows=number)), flush=True) + if not records or len(records) != info.get('rows'): + raise ValueError(f'{filename}: empty data or row count mismatch') + if token_count != info.get('tokens') or target_count != info.get('supervised_tokens'): + raise ValueError(f'{filename}: manifest token totals differ') + result.append(records) + return *result, manifest, digest(manifest_path) + + +def collate_pairs(records, sequence_length, pad_id): + """Adjacent chosen/rejected sequences; empty tail pairs have zero weight.""" + import torch + from posttrain import collate + branches = [record[branch] if record is not None else None + for record in records for branch in ('chosen', 'rejected')] + x, y = collate(branches, sequence_length, pad_id) + return x, y, torch.tensor([record is not None for record in records], dtype=torch.bool) diff --git a/ops/dpo_handoff.py b/ops/dpo_handoff.py new file mode 100644 index 0000000000000000000000000000000000000000..53b3769f16cd6f5917b1c3c42ef5f54cf3b0689a --- /dev/null +++ b/ops/dpo_handoff.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Archive a successfully completed SFT run, then initialize separate BF16 DPO. + +This explicitly authorized follow-on job does not modify or interrupt SFT. Its +archive includes the original full optimizer/RNG checkpoint and a tensor-exact +model-only export. DPO's policy and fixed reference read an independent copy. +""" +from __future__ import annotations + +import argparse +import contextlib +import json +import math +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import tempfile +import time + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +sys.path.insert(0, str(PACKAGE)) +import torch +from ops.launch_training import supervise, training_environment, workspace_path +from ops.posttrain_handoff import (atomic_json, clean_exit, copy_regular, exclusive_lock, + flush_directory, process_status, publish_directory, sha256, tensors_equal, + utc_now, verify_completion as verify_sft_completion, + source_identity as sft_source_identity) + +STOPPED = False +ARCHIVE_FILES = ('checkpoint.pt', 'model.pt', 'snapshot.json', 'tokenizer.model', + 'settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl', + 'sft-execution.json', 'sft-handoff.json', 'PROVENANCE.json', 'ARCHIVE.md') +INIT_FILES = ('model.pt', 'snapshot.json', 'tokenizer.model') + + +def read_json(path): + return json.loads(Path(path).read_text()) + + +def load_plan(path): + plan = read_json(path) + if (plan.get('format') != 'warpstate-dpo-plan-v1' or plan.get('stage') != 'dpo' or + plan.get('after_completion') != 'pause_for_evaluation'): + raise ValueError('Expected a DPO plan that pauses after completion') + options = plan['handoff'] + paths = {key: workspace_path(WORKSPACE / options[key]) for key in ( + 'base_work_dir', 'sft_work_dir', 'sft_plan', 'supervisor_socket', 'archive_dir', + 'work_dir', 'data_dir', 'tokenizer')} + work, base, sft, archive, data = (paths[k] for k in ( + 'work_dir', 'base_work_dir', 'sft_work_dir', 'archive_dir', 'data_dir')) + if work == base or work == WORKSPACE or not work.is_relative_to(base / 'posttrain'): + raise ValueError('DPO output must use its own directory under base work/posttrain') + for protected in (base / 'run', base / 'archives', sft, archive, data): + if work.is_relative_to(protected) or protected.is_relative_to(work): + raise ValueError(f'DPO outputs overlap protected inputs: {protected}') + if not archive.is_relative_to(base / 'archives') or archive == base / 'archives': + raise ValueError('Select a new historical archive directory under base work/archives') + if options['processes'] != 2 or not 1 <= options['poll_seconds'] <= 60: + raise ValueError('This handoff requires two GPUs and a 1..60 second poll interval') + original = read_json(paths['sft_plan']) + if (original['base']['step'] != plan['sft']['base_step'] or + original['base']['tokens_seen'] != plan['sft']['base_tokens_seen'] or + original['base']['tokenizer_sha256'] != plan['sft']['tokenizer_sha256']): + raise ValueError('SFT parent identity differs from the DPO plan') + return plan, paths + + +def source_identity(config, sft_plan): + from dpo import source_identity as dpo_sources + result = {str((PACKAGE / name).relative_to(WORKSPACE)): checksum + for name, checksum in dpo_sources().items()} + result.update(sft_source_identity(sft_plan)) + for path in (config, Path(__file__), PACKAGE / 'ops/dpo_preflight.py', + PACKAGE / 'ops/dpo-handoff.sh'): + result[str(path.relative_to(WORKSPACE))] = sha256(path) + return result + + +def data_readiness(folder, plan, *, verify=False): + manifest = folder / 'manifest.json' + if not manifest.is_file(): + return False + data = read_json(manifest) + if data.get('format') != 'warpstate-posttrain-data-v1' or data.get('complete') is not True: + return False + if (sha256(manifest) != plan['handoff']['data_manifest_sha256'] or + data.get('tokenizer_sha256') != plan['sft']['tokenizer_sha256'] or + data.get('max_length') != plan['training']['sequence_length']): + raise ValueError('Prepared preference data differs from the authorized plan') + for name in ('dpo_train.jsonl', 'dpo_val.jsonl'): + entry = data.get('outputs', {}).get(name, {}) + file = folder / name + if not file.is_file() or file.is_symlink() or entry.get('rows', 0) < 1: + raise ValueError(f'Prepared preference split is empty or missing: {name}') + if verify and sha256(file) != entry.get('sha256'): + raise ValueError(f'Preference split checksum differs: {name}') + return True + + +def validate_sft(checkpoint, evaluation, completion, plan, sft_plan): + expected = plan['sft'] + if (checkpoint.get('format') != 'warpstate-posttrain-v1' or + checkpoint.get('completed') is not True or + checkpoint.get('step') != expected['step'] or + checkpoint.get('total_steps') != expected['step'] or + checkpoint.get('sample_cursor') != expected['train_rows'] or + checkpoint.get('tokens_seen') != expected['tokens_seen'] or + checkpoint.get('assistant_tokens_seen') != expected['assistant_tokens_seen'] or + checkpoint.get('val_step') != expected['step']): + raise ValueError('SFT must complete the exact full epoch and final validation before DPO') + if not math.isfinite(checkpoint.get('val_loss', float('nan'))): + raise ValueError('SFT final validation loss is not finite') + signature = checkpoint['signature'] + if (signature['plan'] != sft_plan or signature['base_sha256'] != expected['raw_archive_sha256'] or + signature['tokenizer_sha256'] != expected['tokenizer_sha256'] or + signature['data_sha256'] != plan['handoff']['data_manifest_sha256'] or + checkpoint['origin']['step'] != expected['base_step'] or + checkpoint['origin']['tokens_seen'] != expected['base_tokens_seen']): + raise ValueError('Completed SFT checkpoint belongs to a different base, tokenizer, data, or plan') + if not checkpoint.get('optimizer') or len(checkpoint.get('rng_by_rank', [])) != 2: + raise ValueError('Full SFT optimizer or per-rank resume state is missing') + if (completion.get('format') != 'warpstate-posttrain-completion-v1' or + completion.get('stage') != 'sft' or completion.get('completed') is not True or + completion.get('step') != expected['step'] or completion.get('total_steps') != expected['step']): + raise ValueError('SFT completion report is missing or inconsistent') + if (evaluation.get('stage') != 'sft' or evaluation.get('step') != expected['step'] or + evaluation.get('full_holdout') is not True or + evaluation.get('evaluated_samples') != expected['validation_rows'] or + evaluation.get('available_validation_samples') != expected['validation_rows'] or + evaluation.get('val_loss') != checkpoint['val_loss'] or + evaluation.get('base_sha256') != expected['raw_archive_sha256']): + raise ValueError('SFT archive requires the matching full-holdout evaluation') + if not checkpoint.get('model'): + raise ValueError('SFT model tensors are missing') + for name, tensor in checkpoint['model'].items(): + if tensor.is_floating_point() and tensor.dtype != torch.float32: + raise ValueError(f'Expected untouched FP32 SFT weights: {name}') + + +def verify_files(folder, names): + if folder.is_symlink() or not folder.is_dir(): + raise ValueError(f'Artifact directory is missing or is a symlink: {folder}') + checksums = folder / 'SHA256SUMS' + if not checksums.is_file() or checksums.is_symlink(): + raise ValueError('Artifact checksum manifest is missing') + entries = {} + for line in checksums.read_text().splitlines(): + checksum, name = line.split(' ', 1) + if name not in names or name in entries: + raise ValueError('Unexpected/duplicate artifact checksum entry') + entries[name] = checksum + if set(entries) != set(names): + raise ValueError('Artifact checksum manifest is incomplete') + for name in (*names, 'SHA256SUMS'): + path = folder / name + if path.is_symlink() or not path.is_file() or stat.S_IMODE(path.stat().st_mode) & 0o222: + raise ValueError(f'Artifact file is missing or not sealed read-only: {name}') + if name in entries and sha256(path) != entries[name]: + raise ValueError(f'Artifact checksum mismatch: {name}') + return entries + + +def seal(folder, names): + checksums = folder / 'SHA256SUMS' + with checksums.open('x') as stream: + for name in names: + stream.write(f'{sha256(folder / name)} {name}\n') + stream.flush() + os.fsync(stream.fileno()) + for name in (*names, 'SHA256SUMS'): + os.chmod(folder / name, 0o444) + os.chmod(folder, 0o555) + flush_directory(folder) + + +def verify_sft_archive(folder, plan, sft_plan, *, expected_checkpoint_hash=None): + entries = verify_files(folder, ARCHIVE_FILES) + if expected_checkpoint_hash and entries['checkpoint.pt'] != expected_checkpoint_hash: + raise ValueError('Archived SFT state differs from the completed source checkpoint') + checkpoint = torch.load(folder / 'checkpoint.pt', map_location='cpu', weights_only=False, mmap=True) + validate_sft(checkpoint, read_json(folder / 'evaluation.json'), read_json(folder / 'completion.json'), plan, sft_plan) + metadata = read_json(folder / 'snapshot.json') + if (metadata.get('format') != 'warpstate-chat-v1' or metadata.get('stage') != 'sft' or + metadata.get('step') != plan['sft']['step'] or + metadata.get('tokens_seen') != checkpoint['tokens_seen'] or + metadata.get('tokenizer_sha256') != plan['sft']['tokenizer_sha256'] or + metadata.get('snapshot_sha256') != entries['model.pt'] or + metadata.get('snapshot_bytes') != (folder / 'model.pt').stat().st_size or + metadata.get('source', {}).get('sha256') != entries['checkpoint.pt'] or + entries['tokenizer.model'] != plan['sft']['tokenizer_sha256']): + raise ValueError('SFT model export metadata differs from the full archive') + exported = torch.load(folder / 'model.pt', map_location='cpu', weights_only=False, mmap=True) + tensors_equal(checkpoint['model'], exported['model']) + return metadata, entries + + +def archive_sft(sft_work, tokenizer, destination, plan, sft_plan, revision): + run = sft_work / 'run' + source = run / 'latest.pt' + if source.is_symlink(): + raise ValueError('SFT checkpoint must be a regular completed file') + before = source.stat() + checkpoint = torch.load(source, map_location='cpu', weights_only=False, mmap=True) + evaluation, completion = read_json(run / 'evaluation.json'), read_json(run / 'completion.json') + validate_sft(checkpoint, evaluation, completion, plan, sft_plan) + full_hash = sha256(source) + if destination.exists() or destination.is_symlink(): + return verify_sft_archive(destination, plan, sft_plan, expected_checkpoint_hash=full_hash) + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix='.sft-final-', dir=destination.parent)) + # Full-state archive is a byte-for-byte copy, preserving Adam/RNG/signature. + copy_regular(source, staging / 'checkpoint.pt') + if sha256(staging / 'checkpoint.pt') != full_hash: + raise ValueError('SFT full-state archive copy differs from its source') + metadata = dict(format='warpstate-chat-v1', inference_only=True, stage='sft', + config=checkpoint['config'], step=checkpoint['step'], tokens_seen=checkpoint['tokens_seen'], + assistant_tokens_seen=checkpoint['assistant_tokens_seen'], val_loss=checkpoint['val_loss'], + tokenizer_sha256=plan['sft']['tokenizer_sha256'], origin=checkpoint['origin'], + captured_at_utc=utc_now(), source=dict(path=str(source), device=before.st_dev, + inode=before.st_ino, size=before.st_size, mtime_ns=before.st_mtime_ns, sha256=full_hash)) + with (staging / 'model.pt').open('xb') as stream: + torch.save(dict(metadata, model=checkpoint['model']), stream) + stream.flush() + os.fsync(stream.fileno()) + metadata.update(snapshot_sha256=sha256(staging / 'model.pt'), + snapshot_bytes=(staging / 'model.pt').stat().st_size) + atomic_json(staging / 'snapshot.json', metadata) + copy_regular(tokenizer, staging / 'tokenizer.model') + for name in ('settings.json', 'evaluation.json', 'completion.json', 'metrics.jsonl'): + copy_regular(run / name, staging / name) + copy_regular(sft_work / 'control/execution.json', staging / 'sft-execution.json') + copy_regular(sft_work / 'control/handoff.json', staging / 'sft-handoff.json') + atomic_json(staging / 'PROVENANCE.json', dict(format='warpstate-sft-archive-v1', + repository_revision=revision, source_checkpoint_sha256=full_hash, + model_sha256=metadata['snapshot_sha256'], sft_step=checkpoint['step'], + sft_input_tokens=checkpoint['tokens_seen'], sft_assistant_tokens=checkpoint['assistant_tokens_seen'], + raw_base_step=plan['sft']['base_step'], raw_base_tokens=plan['sft']['base_tokens_seen'], + raw_base_sha256=plan['sft']['raw_archive_sha256'], data_sha256=checkpoint['signature']['data_sha256'], + tokenizer_sha256=plan['sft']['tokenizer_sha256'], sft_signature=checkpoint['signature'])) + with (staging / 'ARCHIVE.md').open('x') as stream: + stream.write(f'''# KEEP: completed SFT after raw step {plan['sft']['base_step']} + +User-requested historical archive created {utc_now()}. Preserve during routine +cleanup, DPO, and chat refreshes. Remove only on explicit user instruction. + +- Completed SFT update: {checkpoint['step']:,}; SFT input tokens: {checkpoint['tokens_seen']:,}. +- Supervised assistant tokens: {checkpoint['assistant_tokens_seen']:,}. +- Original raw base: step {plan['sft']['base_step']:,}, {plan['sft']['base_tokens_seen']:,} tokens. +- Final full-holdout SFT loss: {checkpoint['val_loss']:.12f}. +- `checkpoint.pt` is the untouched complete SFT checkpoint: FP32 model, Adam, + both ranks' RNG, sample cursor, configuration and source/data signatures. +- `model.pt` is a tensor-exact FP32 model-only export with original tokenizer. +- Original evaluation, completion, settings, metrics and handoff provenance are + included. SFT and pretraining counters refer to distinct training stages. +- Independent physical files, no hardlinks/reflinks; every file is read-only. +- DPO uses an independent model copy and fresh Adam; its reference stays fixed + at this SFT model. The original SFT run checkpoint and raw archive are kept. +- Repository revision at archival time: `{revision}`. + +Verify here with `sha256sum -c SHA256SUMS`. This is an on-device archive. +''') + stream.flush() + os.fsync(stream.fileno()) + after = source.stat() + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns): + raise ValueError('SFT checkpoint changed while archival was in progress') + seal(staging, ARCHIVE_FILES) + verified = verify_sft_archive(staging, plan, sft_plan, expected_checkpoint_hash=full_hash) + publish_directory(staging, destination) + return verified + + +def initialize_copy(archive, destination, metadata): + if not destination.exists() and not destination.is_symlink(): + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix='.dpo-init-', dir=destination.parent)) + for name in INIT_FILES: + copy_regular(archive / name, staging / name) + seal(staging, INIT_FILES) + publish_directory(staging, destination) + entries = verify_files(destination, INIT_FILES) + if entries['model.pt'] != metadata['snapshot_sha256']: + raise ValueError('DPO initialization differs from the sealed SFT model') + for name in INIT_FILES: + original, copied = (archive / name).stat(), (destination / name).stat() + if ((original.st_dev, original.st_ino) == (copied.st_dev, copied.st_ino) or + sha256(archive / name) != entries[name]): + raise ValueError(f'DPO initialization must be an independent identical copy: {name}') + return destination / 'model.pt' + + +def verify_preflight(path, processes): + from dpo import source_identity as dpo_sources + report = read_json(path) + if (report.get('format') != 'warpstate-dpo-preflight-v1' or report.get('accepted') is not True or + report.get('processes') != processes or report.get('source_sha256') != dpo_sources() or + report.get('bf16_compute') is not True or + report.get('fp32_parameters_and_adam') is not True or + report.get('frozen_reference_unchanged') is not True or + report.get('generator_sha256') != sha256(PACKAGE / 'ops/dpo_preflight.py')): + raise ValueError('DPO GPU preflight failed or belongs to different sources') + return report + + +def verify_dpo_completion(run, plan, model_hash): + from dpo import DEFAULTS, reference_identity, validate_cache + completion = read_json(run / 'completion.json') + evaluation = read_json(run / 'evaluation.json') + checkpoint = torch.load(run / 'latest.pt', map_location='cpu', weights_only=False, mmap=True) + manifest_path = WORKSPACE / plan['handoff']['data_dir'] / 'manifest.json' + manifest = read_json(manifest_path) + if sha256(manifest_path) != plan['handoff']['data_manifest_sha256']: + raise ValueError('DPO data manifest changed before completion') + training, validation = (manifest['outputs'][name] for name in ('dpo_train.jsonl', 'dpo_val.jsonl')) + settings = DEFAULTS | plan['training'] + batch = settings['micro_batch'] * settings['grad_accum'] * plan['handoff']['processes'] + total_steps = math.ceil(training['rows'] / batch) * settings['epochs'] + sample_count = training['rows'] * settings['epochs'] + expected_tokens = (training['tokens'] - 2 * training['rows']) * settings['epochs'] + expected_targets = training['supervised_tokens'] * settings['epochs'] + if (checkpoint.get('format') != 'warpstate-dpo-v1' or checkpoint.get('completed') is not True or + checkpoint.get('step') != total_steps or checkpoint.get('total_steps') != total_steps or + checkpoint.get('sample_cursor') != sample_count or checkpoint.get('tokens_seen') != expected_tokens or + checkpoint.get('response_tokens_seen') != expected_targets or + checkpoint.get('val_step') != checkpoint['step'] or + not math.isfinite(checkpoint.get('val_loss', float('nan'))) or + checkpoint['signature']['plan'] != plan or checkpoint['signature']['base_sha256'] != model_hash or + checkpoint['signature']['settings'] != settings or + checkpoint['signature']['data_sha256'] != plan['handoff']['data_manifest_sha256']): + raise ValueError('DPO did not publish a matching fully completed checkpoint') + if (completion.get('format') != 'warpstate-dpo-completion-v1' or completion.get('stage') != 'dpo' or + completion.get('completed') is not True or completion.get('paused_for_evaluation') is not True or + any(completion.get(key) != checkpoint[key] for key in ('step', 'total_steps', 'sample_cursor', + 'tokens_seen', 'response_tokens_seen', 'val_loss'))): + raise ValueError('DPO completion report differs from its checkpoint') + if (evaluation.get('stage') != 'dpo' or evaluation.get('step') != checkpoint['step'] or + evaluation.get('full_holdout') is not True or + evaluation.get('evaluated_samples') != validation['rows'] or + evaluation.get('available_validation_samples') != validation['rows'] or + evaluation.get('val_loss') != checkpoint['val_loss'] or + evaluation.get('base_sha256') != model_hash or + evaluation.get('data_sha256') != checkpoint['signature']['data_sha256'] or + not all(math.isfinite(evaluation.get(key, float('nan'))) for key in + ('reward_margin', 'reward_accuracy', 'chosen_nll'))): + raise ValueError('DPO final full-holdout evaluation is missing') + identity = reference_identity(model_hash, plan['handoff']['data_manifest_sha256'], + plan['sft']['tokenizer_sha256'], settings, torch.device('cuda'), plan['handoff']['processes']) + _, _, reference_hash = validate_cache(run.parent / 'reference-cache', identity, + (training['rows'], validation['rows'])) + if any(value != reference_hash for value in (checkpoint['signature']['reference_sha256'], + completion.get('reference_sha256'), evaluation.get('reference_sha256'))): + raise ValueError('DPO completion no longer matches its fixed SFT reference cache') + return completion + + +def request_stop(signum, frame): + global STOPPED + STOPPED = True + + +def run_handoff(config, *, check=False): + plan, paths = load_plan(config) + work, sft, base = (paths[k] for k in ('work_dir', 'sft_work_dir', 'base_work_dir')) + state_path = work / 'control/handoff.json' + original_plan = read_json(paths['sft_plan']) + def status(): + return process_status(paths['supervisor_socket'], plan['handoff']['sft_program']) + if check: + print(json.dumps(dict(sft_process=status(), data_ready=data_readiness(paths['data_dir'], plan), + handoff=read_json(state_path) if state_path.exists() else None), indent=2)) + return 0 + state_path.parent.mkdir(parents=True, exist_ok=True) + with exclusive_lock(work / 'control/handoff.lock') as own_fd: + identity = source_identity(config, paths['sft_plan']) + if state_path.exists(): + state = read_json(state_path) + if state['source_sha256'] != identity: + raise ValueError('Armed DPO sources/config changed; inspect before explicitly re-arming') + if state.get('phase') == 'complete': + print('DPO already completed; remaining paused for evaluation.', flush=True) + return 0 + else: + state = dict(format='warpstate-dpo-handoff-v1', armed_at=utc_now(), + source_sha256=identity, sft=plan['sft'], + git_revision=subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=WORKSPACE, text=True).strip()) + def record(phase, message, **extra): + changed = state.get('phase') != phase or state.get('message') != message + state.update(phase=phase, message=message, updated_at=utc_now(), **extra) + atomic_json(state_path, state) + if changed: + print(f'[{utc_now()}] {phase}: {message}', flush=True) + for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + signal.signal(sig, request_stop) + stop_file = work / 'control/training-stop-request' + try: + while not STOPPED: + info = status() + ready = data_readiness(paths['data_dir'], plan) + prior = read_json(sft / 'control/handoff.json') + if not clean_exit(info) or prior.get('phase') != 'complete': + record('waiting_for_sft', f'SFT {info["statename"]}; final state is not yet complete', data_ready=ready) + time.sleep(plan['handoff']['poll_seconds']) + continue + with contextlib.ExitStack() as locks: + # Identical outer-to-inner order to the SFT guardian. Any + # surviving child owns a SH pipeline lock and blocks retries. + try: + sft_fd = locks.enter_context(exclusive_lock(sft / 'control/handoff.lock')) + launcher_fd = locks.enter_context(exclusive_lock(base / 'control/training-launcher.lock')) + pipeline = exclusive_lock(base / 'pipeline.lock') + pipeline.__enter__() + except BlockingIOError: + record('waiting_for_locks', 'A training worker or launcher still owns an exclusion lock') + time.sleep(plan['handoff']['poll_seconds']) + continue + try: + if not clean_exit(status()) or read_json(sft / 'control/handoff.json').get('phase') != 'complete': + continue + if source_identity(config, paths['sft_plan']) != identity: + raise ValueError('Armed source identity changed before SFT archival') + verify_sft_completion(sft / 'run', original_plan, plan['sft']['raw_archive_sha256']) + record('archiving_sft', 'Sealing the full final SFT checkpoint and original model tensors') + metadata, checksums = archive_sft(sft, paths['tokenizer'], paths['archive_dir'], + plan, original_plan, state['git_revision']) + record('copying_initialization', 'Creating/verifying independent policy and fixed-reference initialization', + archive_dir=str(paths['archive_dir']), archive_model_sha256=metadata['snapshot_sha256'], + archive_checkpoint_sha256=checksums['checkpoint.pt']) + weights = initialize_copy(paths['archive_dir'], work / 'initialization', metadata) + while not STOPPED and not data_readiness(paths['data_dir'], plan): + record('waiting_for_data', 'Waiting for completed pinned preference data') + time.sleep(plan['handoff']['poll_seconds']) + if STOPPED: + break + data_readiness(paths['data_dir'], plan, verify=True) + if source_identity(config, paths['sft_plan']) != identity: + raise ValueError('Armed source identity changed before DPO launch') + stop_file.unlink(missing_ok=True) + atomic_json(work / 'control/execution.json', dict(plan=plan, source_sha256=identity, + sft_model_sha256=metadata['snapshot_sha256'], sft_checkpoint_sha256=checksums['checkpoint.pt'], + git_revision=state['git_revision'])) + finally: + pipeline.__exit__(None, None, None) + if STOPPED: + break + environment = training_environment(work) + inherited = (own_fd, sft_fd, launcher_fd) + preflight_command = [sys.executable, '-B', str(PACKAGE / 'ops/dpo_preflight.py'), + '--work-dir', str(work / 'preflight'), '--base-work-dir', str(base), '--processes', '2'] + record('gpu_preflight', 'Checking BF16 DPO, frozen reference scoring, pair masks, and resume') + result = supervise(preflight_command, stop_file, environment, pass_fds=inherited) + if result: + raise RuntimeError(f'DPO GPU preflight failed with exit status {result}') + if STOPPED or stop_file.exists(): + record('paused', 'Paused during preflight; full DPO was not started') + return 0 + state['preflight'] = verify_preflight(work / 'preflight/report.json', 2) + command = [sys.executable, '-B', '-m', 'torch.distributed.run', '--standalone', + '--nnodes', '1', '--nproc-per-node', '2', '--log-dir', str(work / 'logs/torchrun'), + str(PACKAGE / 'dpo.py'), '--config', str(config), '--work-dir', str(work / 'run'), + '--data-dir', str(paths['data_dir']), '--weights', str(weights), + '--tokenizer', str(paths['tokenizer']), '--reference-cache-dir', str(work / 'reference-cache'), + '--stop-file', str(stop_file)] + record('dpo', 'Preparing frozen SFT reference scores, then starting/resuming DPO', command=command) + result = supervise(command, stop_file, environment, pass_fds=inherited) + if result: + raise RuntimeError(f'DPO exited with status {result}; independent checkpoints are retained') + if (work / 'run/completion.json').exists(): + completed = verify_dpo_completion(work / 'run', plan, metadata['snapshot_sha256']) + verify_sft_archive(paths['archive_dir'], plan, original_plan, + expected_checkpoint_hash=checksums['checkpoint.pt']) + if sha256(weights) != metadata['snapshot_sha256']: + raise ValueError('DPO changed its original SFT initialization') + record('complete', 'DPO finished and paused for evaluation; the archived SFT state is unchanged', + completion=completed) + else: + record('paused', 'DPO/reference preparation paused; resume state is retained') + return 0 + record('paused', 'DPO handoff paused by request; no further job will start') + return 0 + except BaseException as error: + record('failed', f'{type(error).__name__}: {error}') + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', required=True, type=Path) + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + return run_handoff(workspace_path(args.config), check=args.check) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ops/dpo_preflight.py b/ops/dpo_preflight.py new file mode 100644 index 0000000000000000000000000000000000000000..126d9eaf1e484a84cdca941400700f34b7cdef28 --- /dev/null +++ b/ops/dpo_preflight.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Deferred BF16 DPO gate for numerical continuation and exact bookkeeping. + +Fixture creation is CPU-only. The executable must run after SFT has stopped: +it scores a frozen reference, then checks interrupted and continuous DPO on a +five-pair fixture with a masked final distributed update. No archive is edited. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +from pathlib import Path +import sys + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +if str(PACKAGE) not in sys.path: + sys.path.insert(0, str(PACKAGE)) + +import torch + +from posttrain import atomic_json, digest, shared_pipeline_lock +from ops.launch_training import supervise +from ops.posttrain_data import encode_messages, renderer_sha256 +from ops.posttrain_preflight import compare_tree, inherited_fds + + +def make_fixture(directory, *, device='cpu', compile_core=False, base_work_dir=None): + """Create or verify a source-pinned small real model and preference data.""" + import sentencepiece as spm + import dpo + from warpstate import WarpStateConfig, WarpStateLM + + directory = Path(directory).resolve() + if not directory.is_relative_to(WORKSPACE) or directory == WORKSPACE: + raise ValueError('Preflight fixtures must stay inside the workspace') + if device not in ('cpu', 'cuda') or type(compile_core) is not bool: + raise ValueError('Invalid fixture execution mode') + base = Path(base_work_dir).resolve() if base_work_dir is not None else None + if base is not None and (not base.is_relative_to(WORKSPACE) or not base.is_dir()): + raise ValueError('Invalid base work directory') + if base is not None and (directory == base or directory.is_relative_to(base / 'run')): + raise ValueError('Preflight must not overwrite the base run') + tokenizer = PACKAGE / 'tokenizer.model' + args = argparse.Namespace(config=directory / 'plan.json', data_dir=directory / 'data', + weights=directory / 'base/model.pt', tokenizer=tokenizer, work_dir=directory / 'run', + reference_cache_dir=directory / 'reference', reference_only=False, + max_steps=None, stop_file=None, device=device) + identity = dict(device=device, compile_core=compile_core, + base_work_dir=str(base) if base else None, tokenizer_sha256=digest(tokenizer), + renderer_sha256=renderer_sha256(), generator_sha256=digest(__file__), + runner_sources=dpo.source_identity()) + marker = directory / 'fixture.json' + if marker.exists(): + old = json.loads(marker.read_text()) + if old.get('format') != 'warpstate-dpo-fixture-v1' or old.get('identity') != identity: + raise ValueError('Existing preflight fixture settings/source differ; use a new fixture directory') + for filename, checksum in old['files'].items(): + path = (directory / filename).resolve() + if not path.is_relative_to(directory) or digest(path) != checksum: + raise ValueError(f'Existing preflight input changed: {filename}') + return args + if directory.exists() and any(directory.iterdir()): + raise ValueError('Refusing to create fixtures in a nonempty unrecognized directory') + directory.mkdir(parents=True, exist_ok=True) + args.weights.parent.mkdir() + args.data_dir.mkdir() + sp = spm.SentencePieceProcessor(model_file=str(tokenizer)) + cfg = WarpStateConfig(vocab_size=sp.vocab_size(), d_model=32, n_heads=4, + n_cores=1, logical_depth=1, ffn_hidden=64, chunk_size=16, + parallel_memory=True, gradient_checkpointing=False, dropout=0.0) + with torch.random.fork_rng(devices=[]): + torch.manual_seed(123) + model = WarpStateLM(cfg) + torch.save(dict(format='warpstate-chat-v1', inference_only=True, synthetic_fixture=True, + stage='sft', config=vars(cfg), model=model.state_dict(), step=10312, + tokens_seen=212651345, tokenizer_sha256=identity['tokenizer_sha256'], + origin=dict(base_step=76294, base_tokens_seen=10_000_007_168)), args.weights) + sequence_length = 128 + outputs = {} + for filename, count in [('dpo_train.jsonl', 5), ('dpo_val.jsonl', 2)]: + rows = [] + for index in range(count): + prompt = [{'role': 'system', 'content': 'Be concise.'}] + if index == 0: + # Earlier assistant turns are conditioning, never DPO targets. + prompt += [{'role': 'user', 'content': 'Say hello.'}, + {'role': 'assistant', 'content': 'Hello.'}] + prompt += [{'role': 'user', 'content': f'Question {index}: two plus two?'}] + row = dict(prompt=prompt, + chosen=[{'role': 'assistant', 'content': 'four' if index % 2 else 'The answer is four.'}], + rejected=[{'role': 'assistant', 'content': 'five, or perhaps six.' if index % 2 else 'Three.'}], + tokens={}, supervised_tokens={}) + for branch in ('chosen', 'rejected'): + encoded = encode_messages(prompt + row[branch], sp, max_length=sequence_length) + response = encode_messages(row[branch], sp, max_length=sequence_length) + row['tokens'][branch] = len(encoded['input_ids']) + row['supervised_tokens'][branch] = sum(v != -100 for v in response['labels'][1:]) + rows.append(row) + path = args.data_dir / filename + path.write_text(''.join(json.dumps(row) + '\n' for row in rows)) + outputs[filename] = dict(sha256=digest(path), rows=count, + tokens=sum(sum(row['tokens'].values()) for row in rows), + supervised_tokens=sum(sum(row['supervised_tokens'].values()) for row in rows)) + atomic_json(args.data_dir / 'manifest.json', dict(format='warpstate-posttrain-data-v1', + complete=True, outputs=outputs, tokenizer_sha256=identity['tokenizer_sha256'], + renderer_sha256=identity['renderer_sha256'], max_length=sequence_length)) + plan = dict(format='warpstate-dpo-plan-v1', stage='dpo', base={'step': 10312}, + after_completion='pause_for_evaluation', + training=dict(sequence_length=sequence_length, micro_batch=1, grad_accum=2, + lr=1e-3, min_lr=1e-4, warmup_ratio=0.0, epochs=1, eval_every=2, + save_every=2, eval_batches=2, compile_core=compile_core, workers=1, + beta=5.0, objective='length-normalized-dpo', head_chunk_size=32)) + if base is not None: + plan['handoff'] = {'base_work_dir': str(base.relative_to(WORKSPACE))} + atomic_json(args.config, plan) + files = [args.weights, args.config, args.data_dir / 'manifest.json', + args.data_dir / 'dpo_train.jsonl', args.data_dir / 'dpo_val.jsonl'] + atomic_json(marker, dict(format='warpstate-dpo-fixture-v1', identity=identity, + files={str(path.relative_to(directory)): digest(path) for path in files})) + return args + + +def training_command(args, processes, work_dir, stop_file, max_steps=None, *, reference_only=False): + if type(processes) is not int or processes != 2: + raise ValueError('The preflight requires exactly two ranks to exercise its masked tail') + if reference_only and max_steps is not None: + raise ValueError('Reference-only scoring cannot specify training steps') + command = [sys.executable, '-B', '-m', 'torch.distributed.run', '--standalone', + '--nnodes', '1', '--nproc-per-node', str(processes), str(PACKAGE / 'dpo.py'), + '--config', str(args.config), '--data-dir', str(args.data_dir), + '--weights', str(args.weights), '--tokenizer', str(args.tokenizer), + '--reference-cache-dir', str(args.reference_cache_dir), '--work-dir', str(work_dir), + '--stop-file', str(stop_file), '--device', args.device] + if reference_only: + command.append('--reference-only') + if max_steps is not None: + command += ['--max-steps', str(max_steps)] + return command + + +def reference_artifacts(directory): + """Validate all sealed reference arrays and return immutable content hashes.""" + import numpy as np + + directory = Path(directory).resolve() + manifest_path = directory / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + if manifest.get('format') != 'warpstate-dpo-reference-v1' or manifest.get('complete') is not True: + raise ValueError('Frozen reference cache is incomplete') + files = manifest.get('files', {}) + if set(files) != {'train.npy', 'val.npy'}: + raise ValueError('Frozen reference cache lacks the complete train/validation arrays') + result = {'manifest.json': digest(manifest_path)} + for name, info in files.items(): + path = directory / name + if not path.is_file() or path.is_symlink() or digest(path) != info.get('sha256'): + raise ValueError(f'Frozen reference cache hash differs: {name}') + array = np.load(path, mmap_mode='r', allow_pickle=False) + expected_rows = 5 if name == 'train.npy' else 2 + if array.shape != (expected_rows, 2) or info.get('rows') != expected_rows: + raise ValueError(f'Frozen reference cache shape differs: {name}') + if array.dtype != np.float64 or not np.isfinite(array).all(): + raise ValueError(f'Frozen reference cache contains invalid scores: {name}') + if not (array < 0).all(): + raise ValueError(f'Frozen reference must contain summed log probabilities: {name}') + result[name] = info['sha256'] + return result + + +def verify_fixture_run(work, *, expected_step, completed): + work = Path(work) + checkpoint = torch.load(work / 'latest.pt', map_location='cpu', weights_only=False, mmap=True) + if checkpoint.get('format') != 'warpstate-dpo-v1' or checkpoint.get('step') != expected_step: + raise ValueError('Preflight checkpoint format/step differs') + if checkpoint.get('completed') is not completed or checkpoint['signature'].get('world_size') != 2: + raise ValueError('Preflight completion/distributed state differs') + if len(checkpoint['rng_by_rank']) != 2: + raise ValueError('Preflight lacks both rank RNG states') + for name, value in checkpoint['model'].items(): + if value.dtype != torch.float32 or not torch.isfinite(value).all(): + raise ValueError(f'Invalid FP32 model weight: {name}') + optimizer = checkpoint['optimizer']['state'] + if not optimizer: + raise ValueError('Preflight lacks Adam optimizer state') + for value in optimizer.values(): + if float(value['step']) != expected_step: + raise ValueError('Optimizer step was not restored') + for name in ('exp_avg', 'exp_avg_sq'): + if value[name].dtype != torch.float32 or not torch.isfinite(value[name]).all(): + raise ValueError('Invalid FP32 Adam state') + metrics = [json.loads(line) for line in (work / 'metrics.jsonl').read_text().splitlines()] + updates = [row for row in metrics if row.get('event') == 'train'] + if [row['step'] for row in updates] != list(range(1, expected_step + 1)): + raise ValueError('Preflight skipped or repeated an optimizer update') + for row in updates: + if not all(math.isfinite(row[key]) for key in ('loss', 'grad_norm', 'lr')): + raise ValueError('Non-finite preflight training metric') + initial = [row for row in metrics if row.get('event') == 'validation' and row.get('step') == 0] + if len(initial) != 1 or not math.isclose(initial[0]['val_loss'], math.log(2), rel_tol=1e-5, abs_tol=1e-7): + raise ValueError('Initial policy/reference DPO loss must equal log(2)') + if completed: + completion = json.loads((work / 'completion.json').read_text()) + evaluation = json.loads((work / 'evaluation.json').read_text()) + if not completion.get('completed') or not completion.get('paused_for_evaluation'): + raise ValueError('Preflight did not pause for evaluation') + if checkpoint['sample_cursor'] != 5 or updates[-1]['update_samples'] != 1: + raise ValueError('Preflight did not exercise a nondivisible masked pair tail') + if sum(row['update_samples'] for row in updates) != 5: + raise ValueError('Preflight did not train every preference pair exactly once') + if evaluation.get('step') != expected_step or evaluation.get('full_holdout') is not True: + raise ValueError('Preflight final full evaluation is missing') + if evaluation.get('evaluated_samples') != 2: + raise ValueError('Preflight did not evaluate all held-out preference pairs') + for key in ('val_loss', 'reward_accuracy', 'reward_margin', 'chosen_nll'): + if not math.isfinite(evaluation[key]): + raise ValueError(f'Invalid preflight evaluation metric: {key}') + if not 0 <= evaluation['reward_accuracy'] <= 1: + raise ValueError('Invalid preference reward accuracy') + return checkpoint + + +def verify_resume(resumed, continuous): + """Require close FP32 training state and exact counters and per-rank RNG.""" + comparison = {key: compare_tree(resumed[key], continuous[key], key, tolerant=True) + for key in ('model', 'optimizer')} + for key in ('tokens_seen', 'response_tokens_seen', 'sample_cursor', 'step'): + compare_tree(resumed[key], continuous[key], key) + for rank in range(2): + for key in ('torch', 'cuda', 'python'): + compare_tree(resumed['rng_by_rank'][rank][key], continuous['rng_by_rank'][rank][key], f'rng.{rank}.{key}') + a, b = resumed['rng_by_rank'][rank]['numpy'], continuous['rng_by_rank'][rank]['numpy'] + if a[0] != b[0] or not (a[1] == b[1]).all() or a[2:] != b[2:]: + raise ValueError('NumPy RNG state differs after resume') + if not math.isclose(resumed['val_loss'], continuous['val_loss'], rel_tol=1e-5, abs_tol=1e-7): + raise ValueError('Validation loss differs after resume') + return comparison + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--work-dir', type=Path, required=True) + parser.add_argument('--base-work-dir', type=Path, required=True) + parser.add_argument('--processes', type=int, default=2) + parser.add_argument('--prepare-only', action='store_true', help='Create CPU fixtures without launching GPU work') + args = parser.parse_args() + if args.processes != 2: + parser.error('--processes must be 2 for the masked pair-tail fixture') + work = args.work_dir.resolve() + base = args.base_work_dir.resolve() + if not base.is_relative_to(WORKSPACE) or not base.is_dir(): + parser.error('--base-work-dir must be an existing workspace directory') + stop_file = Path(os.environ.get('WARPSTATE_STOP_FILE', work / 'control/training-stop-request')).resolve() + if not stop_file.is_relative_to(WORKSPACE): + parser.error('Stop file must stay inside the workspace') + # The guardian releases its exclusive pipeline lock before spawning us. + # This shared lock and inherited guardian FDs exclude base/SFT launches + # throughout reference scoring and the gaps between torchrun processes. + with shared_pipeline_lock(base / 'pipeline.lock'): + fixture = make_fixture(work, device='cuda', compile_core=True, base_work_dir=base) + if args.prepare_only: + print(json.dumps({'prepared': True, 'config': str(fixture.config)}), flush=True) + return 0 + stop_file.parent.mkdir(parents=True, exist_ok=True) + if stop_file.exists(): + return 0 + report_path = work / 'report.json' + report_path.unlink(missing_ok=True) + raw_hash = digest(fixture.weights) + if not (fixture.reference_cache_dir / 'manifest.json').exists(): + result = supervise(training_command(fixture, args.processes, work / 'reference-scoring', + stop_file, reference_only=True), stop_file, os.environ.copy(), pass_fds=inherited_fds()) + if stop_file.exists(): + return 0 + if result: + raise RuntimeError(f'Preflight reference scoring failed with status {result}') + reference_hashes = reference_artifacts(fixture.reference_cache_dir) + phases = [('split', 1), ('split', None), ('continuous', None)] + for phase, limit in phases: + if stop_file.exists(): + return 0 + target = work / phase + if (target / 'completion.json').exists(): + continue + if limit == 1 and (target / 'latest.pt').exists(): + continue + result = supervise(training_command(fixture, args.processes, target, stop_file, limit), + stop_file, os.environ.copy(), pass_fds=inherited_fds()) + if stop_file.exists(): + return 0 + if result: + raise RuntimeError(f'DPO preflight {phase} failed with status {result}') + verify_fixture_run(target, expected_step=1 if limit else 2, completed=limit is None) + resumed = verify_fixture_run(work / 'split', expected_step=2, completed=True) + continuous = verify_fixture_run(work / 'continuous', expected_step=2, completed=True) + comparison = verify_resume(resumed, continuous) + if digest(fixture.weights) != raw_hash: + raise ValueError('Raw SFT fixture weights changed') + if reference_artifacts(fixture.reference_cache_dir) != reference_hashes: + raise ValueError('Frozen reference cache changed during DPO') + original = torch.load(fixture.weights, map_location='cpu', weights_only=False, mmap=True) + if not any(not torch.equal(value, resumed['model'][name]) for name, value in original['model'].items()): + raise ValueError('DPO preflight did not update policy parameters') + evaluation = json.loads((work / 'split/evaluation.json').read_text()) + atomic_json(report_path, dict(format='warpstate-dpo-preflight-v1', accepted=True, + processes=2, bf16_compute=True, fp32_parameters_and_adam=True, + exact_resume_bookkeeping=True, numerical_rtol=1e-5, numerical_atol=1e-7, + numerical_comparison=comparison, masked_pair_tail=True, total_pairs=5, + optimizer_steps=2, fixture_sha256=raw_hash, + source_sha256=resumed['signature']['sources'], generator_sha256=digest(__file__), + config_sha256=digest(fixture.config), reference_artifacts=reference_hashes, + reference_manifest_sha256=reference_hashes['manifest.json'], + frozen_reference_unchanged=True, final_validation={key: evaluation[key] + for key in ('val_loss', 'reward_accuracy', 'reward_margin', 'chosen_nll')})) + print(f'DPO preflight accepted: {report_path}', flush=True) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/ops/evaluate_sft_snapshot.py b/ops/evaluate_sft_snapshot.py new file mode 100644 index 0000000000000000000000000000000000000000..3f93a2549b6ca6017296b21257449e24844557cc --- /dev/null +++ b/ops/evaluate_sft_snapshot.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Evaluate a sealed posttraining snapshot on the complete pinned SFT holdout.""" + +import argparse +from datetime import datetime, timezone +import json +import math +from pathlib import Path +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import torch + +from chat import load_model, sha256 +from ops.loss_experiment import apply_loss_execution +from ops.posttrain_data import encode_messages, renderer_sha256 +from ops.residual_experiment import apply_bf16_residuals +from posttrain import evaluate + + +def load_holdout(directory, tokenizer, tokenizer_hash): + manifest_path = directory / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + if (manifest.get('format') != 'warpstate-posttrain-data-v1' + or manifest.get('complete') is not True + or manifest.get('tokenizer_sha256') != tokenizer_hash + or manifest.get('renderer_sha256') != renderer_sha256()): + raise ValueError('Completed data, tokenizer and renderer identities must match') + info = manifest['outputs']['sft_val.jsonl'] + source = directory / 'sft_val.jsonl' + if sha256(source) != info['sha256']: + raise ValueError('SFT holdout checksum mismatch') + records, tokens, targets = [], 0, 0 + with source.open() as stream: + for line in stream: + row = json.loads(line) + if row.get('tools'): + raise ValueError('Tools must be serialized into conversation messages') + record = encode_messages(row['messages'], tokenizer, max_length=manifest['max_length']) + count = sum(value != -100 for value in record['labels'][1:]) + if row['tokens'] != len(record['input_ids']) or row['supervised_tokens'] != count: + raise ValueError('SFT holdout token counts differ from exact encoding') + tokens += len(record['input_ids']) + targets += count + records.append(record) + if (len(records), tokens, targets) != (info['rows'], info['tokens'], info['supervised_tokens']): + raise ValueError('SFT holdout totals differ from manifest') + return records, manifest, sha256(manifest_path) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--snapshot', type=Path, required=True) + parser.add_argument('--data-dir', type=Path, default=ROOT / 'work/posttrain/data') + parser.add_argument('--device', default='cuda:0') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + snapshot, data, output = (path.resolve() for path in (args.snapshot, args.data_dir, args.output)) + if any(not path.is_relative_to(ROOT.parent) for path in (snapshot, data, output)): + parser.error('All paths must stay inside /workspace/WarpState') + if output.exists() or output.is_relative_to(snapshot.parent) or output.is_relative_to(data): + parser.error('Select a new report outside the snapshot and dataset directories') + device = torch.device(args.device) + if device.type != 'cuda' or not torch.cuda.is_available(): + parser.error('This full-size evaluation requires a CUDA GPU') + torch.cuda.set_device(device) + if not torch.cuda.is_bf16_supported(): + parser.error('BF16 support required') + torch.set_num_threads(8) + torch.use_deterministic_algorithms(True) + started = time.perf_counter() + tokenizer = snapshot.with_name('tokenizer.model') + model, sp, metadata = load_model(snapshot, tokenizer) + records, manifest, data_hash = load_holdout(data, sp, metadata['tokenizer_sha256']) + print(f'Validated {len(records):,} SFT holdout conversations; loading {device}', flush=True) + model.to(device) + apply_loss_execution(model, 'full') + apply_bf16_residuals(model) + settings = dict(sequence_length=manifest['max_length'], micro_batch=4, eval_batches=32) + pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id() + torch.cuda.reset_peak_memory_stats(device) + evaluation_started = time.perf_counter() + loss, targets = evaluate(model, records, settings, device, pad_id, full=True) + torch.cuda.synchronize(device) + report = dict(format='warpstate-sft-snapshot-evaluation-v1', + timestamp_utc=datetime.now(timezone.utc).isoformat(), snapshot=str(snapshot), + snapshot_sha256=sha256(snapshot), snapshot_metadata=metadata, + data_sha256=data_hash, holdout_sha256=manifest['outputs']['sft_val.jsonl']['sha256'], + renderer_sha256=renderer_sha256(), full_holdout=True, evaluated_samples=len(records), + assistant_target_tokens=targets, val_loss=loss, val_perplexity=math.exp(loss), + objective='assistant-token-mean', settings=settings, + execution=dict(device=str(device), device_name=torch.cuda.get_device_name(device), + torch_version=torch.__version__, compute_dtype='bfloat16', parameter_dtype='float32', + bf16_residuals=True, compiled=False, deterministic_algorithms=True), + source_sha256={str(path.relative_to(ROOT)): sha256(path) for path in + (Path(__file__), ROOT / 'chat.py', ROOT / 'posttrain.py', + ROOT / 'ops/loss_experiment.py', ROOT / 'ops/residual_experiment.py', + ROOT / 'warpstate/model.py')}, + evaluation_seconds=time.perf_counter() - evaluation_started, + total_seconds=time.perf_counter() - started, + peak_allocated_bytes=torch.cuda.max_memory_allocated(device)) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open('x') as stream: + json.dump(report, stream, indent=2, allow_nan=False) + stream.write('\n') + print(json.dumps({key: report[key] for key in ('evaluated_samples', 'assistant_target_tokens', + 'val_loss', 'val_perplexity', 'evaluation_seconds')}) + f'\nReport: {output}', flush=True) + + +if __name__ == '__main__': + main() diff --git a/ops/gradient_execution.py b/ops/gradient_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..3914041f49ff0f5ffca27644ec950022c4987644 --- /dev/null +++ b/ops/gradient_execution.py @@ -0,0 +1,218 @@ +"""Opt-in capture of weight-gradient conversion and FP32 accumulation. + +Ordinary AOTAutograd compilation ends before leaf ``AccumulateGrad`` nodes. +Compiled Autograd exposes those nodes, casts, and DDP hooks to Inductor so a +BF16 gradient can be converted while adding directly into its FP32 destination. +Master parameters, optimizer, and state-dict keys are unchanged. No custom hook +writes a gradient or bypasses autograd. The isolated linear cast/accumulation +path preserves BF16 GEMM output rounding exactly. Whole-backward compilation +can also fuse a tied-head BF16 matrix product and gradient add into BF16 addmm, +or fuse other BF16 operations across existing region boundaries. These alter +intermediate rounding; complete-model gradients use explicit BF16 tolerances. + +This uses the experimental Compiled Autograd integration in the installed +PyTorch. Select it before compiling the forward or constructing DDP. Benchmark +it on the actual model; larger backward graphs can increase compilation costs. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import asdict, dataclass +import math +from numbers import Real + +import torch + + +GRADIENT_MODES = ("eager", "compiled-autograd", "compiled-autograd-overlap") +MAX_OVERLAP_MEMORY_BUDGET = 0.25 + + +@dataclass(frozen=True) +class OverlapSettings: + """Explicit scheduler choices; precision and collective launch order stay fixed. + + ``peak_memory_budget`` is a fractional allowance in the scheduler's memory + estimate, not a reservation or a substitute for the measured CUDA cap. + Runtime estimates opt into GEMM benchmarking during compilation, and align + estimates across ranks for distributed execution. These may lengthen cold + compilation; they do not benchmark or communicate during configuration. + + ``producer_schedule`` separately experiments with moving a collective's + packing/casting producers earlier. It retains collective order, permits no + estimated peak-memory increase, and initially excludes other overlap knobs. + """ + + swap_with_collectives: bool = False + use_runtime_estimations: bool = False + peak_memory_budget: float = 0.0 + producer_schedule: bool = False + + def __post_init__(self): + for name in ("swap_with_collectives", "use_runtime_estimations", "producer_schedule"): + if not isinstance(getattr(self, name), bool): + raise ValueError(f"{name} must be a boolean") + value = self.peak_memory_budget + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError("peak_memory_budget must be a finite fraction between 0 and 0.25") + try: + value = float(value) + except OverflowError as error: + raise ValueError("peak_memory_budget must be a finite fraction between 0 and 0.25") from error + if not math.isfinite(value) or not 0 <= value <= MAX_OVERLAP_MEMORY_BUDGET: + raise ValueError("peak_memory_budget must be a finite fraction between 0 and 0.25") + object.__setattr__(self, "peak_memory_budget", value) + if self.producer_schedule and (self.swap_with_collectives or self.use_runtime_estimations or value): + raise ValueError("producer_schedule initially requires default wait swapping, " + "runtime estimates, and zero peak-memory budget") + + @property + def metadata(self): + """JSON-compatible immutable choices for a run's execution signature.""" + return asdict(self) + + +def _configure_communication_overlap(options: OverlapSettings, *, distributed: bool) -> None: + """Delay wait/copy groups while preserving collective order and estimated peak. + + The default DDP FX pass only sinks waits until their first consumer, which + is immediately the gradient copy-back. Inductor's iterative scheduler can + move that dependent group together past unrelated gradient computation. + Defaults retain collective ordering, stop before the next collective, and + allow no increase in estimated peak memory. Optional wait-group movement + past later launches still prohibits changing their relative launch order. + """ + from torch._inductor import comms, config, config_comms + + settings = { + "reorder_for_compute_comm_overlap": True, + "reorder_for_compute_comm_overlap_passes": ["sink_waits_iterative"], + } + communication_settings = { + "sink_iterative_peak_memory_budget": options.peak_memory_budget, + "sink_waits_iterative_unsafe_collectives_reorder": False, + "sink_waits_iterative_swap_with_collectives": options.swap_with_collectives, + "sink_iterative_use_runtime_estimations": options.use_runtime_estimations, + } + if options.producer_schedule: + # Installed config.py recommends producer reordering followed by wait + # sinking (then optionally another reorder). Its comms.py docstring + # mentions a stale sink_comms_and_waits prerequisite that is not a + # callable in this version. The ordinary peak-memory scheduler runs + # before these passes. Start with just one guarded producer pass and + # retain the existing final sink. Without runtime estimates this moves + # pack/cast producer groups but stops when a GEMM producer is reached. + settings["reorder_for_compute_comm_overlap_passes"] = [ + "reorder_communication_preserving_peak_memory", "sink_waits_iterative"] + communication_settings.update( + reorder_iterative_unsafe_collectives_reorder=False, + reorder_iterative_peak_memory_budget=0.0, + reorder_iterative_group_with_collectives=False, + reorder_iterative_use_runtime_estimations=False, + ) + if options.use_runtime_estimations: + settings["runtime_estimations_mms_benchmark"] = True + communication_settings["runtime_estimations_align_across_all_distributed_ranks"] = distributed + if distributed and not callable(getattr( + comms, "align_runtime_estimations_across_all_distributed_ranks", None)): + raise RuntimeError("Installed PyTorch lacks distributed runtime-estimate alignment") + for module, values in ((config, settings), (config_comms, communication_settings)): + missing = [key for key in values if not hasattr(module, key)] + if missing: + raise RuntimeError("Installed PyTorch lacks overlap settings: " + ", ".join(missing)) + for name in settings["reorder_for_compute_comm_overlap_passes"]: + if not callable(getattr(comms, name, None)): + raise RuntimeError(f"Installed PyTorch lacks the {name} scheduler pass") + if config.deterministic: + raise ValueError("Inductor deterministic mode disables communication reordering; " + "compiled-autograd-overlap would have no effect") + for module, values in ((config, settings), (config_comms, communication_settings)): + for name, value in values.items(): + setattr(module, name, value) + + +def scale_loss(loss: torch.Tensor, scale: float, mode: str = "eager") -> torch.Tensor: + """Apply the existing microbatch weight without CPU-scalar capture on CUDA. + + PyTorch 2.14 Compiled Autograd can emit invalid CUDA wrapper code for an + eager multiplication's saved CPU scalar. Materializing the same coefficient + in the loss's dtype/device avoids that path without changing the arithmetic. + For repeated fixed weights a caller can equivalently prepare the scalar + tensor once per update and multiply directly. This helper is convenient for + the variable valid-token fraction in the final training update. + """ + if mode not in GRADIENT_MODES: + raise ValueError(f"Unknown gradient execution mode: {mode}") + if mode != "eager" and loss.device.type == "cuda": + return loss * loss.new_tensor(scale) + return loss * scale + + +def _backward(loss: torch.Tensor) -> None: + loss.backward() + + +def make_backward( + mode: str = "eager", *, distributed: bool = False, backend: str | Callable = "inductor", + overlap_settings: OverlapSettings | None = None, +) -> Callable[[torch.Tensor], None]: + """Return the backward operation for a training process. + + Call once, before any ``torch.compile`` or DDP construction. The compiled + mode enables process-wide Dynamo settings; each experiment must use its own + process. Distributed execution uses PyTorch's Python reducer, whose ordinary + post-accumulation hooks respect ``DDP.no_sync()`` and retain FP32 all-reduce. + Its communication fusion uses the DDP bucket capacity. C++ reducer options + such as ``batched_grad_copy`` are separate experiments, not combined modes. + + Leave forward, ``no_sync`` contexts, ``zero_grad``, clipping, and optimizer + stepping with the caller. Use ``scale_loss`` for the existing loss weight, + or multiply by a scalar tensor already on the CUDA device. The callable changes only + how the existing autograd graph is executed. Existing checkpoint loading and + saving require no model or optimizer conversion. A change in this setting + should nevertheless be recorded as an execution reconfiguration. + + ``fullgraph`` applies to the captured autograd graph. PyTorch 2.14 still + requires the outer wrapper to permit the ``Tensor.backward`` graph break. + The first microbatch (no gradients) and later accumulating microbatches may + create different graphs; the final DDP synchronization creates another. + Compilation failures propagate rather than silently changing algorithms. + + ``compiled-autograd-overlap`` additionally enables a conservative scheduler + pass that sinks collective waits and gradient copy-back together. FP32 + collective order is preserved; the default estimated peak-memory budget is + zero. ``overlap_settings`` is accepted only in this mode. Runtime estimates + also benchmark GEMMs during compilation and, when distributed, align timing + estimates through the existing process group. Both ranks must compile the + same collective-bearing graphs; normal DDP construction and no-sync ordering + remain the caller's responsibility. + The separate ``producer_schedule`` experiment first raises pack/cast + producer groups and their collective, then keeps the conservative final + wait sink. It cannot reorder collective launches or increase the scheduler's + estimated peak; actual CUDA memory and throughput still require measurement. + Select and record this separately because scheduling benefits require an + actual distributed benchmark and compatible installed PyTorch internals. + """ + if mode not in GRADIENT_MODES: + raise ValueError(f"Unknown gradient execution mode: {mode}") + if overlap_settings is not None: + if not isinstance(overlap_settings, OverlapSettings): + raise TypeError("overlap_settings must be an OverlapSettings instance") + if mode != "compiled-autograd-overlap": + raise ValueError("overlap_settings requires compiled-autograd-overlap") + if mode == "eager": + return _backward + + from torch._dynamo import config + + if mode == "compiled-autograd-overlap": + _configure_communication_overlap(overlap_settings or OverlapSettings(), distributed=distributed) + config.compiled_autograd = True + config.compiled_autograd_kwargs_override = { + **config.compiled_autograd_kwargs_override, + "fullgraph": True, + } + if distributed: + config.optimize_ddp = "python_reducer" + return torch.compile(_backward, backend=backend, fullgraph=False, dynamic=False) diff --git a/ops/launch_training.py b/ops/launch_training.py new file mode 100644 index 0000000000000000000000000000000000000000..61af76290d187777b8c319f7777e0da63de63d9f --- /dev/null +++ b/ops/launch_training.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Supervise torchrun while allowing workers time to save on a stop request.""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + + +WORKSPACE = Path('/workspace/WarpState').resolve() +PACKAGE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PACKAGE)) +from run_40b import process_lock + + +def workspace_path(value): + path = Path(value).expanduser().resolve() + if not path.is_relative_to(WORKSPACE): + raise ValueError(f'Created paths must stay inside {WORKSPACE}: {path}') + return path + + +def process_info(pid): + """Return parent and creation time; creation time guards against PID reuse.""" + try: + fields = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() + return int(fields[1]), int(fields[19]) + except (OSError, ValueError, IndexError): + return None + + +def remember_descendants(parent, known, parent_created): + processes = {} + for path in Path('/proc').iterdir(): + if path.name.isdigit(): + info = process_info(int(path.name)) + if info is not None: + processes[int(path.name)] = info + ancestors = {parent} if processes.get(parent, (None, None))[1] == parent_created else set() + ancestors.update(pid for pid, created in known.items() + if processes.get(pid, (None, None))[1] == created) + while True: + added = {pid for pid, (ppid, _) in processes.items() if ppid in ancestors} - ancestors + if not added: + break + ancestors.update(added) + for pid in ancestors - {parent}: + if pid in processes: + known[pid] = processes[pid][1] + + +def kill_descendants(known): + """Only signal processes whose recorded identities still match.""" + for pid, created in known.items(): + current = process_info(pid) + if current is not None and current[1] == created: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, stop_path, environment, *, grace_seconds=820, terminate_seconds=45, + pass_fds=()): + """A stop file reaches workers even though torchrun gives each a new session.""" + stop_path = workspace_path(stop_path) + stopped_at = None + handlers = {} + child = None + child_created = None + descendants = {} + + def request_stop(signum, frame): + nonlocal stopped_at + if stopped_at is None: + stopped_at = time.monotonic() + descriptor = os.open(stop_path, os.O_CREAT | os.O_WRONLY, 0o600) + os.close(descriptor) + + for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + handlers[sig] = signal.signal(sig, request_stop) + try: + if stopped_at is not None or stop_path.exists(): + return 0 + # Handoff jobs pass their exclusion locks to torchrun, which retains them + # even if the supervising Python process is unexpectedly killed. + child = subprocess.Popen(command, env=environment, start_new_session=True, + pass_fds=pass_fds) + identity = process_info(child.pid) + child_created = identity[1] if identity is not None else None + print(f'Training launcher started PID {child.pid}; stop requests use {stop_path}', flush=True) + last_scan = 0.0 + reported_stop = False + termination_sent = None + while child.poll() is None: + now = time.monotonic() + if now - last_scan >= 2: + remember_descendants(child.pid, descendants, child_created) + last_scan = now + if stopped_at is not None: + if not reported_stop: + print('Stop requested; waiting for the current update and checkpoint.', flush=True) + reported_stop = True + if termination_sent is None and now - stopped_at >= grace_seconds: + remember_descendants(child.pid, descendants, child_created) + print('Graceful-stop deadline exceeded; asking torchrun to terminate its workers.', flush=True) + try: + child.send_signal(signal.SIGTERM) + except ProcessLookupError: + pass + termination_sent = now + elif termination_sent is not None and now - termination_sent >= terminate_seconds: + remember_descendants(child.pid, descendants, child_created) + print('Termination deadline exceeded; killing remaining training processes.', flush=True) + kill_descendants(descendants) + child.kill() + child.wait(timeout=5) + break + try: + child.wait(timeout=0.2) + except subprocess.TimeoutExpired: + pass + return child.returncode + finally: + if child is not None: + remember_descendants(child.pid, descendants, child_created) + if child.poll() is None: + kill_descendants(descendants) + child.kill() + child.wait(timeout=5) + kill_descendants(descendants) + for sig, previous in handlers.items(): + signal.signal(sig, previous) + + +def training_environment(work): + cache = workspace_path(WORKSPACE / '.cache') + hf = workspace_path(work / 'hf-cache') + paths = { + 'TMPDIR': WORKSPACE / 'tmp', + 'XDG_CACHE_HOME': cache, 'UV_CACHE_DIR': cache / 'uv', + 'PIP_CACHE_DIR': cache / 'pip', 'TORCH_HOME': cache / 'torch', + 'CUDA_CACHE_PATH': cache / 'cuda', 'TRITON_CACHE_DIR': cache / 'triton', + 'TORCHINDUCTOR_CACHE_DIR': cache / 'torchinductor', + 'TORCH_EXTENSIONS_DIR': cache / 'torch-extensions', + 'HF_HOME': hf, 'HF_DATASETS_CACHE': hf / 'datasets', 'HF_XET_CACHE': hf / 'xet', + } + environment = os.environ.copy() + for key, value in paths.items(): + path = workspace_path(value) + path.mkdir(parents=True, exist_ok=True) + environment[key] = str(path) + environment['PYTHONDONTWRITEBYTECODE'] = '1' + environment['WARPSTATE_STOP_FILE'] = str(workspace_path(work / 'control/training-stop-request')) + return environment + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--processes', type=int, default=2) + parser.add_argument('--work-dir', type=Path, default=PACKAGE / 'work') + parser.add_argument('runner_args', nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.processes < 1: + parser.error('--processes must be positive') + extra = args.runner_args[1:] if args.runner_args[:1] == ['--'] else args.runner_args + if any(value == '--work-dir' or value.startswith('--work-dir=') for value in extra): + parser.error('Pass --work-dir to the launcher, before the -- separator') + try: + work = workspace_path(args.work_dir) + control = workspace_path(work / 'control') + logs = workspace_path(work / 'logs/torchrun-training') + stop_path = workspace_path(control / 'training-stop-request') + lock_path = workspace_path(control / 'training-launcher.lock') + except ValueError as error: + parser.error(str(error)) + control.mkdir(parents=True, exist_ok=True) + with process_lock(lock_path): + stop_path.unlink(missing_ok=True) + logs.mkdir(parents=True, exist_ok=True) + environment = training_environment(work) + command = [sys.executable] + if args.processes > 1: + command += ['-m', 'torch.distributed.run', '--standalone', '--nnodes', '1', + '--nproc-per-node', str(args.processes), '--log-dir', str(logs)] + command += [str(PACKAGE / 'run_40b.py'), '--work-dir', str(work), *extra] + result = supervise(command, stop_path, environment) + return 128 - result if result < 0 else result + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ops/loss_curve_forecast.py b/ops/loss_curve_forecast.py new file mode 100644 index 0000000000000000000000000000000000000000..0867e8e4973311fb51f4de089007d1b395bf8140 --- /dev/null +++ b/ops/loss_curve_forecast.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Illustrative loss-curve sensitivity; CPU only, no training/checkpoint access. + +Use a frozen validation JSONL snapshot. NumPy is required; --plot also requires +Matplotlib. Fits describe the observed schedule and do not establish a loss +floor, confidence interval, or ETA for a different training schedule. +""" + +import argparse +import hashlib +import json +import math +import os +import platform +from pathlib import Path + +import numpy as np + + +BIN_WIDTH = 0.25 # Billions of tokens. +WINDOW_STARTS = (1, 2, 3, 4, 5) +FIXED_FLOORS = (1.5, 1.8, 1.9, 2.0, 2.1) +BUDGETS = {"268.435456B": 268.435456, "1073.741824B": 1073.741824} +MODELS = ("log", "power_free") + tuple(f"power_fixed:{f}" for f in FIXED_FLOORS) + + +def power_parameters(x, y, alpha, fixed_floor): + z = x ** (-alpha) + if fixed_floor is None: + centered = z - z.mean() + amplitude = float(centered @ (y - y.mean()) / (centered @ centered)) + floor = float(np.clip(y.mean() - amplitude * z.mean(), 0, y.min())) + else: + floor = fixed_floor + amplitude = max(0.0, float(z @ (y - floor) / (z @ z))) + mse = float(np.mean((floor + amplitude * z - y) ** 2)) + return mse, floor, amplitude + + +def fit(x, y, model): + if len(x) < 3: + raise ValueError("Each fitting window needs at least three complete bins") + if model == "log": + z = np.log(x) + centered = z - z.mean() + slope = float(centered @ (y - y.mean()) / (centered @ centered)) + intercept = float(y.mean() - slope * z.mean()) + mse = float(np.mean((intercept + slope * z - y) ** 2)) + return {"model": model, "intercept": intercept, "slope": slope, + "rmse": math.sqrt(mse)} + fixed = None if model == "power_free" else float(model.split(":")[1]) + grid = np.geomspace(0.0001, 3, 1000) + errors = [power_parameters(x, y, a, fixed)[0] for a in grid] + index = int(np.argmin(errors)) + low, high = grid[max(index - 1, 0)], grid[min(index + 1, len(grid) - 1)] + ratio = (math.sqrt(5) - 1) / 2 + for _ in range(55): + p, q = high - ratio * (high - low), low + ratio * (high - low) + if power_parameters(x, y, p, fixed)[0] < power_parameters(x, y, q, fixed)[0]: + high = q + else: + low = p + alpha = float((low + high) / 2) + mse, floor, amplitude = power_parameters(x, y, alpha, fixed) + return {"model": model, "floor": floor, "amplitude": amplitude, + "alpha": alpha, "rmse": math.sqrt(mse), + "alpha_at_search_boundary": index in (0, len(grid) - 1)} + + +def predict(parameters, tokens_b): + x = np.asarray(tokens_b) + if parameters["model"] == "log": + return parameters["intercept"] + parameters["slope"] * np.log(x) + return parameters["floor"] + parameters["amplitude"] * x ** (-parameters["alpha"]) + + +def crossing(parameters, target=2.0): + if parameters["model"] == "log": + if parameters["slope"] >= 0: + return None + exponent = (target - parameters["intercept"]) / parameters["slope"] + else: + if parameters["floor"] >= target or parameters["amplitude"] <= 0: + return None + exponent = math.log(parameters["amplitude"] / (target - parameters["floor"])) + exponent /= parameters["alpha"] + return math.exp(exponent) if exponent < 700 else None + + +def analyze(raw): + rows = [json.loads(line) for line in raw.splitlines() if line.strip()] + if not rows or any(b["tokens"] <= a["tokens"] for a, b in zip(rows, rows[1:])): + raise ValueError("Validation snapshot must have strictly increasing token counts") + right = math.floor(rows[-1]["tokens"] / 1e9 / BIN_WIDTH) * BIN_WIDTH + bins = [] + for left in np.arange(1, right, BIN_WIDTH): + selected = [r for r in rows if left <= r["tokens"] / 1e9 < left + BIN_WIDTH] + if not selected: + raise ValueError(f"Missing validation data in bin starting at {left}B") + bins.append({"left_B": float(left), "right_B": float(left + BIN_WIDTH), + "tokens_B": float(np.mean([r["tokens"] / 1e9 for r in selected])), + "loss": float(np.mean([r["loss"] for r in selected])), + "count": len(selected)}) + x = np.array([b["tokens_B"] for b in bins]) + y = np.array([b["loss"] for b in bins]) + fits = [] + for start in WINDOW_STARTS: + selected = x >= start + for model in MODELS: + parameters = fit(x[selected], y[selected], model) + fits.append({"start_B": start, "end_B": right, "bins": int(selected.sum()), + "parameters": parameters, "cross_2_tokens_B": crossing(parameters), + "endpoint_losses": {k: float(predict(parameters, v)) + for k, v in {"40B": 40, **BUDGETS}.items()}}) + holdouts = [] + for end in (4, 5, 6): + train, test = (x >= 2) & (x < end), (x >= end) & (x < end + 1) + if not test.any() or right < end + 1: + raise ValueError("Requested holdouts require complete validation bins through 7B") + for model in MODELS: + parameters = fit(x[train], y[train], model) + difference = predict(parameters, x[test]) - y[test] + holdouts.append({"train_start_B": 2, "train_end_B": end, "test_end_B": end + 1, + "parameters": parameters, "actual_mean_loss": float(y[test].mean()), + "predicted_mean_loss": float(predict(parameters, x[test]).mean()), + "mean_bias": float(difference.mean()), + "rmse": float(np.sqrt(np.mean(difference ** 2)))}) + return {"method": "Equal-weight least squares in loss space on completed 0.25B token bins; " + "x is each bin's actual mean token count. Log: L=a+b*ln(t). Power: L=f+A*t^-alpha. " + "Tokens t are in billions. Free floor constrained to [0, minimum binned loss], A>=0; " + "alpha searched on [0.0001,3] with 1000 logarithmic points and 55 golden refinements. " + "Fixed floors are sensitivity assumptions, not measured irreducible losses. " + "Holdouts fit 2B to 4/5/6B and predict the next complete 1B.", + "caveats": ["Illustrative extrapolations, not confidence intervals or reliable ETAs.", + "Short in-distribution holdouts do not validate a 38x or 151x token extrapolation.", + "Current schedule, changing LR, data repetition, and distribution changes are not modeled.", + "A fitted floor at or above two does not prove an architectural limit.", + "Null crossing means this fitted curve has no finite crossing or exceeds numeric range."], + "source_sha256": hashlib.sha256(raw).hexdigest(), "source_rows": len(rows), + "latest_validation": rows[-1], "complete_bins_end_B": right, + "observed_validation": [{"tokens_B": r["tokens"] / 1e9, "loss": r["loss"]} + for r in rows if r["tokens"] >= 1e9], + "budget_steps": {"268.435456B": 8000 * 256, "1073.741824B": 8000 * 1024}, + "tokens_per_update": 131072, "bins": bins, "fits": fits, "holdouts": holdouts} + + +def plot(report, output): + os.environ.setdefault("MPLCONFIGDIR", str(output.parent / "matplotlib-cache")) + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + x = np.array([b["tokens_B"] for b in report["bins"]]) + y = np.array([b["loss"] for b in report["bins"]]) + end = report["complete_bins_end_B"] + maximum = max(1100, end * 1.25) + fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained") + scenarios = (("power_free", 1, "#2469aa", "Power, fitted floor"), + ("power_free", 2, "#8051a5", "Power, fitted floor"), + ("log", 3, "#ca6d12", "Log-token"), + ("log", 5, "#cfa323", "Log-token"), + ("power_fixed:1.8", 1, "#238456", "Power, ASSUMED floor 1.8")) + raw_x = [r["tokens_B"] for r in report["observed_validation"]] + raw_y = [r["loss"] for r in report["observed_validation"]] + axes[0].scatter(raw_x, raw_y, color="gray", alpha=.35, s=6, label="Individual validations") + for axis in axes: + axis.plot(x, y, ".-", color="black", lw=1.1, ms=4, label="Complete 0.25B-bin means") + axis.set_xlabel("Tokens trained (billions)") + axis.set_ylabel("Validation cross-entropy loss") + axis.grid(alpha=.2) + for model, start, color, label in scenarios: + label += f"; fit {start}–{end:g}B" + parameters = next(r["parameters"] for r in report["fits"] + if r["start_B"] == start and r["parameters"]["model"] == model) + observed = np.linspace(start, end, 150) + future = np.geomspace(end, maximum, 350) + if model == "power_free": + axes[0].plot(observed, predict(parameters, observed), color=color, lw=1, label=label) + axes[1].plot(observed, predict(parameters, observed), color=color, lw=1, alpha=.6) + axes[1].plot(future, predict(parameters, future), color=color, ls="--", lw=1.5, label=label) + axes[0].set_title("Observed losses and fitted power curves") + axes[0].set_xlim(1, max(raw_x) + BIN_WIDTH * .15) + axes[0].set_ylim(min(raw_y) - .05, max(raw_y) + .05) + axes[0].legend(loc="upper right", fontsize=8) + axes[1].set_xscale("log") + axes[1].set_xlim(1, maximum) + axes[1].set_ylim(1.1, 3) + axes[1].axhline(2, color="#8f3333", ls="--", lw=1, label="Loss 2.0") + axes[1].axvspan(end, maximum, color="gray", alpha=.05) + for value, label in ((40, "40B schedule"), (268.435456, "268B budget"), + (1073.741824, "1.074T budget")): + axes[1].axvline(value, color="gray", ls=":", lw=1) + axes[1].text(value, 2.9, label, rotation=90, ha="right", va="top", fontsize=8) + axes[1].set_title("Long extrapolation: strongly dependent on assumed curve") + axes[1].legend(loc="lower left", fontsize=7.5) + fig.suptitle("WarpState: illustrative loss scenarios, NOT confidence intervals or reliable ETAs", + fontsize=12) + output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output, dpi=180) + plt.close(fig) + return matplotlib.__version__ + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path, help="Frozen validation JSONL snapshot") + parser.add_argument("--output", required=True, type=Path, help="Generated JSON analysis") + parser.add_argument("--plot", type=Path, help="Optional generated PNG/PDF/SVG chart") + args = parser.parse_args() + paths = [args.input.resolve(), args.output.resolve()] + if args.plot: + paths.append(args.plot.resolve()) + if len(paths) != len(set(paths)): + parser.error("Input snapshot, JSON output, and plot must use distinct file paths") + report = analyze(args.input.read_bytes()) + report["source_path"] = str(args.input) + report["analysis_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + report["versions"] = {"python": platform.python_version(), "numpy": np.__version__} + if args.plot: + report["versions"]["matplotlib"] = plot(report, args.plot) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + print(f"Analyzed {report['source_rows']} validation rows through " + f"{report['complete_bins_end_B']:g}B complete bins; wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/ops/loss_experiment.py b/ops/loss_experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..0f964cdc1ed9ddbe7f6feb203e456b50944176e9 --- /dev/null +++ b/ops/loss_experiment.py @@ -0,0 +1,52 @@ +"""Disposable full-vocabulary loss execution options for throughput experiments. + +These options retain the existing model, tied embedding/head parameter, objective, +and canonical state dictionary. They trade additional logits/softmax memory for +avoiding the default chunked head's activation recomputation. Apply before DDP. +""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F + + +def _full_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + labels = labels.reshape(-1) + values = logits.reshape(-1, logits.size(-1)).float() + # Sum/valid-count also gives finite zero on a rank with no valid final tokens. + total = F.cross_entropy(values, labels, ignore_index=-100, reduction="sum") + return total / labels.ne(-100).sum().clamp_min(1) + + +def apply_loss_execution(model: torch.nn.Module, mode: str) -> None: + """Select checkpointed, full, or compiled-full loss without replacing weights. + + Only calls with targets and ``return_logits=False`` use the experiment. + Inference and explicit-logit calls retain the model's public behavior. The + original forward owns every core operation and the tied output projection, + so the experiment does not duplicate or diverge from the model architecture. + ``compiled-full`` compiles the loss reduction, using eager loss in evaluation + to avoid compiling another graph solely for validation's smaller microbatch. + """ + if mode not in {"checkpointed", "full", "compiled-full"}: + raise ValueError(f"Unknown loss execution mode: {mode}") + if getattr(model, "_loss_execution_mode", "checkpointed") != "checkpointed": + raise ValueError("Loss execution has already been patched on this model") + if mode == "checkpointed": + return + + original_forward = model.forward + training_loss = _full_loss + if mode == "compiled-full": + training_loss = torch.compile(_full_loss, fullgraph=True, dynamic=False) + + def forward(input_ids, targets=None, *, return_logits=True): + if targets is None or return_logits: + return original_forward(input_ids, targets, return_logits=return_logits) + logits = original_forward(input_ids, targets=None, return_logits=True) + loss_fn = training_loss if model.training and torch.is_grad_enabled() else _full_loss + return None, loss_fn(logits, targets) + + model.forward = forward + model._loss_execution_mode = mode diff --git a/ops/matmul_epilogue.py b/ops/matmul_epilogue.py new file mode 100644 index 0000000000000000000000000000000000000000..cfa42cbd2d3b5e0cc69343d839e48de64a772ac1 --- /dev/null +++ b/ops/matmul_epilogue.py @@ -0,0 +1,142 @@ +"""BF16 paired FFN projections with a SwiGLU epilogue. + +This module launches kernels only. The training adapter owns autograd and keeps +the original FP32 master parameters. Both projection halves are rounded to BF16 +before activation, and the packed projection is returned for ordinary backward. +Fusing the forward therefore saves a projection read, not its required storage. +""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from triton.language.extra.cuda import libdevice +from torch.library import wrap_triton + + +SUPPORTED_TILES = ( + (64, 64, 32, 4, 3), + (64, 64, 64, 4, 2), + (128, 64, 32, 8, 3), + (64, 128, 32, 8, 3), + (128, 64, 32, 4, 3), + (64, 128, 32, 4, 3), + (128, 128, 32, 8, 3), +) + + +@triton.jit +def _paired_swiglu_mm_kernel( + A, B, PACKED, OUTPUT, + M: tl.constexpr, H: tl.constexpr, K: tl.constexpr, + STRIDE_AM: tl.constexpr, STRIDE_AK: tl.constexpr, + STRIDE_BK: tl.constexpr, STRIDE_BN: tl.constexpr, + BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, + NUM_PROGRAMS: tl.constexpr, GROUP_M: tl.constexpr, ROUND: tl.constexpr, +): + tiles_m = tl.cdiv(M, BLOCK_M) + tiles_n = tl.cdiv(H, BLOCK_N) + total_tiles = tiles_m * tiles_n + for tile_id in range(tl.program_id(0), total_tiles, NUM_PROGRAMS): + # Group neighboring row tiles to reuse the two weight panels in L2. + tiles_per_group = GROUP_M * tiles_n + group = tile_id // tiles_per_group + first_m = group * GROUP_M + group_m = tl.minimum(tiles_m - first_m, GROUP_M) + inner = tile_id % tiles_per_group + pid_m = first_m + inner % group_m + pid_n = inner // group_m + rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + reduction = tl.arange(0, BLOCK_K) + # Adjacent logical columns pair up/gate from different packed halves. + # One wider dot shares A and lets the MMA scheduler handle both halves; + # split the accumulator in registers before the rounded epilogue. + pair_columns = tl.arange(0, 2 * BLOCK_N) + paired_cols = pid_n * BLOCK_N + pair_columns // 2 + paired_offsets = paired_cols + (pair_columns % 2) * H + accumulator = tl.zeros((BLOCK_M, 2 * BLOCK_N), tl.float32) + for start in range(tl.cdiv(K, BLOCK_K)): + ks = start * BLOCK_K + reduction + x = tl.load(A + rows[:, None] * STRIDE_AM + ks[None, :] * STRIDE_AK, + (rows[:, None] < M) & (ks[None, :] < K), 0) + paired_weights = tl.load( + B + ks[:, None] * STRIDE_BK + paired_offsets[None, :] * STRIDE_BN, + (ks[:, None] < K) & (paired_cols[None, :] < H), 0) + accumulator = tl.dot(x, paired_weights, accumulator) + up_acc, gate_acc = tl.split(tl.reshape(accumulator, (BLOCK_M, BLOCK_N, 2))) + # These casts are essential, independently of ROUND. SwiGLU consumes + # the BF16 GEMM output in both the eager and existing compiled model. + up_bf16 = up_acc.to(tl.bfloat16) + gate_bf16 = gate_acc.to(tl.bfloat16) + mask = (rows[:, None] < M) & (cols[None, :] < H) + packed_offsets = rows[:, None] * (2 * H) + cols[None, :] + tl.store(PACKED + packed_offsets, up_bf16, mask) + tl.store(PACKED + packed_offsets + H, gate_bf16, mask) + up = up_bf16.to(tl.float32) + gate = gate_bf16.to(tl.float32) + silu = gate / (1.0 + libdevice.exp(-gate)) + if ROUND: + silu = silu.to(tl.bfloat16).to(tl.float32) + tl.store(OUTPUT + rows[:, None] * H + cols[None, :], silu * up, mask) + + +def launch_swiglu_mm(a: torch.Tensor, b: torch.Tensor, algorithm: str = "tiled", + tile=SUPPORTED_TILES[0], eager_rounding: bool = False + ) -> tuple[torch.Tensor, torch.Tensor]: + """Return packed ``a @ b`` and SwiGLU, both BF16 and row-contiguous. + + ``a`` has shape [M,K], ``b`` [K,2H]. Positive-stride layouts, including + transposed FP32-master weight casts, are supported. ``tile`` is + (block_m, block_n, block_k, num_warps, num_stages); block_n counts columns + in *each* projection half. ``persistent`` assigns one program per SM and + strides over tiles; ``tiled`` launches one program per paired output tile. + + ``eager_rounding`` controls only the SiLU-output BF16 rounding before the + multiply. Projection rounding always remains in place. ``aten`` is a + reference execution path using the same explicit rounding choices. + """ + from ops.matmul_kernels import normalize_tile + + if algorithm not in ("aten", "tiled", "persistent"): + raise ValueError(f"Unknown SwiGLU matmul algorithm: {algorithm}") + if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]: + raise ValueError("SwiGLU matmul expects compatible [M,K] and [K,2H] matrices") + if min(a.shape) <= 0 or b.shape[1] <= 0 or b.shape[1] % 2: + raise ValueError("SwiGLU matmul requires positive dimensions and an even packed width") + if a.dtype != torch.bfloat16 or b.dtype != torch.bfloat16: + raise ValueError("SwiGLU matmul requires BF16 operands") + if a.device != b.device or any(stride <= 0 for tensor in (a, b) for stride in tensor.stride()): + raise ValueError("SwiGLU matmul requires the same device and positive strides") + block_m, block_n, block_k, warps, stages = normalize_tile(tile) + # Two independent B panels share one A panel. Avoid configurations whose + # pipeline alone exceeds the conservative SM120 96 KiB shared-memory cap. + shared_bytes = (block_m + 2 * block_n) * block_k * 2 * stages + if shared_bytes > 96 * 1024: + raise ValueError("Paired SwiGLU tile exceeds the conservative 96 KiB shared-memory budget") + if algorithm == "aten": + packed = torch.mm(a, b) + up, gate = packed.chunk(2, dim=-1) + if eager_rounding: + output = F.silu(gate) * up + else: + output = (F.silu(gate.float()) * up.float()).to(torch.bfloat16) + return packed, output + if a.device.type != "cuda": + raise ValueError("Custom SwiGLU matmul kernels require CUDA") + rows, reduction = a.shape + hidden = b.shape[1] // 2 + packed = torch.empty((rows, 2 * hidden), dtype=torch.bfloat16, device=a.device) + output = torch.empty((rows, hidden), dtype=torch.bfloat16, device=a.device) + total_tiles = triton.cdiv(rows, block_m) * triton.cdiv(hidden, block_n) + programs = total_tiles + if algorithm == "persistent": + programs = min(total_tiles, torch.cuda.get_device_properties(a.device).multi_processor_count) + wrap_triton(_paired_swiglu_mm_kernel)[(programs,)]( + a, b, packed, output, rows, hidden, reduction, + a.stride(0), a.stride(1), b.stride(0), b.stride(1), + block_m, block_n, block_k, programs, 8, eager_rounding, + num_warps=warps, num_stages=stages, enable_fp_fusion=not eager_rounding) + return packed, output diff --git a/ops/matmul_execution.py b/ops/matmul_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..6f76a201d91891336eb03ed124936de1e9fb1ae0 --- /dev/null +++ b/ops/matmul_execution.py @@ -0,0 +1,388 @@ +"""Opt-in, shape-specific BF16 matrix execution with ordinary autograd ownership. + +Plans are validated and registered once, before compiling cores or constructing +DDP. FP32 parameter casts stay inside each visible forward graph. CUDA kernels +return BF16 values and gradients, and ordinary differentiable casts return the +gradients to the original FP32 leaves. No hook writes a parameter gradient. +Unsupported/unselected shapes use ATen; CPU and evaluation model paths stay +original. Checkpoint keys and archived model layouts are not changed. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path + +import torch +from torch.library import custom_op, triton_op + + +PLAN_FORMAT = "warpstate-matmul-plan-v1" +_DEFAULT_TILE = (64, 64, 32, 4, 3) +_TILE_FIELDS = ("block_m", "block_n", "block_k", "num_warps", "num_stages") +_PLANS: dict[str, "MatmulPlan"] = {} + + +@dataclass(frozen=True) +class MatmulEntry: + kind: str + m: int + n: int + k: int + a_stride: tuple[int, int] + b_stride: tuple[int, int] + algorithm: str + tile: tuple[int, int, int, int, int] + name: str = "" + + @property + def key(self): + return (self.kind, self.m, self.n, self.k, *self.a_stride, *self.b_stride) + + def as_dict(self): + return {"kind": self.kind, "m": self.m, "n": self.n, "k": self.k, + "a_stride": list(self.a_stride), "b_stride": list(self.b_stride), + "dtype": "bfloat16", "algorithm": self.algorithm, + "tile": dict(zip(_TILE_FIELDS, self.tile, strict=True)), "name": self.name} + + +@dataclass(frozen=True) +class MatmulPlan: + entries: tuple[MatmulEntry, ...] + sha256: str + source_hashes: tuple[tuple[str, str], ...] + registry_key: str + description: str = "" + + @property + def metadata(self): + """Serializable provenance for runtime records and resume signatures.""" + return {"format": PLAN_FORMAT, "sha256": self.sha256, + "source_sha256": dict(self.source_hashes), + "entries": [entry.as_dict() for entry in self.entries]} + + def select(self, kind, m, n, k, a_stride, b_stride): + """Pure shape/layout lookup; no device work or filesystem access.""" + key = (kind, m, n, k, *a_stride, *b_stride) + for entry in self.entries: + if entry.key == key: + return entry + return None + + +def _positive_integer(value, name): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{name} must be a positive integer") + return value + + +def _stride(value, name): + if not isinstance(value, list) or len(value) != 2: + raise ValueError(f"{name} must contain two positive integer strides") + return tuple(_positive_integer(item, name) for item in value) + + +def _no_duplicate_json_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + +def _implementation_hashes(has_fused_swiglu): + names = ["matmul_execution.py", "matmul_kernels.py"] + if has_fused_swiglu: + names += ["matmul_epilogue.py", "activation_execution.py"] + directory = Path(__file__).resolve().parent + return tuple(("ops/" + name, hashlib.sha256((directory / name).read_bytes()).hexdigest()) + for name in sorted(names)) + + +def matmul_plan_from_dict(document: dict) -> MatmulPlan: + """Validate a source recipe; subsequent selection only uses immutable data.""" + from ops.matmul_kernels import normalize_tile + + if not isinstance(document, dict): + raise ValueError("Matmul plan must be a JSON object") + if set(document) - {"format", "description", "entries"}: + raise ValueError("Unknown matmul plan fields") + if document.get("format") != PLAN_FORMAT: + raise ValueError(f"Matmul plan format must be {PLAN_FORMAT}") + description = document.get("description", "") + if not isinstance(description, str): + raise ValueError("Matmul plan description must be a string") + if not isinstance(document.get("entries"), list): + raise ValueError("Matmul plan entries must be a list") + entries = [] + seen = set() + fields = {"kind", "m", "n", "k", "a_stride", "b_stride", "dtype", "algorithm", "tile", "name"} + for index, row in enumerate(document["entries"]): + if not isinstance(row, dict) or set(row) - fields: + raise ValueError(f"Unknown fields in matmul entry {index}") + if row.get("kind") not in {"mm", "swiglu"}: + raise ValueError("Matmul entry kind must be mm or swiglu") + if row.get("dtype") != "bfloat16": + raise ValueError("Matmul plans support BF16 operands only") + if row.get("algorithm") not in {"aten", "tiled", "persistent", "persistent_tma"}: + raise ValueError("Unknown matmul algorithm") + if row["kind"] == "swiglu" and row["algorithm"] == "persistent_tma": + raise ValueError("Fused SwiGLU does not yet support the persistent_tma algorithm") + dimensions = tuple(_positive_integer(row.get(name), name) for name in ("m", "n", "k")) + if row["kind"] == "swiglu" and dimensions[1] % 2: + raise ValueError("Fused SwiGLU output width must be even") + name = row.get("name", "") + if not isinstance(name, str): + raise ValueError("Matmul entry name must be a string") + tile_document = row.get("tile", dict(zip(_TILE_FIELDS, _DEFAULT_TILE, strict=True))) + if not isinstance(tile_document, dict) or set(tile_document) != set(_TILE_FIELDS): + raise ValueError("Matmul tile must specify block_m/block_n/block_k/num_warps/num_stages") + for field, value in tile_document.items(): + _positive_integer(value, field) + tile = tuple(normalize_tile(tile_document)) + if row["kind"] == "swiglu": + bm, bn, bk, _warps, stages = tile + if (bm + 2 * bn) * bk * 2 * stages > 96 * 1024: + raise ValueError("Fused SwiGLU tile exceeds the conservative shared-memory budget") + entry = MatmulEntry(row["kind"], *dimensions, + _stride(row.get("a_stride"), "a_stride"), + _stride(row.get("b_stride"), "b_stride"), + row["algorithm"], tile, name) + if entry.key in seen: + raise ValueError("Duplicate matmul shape/layout entry") + seen.add(entry.key) + entries.append(entry) + canonical = {"format": PLAN_FORMAT, "description": description, + "entries": [entry.as_dict() for entry in entries]} + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(payload.encode()).hexdigest() + hashes = _implementation_hashes(any(entry.kind == "swiglu" for entry in entries)) + registry_key = hashlib.sha256((digest + json.dumps(hashes)).encode()).hexdigest() + plan = MatmulPlan(tuple(entries), digest, hashes, registry_key, description) + existing = _PLANS.get(registry_key) + if existing is not None and existing != plan: + raise RuntimeError("Matmul plan registry identity collision") + _PLANS[registry_key] = plan + return plan + + +def load_matmul_plan(path: str | Path) -> MatmulPlan: + """Load once before training; no recipes or hashes are read during forward.""" + path = Path(path).expanduser().resolve() + workspace = Path(__file__).resolve().parents[2] + if not path.is_relative_to(workspace): + raise ValueError("Matmul plan must remain inside the WarpState workspace") + document = json.loads(path.read_text(), object_pairs_hook=_no_duplicate_json_keys) + return matmul_plan_from_dict(document) + + +def _choice(a, b, plan_key, kind="mm"): + if (a.ndim != 2 or b.ndim != 2 or not a.is_cuda or not b.is_cuda + or a.device != b.device or a.dtype != torch.bfloat16 + or b.dtype != torch.bfloat16 or a.shape[1] != b.shape[0] + or a.numel() == 0 or b.numel() == 0 + or any(stride <= 0 for stride in (*a.stride(), *b.stride()))): + return None + plan = _PLANS[plan_key] + entry = plan.select(kind, a.shape[0], b.shape[1], a.shape[1], a.stride(), b.stride()) + if entry is not None and entry.algorithm == "persistent_tma": + # Torch allocations have an aligned base; sliced views may not retain + # descriptor alignment even when their shape/stride matches a plan. + if (min(a.stride()) != 1 or min(b.stride()) != 1 + or max(a.stride()) * 2 % 16 or max(b.stride()) * 2 % 16 + or a.storage_offset() * 2 % 16 or b.storage_offset() * 2 % 16): + return None + return entry + + +@custom_op("warpstate::selected_matrix_kernel", mutates_args=()) +def _selected_matrix_kernel(a: torch.Tensor, b: torch.Tensor, plan_key: str) -> torch.Tensor: + """Opaque launch, retaining the outer operator's registered autograd. + + PyTorch 2.14 Compiled Autograd retraces the AOT backward graph, but Dynamo + cannot retrace its functionalized Triton higher-order launch operator. + Keep selected launches opaque in that graph; ordinary ATen fallbacks stay + visible to the compiler. This also covers activation recomputation. + """ + from ops.matmul_kernels import launch_mm + + entry = _choice(a, b, plan_key) + if entry is None or entry.algorithm == "aten": + return torch.mm(a, b) + return launch_mm(a, b, entry.algorithm, entry.tile) + + +@_selected_matrix_kernel.register_fake +def _selected_matrix_kernel_fake(a, b, plan_key): + return torch.empty((a.shape[0], b.shape[1]), dtype=a.dtype, device=a.device) + + +@triton_op("warpstate::planned_mm", mutates_args={}) +def _planned_mm(a: torch.Tensor, b: torch.Tensor, plan_key: str) -> torch.Tensor: + entry = _choice(a, b, plan_key) + if entry is None or entry.algorithm == "aten": + return torch.mm(a, b) + return _selected_matrix_kernel(a, b, plan_key) + + +def _mm_setup(ctx, inputs, output): + ctx.save_for_backward(inputs[0], inputs[1]) + ctx.plan_key = inputs[2] + + +def _mm_backward(ctx, grad): + a, b = ctx.saved_tensors + grad = grad.contiguous() + da = _planned_mm(grad, b.T, ctx.plan_key) if ctx.needs_input_grad[0] else None + # Match Linear's production dW orientation (dY.T @ X), then return the + # gradient of the transposed BF16 weight operand. This avoids an extra + # gradient-layout copy and lets the plan reuse measured dW shapes/strides. + db = _planned_mm(grad.T, a, ctx.plan_key).T if ctx.needs_input_grad[1] else None + return da, db, None + + +_planned_mm.register_autograd(_mm_backward, setup_context=_mm_setup) + + +def planned_matmul(a: torch.Tensor, b: torch.Tensor, plan: MatmulPlan) -> torch.Tensor: + """Autograd-enabled entry point, also useful for isolated correctness checks.""" + return _planned_mm(a, b, plan.registry_key) + + +@custom_op("warpstate::selected_swiglu_kernel", mutates_args=()) +def _selected_swiglu_kernel(a: torch.Tensor, weight: torch.Tensor, plan_key: str, + eager_rounding: bool) -> tuple[torch.Tensor, torch.Tensor]: + """Accept native-layout BF16 W; form its projection view only at launch. + + Passing W.T across the opaque boundary makes Inductor exact-stride + constraints materialize a second cast buffer alongside the W saved for + backward. A native-layout input lets both consumers share the same visible + FP32-to-BF16 cast. Autograd ownership and the plan still use the outer B=W.T. + """ + from ops.matmul_epilogue import launch_swiglu_mm + + b = weight.T + entry = _choice(a, b, plan_key, "swiglu") + if entry is None or entry.algorithm == "aten": + return launch_swiglu_mm(a, b, "aten", _DEFAULT_TILE, eager_rounding) + return launch_swiglu_mm(a, b, entry.algorithm, entry.tile, eager_rounding) + + +@_selected_swiglu_kernel.register_fake +def _selected_swiglu_kernel_fake(a, weight, plan_key, eager_rounding): + packed = torch.empty((a.shape[0], weight.shape[0]), dtype=a.dtype, device=a.device) + activation = torch.empty((a.shape[0], weight.shape[0] // 2), dtype=a.dtype, device=a.device) + return packed, activation + + +@custom_op("warpstate::selected_swiglu_gradient_kernel", mutates_args=()) +def _selected_swiglu_gradient_kernel(packed: torch.Tensor, grad: torch.Tensor, + eager_rounding: bool) -> torch.Tensor: + from ops.activation_execution import _swiglu_backward + + return _swiglu_backward(packed, grad, eager_rounding) + + +@_selected_swiglu_gradient_kernel.register_fake +def _selected_swiglu_gradient_kernel_fake(packed, grad, eager_rounding): + return torch.empty_like(packed) + + +@triton_op("warpstate::planned_swiglu_mm", mutates_args={}) +def _planned_swiglu_mm(a: torch.Tensor, b: torch.Tensor, plan_key: str, + eager_rounding: bool) -> tuple[torch.Tensor, torch.Tensor]: + from ops.matmul_epilogue import launch_swiglu_mm + + entry = _choice(a, b, plan_key, "swiglu") + if entry is not None and entry.algorithm != "aten": + return _selected_swiglu_kernel(a, b.T, plan_key, eager_rounding) + algorithm = entry.algorithm if entry is not None else "aten" + tile = entry.tile if entry is not None else _DEFAULT_TILE + return launch_swiglu_mm(a, b, algorithm, tile, eager_rounding) + + +def _fused_setup(ctx, inputs, output): + ctx.save_for_backward(inputs[0], inputs[1], output[0]) + ctx.plan_key = inputs[2] + ctx.eager_rounding = inputs[3] + ctx.mark_non_differentiable(output[0]) + + +def _fused_backward(ctx, _packed_grad, grad): + a, b, packed = ctx.saved_tensors + projection_grad = _selected_swiglu_gradient_kernel(packed, grad, ctx.eager_rounding) + da = _planned_mm(projection_grad, b.T, ctx.plan_key) if ctx.needs_input_grad[0] else None + db = _planned_mm(projection_grad.T, a, ctx.plan_key).T if ctx.needs_input_grad[1] else None + return da, db, None, None + + +_planned_swiglu_mm.register_autograd(_fused_backward, setup_context=_fused_setup) + + +def _linear_has_choices(module, plan): + width, output = module.in_features, module.out_features + return any(entry.kind == "mm" and ( + (entry.n == output and entry.k == width) # Forward. + or (entry.n == width and entry.k == output) # dX. + or (entry.m == output and entry.n == width) # dW. + ) for entry in plan.entries) + + +def apply_matmul_execution(model: torch.nn.Module, plan: MatmulPlan) -> None: + """Apply before core compilation and DDP; CPU/evaluation paths stay original. + + Selected matrix calls do not replace parameters with externally cached BF16 + views. The ordinary FP32-to-BF16 casts are part of each visible forward graph + so AOTAutograd/DDP retain their original parameter dependencies and hooks. + """ + from warpstate.model import SwiGLU + + if not isinstance(plan, MatmulPlan): + raise TypeError("Load a MatmulPlan before applying matrix execution") + if getattr(model, "_matmul_execution_plan", None) is not None: + raise ValueError("Matmul execution has already been applied") + if any(parameter.dtype != torch.float32 for parameter in model.parameters()): + raise ValueError("Matmul execution requires FP32 master parameters") + if plan.registry_key not in _PLANS: + raise ValueError("Matmul plan is not registered in this process") + model._matmul_execution_plan = plan + if not plan.entries: + return + has_fused = any(entry.kind == "swiglu" and entry.algorithm != "aten" for entry in plan.entries) + for core in model.cores: + for module in core.modules(): + original = module.forward + if (isinstance(module, torch.nn.Linear) and module.bias is None + and _linear_has_choices(module, plan)): + def linear_forward(x, _module=module, _original=original): + if (not _module.training or not torch.is_grad_enabled() or not x.is_cuda + or not torch.is_autocast_enabled("cuda") + or torch.get_autocast_dtype("cuda") != torch.bfloat16): + return _original(x) + a = x.reshape(-1, x.shape[-1]).to(torch.bfloat16) + b = _module.weight.to(torch.bfloat16).T + output = _planned_mm(a, b, plan.registry_key) + return output.reshape(*x.shape[:-1], _module.out_features) + module.forward = linear_forward + elif (isinstance(module, SwiGLU) and has_fused and any( + entry.kind == "swiglu" and entry.n == module.up_gate.out_features + and entry.k == module.up_gate.in_features for entry in plan.entries)): + def swiglu_forward(x, _module=module, _original=original): + if (not _module.training or not torch.is_grad_enabled() or not x.is_cuda + or not torch.is_autocast_enabled("cuda") + or torch.get_autocast_dtype("cuda") != torch.bfloat16 + or _module.up_gate.bias is not None): + return _original(x) + a = x.reshape(-1, x.shape[-1]).to(torch.bfloat16) + b = _module.up_gate.weight.to(torch.bfloat16).T + entry = _choice(a, b, plan.registry_key, "swiglu") + if entry is None or entry.algorithm == "aten": + return _original(x) + _, activation = _planned_swiglu_mm( + a, b, plan.registry_key, not torch.compiler.is_compiling()) + activation = activation.reshape(*x.shape[:-1], _module.down.in_features) + return _module.down(activation) + module.forward = swiglu_forward diff --git a/ops/matmul_kernels.py b/ops/matmul_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..5b0926874e71b5b4ff5dbf9a225c432d6ddae214 --- /dev/null +++ b/ops/matmul_kernels.py @@ -0,0 +1,205 @@ +"""Bounded BF16 matrix kernels; autograd and policy selection belong to callers. + +Triton variants accumulate dot products in FP32 and return BF16, without split-K +atomics or reduced-precision operands. ``aten`` preserves the process's existing +BLAS settings. Persistent scheduling assigns multiple output tiles to each +resident program; ``persistent_tma`` also uses TensorDescriptor loads. These +kernels do not use architecture-specific warp-specialized execution. +""" + +from collections.abc import Mapping +import json + +import torch +from torch.library import wrap_triton +import triton +import triton.language as tl + + +ALGORITHMS = ("aten", "tiled", "persistent", "persistent_tma") +TILE_FIELDS = ("block_m", "block_n", "block_k", "num_warps", "num_stages") +DEFAULT_TILE = (128, 128, 32, 4, 3) +TUNING_TILES = ( + (64, 128, 64, 4, 3), + DEFAULT_TILE, + (128, 256, 32, 8, 3), + (256, 128, 32, 8, 3), + (128, 128, 64, 4, 2), +) + + +def normalize_tile(tile=None): + """Normalize a five-integer tile tuple or named JSON object.""" + if tile is None: + tile = DEFAULT_TILE + if isinstance(tile, Mapping): + if set(tile) != set(TILE_FIELDS): + raise ValueError("Tile must contain exactly " + ", ".join(TILE_FIELDS)) + tile = tuple(tile[key] for key in TILE_FIELDS) + if not isinstance(tile, (tuple, list)) or len(tile) != 5: + raise ValueError("Tile must be (block_m, block_n, block_k, num_warps, num_stages)") + if any(type(value) is not int for value in tile): + raise ValueError("Tile values must be integers") + bm, bn, bk, warps, stages = tile + if any(value < 16 or value > 256 or value & (value - 1) for value in (bm, bn, bk)): + raise ValueError("Matrix tile dimensions must be powers of two between 16 and 256") + if warps not in (4, 8) or stages not in (1, 2, 3, 4): + raise ValueError("Supported launch settings are 4/8 warps and 1–4 stages") + if (bm + bn) * bk * 2 * stages > 96 * 1024: + raise ValueError("BF16 operand pipeline exceeds the conservative 96 KiB shared-memory budget") + return (bm, bn, bk, warps, stages) + + +def tile_dict(tile): + return dict(zip(TILE_FIELDS, normalize_tile(tile), strict=True)) + + +def matrix_signature(a, b): + """Serializable dispatch signature; use outside captured execution.""" + if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]: + raise ValueError("Expected compatible two-dimensional matrix operands") + if a.dtype != b.dtype: + raise ValueError("Matrix operands must have matching dtypes") + return {"m": a.shape[0], "n": b.shape[1], "k": a.shape[1], + "a_stride": list(a.stride()), "b_stride": list(b.stride()), + "dtype": str(a.dtype).removeprefix("torch.")} + + +def policy_key(a, b): + return json.dumps(matrix_signature(a, b), sort_keys=True, separators=(",", ":")) + + +@triton.jit +def _tile_position(tile_id, tiles_m, tiles_n, GROUP_M: tl.constexpr): + group_size = GROUP_M * tiles_n + first_m = (tile_id // group_size) * GROUP_M + actual_m = tl.minimum(tiles_m - first_m, GROUP_M) + within = tile_id % group_size + return first_m + within % actual_m, within // actual_m + + +@triton.jit +def _matmul_kernel(A, B, C, M: tl.constexpr, N: tl.constexpr, K: tl.constexpr, + A0: tl.constexpr, A1: tl.constexpr, B0: tl.constexpr, B1: tl.constexpr, + BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr, + GROUP_M: tl.constexpr, PROGRAMS: tl.constexpr, + PERSISTENT: tl.constexpr): + tiles_m = tl.cdiv(M, BM) + tiles_n = tl.cdiv(N, BN) + first = tl.program_id(0) + # A tiled launch has one iteration. Persistent programs advance through + # multiple output tiles, keeping the launch bounded by the SM count. + stop = tiles_m * tiles_n if PERSISTENT else first + 1 + step = PROGRAMS if PERSISTENT else 1 + for tile_id in range(first, stop, step): + tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, GROUP_M) + rows = tile_m * BM + tl.arange(0, BM) + cols = tile_n * BN + tl.arange(0, BN) + reduction = tl.arange(0, BK) + accumulator = tl.full((BM, BN), 0, tl.float32) + for block in range(tl.cdiv(K, BK)): + kk = block * BK + reduction + av = tl.load(A + rows[:, None] * A0 + kk[None, :] * A1, + (rows[:, None] < M) & (kk[None, :] < K), 0) + bv = tl.load(B + kk[:, None] * B0 + cols[None, :] * B1, + (kk[:, None] < K) & (cols[None, :] < N), 0) + accumulator = tl.dot(av, bv, accumulator, input_precision="ieee") + tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16), + (rows[:, None] < M) & (cols[None, :] < N)) + + +@triton.jit +def _persistent_tma_kernel(A_DESC, B_DESC, C, + M: tl.constexpr, N: tl.constexpr, K: tl.constexpr, + BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr, + PROGRAMS: tl.constexpr, A_ROW: tl.constexpr, B_ROW: tl.constexpr): + tiles_m = tl.cdiv(M, BM) + tiles_n = tl.cdiv(N, BN) + for tile_id in tl.range(tl.program_id(0), tiles_m * tiles_n, PROGRAMS): + tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, 8) + row = (tile_m * BM).to(tl.int32) + col = (tile_n * BN).to(tl.int32) + accumulator = tl.full((BM, BN), 0, tl.float32) + for block in tl.range(tl.cdiv(K, BK)): + kk = (block * BK).to(tl.int32) + av = A_DESC.load([row, kk] if A_ROW else [kk, row]) + bv = B_DESC.load([kk, col] if B_ROW else [col, kk]) + accumulator = tl.dot(av if A_ROW else av.T, bv if B_ROW else bv.T, + accumulator, input_precision="ieee") + rows = row + tl.arange(0, BM) + cols = col + tl.arange(0, BN) + tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16), + (rows[:, None] < M) & (cols[None, :] < N)) + + +@torch.library.custom_op("warpstate::persistent_tma_mm", mutates_args=()) +def _tma_mm(a: torch.Tensor, b: torch.Tensor, bm: int, bn: int, bk: int, + warps: int, stages: int) -> torch.Tensor: + """Opaque launch boundary for the host descriptor API in PyTorch 2.14. + + That release's triton_op/AOT path does not preserve host descriptor metadata. + This custom-op boundary keeps the TMA implementation compilable without + substituting a different algorithm or making inputs appear mutable. + """ + from triton.tools.tensor_descriptor import TensorDescriptor + + m, k = a.shape + n = b.shape[1] + a_row, b_row = a.stride(1) == 1, b.stride(1) == 1 + a_desc = TensorDescriptor.from_tensor(a if a_row else a.T, + [bm, bk] if a_row else [bk, bm]) + b_desc = TensorDescriptor.from_tensor(b if b_row else b.T, + [bk, bn] if b_row else [bn, bk]) + output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16) + programs = min(triton.cdiv(m, bm) * triton.cdiv(n, bn), + torch.cuda.get_device_properties(a.device).multi_processor_count) + _persistent_tma_kernel[(programs,)]( + a_desc, b_desc, output, m, n, k, bm, bn, bk, programs, a_row, b_row, + num_warps=warps, num_stages=stages) + return output + + +@_tma_mm.register_fake +def _tma_mm_fake(a, b, bm, bn, bk, warps, stages): + return torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=torch.bfloat16) + + +def launch_mm(a, b, algorithm="aten", tile=None): + """Return CUDA BF16 ``a @ b``; caller supplies any autograd registration. + + Supports positive 2D strides, including ordinary transposed weights and + transposed output gradients. No inputs or parameters are modified. The + conservative tile footprint check is not a guarantee of compiler resource + usage; compilation failures remain explicit and are recorded by the tuner. + """ + if algorithm not in ALGORITHMS: + raise ValueError("Unknown matrix algorithm: " + str(algorithm)) + if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]: + raise ValueError("Expected compatible two-dimensional matrix operands") + if a.device != b.device or not a.is_cuda: + raise ValueError("Matrix kernels require operands on the same CUDA device") + if a.dtype != torch.bfloat16 or b.dtype != torch.bfloat16: + raise ValueError("Matrix kernels require BF16 inputs") + if min(a.shape[0], a.shape[1], b.shape[1]) < 1 or min(*a.stride(), *b.stride()) < 1: + raise ValueError("Matrix kernels require positive dimensions and strides") + if algorithm == "aten": + return torch.mm(a, b) + bm, bn, bk, warps, stages = normalize_tile(tile) + m, k = a.shape + n = b.shape[1] + if algorithm == "persistent_tma": + if (min(a.stride()) != 1 or min(b.stride()) != 1 + or max(a.stride()) * 2 % 16 or max(b.stride()) * 2 % 16): + raise ValueError("TMA operands require an innermost stride of one and 16-byte-aligned rows") + if a.storage_offset() * 2 % 16 or b.storage_offset() * 2 % 16: + raise ValueError("TMA operands require 16-byte-aligned storage offsets") + return _tma_mm(a, b, bm, bn, bk, warps, stages) + output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16) + tiles = triton.cdiv(m, bm) * triton.cdiv(n, bn) + programs = min(tiles, torch.cuda.get_device_properties(a.device).multi_processor_count) \ + if algorithm == "persistent" else tiles + wrap_triton(_matmul_kernel)[(programs,)]( + a, b, output, m, n, k, a.stride(0), a.stride(1), b.stride(0), b.stride(1), + bm, bn, bk, 8, programs, algorithm == "persistent", + num_warps=warps, num_stages=stages) + return output diff --git a/ops/posttrain-handoff.sh b/ops/posttrain-handoff.sh new file mode 100644 index 0000000000000000000000000000000000000000..3cbffe5e929fcf6ec779b162bbd3bb2ac75a0941 --- /dev/null +++ b/ops/posttrain-handoff.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +exec rtx6000_40b/.server-venv/bin/python -B \ + rtx6000_40b/ops/posttrain_handoff.py \ + --config rtx6000_40b/configs/posttrain_10b.json "$@" diff --git a/ops/posttrain-supervisord.conf b/ops/posttrain-supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..70c65966b76976b4fc525077f4da64918f8dab94 --- /dev/null +++ b/ops/posttrain-supervisord.conf @@ -0,0 +1,35 @@ +# Explicitly armed once by the user-authorized 10B -> SFT handoff. +# Existing base training is never restarted by either of these programs. +[program:warpstate-posttrain] +command=/workspace/WarpState/rtx6000_40b/ops/posttrain-handoff.sh +directory=/workspace/WarpState +autostart=false +autorestart=false +startsecs=5 +startretries=0 +exitcodes=0 +stopsignal=USR1 +stopwaitsecs=900 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/posttrain.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=5 +redirect_stderr=true + +[program:warpstate-posttrain-data] +command=/workspace/WarpState/rtx6000_40b/ops/prepare-posttrain-data.sh +directory=/workspace/WarpState +autostart=false +autorestart=false +startsecs=5 +startretries=0 +exitcodes=0 +stopsignal=TERM +stopwaitsecs=60 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/posttrain-data.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=3 +redirect_stderr=true diff --git a/ops/posttrain_chat.py b/ops/posttrain_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..88f8c8d3e7aea127a0ff965b5409dfd9112b4389 --- /dev/null +++ b/ops/posttrain_chat.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +"""Chat with a sealed SFT/DPO export using its original training serialization. + +FP32 master parameters remain unchanged. BF16 compute views and bounded chunk +caches reproduce parallel-memory inference without changing training sources. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field +import json +import math +from pathlib import Path +import sys +import time + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +if str(PACKAGE) not in sys.path: + sys.path.insert(0, str(PACKAGE)) + +import torch +import torch.nn.functional as F +from torch.func import functional_call + +from chat import load_model, sample_token +from ops.posttrain_data import RENDER_FORMAT, ROLES, renderer_sha256 +from posttrain import atomic_json, digest +from warpstate.model import rotate_half + +DEFAULT_SNAPSHOT = PACKAGE / 'work/chat/dpo-final/model.pt' + + +def encode_prompt(messages, tokenizer): + """Match segmented SFT tokens, ending immediately after an assistant header.""" + if not isinstance(messages, list) or not messages: + raise ValueError('Prompt requires nonempty messages') + if not isinstance(messages[-1], dict) or messages[-1].get('role') not in ('user', 'tool'): + raise ValueError('The conditioning conversation must end in a user/tool message') + vocab, eos, bos = tokenizer.vocab_size(), tokenizer.eos_id(), tokenizer.bos_id() + if not 0 <= eos < vocab: + raise ValueError('Tokenizer requires a real EOS token') + ids = [bos] if 0 <= bos < vocab else [] + + def append(text): + encoded = tokenizer.encode(text, out_type=int) + if any(type(token) is not int or not 0 <= token < vocab for token in encoded): + raise ValueError('Tokenizer returned invalid token IDs') + ids.extend(encoded) + + for message in messages: + if not isinstance(message, dict) or message.get('role') not in ROLES: + raise ValueError('Unsupported prompt role') + if not isinstance(message.get('content'), str) or not message['content'].strip(): + raise ValueError('Messages require nonempty text content') + if message.get('tool_calls'): + raise ValueError('Tool calls must be serialized into message content') + append(f"<|{message['role']}|>\n") + append(message['content']) + if message['role'] == 'assistant': + ids.append(eos) + append('\n') + append('<|assistant|>\n') + return ids + + +@dataclass +class DecodeCache: + keys: torch.Tensor + values: torch.Tensor + memory_values: torch.Tensor + fast: torch.Tensor + slow: torch.Tensor + writes: list[torch.Tensor] = field(default_factory=list) + position: int = 0 + + +def init_caches(model, device, dtype): + caches = [] + for depth in range(model.cfg.logical_depth): + core = model.cores[depth % model.cfg.n_cores] + state_shape = (1, core.h, core.hd, core.hd) + kv_shape = (1, core.h, core.chunk, core.hd) + caches.append(DecodeCache( + keys=torch.zeros(kv_shape, device=device, dtype=dtype), + values=torch.zeros(kv_shape, device=device, dtype=dtype), + memory_values=torch.zeros(kv_shape, device=device, dtype=dtype), + fast=torch.zeros(state_shape, device=device, dtype=dtype), + slow=torch.zeros(state_shape, device=device, dtype=dtype))) + return caches + + +def _next_states(core, cache): + # Form the chunk write in one BF16 GEMM, as forward() does. Accumulating + # separately rounded token outer products would alter the recurrent memory. + write = (torch.tanh(cache.keys).transpose(-2, -1) @ torch.tanh(cache.memory_values)) / core.chunk + if core.cfg.parallel_memory: + cache.writes.append(write) + values = torch.stack(cache.writes, dim=2).flatten(-2) # B,H,N,Dh² + lag = torch.arange(len(cache.writes) - 1, -1, -1, device=write.device) + + def state(logit): + decay = logit.float().sigmoid()[:, None, None] + weights = ((1 - decay) * decay.pow(lag[None, None])).to(values.dtype) + return (weights @ values).reshape_as(cache.fast) + + cache.fast = state(core.fast_decay_logit) + cache.slow = state(core.slow_decay_logit) + else: + fast_decay = core.fast_decay_logit.sigmoid().to(write.dtype).view(1, core.h, 1, 1) + slow_decay = core.slow_decay_logit.sigmoid().to(write.dtype).view(1, core.h, 1, 1) + cache.fast = fast_decay * cache.fast + (1 - fast_decay) * write + cache.slow = slow_decay * cache.slow + (1 - slow_decay) * write + cache.position = 0 + + +def _core_step(core, x, cache): + q, k, v, g, u = (value.permute(0, 2, 1, 3) for value in core._split(core.norm1(x))) + pos = cache.position + cos = core.rope_cos[pos:pos + 1].to(q.dtype).view(1, 1, 1, core.hd) + sin = core.rope_sin[pos:pos + 1].to(q.dtype).view(1, 1, 1, core.hd) + q, k = q * cos + rotate_half(q) * sin, k * cos + rotate_half(k) * sin + cache.keys[:, :, pos:pos + 1].copy_(k) + cache.values[:, :, pos:pos + 1].copy_(v) + cache.memory_values[:, :, pos:pos + 1].copy_(u) + local = F.scaled_dot_product_attention(q, cache.keys[:, :, :pos + 1], + cache.values[:, :, :pos + 1], is_causal=False, dropout_p=0.0) + mix = core.memory_mix_logit.sigmoid().to(q.dtype).view(1, core.h, 1, 1) + if core.cfg.parallel_memory: + memory = (q @ (mix * cache.fast + (1 - mix) * cache.slow)) / math.sqrt(core.hd) + else: + memory = mix * ((q @ cache.fast) / math.sqrt(core.hd)) + memory = memory + (1 - mix) * ((q @ cache.slow) / math.sqrt(core.hd)) + gate = torch.sigmoid(g) + y = (gate * local + (1 - gate) * memory).permute(0, 2, 1, 3).reshape(1, 1, core.cfg.d_model) + x = x + core.out_proj(y) + x = x + core.ffn(core.norm2(x)) + cache.position += 1 + if cache.position == core.chunk: + _next_states(core, cache) + return x + + +class _Step(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.wrapped = model + + def forward(self, token, caches): + model = self.wrapped + x = model.embed(token) + for depth in range(model.cfg.logical_depth): + x = x * (1 + model.depth_scale[depth]) + model.depth_bias[depth] + x = _core_step(model.cores[depth % model.cfg.n_cores], x, caches[depth]) + return F.linear(model.final_norm(x), model.embed.weight) + + +class CachedDecoder: + """Inference-only compute views, retaining the model's original FP32 masters.""" + def __init__(self, model, *, dtype=torch.bfloat16): + if dtype not in (torch.float32, torch.bfloat16): + raise ValueError('Only BF16 compute or FP32 correctness fixtures are supported') + if model.training or any(parameter.requires_grad for parameter in model.parameters()): + raise ValueError('Decode requires an eval-only frozen model') + parameters = dict(model.named_parameters()) + if any(parameter.dtype != torch.float32 for parameter in parameters.values()): + raise ValueError('Model masters must retain original FP32 values') + self.device = next(model.parameters()).device + if any(parameter.device != self.device for parameter in parameters.values()): + raise ValueError('The model must occupy one device') + self.dtype, self.model, self.wrapper = dtype, model, _Step(model) + # Decay/mix sigmoids must use original FP32 logits before rounding. + self.views = {'wrapped.' + name: parameter.to(dtype) + for name, parameter in parameters.items() + if not name.endswith(('fast_decay_logit', 'slow_decay_logit', 'memory_mix_logit'))} + + @torch.inference_mode() + def step(self, token, caches): + if token.shape != (1, 1) or token.device != self.device: + raise ValueError('Decode expects one token on the model device') + with torch.autocast(self.device.type, dtype=torch.bfloat16, + enabled=self.dtype == torch.bfloat16, cache_enabled=False): + return functional_call(self.wrapper, self.views, (token, caches), strict=False) + + def new_caches(self): + return init_caches(self.model, self.device, self.dtype) + + +@torch.inference_mode() +def generate(decoder, tokenizer, messages, *, max_new_tokens=256, max_context=2048, + temperature=0.0, seed=42, on_token=None): + if type(max_new_tokens) is not int or max_new_tokens < 1: + raise ValueError('max_new_tokens must be positive') + if type(max_context) is not int or max_context < 2: + raise ValueError('max_context must exceed one token') + if not math.isfinite(temperature) or temperature < 0 or type(seed) is not int or seed < 0: + raise ValueError('Temperature and seed must be nonnegative') + ids = encode_prompt(messages, tokenizer) + if len(ids) >= max_context: + raise ValueError(f'Prompt has {len(ids)} tokens; context limit is {max_context}') + limit = min(max_new_tokens, max_context - len(ids)) + caches = decoder.new_caches() + token = torch.empty((1, 1), dtype=torch.long, device=decoder.device) + generator = torch.Generator(device=decoder.device).manual_seed(seed) + if decoder.device.type == 'cuda': + torch.cuda.synchronize(decoder.device) + begin = time.perf_counter() + for value in ids: + token.fill_(value) + logits = decoder.step(token, caches)[0, -1] + generated, first = [], None + stop_reason = 'context_limit' if limit < max_new_tokens else 'length' + sampled_tokens = 0 + for index in range(limit): + value = sample_token(logits, temperature, 0, 1.0, generator) + sampled_tokens += 1 + if first is None: + first = time.perf_counter() - begin + if value == tokenizer.eos_id(): + stop_reason = 'eos' + break + if value is None: + raise RuntimeError('No legal continuation token') + generated.append(value) + if on_token is not None: + on_token(tuple(generated)) + if index + 1 < limit: + token.fill_(value) + logits = decoder.step(token, caches)[0, -1] + if decoder.device.type == 'cuda': + torch.cuda.synchronize(decoder.device) + elapsed = time.perf_counter() - begin + return dict(response=tokenizer.decode(generated), token_ids=generated, + prompt_tokens=len(ids), generated_tokens=len(generated), sampled_tokens=sampled_tokens, + elapsed_seconds=elapsed, first_token_seconds=first, + tokens_per_second=len(generated) / elapsed, stop_reason=stop_reason, + temperature=temperature, seed=seed) + + +def read_prompts(path): + value = json.loads(Path(path).read_text()) + rows = value.get('prompts') if isinstance(value, dict) else value + if not isinstance(rows, list) or not rows: + raise ValueError('Prompt suite must be a nonempty list or an object with prompts') + seen = set() + for index, row in enumerate(rows): + if not isinstance(row, dict) or not isinstance(row.get('prompt'), str) or not row['prompt'].strip(): + raise ValueError(f'Invalid prompt suite row {index}') + identity = row.get('id', str(index)) + if not isinstance(identity, (str, int)) or identity in seen: + raise ValueError('Prompt suite IDs must be distinct strings or integers') + seen.add(identity) + return rows + + +def inside_workspace(value, *, must_exist=True): + path = Path(value).expanduser().resolve() + if not path.is_relative_to(WORKSPACE) or path == WORKSPACE: + raise ValueError('Every input and output must stay inside /workspace/WarpState') + if must_exist and not path.is_file(): + raise ValueError(f'Missing input file: {path}') + return path + + +def interactive_chat(reply, messages, *, seed): + history = [] + print('Enter a message; Ctrl-C cancels input/reply, /reset clears history, /quit exits.', flush=True) + while True: + try: + prompt = input('You: ').strip() + if prompt in ('/quit', '/exit'): + break + if prompt == '/reset': + history = [] + continue + if not prompt: + continue + conversation = history + [{'role': 'user', 'content': prompt}] if history else messages(prompt) + try: + result = reply(conversation, seed=seed) + except ValueError as error: + print(f'{error}; use /reset to start a new conversation.', flush=True) + continue + if result['response'].strip(): + history = conversation + [{'role': 'assistant', 'content': result['response']}] + except KeyboardInterrupt: + # Discard this input/unfinished reply, retaining completed turns. + # Generation's context managers unwind and restore parameter views. + print('\nCancelled.', flush=True) + continue + except EOFError: + print() + break + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--snapshot', type=Path, default=DEFAULT_SNAPSHOT) + parser.add_argument('--tokenizer', type=Path, help='Defaults to tokenizer.model beside the snapshot') + parser.add_argument('--device', default='cuda:0', help='cuda:0, another CUDA device, or cpu') + mode = parser.add_mutually_exclusive_group() + mode.add_argument('--prompt') + mode.add_argument('--prompts-json', type=Path) + parser.add_argument('--output-json', type=Path, help='Atomic output report for a prompt or batch suite') + parser.add_argument('--system', default='You are a helpful assistant.') + parser.add_argument('--max-new-tokens', type=int, default=256) + parser.add_argument('--max-context', type=int, default=2048) + parser.add_argument('--temperature', type=float, default=0.0) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--threads', type=int, default=8) + args = parser.parse_args() + try: + snapshot = inside_workspace(args.snapshot) + tokenizer = inside_workspace(args.tokenizer or snapshot.with_name('tokenizer.model')) + rows = read_prompts(inside_workspace(args.prompts_json)) if args.prompts_json else None + if args.prompt is not None: + if not args.prompt.strip(): + raise ValueError('Prompt cannot be empty') + rows = [dict(id='prompt', prompt=args.prompt)] + output = inside_workspace(args.output_json, must_exist=False) if args.output_json else None + if output is not None and (rows is None or output.exists() + or not output.is_relative_to(PACKAGE / 'work') + or output.is_relative_to(snapshot.parent) + or output.is_relative_to(PACKAGE / 'work/archives') + or output == tokenizer + or (args.prompts_json and output == args.prompts_json.resolve())): + raise ValueError('Select a new batch report under work/, separate from all model and input files') + device = torch.device(args.device) + if device.type not in ('cuda', 'cpu'): + raise ValueError('Device must be CUDA or CPU') + if args.threads < 1 or not 0 < args.max_new_tokens < args.max_context <= 2048: + raise ValueError('Require threads >= 1 and 0 < max_new_tokens < max_context <= 2048') + if not math.isfinite(args.temperature) or args.temperature < 0 or args.seed < 0: + raise ValueError('Temperature and seed must be nonnegative') + if not isinstance(args.system, str): + raise ValueError('System instruction must be text') + if device.type == 'cuda': + torch.cuda.set_device(device) + if not torch.cuda.is_bf16_supported(): + raise ValueError('Selected GPU must support BF16') + torch.set_num_threads(args.threads) + print(f'Loading sealed snapshot {snapshot}', flush=True) + model, sp, metadata = load_model(snapshot, tokenizer) + model.to(device=device, dtype=torch.float32) + decoder = CachedDecoder(model) + except (OSError, ValueError, RuntimeError) as error: + parser.error(str(error)) + print(f'{metadata.get("stage", "posttrain")} step {metadata["step"]:,} | ' + f'{model.num_parameters() / 1e9:.3f}B | {device} BF16 compute, FP32 master weights', flush=True) + + def messages(prompt, system): + prefix = [{'role': 'system', 'content': system}] if system.strip() else [] + return prefix + [{'role': 'user', 'content': prompt}] + + def reply(conversation, *, seed): + result = generate(decoder, sp, conversation, max_new_tokens=args.max_new_tokens, + max_context=args.max_context, temperature=args.temperature, seed=seed) + print(result['response'], flush=True) + print(f'[{result["generated_tokens"]} tokens; {result["elapsed_seconds"]:.2f}s; ' + f'stop={result["stop_reason"]}]', flush=True) + return result + + if rows is not None: + report = dict(format='warpstate-posttrain-generation-v1', snapshot=str(snapshot), + snapshot_sha256=json.loads(snapshot.with_name('snapshot.json').read_text())['snapshot_sha256'], + metadata=metadata, tokenizer_sha256=digest(tokenizer), renderer_format=RENDER_FORMAT, + renderer_sha256=renderer_sha256(), inference_sha256=digest(__file__), + device=str(device), compute_dtype='bfloat16', parameter_dtype='float32', + max_context=args.max_context, max_new_tokens=args.max_new_tokens, + complete=False, results=[]) + if output: + output.parent.mkdir(parents=True, exist_ok=True) + for index, row in enumerate(rows): + identity = row.get('id', str(index)) + system = row.get('system', args.system) + if not isinstance(system, str): + raise ValueError('Prompt system override must be text') + print(f'\n[{identity}] {row["prompt"]}', flush=True) + result = reply(messages(row['prompt'], system), seed=args.seed + index) + report['results'].append(dict(id=identity, prompt=row['prompt'], system=system, **result)) + if output: + atomic_json(output, report) + report['complete'] = True + if output: + atomic_json(output, report) + print(f'Results saved: {output}', flush=True) + return 0 + + return interactive_chat(reply, lambda prompt: messages(prompt, args.system), seed=args.seed) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/ops/posttrain_data.py b/ops/posttrain_data.py new file mode 100644 index 0000000000000000000000000000000000000000..ccf17f6bc3d29f4db37e4dd7dfeac3568dee4b67 --- /dev/null +++ b/ops/posttrain_data.py @@ -0,0 +1,71 @@ +"""Frozen, vocabulary-preserving SFT serialization shared by preparation/training. + +Segments are encoded separately: the textual role header, content, and separator. +This is intentional (SentencePiece boundary normalization is not compositional). +Inference must use the same segments. Assistant content and the real EOS token +are targets; role headers, system/user/tool content and separators are masked. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + + +RENDER_FORMAT = 'warpstate-segmented-roles-v1' +IGNORE_INDEX = -100 +ROLES = frozenset({'system', 'user', 'assistant', 'tool'}) + + +def renderer_sha256(): + return hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + + +def encode_messages(messages, tokenizer, max_length=None): + """Return unshifted token IDs/labels, never truncating a conversation. + +Training shifts with ``x = input_ids[:-1], y = labels[1:]``. A valid optional +SentencePiece BOS begins the conversation; only assistant ends receive EOS. +``max_length`` bounds the entire serialized record including BOS/EOS. +""" + if not isinstance(messages, list) or not messages: + raise ValueError('messages must be a nonempty list') + if max_length is not None and (type(max_length) is not int or max_length < 2): + raise ValueError('max_length must be an integer >= 2') + eos = tokenizer.eos_id() + bos = tokenizer.bos_id() + vocab = tokenizer.vocab_size() + if not 0 <= eos < vocab: + raise ValueError('Tokenizer must define a real EOS token') + ids, labels = [], [] + if 0 <= bos < vocab: + ids.append(bos) + labels.append(IGNORE_INDEX) + + def append(text, supervised=False): + tokens = tokenizer.encode(text, out_type=int) + if any(type(token) is not int or not 0 <= token < vocab for token in tokens): + raise ValueError('Tokenizer returned invalid token IDs') + ids.extend(tokens) + labels.extend(tokens if supervised else [IGNORE_INDEX] * len(tokens)) + + for message in messages: + if not isinstance(message, dict) or message.get('role') not in ROLES: + raise ValueError('Unsupported conversation role') + content = message.get('content') + if not isinstance(content, str) or not content.strip(): + raise ValueError('Each message requires nonempty text content') + if message.get('tool_calls'): + raise ValueError('Structured tool calls must be serialized into content by preparation') + assistant = message['role'] == 'assistant' + append(f"<|{message['role']}|>\n") + append(content, assistant) + if assistant: + ids.append(eos) + labels.append(eos) + append('\n') + if max_length is not None and len(ids) > max_length: + raise ValueError(f'Conversation has {len(ids)} tokens, exceeds max_length={max_length}') + if len(ids) < 2 or not any(label != IGNORE_INDEX for label in labels[1:]): + raise ValueError('Conversation has no supervised next-token targets') + return {'input_ids': ids, 'labels': labels} diff --git a/ops/posttrain_handoff.py b/ops/posttrain_handoff.py new file mode 100644 index 0000000000000000000000000000000000000000..4c5cb88ad3fdca35a747e25158589c6b3d7e2f3c --- /dev/null +++ b/ops/posttrain_handoff.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Wait for a clean exact-step base exit, seal its weights, then run SFT once. + +This is an explicitly armed Supervisor job. It never starts/stops base training, +changes its checkpoint, or runs DPO. All checkpoints loaded here are trusted +local artifacts. No CUDA calls are made by this orchestration process. +""" +from __future__ import annotations + +import argparse +import contextlib +import ctypes +import fcntl +import http.client +import json +import math +import os +from pathlib import Path +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import time +import xmlrpc.client + +WORKSPACE = Path('/workspace/WarpState') +PACKAGE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PACKAGE)) +sys.path.insert(0, str(PACKAGE / 'ops')) +import torch +from chat import capture_latest, sha256 +from launch_training import supervise, training_environment, workspace_path + +ARCHIVE_FILES = ('model.pt', 'snapshot.json', 'tokenizer.model', 'ARCHIVE.md') +STOPPED = False + + +def utc_now(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def flush_directory(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def atomic_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + '.tmp') + with temporary.open('w') as stream: + json.dump(value, stream, indent=2, sort_keys=True, allow_nan=False) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + flush_directory(path.parent) + + +def publish_directory(staging, destination): + """Atomic no-replace publication, including when an empty target exists.""" + libc = ctypes.CDLL(None, use_errno=True) + rename = libc.renameat2 + rename.argtypes = (ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, + ctypes.c_uint) + rename.restype = ctypes.c_int + if rename(-100, os.fsencode(staging), -100, os.fsencode(destination), 1): + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), str(destination)) + flush_directory(Path(destination).parent) + + +@contextlib.contextmanager +def exclusive_lock(path): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('a+b') as stream: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + yield stream.fileno() + + +class UnixHTTPConnection(http.client.HTTPConnection): + def __init__(self, path): + super().__init__('localhost', timeout=10) + self.path = str(path) + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect(self.path) + + +class UnixTransport(xmlrpc.client.Transport): + def __init__(self, path): + super().__init__() + self.path = path + + def make_connection(self, host): + return UnixHTTPConnection(self.path) + + +def process_status(socket_path, program): + with xmlrpc.client.ServerProxy('http://localhost/RPC2', + transport=UnixTransport(socket_path)) as rpc: + return rpc.supervisor.getProcessInfo(program) + + +def clean_exit(info): + return info.get('statename') == 'EXITED' and info.get('exitstatus') == 0 + + +def validate_boundary(checkpoint, base, tokenizer_hash): + if checkpoint.get('format') != 'warpstate-server-v1': + raise ValueError('Expected a full base training checkpoint') + for key in ('step', 'tokens_seen'): + if checkpoint.get(key) != base[key]: + raise ValueError(f'Base {key} must equal {base[key]}, got {checkpoint.get(key)}') + if (checkpoint.get('val_step') != base['step'] or + checkpoint.get('val_tokens') != base['tokens_seen']): + raise ValueError('Base checkpoint must contain final validation at the exact stop boundary') + if not math.isfinite(checkpoint.get('val_loss', float('nan'))): + raise ValueError('Base validation loss is not finite') + actual = checkpoint['signature']['data']['signature']['tokenizer_sha256'] + if actual != tokenizer_hash or actual != base['tokenizer_sha256']: + raise ValueError('Base tokenizer identity differs from the pinned posttraining plan') + if not checkpoint.get('model'): + raise ValueError('Base checkpoint contains no model tensors') + for name, tensor in checkpoint['model'].items(): + if tensor.is_floating_point() and tensor.dtype != torch.float32: + raise ValueError(f'Expected original FP32 model weights: {name}: {tensor.dtype}') + + +def copy_regular(source, destination): + """Physical byte copy; never a hardlink, reflink, or mutable shared snapshot.""" + if source.is_symlink() or not source.is_file(): + raise ValueError(f'Expected regular source file: {source}') + with source.open('rb') as reader, destination.open('xb') as writer: + shutil.copyfileobj(reader, writer, length=8 * 1024**2) + writer.flush() + os.fsync(writer.fileno()) + os.chmod(destination, 0o444) + + +def verify_archive(folder, base, expected_model_hash=None, *, sealed=True): + if folder.is_symlink() or not folder.is_dir(): + raise ValueError(f'Archive is not a regular directory: {folder}') + checksums = folder / 'SHA256SUMS' + if checksums.is_symlink() or not checksums.is_file(): + raise ValueError('Archive checksum manifest is missing') + entries = {} + for line in checksums.read_text().splitlines(): + digest, name = line.split(' ', 1) + if name not in ARCHIVE_FILES or name in entries: + raise ValueError('Unexpected or duplicate archive checksum entry') + entries[name] = digest + if set(entries) != set(ARCHIVE_FILES): + raise ValueError('Archive checksum manifest is incomplete') + for name in (*ARCHIVE_FILES, 'SHA256SUMS'): + path = folder / name + if path.is_symlink() or not path.is_file(): + raise ValueError(f'Archive file missing or not regular: {name}') + if sealed and stat.S_IMODE(path.stat().st_mode) & 0o222: + raise ValueError(f'Archive is not sealed read-only: {name}') + if name in entries and sha256(path) != entries[name]: + raise ValueError(f'Archive checksum mismatch: {name}') + metadata = json.loads((folder / 'snapshot.json').read_text()) + if (metadata.get('format') != 'warpstate-chat-v1' or + metadata.get('step') != base['step'] or + metadata.get('tokens_seen') != base['tokens_seen'] or + metadata.get('tokenizer_sha256') != base['tokenizer_sha256']): + raise ValueError('Archive metadata does not match the requested base boundary') + if (metadata.get('snapshot_sha256') != entries['model.pt'] or + sha256(folder / 'tokenizer.model') != base['tokenizer_sha256'] or + metadata.get('snapshot_bytes') != (folder / 'model.pt').stat().st_size): + raise ValueError('Archive model/tokenizer metadata mismatch') + if expected_model_hash and entries['model.pt'] != expected_model_hash: + raise ValueError('Existing archive differs from its recorded immutable identity') + return metadata + + +def tensors_equal(original, exported): + if original.keys() != exported.keys(): + raise ValueError('Archive model parameter names differ from the raw checkpoint') + for name, value in original.items(): + other = exported[name] + if value.dtype != other.dtype or value.shape != other.shape or not torch.equal(value, other): + raise ValueError(f'Archive changed a raw model tensor: {name}') + + +def archive_checkpoint(source, tokenizer, destination, base, revision): + """Called only while both base exclusion locks are held after clean exit.""" + source = Path(source) + if source.is_symlink(): + raise ValueError('The base checkpoint must not be a symlink') + before = source.stat() + checkpoint = torch.load(source, map_location='cpu', weights_only=False, mmap=True) + validate_boundary(checkpoint, base, sha256(tokenizer)) + if destination.exists() or destination.is_symlink(): + metadata = verify_archive(destination, base) + exported = torch.load(destination / 'model.pt', map_location='cpu', + weights_only=False, mmap=True) + tensors_equal(checkpoint['model'], exported['model']) + return metadata + destination.parent.mkdir(parents=True, exist_ok=True) + staging_root = Path(tempfile.mkdtemp(prefix='.archive-76294-', dir=destination.parent)) + # On failure retain the hidden staging directory for diagnosis; never expose + # a partially sealed archive under the requested historical archive name. + exported_path = capture_latest(source, tokenizer, staging_root) + staging = exported_path.parent + exported = torch.load(exported_path, map_location='cpu', weights_only=False, mmap=True) + tensors_equal(checkpoint['model'], exported['model']) + full_hash = sha256(source) + after = source.stat() + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns): + raise RuntimeError('Base checkpoint changed while it was being archived') + copy_regular(tokenizer, staging / 'tokenizer.model') + note = f'''# KEEP: historical WarpState step {base['step']} weights + +User-requested raw model archive created {utc_now()} before SFT. +Preserve this directory during cleanup and chat refreshes. Remove only when +the user explicitly requests removal of this historical archive. + +- Completed update: {base['step']:,}; tokens trained: {base['tokens_seen']:,}. +- Base validation loss: {checkpoint['val_loss']:.12f}. +- Original FP32 model tensors exported by `chat.capture_latest`; every tensor + was compared exactly with the stopped base checkpoint before publication. +- Inference/model weights only: no Adam or RNG state. The full base resume + checkpoint is retained separately at `{source}`. +- Full source checkpoint SHA256: `{full_hash}`. +- Repository revision at archival time: `{revision}`. +- Independent physical files: no hardlinks or reflinks. All archive files + are read-only. SFT reads an independent copy and writes its own run directory. +- This is an on-device archive, not an off-device backup. + +Verify from this directory: `sha256sum -c SHA256SUMS`. +Files: model.pt, original snapshot.json, exact tokenizer.model, ARCHIVE.md, +and SHA256SUMS. The step 8K, 16K and 40K archives are preserved separately. +''' + with (staging / 'ARCHIVE.md').open('x') as stream: + stream.write(note) + stream.flush() + os.fsync(stream.fileno()) + with (staging / 'SHA256SUMS').open('x') as stream: + for name in ARCHIVE_FILES: + stream.write(f'{sha256(staging / name)} {name}\n') + stream.flush() + os.fsync(stream.fileno()) + for name in (*ARCHIVE_FILES, 'SHA256SUMS'): + os.chmod(staging / name, 0o444) + os.chmod(staging, 0o555) + flush_directory(staging) + metadata = verify_archive(staging, base) + publish_directory(staging, destination) + staging_root.rmdir() + return metadata + + +def independent_initialization(archive, destination, base, model_hash): + verify_archive(archive, base, model_hash) + if not destination.exists() and not destination.is_symlink(): + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix='.initialization-', dir=destination.parent)) + for name in (*ARCHIVE_FILES, 'SHA256SUMS'): + copy_regular(archive / name, staging / name) + verify_archive(staging, base, model_hash) + flush_directory(staging) + publish_directory(staging, destination) + verify_archive(destination, base, model_hash) + for name in (*ARCHIVE_FILES, 'SHA256SUMS'): + original, copied = (archive / name).stat(), (destination / name).stat() + if (original.st_dev, original.st_ino) == (copied.st_dev, copied.st_ino): + raise ValueError(f'SFT initialization must not hardlink its raw archive: {name}') + return destination / 'model.pt' + + +def data_readiness(folder, plan, *, verify=False): + path = folder / 'manifest.json' + if not path.is_file(): + return False, 'waiting for data manifest' + manifest = json.loads(path.read_text()) + if manifest.get('format') != 'warpstate-posttrain-data-v1' or not manifest.get('complete'): + return False, 'data preparation is incomplete' + if manifest.get('tokenizer_sha256') != plan['base']['tokenizer_sha256']: + raise ValueError('Prepared data uses a different tokenizer') + if manifest.get('max_length') != plan['training']['sequence_length']: + raise ValueError('Prepared data context length differs from the SFT plan') + for name in ('sft_train.jsonl', 'sft_val.jsonl'): + entry = manifest['outputs'].get(name, {}) + if entry.get('rows', 0) < 1 or not (folder / name).is_file(): + raise ValueError(f'Prepared SFT split is empty or missing: {name}') + if verify and sha256(folder / name) != entry.get('sha256'): + raise ValueError(f'Prepared SFT file checksum mismatch: {name}') + return True, 'data manifest complete' if not verify else 'SFT data hashes verified' + + +def source_identity(config): + paths = [config, PACKAGE / 'posttrain.py', PACKAGE / 'run_40b.py', + PACKAGE / 'ops/posttrain_data.py', PACKAGE / 'ops/posttrain_handoff.py', + PACKAGE / 'ops/launch_training.py', PACKAGE / 'ops/loss_experiment.py', + PACKAGE / 'ops/residual_experiment.py', PACKAGE / 'ops/checkpoint_writer.py', + PACKAGE / 'ops/posttrain_preflight.py', PACKAGE / 'ops/training-env.sh', + PACKAGE / 'chat.py', + *sorted((PACKAGE / 'warpstate').glob('*.py'))] + return {str(path.relative_to(WORKSPACE)): sha256(path) for path in paths} + + +def verify_completion(work, plan, model_hash): + """A zero exit can mean pause; only a matching durable final state completes.""" + completion = json.loads((work / 'completion.json').read_text()) + checkpoint = torch.load(work / 'latest.pt', map_location='cpu', weights_only=False, mmap=True) + report = json.loads((work / 'evaluation.json').read_text()) + step, total = checkpoint['step'], checkpoint['total_steps'] + if (checkpoint.get('format') != 'warpstate-posttrain-v1' or + checkpoint.get('completed') is not True or step != total or total < 1 or + checkpoint.get('val_step') != step or not math.isfinite(checkpoint['val_loss'])): + raise ValueError('SFT completion does not have a durable, fully evaluated final checkpoint') + if (checkpoint['signature']['plan'] != plan or + checkpoint['signature']['base_sha256'] != model_hash or + checkpoint['origin']['step'] != plan['base']['step']): + raise ValueError('SFT completion belongs to a different plan or initialization') + if (completion.get('format') != 'warpstate-posttrain-completion-v1' or + completion.get('stage') != 'sft' or completion.get('completed') is not True or + completion.get('paused_for_evaluation') is not True or + completion.get('step') != step or completion.get('total_steps') != total or + completion.get('base_step') != plan['base']['step']): + raise ValueError('SFT completion report differs from its checkpoint') + if (report.get('stage') != 'sft' or report.get('step') != step or + report.get('base_sha256') != model_hash or report.get('val_loss') != checkpoint['val_loss']): + raise ValueError('SFT evaluation report differs from its completed checkpoint') + return completion + + +def verify_preflight(path, processes): + from posttrain import source_identity as training_sources + report = json.loads(path.read_text()) + if (report.get('format') != 'warpstate-posttrain-preflight-v1' or + report.get('accepted') is not True or report.get('processes') != processes or + report.get('bf16_compute') is not True or + report.get('fp32_parameters_and_adam') is not True or + report.get('source_sha256') != training_sources() or + report.get('generator_sha256') != sha256(PACKAGE / 'ops/posttrain_preflight.py')): + raise ValueError('GPU preflight report is missing, failed, or belongs to different sources') + return report + + +def load_plan(path): + plan = json.loads(path.read_text()) + if (plan.get('format') != 'warpstate-posttrain-plan-v1' or + plan.get('stage') != 'sft' or plan.get('after_completion') != 'pause_for_evaluation'): + raise ValueError('This handoff supports SFT followed by an evaluation pause only') + options = plan['handoff'] + paths = {key: workspace_path(WORKSPACE / options[key]) for key in ( + 'base_work_dir', 'supervisor_socket', 'archive_dir', 'work_dir', 'data_dir', 'tokenizer')} + base, work, archive = paths['base_work_dir'], paths['work_dir'], paths['archive_dir'] + if (work == base or work.is_relative_to(base / 'run') or + work.is_relative_to(base / 'archives') or work.is_relative_to(archive)): + raise ValueError('Posttraining must have a separate run directory outside the base run/archive') + if (archive.is_relative_to(work) or paths['data_dir'].is_relative_to(work) or + work.is_relative_to(paths['data_dir'])): + raise ValueError('Archive and prepared data must stay outside the mutable posttraining run') + if options['processes'] != 2 or not 1 <= options['poll_seconds'] <= 60: + raise ValueError('Invalid process count or polling interval') + return plan, paths + + +def request_stop(signum, frame): + global STOPPED + STOPPED = True + + +def run_handoff(config, *, check=False): + plan, paths = load_plan(config) + options, base = plan['handoff'], plan['base'] + work, base_work = paths['work_dir'], paths['base_work_dir'] + state_path = work / 'control/handoff.json' + def status(): + return process_status(paths['supervisor_socket'], options['base_program']) + if check: + info = status() + ready, reason = data_readiness(paths['data_dir'], plan) + print(json.dumps(dict(base_process=info, target=base, data_ready=ready, + data_status=reason, state_path=str(state_path), + handoff=json.loads(state_path.read_text()) if state_path.exists() else None), + indent=2)) + return 0 + state_path.parent.mkdir(parents=True, exist_ok=True) + with exclusive_lock(work / 'control/handoff.lock') as handoff_fd: + identity = source_identity(config) + if state_path.exists(): + state = json.loads(state_path.read_text()) + if state['source_sha256'] != identity: + raise ValueError('Armed handoff source/config changed; inspect and explicitly re-arm') + if state.get('phase') == 'complete': + print('SFT already completed; remaining paused for evaluation.', flush=True) + return 0 + else: + revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=WORKSPACE, + text=True).strip() + state = dict(format='warpstate-posttrain-handoff-v1', armed_at=utc_now(), + source_sha256=identity, git_revision=revision, target=base, + archive_dir=str(paths['archive_dir']), work_dir=str(work)) + + def record(phase, message, **extra): + changed = state.get('phase') != phase or state.get('message') != message + state.update(phase=phase, message=message, updated_at=utc_now(), **extra) + atomic_json(state_path, state) + if changed: + print(f'[{utc_now()}] {phase}: {message}', flush=True) + + stop_path = work / 'control/training-stop-request' + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGUSR1): + signal.signal(sig, request_stop) + try: + while not STOPPED: + info = status() + ready, reason = data_readiness(paths['data_dir'], plan) + if not clean_exit(info): + record('waiting_for_base', f'Base {info["statename"]}; {reason}', + data_ready=ready) + time.sleep(options['poll_seconds']) + continue + # Match the base launcher's outer -> inner lock order. Surviving + # SFT ranks hold a SH pipeline lock, so a retry fails closed. + try: + launcher_context = exclusive_lock(base_work / 'control/training-launcher.lock') + launcher_fd = launcher_context.__enter__() + except BlockingIOError: + record('waiting_for_locks', 'Base launcher is still occupied') + time.sleep(options['poll_seconds']) + continue + try: + with exclusive_lock(base_work / 'pipeline.lock'): + if not clean_exit(status()): + continue + checkpoint_path = base_work / 'run/latest.pt' + raw = torch.load(checkpoint_path, map_location='cpu', + weights_only=False, mmap=True) + if raw.get('step', -1) < base['step']: + record('waiting_for_base', f'Base exited early at step {raw.get("step")}') + del raw + time.sleep(options['poll_seconds']) + continue + validate_boundary(raw, base, sha256(paths['tokenizer'])) + del raw + if source_identity(config) != identity: + raise ValueError('Armed source/config changed before handoff') + record('archiving', 'Sealing original raw FP32 weights at the exact final step') + metadata = archive_checkpoint(checkpoint_path, paths['tokenizer'], + paths['archive_dir'], base, state['git_revision']) + model_hash = metadata['snapshot_sha256'] + record('copying_initialization', 'Creating/verifying independent SFT initialization', + archive_sha256=model_hash) + weights = independent_initialization(paths['archive_dir'], work / 'initialization', + base, model_hash) + while not STOPPED: + ready, reason = data_readiness(paths['data_dir'], plan) + if ready: + break + record('waiting_for_data', reason) + time.sleep(options['poll_seconds']) + if STOPPED: + break + data_readiness(paths['data_dir'], plan, verify=True) + if source_identity(config) != identity: + raise ValueError('Armed source/config changed before SFT launch') + # Only clear a previous stop after proving there are no + # surviving workers that might still need to observe it. + stop_path.unlink(missing_ok=True) + atomic_json(work / 'control/execution.json', dict( + plan=plan, source_sha256=identity, git_revision=state['git_revision'], + archive_sha256=model_hash, data_manifest_sha256=sha256( + paths['data_dir'] / 'manifest.json'))) + # Release pipeline EX before workers acquire SH. Launcher EX + # prevents a base launch in this transition. Pass both outer + # locks to torchrun so watcher death cannot duplicate launch. + if STOPPED: + break + environment = training_environment(work) + preflight_command = [sys.executable, '-B', str(PACKAGE / 'ops/posttrain_preflight.py'), + '--work-dir', str(work / 'preflight'), '--base-work-dir', + str(base_work), '--processes', str(options['processes'])] + record('gpu_preflight', 'Checking two-GPU BF16 SFT, masked tails, and checkpoint resume') + result = supervise(preflight_command, stop_path, environment, + pass_fds=(handoff_fd, launcher_fd)) + if result: + raise RuntimeError(f'GPU preflight exited with status {result}; full SFT was not started') + if stop_path.exists() or STOPPED: + record('paused', 'Handoff stopped during GPU preflight; full SFT was not started') + return 0 + preflight = verify_preflight(work / 'preflight/report.json', options['processes']) + state['preflight'] = preflight + command = [sys.executable, '-B', '-m', 'torch.distributed.run', + '--standalone', '--nnodes', '1', '--nproc-per-node', + str(options['processes']), '--log-dir', str(work / 'logs/torchrun'), + str(PACKAGE / 'posttrain.py'), '--config', str(config), + '--work-dir', str(work / 'run'), '--data-dir', str(paths['data_dir']), + '--weights', str(weights), '--tokenizer', str(paths['tokenizer']), + '--stop-file', str(stop_path)] + record('sft', 'Starting/resuming supervised fine-tuning; fresh optimizer on first start', + command=command) + result = supervise(command, stop_path, environment, + pass_fds=(handoff_fd, launcher_fd)) + if result: + raise RuntimeError(f'SFT child exited with status {result}; preserved checkpoints for diagnosis') + # Completion is a trainer-published artifact, never inferred + # from return code alone (graceful early stops also return 0). + completion = work / 'run/completion.json' + if completion.is_file(): + completed = verify_completion(work / 'run', plan, model_hash) + verify_archive(paths['archive_dir'], base, model_hash) + if sha256(weights) != model_hash: + raise ValueError('SFT initialization changed during training') + record('complete', 'SFT finished; paused for evaluation. DPO has not been started.', + completion=completed) + else: + record('paused', 'SFT stopped before completion; its independent resume checkpoint is retained') + return 0 + finally: + launcher_context.__exit__(None, None, None) + record('paused', 'Handoff paused by request; no further training will start automatically') + return 0 + except BaseException as error: + record('failed', f'{type(error).__name__}: {error}') + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', type=Path, required=True) + parser.add_argument('--check', action='store_true', help='Read-only status; do not arm or launch') + args = parser.parse_args() + return run_handoff(workspace_path(args.config), check=args.check) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ops/posttrain_preflight.py b/ops/posttrain_preflight.py new file mode 100644 index 0000000000000000000000000000000000000000..d39bd5e33b0aa732831c211467cdaff8dc6e5673 --- /dev/null +++ b/ops/posttrain_preflight.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Run a tiny, independent two-rank BF16 SFT/resume gate before production SFT. + +Fixture creation is CPU-only. The default executable then launches actual CUDA +training, so orchestration must wait until pretraining has exited. All generated +weights/data belong to this disposable preflight directory, never to an archive. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +from pathlib import Path +import sys + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +if str(PACKAGE) not in sys.path: + sys.path.insert(0, str(PACKAGE)) + +import torch + +from posttrain import atomic_json, digest, shared_pipeline_lock, source_identity +from ops.posttrain_data import encode_messages, renderer_sha256 +from ops.launch_training import supervise + + +def make_fixture(directory, *, device='cpu', compile_core=False, base_work_dir=None): + """Create/reuse a source-hashed small real model with a masked distributed tail.""" + import sentencepiece as spm + from warpstate import WarpStateConfig, WarpStateLM + directory = Path(directory).resolve() + if not directory.is_relative_to(WORKSPACE) or directory == WORKSPACE: + raise ValueError('Preflight fixtures must stay inside the workspace') + if device not in ('cpu', 'cuda') or type(compile_core) is not bool: + raise ValueError('Invalid fixture execution mode') + tokenizer = PACKAGE / 'tokenizer.model' + base = Path(base_work_dir).resolve() if base_work_dir is not None else None + if base is not None and (not base.is_relative_to(WORKSPACE) or not base.is_dir()): + raise ValueError('Invalid base work directory') + if base is not None and (directory == base or directory.is_relative_to(base / 'run')): + raise ValueError('Preflight must not overwrite the base run') + args = argparse.Namespace(config=directory / 'plan.json', data_dir=directory / 'data', + weights=directory / 'base/model.pt', tokenizer=tokenizer, + work_dir=directory / 'run', max_steps=None, stop_file=None, device=device) + identity = dict(device=device, compile_core=compile_core, base_work_dir=str(base) if base else None, + tokenizer_sha256=digest(tokenizer), renderer_sha256=renderer_sha256(), + generator_sha256=digest(__file__), runner_sources=source_identity()) + marker = directory / 'fixture.json' + if marker.exists(): + old = json.loads(marker.read_text()) + if old['identity'] != identity: + raise ValueError('Existing preflight fixture settings/source differ; use a new fixture directory') + for filename, checksum in old['files'].items(): + if digest(directory / filename) != checksum: + raise ValueError(f'Existing preflight input changed: {filename}') + return args + if directory.exists() and any(directory.iterdir()): + raise ValueError('Refusing to create fixtures in a nonempty unrecognized directory') + directory.mkdir(parents=True, exist_ok=True) + args.weights.parent.mkdir() + args.data_dir.mkdir() + sp = spm.SentencePieceProcessor(model_file=str(tokenizer)) + cfg = WarpStateConfig(vocab_size=sp.vocab_size(), d_model=32, n_heads=4, + n_cores=1, logical_depth=1, ffn_hidden=64, + chunk_size=16, parallel_memory=True, + gradient_checkpointing=False, dropout=0.1 if device == 'cpu' else 0.0) + with torch.random.fork_rng(devices=[]): + torch.manual_seed(123) + model = WarpStateLM(cfg) + torch.save(dict(format='warpstate-chat-v1', inference_only=True, synthetic_fixture=True, + config=vars(cfg), model=model.state_dict(), step=76294, tokens_seen=10_000_007_168, + tokenizer_sha256=identity['tokenizer_sha256']), args.weights) + sequence_length = 128 + outputs = {} + for filename, count in [('sft_train.jsonl', 5), ('sft_val.jsonl', 2)]: + rows = [] + for index in range(count): + messages = [{'role': 'system', 'content': 'Be concise.'}, + {'role': 'user', 'content': 'Two plus two?'}, + {'role': 'assistant', 'content': 'four' if index % 2 else 'The answer is four.'}] + encoded = encode_messages(messages, sp, max_length=sequence_length) + rows.append(dict(messages=messages, tokens=len(encoded['input_ids']), + supervised_tokens=sum(v != -100 for v in encoded['labels'][1:]))) + path = args.data_dir / filename + path.write_text(''.join(json.dumps(row) + '\n' for row in rows)) + outputs[filename] = dict(sha256=digest(path), rows=count, + tokens=sum(row['tokens'] for row in rows), + supervised_tokens=sum(row['supervised_tokens'] for row in rows)) + atomic_json(args.data_dir / 'manifest.json', dict(format='warpstate-posttrain-data-v1', + complete=True, outputs=outputs, tokenizer_sha256=identity['tokenizer_sha256'], + renderer_sha256=identity['renderer_sha256'], max_length=sequence_length)) + plan = dict(format='warpstate-posttrain-plan-v1', stage='sft', base={'step': 76294}, + training=dict(sequence_length=sequence_length, micro_batch=1, grad_accum=2, lr=1e-3, + min_lr=1e-4, warmup_ratio=0., eval_every=2, save_every=2, eval_batches=2, + compile_core=compile_core, loss_mode='compiled-full' if compile_core else 'full', workers=1)) + if base is not None: + plan['handoff'] = {'base_work_dir': str(base.relative_to(WORKSPACE))} + atomic_json(args.config, plan) + files = [args.weights, args.config, args.data_dir / 'manifest.json', + args.data_dir / 'sft_train.jsonl', args.data_dir / 'sft_val.jsonl'] + atomic_json(marker, dict(format='warpstate-posttrain-fixture-v1', identity=identity, + files={str(path.relative_to(directory)): digest(path) for path in files})) + return args + + +def training_command(args, processes, work_dir, stop_file, max_steps=None): + if type(processes) is not int or processes != 2: + raise ValueError('The preflight requires exactly two ranks to exercise its masked tail') + command = [sys.executable, '-B', '-m', 'torch.distributed.run', '--standalone', + '--nnodes', '1', '--nproc-per-node', str(processes), str(PACKAGE / 'posttrain.py'), + '--config', str(args.config), '--data-dir', str(args.data_dir), + '--weights', str(args.weights), '--tokenizer', str(args.tokenizer), + '--work-dir', str(work_dir), '--stop-file', str(stop_file), '--device', args.device] + if max_steps is not None: + command += ['--max-steps', str(max_steps)] + return command + + +def inherited_fds(): + """Preserve the explicit inheritable exclusion locks passed by the guardian.""" + descriptors = [] + for name in os.listdir('/proc/self/fd'): + fd = int(name) + try: + if fd > 2 and os.get_inheritable(fd): + descriptors.append(fd) + except OSError: + pass + return tuple(descriptors) + + +def compare_tree(a, b, path='state', *, tolerant=False, report=None): + if report is None: + report = {'bitwise_equal': True, 'max_absolute_difference': 0., 'tensor_count': 0} + if isinstance(a, torch.Tensor): + if not isinstance(b, torch.Tensor) or a.shape != b.shape or a.dtype != b.dtype: + raise ValueError(f'Stop/resume tensor metadata mismatch at {path}') + report['tensor_count'] += 1 + equal = torch.equal(a, b) + report['bitwise_equal'] &= equal + if a.numel(): + report['max_absolute_difference'] = max(report['max_absolute_difference'], float((a.double() - b.double()).abs().max())) + if not equal and (not tolerant or not torch.allclose(a, b, rtol=1e-5, atol=1e-7)): + raise ValueError(f'Stop/resume mismatch exceeds allowed tolerance at {path}') + elif isinstance(a, dict): + if a.keys() != b.keys(): + raise ValueError(f'State keys differ at {path}') + for key in a: + compare_tree(a[key], b[key], f'{path}.{key}', tolerant=tolerant, report=report) + elif isinstance(a, (tuple, list)): + if len(a) != len(b): + raise ValueError(f'State lengths differ at {path}') + for index, (x, y) in enumerate(zip(a, b)): + compare_tree(x, y, f'{path}.{index}', tolerant=tolerant, report=report) + elif a != b: + raise ValueError(f'State differs at {path}') + return report + + +def verify_fixture_run(work, *, expected_step, completed): + checkpoint = torch.load(work / 'latest.pt', map_location='cpu', weights_only=False, mmap=True) + if checkpoint.get('format') != 'warpstate-posttrain-v1' or checkpoint['step'] != expected_step: + raise ValueError('Preflight checkpoint format/step differs') + if checkpoint['completed'] is not completed or checkpoint['signature']['world_size'] != 2: + raise ValueError('Preflight completion/distributed state differs') + if len(checkpoint['rng_by_rank']) != 2: + raise ValueError('Preflight lacks both rank RNG states') + for name, value in checkpoint['model'].items(): + if value.dtype != torch.float32 or not torch.isfinite(value).all(): + raise ValueError(f'Invalid FP32 model weight: {name}') + for value in checkpoint['optimizer']['state'].values(): + if float(value['step']) != expected_step: + raise ValueError('Optimizer step was not restored') + for name in ('exp_avg', 'exp_avg_sq'): + if value[name].dtype != torch.float32 or not torch.isfinite(value[name]).all(): + raise ValueError('Invalid FP32 Adam state') + metrics = [json.loads(line) for line in (work / 'metrics.jsonl').read_text().splitlines()] + updates = [row for row in metrics if row['event'] == 'train'] + for row in updates: + if not all(math.isfinite(row[key]) for key in ('loss', 'grad_norm', 'lr')): + raise ValueError('Non-finite preflight training metric') + if completed: + completion = json.loads((work / 'completion.json').read_text()) + evaluation = json.loads((work / 'evaluation.json').read_text()) + if not completion['completed'] or not completion['paused_for_evaluation']: + raise ValueError('Preflight did not pause for evaluation') + if checkpoint['sample_cursor'] != 5 or updates[-1]['update_samples'] != 1: + raise ValueError('Preflight did not exercise a nondivisible masked tail') + if evaluation['step'] != expected_step or not math.isfinite(evaluation['val_loss']): + raise ValueError('Preflight final evaluation is missing') + return checkpoint + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--work-dir', type=Path, required=True) + parser.add_argument('--base-work-dir', type=Path, required=True) + parser.add_argument('--processes', type=int, default=2) + parser.add_argument('--prepare-only', action='store_true', help='CPU fixture creation only; never launches training') + args = parser.parse_args() + if args.processes != 2: + parser.error('--processes must be 2 for the masked-tail fixture') + work = args.work_dir.resolve() + base = args.base_work_dir.resolve() + if not base.is_relative_to(WORKSPACE) or not base.is_dir(): + parser.error('--base-work-dir must be an existing workspace directory') + stop_file = Path(os.environ.get('WARPSTATE_STOP_FILE', work / 'control/training-stop-request')).resolve() + if not stop_file.is_relative_to(WORKSPACE): + parser.error('Stop file must stay inside the workspace') + # Keep the base excluded throughout the gaps between the three torchrun jobs. + with shared_pipeline_lock(base / 'pipeline.lock'): + fixture = make_fixture(work, device='cuda', compile_core=True, base_work_dir=base) + if args.prepare_only: + print(json.dumps({'prepared': True, 'config': str(fixture.config)}), flush=True) + return 0 + stop_file.parent.mkdir(parents=True, exist_ok=True) + raw_hash = digest(fixture.weights) + if stop_file.exists(): + return 0 + report_path = work / 'report.json' + report_path.unlink(missing_ok=True) + phases = [('split', 1), ('split', None), ('continuous', None)] + for phase, limit in phases: + if stop_file.exists(): + return 0 + target = work / phase + # A previous successful phase is not rerun, preserving exact metrics. + if (target / 'completion.json').exists(): + continue + if limit == 1 and (target / 'latest.pt').exists(): + continue + command = training_command(fixture, args.processes, target, stop_file, limit) + result = supervise(command, stop_file, os.environ.copy(), pass_fds=inherited_fds()) + if stop_file.exists(): + return 0 + if result: + raise RuntimeError(f'Preflight {phase} failed with status {result}') + verify_fixture_run(target, expected_step=1 if limit else 2, completed=limit is None) + resumed = verify_fixture_run(work / 'split', expected_step=2, completed=True) + continuous = verify_fixture_run(work / 'continuous', expected_step=2, completed=True) + comparison = {key: compare_tree(resumed[key], continuous[key], key, tolerant=True) + for key in ('model', 'optimizer')} + for key in ('tokens_seen', 'assistant_tokens_seen', 'sample_cursor', 'step'): + compare_tree(resumed[key], continuous[key], key) + for rank in range(2): + for key in ('torch', 'cuda', 'python'): + compare_tree(resumed['rng_by_rank'][rank][key], continuous['rng_by_rank'][rank][key], f'rng.{rank}.{key}') + a, b = resumed['rng_by_rank'][rank]['numpy'], continuous['rng_by_rank'][rank]['numpy'] + if a[0] != b[0] or not (a[1] == b[1]).all() or a[2:] != b[2:]: + raise ValueError('NumPy RNG state differs after resume') + if not math.isclose(resumed['val_loss'], continuous['val_loss'], rel_tol=1e-5, abs_tol=1e-7): + raise ValueError('Validation loss differs after resume') + if digest(fixture.weights) != raw_hash: + raise ValueError('Raw preflight fixture weights changed') + if not any(not torch.equal(value, resumed['model'][name]) for name, value in + torch.load(fixture.weights, map_location='cpu', weights_only=False, mmap=True)['model'].items()): + raise ValueError('Preflight did not update model parameters') + atomic_json(report_path, dict(format='warpstate-posttrain-preflight-v1', accepted=True, + processes=2, bf16_compute=True, fp32_parameters_and_adam=True, + exact_resume_bookkeeping=True, numerical_rtol=1e-5, numerical_atol=1e-7, + numerical_comparison=comparison, masked_tail=True, total_samples=5, optimizer_steps=2, + fixture_sha256=raw_hash, source_sha256=resumed['signature']['sources'], + generator_sha256=digest(__file__), final_validation_loss=resumed['val_loss'])) + print(f'Posttraining preflight accepted: {report_path}', flush=True) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/ops/prepare-data.sh b/ops/prepare-data.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed80a7fb0a57dcfd215e61f60eb1d030c3b91593 --- /dev/null +++ b/ops/prepare-data.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +ulimit -c 0 + +export TMPDIR=/workspace/WarpState/tmp +export XDG_CACHE_HOME=/workspace/WarpState/.cache +export HF_HOME=/workspace/WarpState/rtx6000_40b/work/hf-cache +export HF_DATASETS_CACHE="$HF_HOME/datasets" +export HF_XET_CACHE="$HF_HOME/xet" +export HF_HUB_DOWNLOAD_TIMEOUT=120 +export HF_HUB_ETAG_TIMEOUT=30 +export PYTHONDONTWRITEBYTECODE=1 +export CUDA_VISIBLE_DEVICES="" +export OMP_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +cd /workspace/WarpState/rtx6000_40b +exec /usr/bin/nice -n 10 .data-venv/bin/python -u ops/prepare_data.py \ + --no-bootstrap \ + --stage prepare \ + --work-dir /workspace/WarpState/rtx6000_40b/work \ + --train-tokens 40000000000 \ + --val-tokens 1048576 \ + --workers 16 diff --git a/ops/prepare-posttrain-data.sh b/ops/prepare-posttrain-data.sh new file mode 100644 index 0000000000000000000000000000000000000000..8aa3b14cc49c4bff7d1e16185a036e1658e4dfec --- /dev/null +++ b/ops/prepare-posttrain-data.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +exec rtx6000_40b/work/posttrain-data-venv/bin/python -u -B \ + prepare_posttrain_data.py \ + --recipe rtx6000_40b/configs/posttrain_data.json \ + --out rtx6000_40b/work/posttrain/data "$@" diff --git a/ops/prepare_data.py b/ops/prepare_data.py new file mode 100644 index 0000000000000000000000000000000000000000..35229b8d4c2030155f8e0c6d2c7525253256bb6a --- /dev/null +++ b/ops/prepare_data.py @@ -0,0 +1,33 @@ +"""Run data preparation, then exit after all pipeline files are committed.""" +from pathlib import Path +import os +import sys +import traceback + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run_40b + + +if __name__ == "__main__": + exit_code = 0 + try: + run_40b.main() + except SystemExit as exc: + if isinstance(exc.code, int): + exit_code = exc.code + elif exc.code is not None: + print(exc.code, file=sys.stderr) + exit_code = 1 + except BaseException: + traceback.print_exc() + exit_code = 1 + + # main() has unwound its file/lock contexts. Successful preparation and + # graceful stops explicitly flush and fsync token files and atomic state. + # PyArrow 25.0.1 still crashes during interpreter finalization on this + # host; gc.collect() alone did not fix it. Exit only after main returns + # (or its failure is reported), so native finalizers cannot race Python. + # Failures stay nonzero, allowing Supervisor to resume committed state. + sys.stdout.flush() + sys.stderr.flush() + os._exit(exit_code) diff --git a/ops/prepare_dpo_chat.py b/ops/prepare_dpo_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..99ec8c46be32d7a3b0bc3d1e461b13d778f81aab --- /dev/null +++ b/ops/prepare_dpo_chat.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Verify the final DPO archive and create an independent sealed chat copy.""" + +import json +from pathlib import Path +import sys + +PACKAGE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PACKAGE)) + +from ops.archive_dpo import verify_dpo_archive +from ops.dpo_handoff import initialize_copy + + +def main(): + archive = PACKAGE / 'work/archives/step-00076294-dpo-final' + destination = PACKAGE / 'work/chat/dpo-final' + metadata, _ = verify_dpo_archive(archive) + model = initialize_copy(archive, destination, metadata) + print(json.dumps(dict(snapshot=str(model), archive=str(archive), + model_sha256=metadata['snapshot_sha256'], independent_copy=True), indent=2)) + + +if __name__ == '__main__': + main() diff --git a/ops/requirements-posttrain-data.lock b/ops/requirements-posttrain-data.lock new file mode 100644 index 0000000000000000000000000000000000000000..06bf65a14ecb750e8ca592d05c46dc1579443d02 --- /dev/null +++ b/ops/requirements-posttrain-data.lock @@ -0,0 +1,44 @@ +# Isolated CPU preparation environment; Python 3.12.3. +# Resolved from huggingface_hub==1.31.0, pyarrow==25.0.1, sentencepiece==0.2.2, +# datasets==5.0.1 and pytest==9.1.1 on 2026-09-14. Dataset recipe directly +# requires the first three; the remaining packages reproduce the provisioned environment. +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.15.1 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +datasets==5.0.1 +dill==0.4.1 +filelock==3.32.6 +frozenlist==1.8.0 +fsspec==2026.6.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.31.0 +idna==3.19 +iniconfig==2.3.0 +multidict==6.8.0 +multiprocess==0.70.19 +numpy==2.5.3 +packaging==26.3 +pandas==3.0.5 +pluggy==1.6.0 +propcache==0.5.2 +pyarrow==25.0.1 +Pygments==2.21.0 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +PyYAML==6.0.3 +requests==2.34.2 +sentencepiece==0.2.2 +six==1.17.0 +tqdm==4.70.1 +typing_extensions==4.16.0 +urllib3==2.7.0 +xxhash==4.0.1 +yarl==1.24.5 diff --git a/ops/residual_experiment.py b/ops/residual_experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..5e92c047eb99adbe2bbf6a6e2190fcd942929ee6 --- /dev/null +++ b/ops/residual_experiment.py @@ -0,0 +1,58 @@ +"""BF16 residual-stream experiment with unchanged FP32 trainable parameters. + +Only embedding and depth modulation use temporary, differentiable BF16 views. +The existing forward owns every model operation. Core matrix weights, parameter +gradients, normalization reductions, optimizer state, and checkpoint names retain +their existing representation. Residual rounding changes; this is experimental. +""" + +from __future__ import annotations + +import torch +from torch.func import functional_call + + +class _OriginalForward(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.wrapped = model + self.original_forward = model.forward + + def forward(self, *args, **kwargs): + return self.original_forward(*args, **kwargs) + + +def apply_bf16_residuals(model: torch.nn.Module) -> None: + """Apply after loading weights/placing the model and before wrapping in DDP. + + ``functional_call`` restores all master parameter references after each + forward, including on errors. Casts remain differentiable, so gradients flow + to the original FP32 leaves and the existing optimizer continues to own them. + The temporary wrapper is held in the forward closure, never registered as a + child of the model, which preserves the canonical state dictionary. + + BF16 views are enabled only inside BF16 autocast. Inference calls outside + autocast retain the original forward's behavior. + """ + if getattr(model, "_bf16_residuals", False): + raise ValueError("BF16 residual execution has already been applied") + parameters = { + "wrapped.embed.weight": model.embed.weight, + "wrapped.depth_scale": model.depth_scale, + "wrapped.depth_bias": model.depth_bias, + } + if any(parameter.dtype != torch.float32 for parameter in parameters.values()): + raise ValueError("BF16 residual execution requires FP32 master parameters") + wrapper = _OriginalForward(model) + + def forward(input_ids, targets=None, *, return_logits=True): + device_type = input_ids.device.type + kwargs = {"targets": targets, "return_logits": return_logits} + if (not torch.is_autocast_enabled(device_type) + or torch.get_autocast_dtype(device_type) != torch.bfloat16): + return wrapper.original_forward(input_ids, **kwargs) + views = {name: parameter.to(torch.bfloat16) for name, parameter in parameters.items()} + return functional_call(wrapper, views, (input_ids,), kwargs, strict=False) + + model.forward = forward + model._bf16_residuals = True diff --git a/ops/supervisord.conf b/ops/supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..ac4f800f95ddb54aa038f60de74964cf46382a27 --- /dev/null +++ b/ops/supervisord.conf @@ -0,0 +1,59 @@ +[unix_http_server] +file=/workspace/WarpState/rtx6000_40b/work/control/supervisor.sock +chmod=0700 + +[supervisord] +logfile=/workspace/WarpState/rtx6000_40b/work/logs/supervisor.log +logfile_maxbytes=10MB +logfile_backups=3 +pidfile=/workspace/WarpState/rtx6000_40b/work/control/supervisord.pid +childlogdir=/workspace/WarpState/rtx6000_40b/work/logs +directory=/workspace/WarpState/rtx6000_40b +user=root +umask=022 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///workspace/WarpState/rtx6000_40b/work/control/supervisor.sock + +[include] +files=/workspace/WarpState/rtx6000_40b/ops/benchmark-supervisord.conf /workspace/WarpState/rtx6000_40b/ops/posttrain-supervisord.conf /workspace/WarpState/rtx6000_40b/ops/dpo-supervisord.conf + +[program:warpstate-prepare] +command=/workspace/WarpState/rtx6000_40b/ops/prepare-data.sh +directory=/workspace/WarpState/rtx6000_40b +# The corpus is complete; preparation is an explicit maintenance action. +autostart=false +autorestart=false +exitcodes=0 +startsecs=5 +startretries=5 +stopsignal=TERM +stopwaitsecs=600 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/prepare.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=5 +redirect_stderr=true + +[program:warpstate-train] +# User-requested pause at the first completed update past 10B tokens. +# Keep the absolute boundary across restarts until a later resume is requested. +command=/workspace/WarpState/rtx6000_40b/ops/train-local-resume.sh --stop-at-step 76294 +directory=/workspace/WarpState/rtx6000_40b +autostart=false +# A failed memory check must not become a restart loop. +autorestart=false +startsecs=5 +startretries=0 +stopsignal=USR1 +stopwaitsecs=900 +stopasgroup=true +killasgroup=true +stdout_logfile=/workspace/WarpState/rtx6000_40b/work/logs/train.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=5 +redirect_stderr=true diff --git a/ops/tmux-workspace-install.md b/ops/tmux-workspace-install.md new file mode 100644 index 0000000000000000000000000000000000000000..c4ee6b548550ba5685e0827582e04885becc6b58 --- /dev/null +++ b/ops/tmux-workspace-install.md @@ -0,0 +1,56 @@ +# Workspace-local tmux + +The B200 host uses Ubuntu 24.04 amd64 packages extracted into the ignored +`/workspace/WarpState/.tools` directory. Downloads stay in ignored +`/workspace/WarpState/tmp/tmux-install`. No system package installation is +required. [tmux-workspace.sh](tmux-workspace.sh) supplies the local library +path and uses `work/control/b200-tmux.sock` with an empty tmux configuration. +It sources the usual environment scripts without changing their contents. + +Package provenance comes from the host's authenticated Ubuntu APT indexes; +the downloaded package SHA256 hashes matched those indexes. Exact sources: + +| Package | Version | Download | +| --- | --- | --- | +| `tmux` | `3.4-1ubuntu0.1` | [Ubuntu archive](http://archive.ubuntu.com/ubuntu/pool/main/t/tmux/tmux_3.4-1ubuntu0.1_amd64.deb) | +| `libevent-core-2.1-7t64` | `2.1.12-stable-9ubuntu2.1` | [Ubuntu archive](http://archive.ubuntu.com/ubuntu/pool/main/libe/libevent/libevent-core-2.1-7t64_2.1.12-stable-9ubuntu2.1_amd64.deb) | +| `libutempter0` | `1.2.1-3build1` | [Ubuntu archive](http://archive.ubuntu.com/ubuntu/pool/main/libu/libutempter/libutempter0_1.2.1-3build1_amd64.deb) | + +The extracted dependencies are `libevent-core-2.1-7t64` and `libutempter0`. +The host already supplies the remaining direct tmux dependencies: `libc6` +(>= 2.38), `libsystemd0` (>= 221), and `libtinfo6` +(>= 6.4+20230603, < 6.5), together with their dependencies. + +To reproduce the installation from the exact package URLs and verify the +expected bytes before extraction: + +```bash +set -euo pipefail +cd /workspace/WarpState +source rtx6000_40b/ops/training-env.sh +source rtx6000_40b/ops/workspace-tools.sh +mkdir -p tmp/tmux-install .tools +cd tmp/tmux-install +curl --fail --location --output tmux_3.4-1ubuntu0.1_amd64.deb \ + http://archive.ubuntu.com/ubuntu/pool/main/t/tmux/tmux_3.4-1ubuntu0.1_amd64.deb +curl --fail --location --output libevent-core-2.1-7t64_2.1.12-stable-9ubuntu2.1_amd64.deb \ + http://archive.ubuntu.com/ubuntu/pool/main/libe/libevent/libevent-core-2.1-7t64_2.1.12-stable-9ubuntu2.1_amd64.deb +curl --fail --location --output libutempter0_1.2.1-3build1_amd64.deb \ + http://archive.ubuntu.com/ubuntu/pool/main/libu/libutempter/libutempter0_1.2.1-3build1_amd64.deb +sha256sum --check <<'SHA256' +2913e17aa61879d1f905aab6f374f153f1ece63aefba1b996e387325d1d5338d tmux_3.4-1ubuntu0.1_amd64.deb +f8af913312f8de9cab6efcd2259918b3df92f5b7d501aba7ce9a55699fa452b8 libevent-core-2.1-7t64_2.1.12-stable-9ubuntu2.1_amd64.deb +46c0dfbfeca19ba0773398d654b7e7225b178f26b414f0e2f8250519551bfb32 libutempter0_1.2.1-3build1_amd64.deb +SHA256 +for tmux_package in \ + tmux_3.4-1ubuntu0.1_amd64.deb \ + libevent-core-2.1-7t64_2.1.12-stable-9ubuntu2.1_amd64.deb \ + libutempter0_1.2.1-3build1_amd64.deb; do + dpkg-deb -x "$tmux_package" /workspace/WarpState/.tools +done +/workspace/WarpState/rtx6000_40b/ops/tmux-workspace.sh -V +``` + +The verified binary reports `tmux 3.4`. To inspect sessions, run +`rtx6000_40b/ops/tmux-workspace.sh list-sessions`; attach with +`rtx6000_40b/ops/tmux-workspace.sh attach -t SESSION_NAME`. diff --git a/ops/tmux-workspace.sh b/ops/tmux-workspace.sh new file mode 100644 index 0000000000000000000000000000000000000000..3cc92726e0bb33b8507a54599f773bcb9c370fe8 --- /dev/null +++ b/ops/tmux-workspace.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +source /workspace/WarpState/rtx6000_40b/ops/workspace-tools.sh +export TMUX_TMPDIR=/workspace/WarpState/rtx6000_40b/work/control +mkdir -p "$TMUX_TMPDIR" +LD_LIBRARY_PATH="/workspace/WarpState/.tools/usr/lib/x86_64-linux-gnu${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + exec /workspace/WarpState/.tools/usr/bin/tmux \ + -S "$TMUX_TMPDIR/b200-tmux.sock" -f /dev/null "$@" diff --git a/ops/train-b200.sh b/ops/train-b200.sh new file mode 100644 index 0000000000000000000000000000000000000000..588569c90823d5fb4986e3ed900e32314cf9f892 --- /dev/null +++ b/ops/train-b200.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +ulimit -c 0 +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +if [[ -f /workspace/WarpState/tmp/remote-setup/toolchain-env.sh ]]; then + source /workspace/WarpState/tmp/remote-setup/toolchain-env.sh +fi +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES-0}" +export TORCHINDUCTOR_COMPILE_THREADS="${TORCHINDUCTOR_COMPILE_THREADS:-8}" +cd /workspace/WarpState/rtx6000_40b +exec .server-venv/bin/python -u ops/launch_training.py \ + --processes 1 --work-dir /workspace/WarpState/rtx6000_40b/work -- \ + --no-bootstrap --stage train --reconfigure \ + --seq-len 2048 --micro-batch 16 --grad-accum 4 \ + --compile-core --compile-mode max-autotune-no-cudagraphs \ + --no-activation-checkpointing --autocast-cache \ + --loss-mode compiled-full --loss-chunk-size 1024 --async-checkpoint \ + --vram-limit-gib 150 --vram-headroom-gib 3 \ + --workers 4 --eval-batches 8 --eval-micro-batch 1 --eval-every 100 --save-every 50 \ + "$@" diff --git a/ops/train-fast.sh b/ops/train-fast.sh new file mode 100644 index 0000000000000000000000000000000000000000..2b2091b895b6c38fd605ca0825b05427b1beb2e7 --- /dev/null +++ b/ops/train-fast.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +ulimit -c 0 +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +export CUDA_VISIBLE_DEVICES=GPU-7091a369-8eef-e6c9-81b2-679a8e6a78d6,GPU-a32efffc-8eb9-fb3e-b951-6f8cea2edd91 +cd /workspace/WarpState/rtx6000_40b +exec .server-venv/bin/python -u ops/launch_training.py \ + --processes 2 --work-dir /workspace/WarpState/rtx6000_40b/work -- \ + --no-bootstrap --stage train --reconfigure \ + --seq-len 2048 --micro-batch 4 --grad-accum 8 \ + --compile-core --no-activation-checkpointing --autocast-cache --loss-chunk-size 1024 \ + --vram-limit-gib 88 --vram-headroom-gib 3 \ + --workers 4 --eval-batches 8 --eval-micro-batch 1 --eval-every 100 --save-every 50 \ + "$@" diff --git a/ops/train-local-resume.sh b/ops/train-local-resume.sh new file mode 100644 index 0000000000000000000000000000000000000000..c4b40ac32ee0a36dcae4334e0057c808444feec5 --- /dev/null +++ b/ops/train-local-resume.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +ulimit -c 0 +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +# Both workstation GPUs must have room for the validated two-GPU preset. +export CUDA_VISIBLE_DEVICES=GPU-7091a369-8eef-e6c9-81b2-679a8e6a78d6,GPU-a32efffc-8eb9-fb3e-b951-6f8cea2edd91 +cd /workspace/WarpState/rtx6000_40b +if [[ ! -s work/run/latest.pt ]]; then + printf '%s\n' 'Resume requires the verified full checkpoint at work/run/latest.pt.' >&2 + exit 1 +fi +exec .server-venv/bin/python -u ops/launch_training.py \ + --processes 2 --work-dir /workspace/WarpState/rtx6000_40b/work -- \ + --no-bootstrap --stage train --reconfigure \ + --seq-len 2048 --micro-batch 4 --grad-accum 8 \ + --compile-core --compile-mode default --no-activation-checkpointing --autocast-cache \ + --loss-mode compiled-full --loss-chunk-size 1024 --bf16-residuals \ + --gradient-mode compiled-autograd-overlap --backward-layout paired \ + --async-checkpoint \ + --vram-limit-gib 88 --vram-headroom-gib 3 \ + --workers 4 --eval-batches 8 --eval-micro-batch 1 --eval-every 100 --save-every 100 \ + "$@" diff --git a/ops/train-shared.sh b/ops/train-shared.sh new file mode 100644 index 0000000000000000000000000000000000000000..c04824404e7be46f5e157109286ec9ec4075c40a --- /dev/null +++ b/ops/train-shared.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +ulimit -c 0 +source /workspace/WarpState/rtx6000_40b/ops/training-env.sh +# Select physical GPU 1 by UUID; it appears as cuda:0 inside this process. +export CUDA_VISIBLE_DEVICES="${WARPSTATE_CUDA_DEVICE:-GPU-a32efffc-8eb9-fb3e-b951-6f8cea2edd91}" +cd /workspace/WarpState/rtx6000_40b +exec .server-venv/bin/python -u run_40b.py \ + --no-bootstrap --stage train --from-scratch \ + --work-dir /workspace/WarpState/rtx6000_40b/work \ + --seq-len 2048 --micro-batch 1 --grad-accum 64 \ + --activation-checkpointing --no-autocast-cache --loss-chunk-size 256 \ + --vram-limit-gib 33.5 --vram-headroom-gib 3 \ + --workers 4 --eval-batches 8 --eval-every 100 --save-every 50 \ + "$@" diff --git a/ops/training-env.sh b/ops/training-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..e7e01f9da5d959382cb608361ec8e05d02267965 --- /dev/null +++ b/ops/training-env.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Source this file before installing dependencies, running tests, or training. +export TMPDIR=/workspace/WarpState/tmp +export XDG_CACHE_HOME=/workspace/WarpState/.cache +export UV_CACHE_DIR="$XDG_CACHE_HOME/uv" +export PIP_CACHE_DIR="$XDG_CACHE_HOME/pip" +export HF_HOME=/workspace/WarpState/rtx6000_40b/work/hf-cache +export HF_DATASETS_CACHE="$HF_HOME/datasets" +export HF_XET_CACHE="$HF_HOME/xet" +export TORCH_HOME="$XDG_CACHE_HOME/torch" +export CUDA_CACHE_PATH="$XDG_CACHE_HOME/cuda" +export TRITON_CACHE_DIR="$XDG_CACHE_HOME/triton" +export TORCHINDUCTOR_CACHE_DIR="$XDG_CACHE_HOME/torchinductor" +export TORCH_EXTENSIONS_DIR="$XDG_CACHE_HOME/torch-extensions" +export PYTHONDONTWRITEBYTECODE=1 +export UV_PYTHON_DOWNLOADS=never +export OMP_NUM_THREADS=4 +export MKL_NUM_THREADS=4 +export OPENBLAS_NUM_THREADS=1 +export PYTORCH_ALLOC_CONF=backend:native +export CUBLAS_WORKSPACE_CONFIG=:4096:8 +mkdir -p "$TMPDIR" "$XDG_CACHE_HOME" "$CUDA_CACHE_PATH" diff --git a/ops/training-requirements-resolved.txt b/ops/training-requirements-resolved.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff5e550daf23b7409fba92bfe55794e88b673fe5 --- /dev/null +++ b/ops/training-requirements-resolved.txt @@ -0,0 +1,37 @@ +cuda-bindings==13.3.1 +cuda-pathfinder==1.6.0 +cuda-toolkit==13.0.3.0 +filelock==3.32.3 +fsspec==2026.7.0 +iniconfig==2.3.0 +jinja2==3.1.6 +markupsafe==3.0.3 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.5.3 +nvidia-cublas==13.1.1.3 +nvidia-cuda-cupti==13.0.85 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-runtime==13.0.96 +nvidia-cudnn-cu13==9.24.0.43 +nvidia-cufft==12.0.0.61 +nvidia-cufile==1.15.1.6 +nvidia-curand==10.4.0.35 +nvidia-cusolver==12.0.4.66 +nvidia-cusparse==12.6.3.3 +nvidia-cusparselt-cu13==0.8.1 +nvidia-nccl-cu13==2.30.7 +nvidia-nvjitlink==13.3.33 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvtx==13.0.85 +packaging==26.3 +pip==24.0 +pluggy==1.6.0 +pygments==2.21.0 +pytest==9.1.1 +sentencepiece==0.2.2 +setuptools==78.1.0 +sympy==1.14.0 +torch==2.14.0+cu130 +triton==3.8.0 +typing-extensions==4.16.0 diff --git a/ops/transfer_artifacts.py b/ops/transfer_artifacts.py new file mode 100644 index 0000000000000000000000000000000000000000..1447bd784ba547f009f5fd7c83b4483ae87f8da6 --- /dev/null +++ b/ops/transfer_artifacts.py @@ -0,0 +1,660 @@ +#!/usr/bin/env python3 +"""Resume large artifact uploads or downloads over independent SSH connections. + +Both endpoints stay inside /workspace/WarpState. Receivers need only Python 3 +on Linux. No daemon, additional port, password file, or third-party package is +required. Completed chunks are read and checked on resume; the final file is +published only after full SHA256 verification and never replaces another file. +""" + +import argparse +import concurrent.futures +import hashlib +import inspect +import json +import os +from pathlib import Path +import re +import shlex +import stat +import subprocess +import sys +import threading +import time + + +ROOT = Path('/workspace/WarpState') +BLOCK = 1024 * 1024 + + +def workspace_path(value): + """Reject lexical escapes and symlinks, including intermediate components.""" + path = Path(os.path.abspath(value)) + if not path.is_relative_to(ROOT) or path == ROOT: + raise ValueError(f'Path must be below {ROOT}: {path}') + current = Path('/') + for part in path.parts[1:]: + current /= part + if current.is_symlink(): + raise ValueError(f'Symlink is not allowed: {current}') + return path + + +def remote_main(): + # This function is sent to Python over SSH; keep it self-contained. + import ctypes + import fcntl + import hashlib + import json + import os + from pathlib import Path + import stat + import sys + import uuid + + root = Path('/workspace/WarpState') + block = 1024 * 1024 + action, destination, expected_sha = sys.argv[1:4] + + def checked(value): + path = Path(os.path.abspath(value)) + if not path.is_relative_to(root): + raise ValueError(f'Path must be below {root}: {path}') + current = Path('/') + for part in path.parts[1:]: + current /= part + if current.is_symlink(): + raise ValueError(f'Symlink is not allowed: {current}') + return path + + def open_regular(path, flags, mode=0o600): + checked(path) + fd = os.open(path, flags | os.O_NOFOLLOW, mode) + if not stat.S_ISREG(os.fstat(fd).st_mode): + os.close(fd) + raise ValueError(f'Not a regular file: {path}') + return fd + + def sync_directory(path): + fd = os.open(path, os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + def atomic_json(path, value): + checked(path) + temporary = checked(str(path) + '.' + uuid.uuid4().hex + '.tmp') + fd = open_regular(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + try: + with os.fdopen(fd, 'w') as stream: + json.dump(value, stream, sort_keys=True) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + sync_directory(path.parent) + finally: + if temporary.exists(): + temporary.unlink() + + def read_json(path): + fd = open_regular(path, os.O_RDONLY) + with os.fdopen(fd, 'r') as stream: + return json.load(stream) + + def digest(path, offset=0, count=None): + fd = open_regular(path, os.O_RDONLY) + checksum = hashlib.sha256() + with os.fdopen(fd, 'rb') as stream: + stream.seek(offset) + while count is None or count > 0: + data = stream.read(block if count is None else min(block, count)) + if not data: + if count: + raise ValueError('Unexpected end of file during verification') + break + checksum.update(data) + if count is not None: + count -= len(data) + return checksum.hexdigest() + + target = checked(destination) + if target == root: + raise ValueError('Destination must be a file below the workspace root') + target.parent.mkdir(parents=True, exist_ok=True) + checked(target.parent) + partial = checked(str(target) + '.uploading') + state_path = checked(str(target) + '.upload-state.json') + receipts = checked(str(target) + '.upload-chunks') + lock_fd = open_regular(str(target) + '.upload.lock', os.O_RDWR | os.O_CREAT) + fcntl.flock(lock_fd, fcntl.LOCK_SH if action == 'chunk' else fcntl.LOCK_EX) + + def target_matches(size): + if not target.exists(): + return False + if target.stat().st_size != size or digest(target) != expected_sha: + raise ValueError(f'Existing destination differs; refusing overwrite: {target}') + return True + + if action == 'prepare': + incoming = json.load(sys.stdin) + if incoming['sha256'] != expected_sha: + raise ValueError('Source identity mismatch') + if target_matches(incoming['size']): + print(json.dumps({'complete': True, 'sha256': expected_sha})) + return + if state_path.exists(): + state = read_json(state_path) + if state != incoming: + raise ValueError('Partial upload belongs to a different source or chunk size') + else: + if partial.exists() or receipts.exists(): + raise ValueError('Unclaimed upload sidecar exists; refusing to replace it') + # Persist identity before creating the sparse file, so interruption + # between these operations can safely resume. + atomic_json(state_path, incoming) + state = incoming + if not partial.exists(): + fd = open_regular(partial, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + try: + os.ftruncate(fd, state['size']) + os.fsync(fd) + finally: + os.close(fd) + if partial.stat().st_size != state['size']: + raise ValueError('Partial upload has an unexpected size') + receipts.mkdir(exist_ok=True) + completed = [] + for index, expected in enumerate(state['chunk_sha256']): + receipt = checked(receipts / f'{index:08d}.json') + if not receipt.exists(): + continue + offset = index * state['chunk_size'] + length = min(state['chunk_size'], state['size'] - offset) + try: + valid = read_json(receipt) == {'sha256': expected, 'size': length} + valid = valid and digest(partial, offset, length) == expected + except (ValueError, OSError): + valid = False + if valid: + completed.append(index) + else: + receipt.unlink() + sync_directory(receipts) + print(json.dumps({'complete': False, 'completed': completed})) + return + + state = read_json(state_path) + if state['sha256'] != expected_sha: + raise ValueError('Upload source identity mismatch') + if action == 'chunk': + index = int(sys.argv[4]) + if not 0 <= index < len(state['chunk_sha256']): + raise ValueError('Invalid chunk index') + offset = index * state['chunk_size'] + length = min(state['chunk_size'], state['size'] - offset) + checksum = hashlib.sha256() + fd = open_regular(partial, os.O_WRONLY) + try: + remaining = length + position = offset + while remaining: + data = sys.stdin.buffer.read(min(block, remaining)) + if not data: + raise ValueError('Upload stream ended early') + checksum.update(data) + view = memoryview(data) + while view: + count = os.pwrite(fd, view, position) + if count <= 0: + raise OSError('pwrite made no progress') + view = view[count:] + position += count + remaining -= len(data) + if sys.stdin.buffer.read(1): + raise ValueError('Upload stream exceeded its chunk length') + actual = checksum.hexdigest() + if actual != state['chunk_sha256'][index]: + raise ValueError('Chunk SHA256 mismatch') + os.fsync(fd) + finally: + os.close(fd) + atomic_json(checked(receipts / f'{index:08d}.json'), + {'sha256': actual, 'size': length}) + print(json.dumps({'index': index, 'sha256': actual, 'bytes': length})) + return + + if action != 'finalize': + raise ValueError('Unknown receiver action') + if target_matches(state['size']): + print(json.dumps({'complete': True, 'sha256': expected_sha, 'bytes': state['size']})) + return + for index, expected in enumerate(state['chunk_sha256']): + length = min(state['chunk_size'], state['size'] - index * state['chunk_size']) + if read_json(checked(receipts / f'{index:08d}.json')) != { + 'sha256': expected, 'size': length}: + raise ValueError(f'Missing or invalid receipt for chunk {index}') + if partial.stat().st_size != state['size'] or digest(partial) != expected_sha: + raise ValueError('Full destination SHA256 mismatch; partial upload retained') + # Linux renameat2(RENAME_NOREPLACE) atomically publishes the verified file + # while protecting a destination created by another process during hashing. + libc = ctypes.CDLL(None, use_errno=True) + rename = libc.renameat2 + rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, + ctypes.c_char_p, ctypes.c_uint] + rename.restype = ctypes.c_int + if rename(-100, os.fsencode(partial), -100, os.fsencode(target), 1): + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), str(target)) + sync_directory(target.parent) + print(json.dumps({'complete': True, 'sha256': expected_sha, 'bytes': state['size']})) + + +def identity(path): + info = path.stat() + if not stat.S_ISREG(info.st_mode): + raise ValueError(f'Not a regular file: {path}') + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def source_manifest(path, chunk_size): + before = identity(path) + whole = hashlib.sha256() + chunks = [] + with path.open('rb') as stream: + while True: + checksum = hashlib.sha256() + consumed = 0 + while consumed < chunk_size: + data = stream.read(min(BLOCK, chunk_size - consumed)) + if not data: + break + whole.update(data) + checksum.update(data) + consumed += len(data) + if not consumed: + break + chunks.append(checksum.hexdigest()) + if identity(path) != before: + raise ValueError('Source changed while computing its SHA256') + return {'version': 1, 'size': before[2], 'sha256': whole.hexdigest(), + 'chunk_size': chunk_size, 'chunk_sha256': chunks}, before + + +def remote_source_main(): + # Sent over SSH without installing or changing any remote source files. + import hashlib + import json + import os + from pathlib import Path + import stat + import sys + + action, name = sys.argv[1:3] + root = Path('/workspace/WarpState') + path = Path(os.path.abspath(name)) + if not path.is_relative_to(root) or path == root: + raise ValueError(f'Path must be below {root}: {path}') + current = Path('/') + for part in path.parts[1:]: + current /= part + if current.is_symlink(): + raise ValueError(f'Symlink is not allowed: {current}') + + def identity(info): + if not stat.S_ISREG(info.st_mode): + raise ValueError('Source is not a regular file') + return [info.st_dev, info.st_ino, info.st_size, + info.st_mtime_ns, info.st_ctime_ns] + + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, 'rb') as stream: + before = identity(os.fstat(stream.fileno())) + + def unchanged(): + if (identity(os.fstat(stream.fileno())) != before or + identity(path.stat(follow_symlinks=False)) != before): + raise ValueError('Source changed during transfer') + + unchanged() + if action == 'manifest': + chunk_size = int(sys.argv[3]) + if chunk_size < 1: + raise ValueError('Chunk size must be positive') + whole = hashlib.sha256() + chunks = [] + while True: + checksum = hashlib.sha256() + consumed = 0 + while consumed < chunk_size: + data = stream.read(min(1024 * 1024, chunk_size - consumed)) + if not data: + break + whole.update(data) + checksum.update(data) + consumed += len(data) + if not consumed: + break + chunks.append(checksum.hexdigest()) + unchanged() + print(json.dumps({'manifest': { + 'version': 1, 'size': before[2], 'sha256': whole.hexdigest(), + 'chunk_size': chunk_size, 'chunk_sha256': chunks}, + 'identity': before})) + return + + if before != json.loads(sys.argv[3]): + raise ValueError('Source changed during transfer') + if action == 'identity': + unchanged() + print(json.dumps({'identity': before})) + return + if action != 'chunk': + raise ValueError('Unknown source action') + offset, remaining = map(int, sys.argv[4:6]) + if offset < 0 or remaining < 0 or offset + remaining > before[2]: + raise ValueError('Invalid source chunk range') + stream.seek(offset) + while remaining: + data = stream.read(min(1024 * 1024, remaining)) + if not data: + raise ValueError('Source ended early') + sys.stdout.buffer.write(data) + remaining -= len(data) + sys.stdout.buffer.flush() + unchanged() + + +def ssh_command(args): + command = ['ssh', '-F', '/dev/null', '-p', str(args.port), + '-i', str(args.key), '-o', 'BatchMode=yes', + '-o', 'IdentitiesOnly=yes', '-o', 'StrictHostKeyChecking=yes', + '-o', f'UserKnownHostsFile={args.known_hosts}', + '-o', 'GlobalKnownHostsFile=/dev/null', + '-o', 'ConnectTimeout=20', '-o', 'ServerAliveInterval=15', + '-o', 'ServerAliveCountMax=4', '-o', 'ControlMaster=no', + '-o', 'ControlPath=none'] + if args.compress: + command.append('-C') + return command + [args.host] + + +class SSHSource: + def __init__(self, args, source): + self.source = source + self.ssh = ssh_command(args) + self.code = inspect.getsource(remote_source_main) + '\nremote_source_main()\n' + + def command(self, action, *parameters): + remote = ['python3', '-c', self.code, action, str(self.source), + *(str(value) for value in parameters)] + return self.ssh + [shlex.join(remote)] + + def call(self, action, *parameters): + result = subprocess.run(self.command(action, *parameters), + stdin=subprocess.DEVNULL, capture_output=True) + if result.returncode: + raise RuntimeError(f'SSH source failed ({result.returncode}): ' + + result.stderr.decode(errors='replace').strip()) + return json.loads(result.stdout) + + def manifest(self, chunk_size): + result = self.call('manifest', chunk_size) + return result['manifest'], result['identity'] + + def check_identity(self, expected): + if self.call('identity', json.dumps(expected))['identity'] != expected: + raise ValueError('Source changed during transfer') + + +class SSHReceiver: + def __init__(self, args, destination, manifest): + self.destination = destination + self.manifest = manifest + self.ssh = ssh_command(args) + self.code = inspect.getsource(remote_main) + '\nremote_main()\n' + self.processes = set() + self.lock = threading.Lock() + self.cancelled = threading.Event() + + def command(self, action, index): + remote = ['python3', '-c', self.code, action, str(self.destination), + self.manifest['sha256']] + if index is not None: + remote.append(str(index)) + return self.ssh + [shlex.join(remote)] + + def start_process(self, command, **kwargs): + if self.cancelled.is_set(): + raise RuntimeError('Transfer cancelled') + process = subprocess.Popen(command, **kwargs) + with self.lock: + self.processes.add(process) + if self.cancelled.is_set(): + process.terminate() + return process + + def call(self, action, payload=None, source=None, index=None): + process = self.start_process(self.command(action, index), + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + if source is None: + out, err = process.communicate(payload) + else: + offset = index * self.manifest['chunk_size'] + remaining = min(self.manifest['chunk_size'], self.manifest['size'] - offset) + with source.open('rb') as stream: + stream.seek(offset) + while remaining: + data = stream.read(min(BLOCK, remaining)) + if not data: + raise ValueError('Source ended early') + process.stdin.write(data) + remaining -= len(data) + process.stdin.close() + process.stdin = None + out, err = process.communicate() + if process.returncode: + raise RuntimeError(f'SSH receiver failed ({process.returncode}): ' + + err.decode(errors='replace').strip()) + return json.loads(out) + except BaseException: + if process.poll() is None: + process.terminate() + if process.stdin is not None: + try: + process.stdin.close() + except BrokenPipeError: + pass + process.stdin = None + process.communicate() + raise + finally: + with self.lock: + self.processes.discard(process) + + def cancel(self): + self.cancelled.set() + with self.lock: + for process in self.processes: + if process.poll() is None: + process.terminate() + + +class PullReceiver(SSHReceiver): + """Run the same receiver locally, with each SSH stdout piped to its stdin.""" + + def __init__(self, args, destination, manifest, remote_source, initial_identity): + super().__init__(args, destination, manifest) + self.remote_source = remote_source + self.initial_identity = initial_identity + + def command(self, action, index): + command = [sys.executable, '-c', self.code, action, str(self.destination), + self.manifest['sha256']] + return command + ([] if index is None else [str(index)]) + + def call(self, action, payload=None, source=None, index=None): + if action != 'chunk': + return super().call(action, payload, index=index) + offset = index * self.manifest['chunk_size'] + length = min(self.manifest['chunk_size'], self.manifest['size'] - offset) + processes = [] + communicated = set() + try: + sender = self.start_process(self.remote_source.command( + 'chunk', json.dumps(self.initial_identity), offset, length), + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + processes.append(sender) + receiver = self.start_process(self.command('chunk', index), + stdin=sender.stdout, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + processes.append(receiver) + # Only the receiver owns the pipe's read end; data never accumulates + # in this process, and receiver failure breaks the sender's pipe. + sender.stdout.close() + sender.stdout = None + out, receiver_error = receiver.communicate() + communicated.add(receiver) + _, sender_error = sender.communicate() + communicated.add(sender) + if sender.returncode: + raise RuntimeError(f'SSH source failed ({sender.returncode}): ' + + sender_error.decode(errors='replace').strip()) + if receiver.returncode: + raise RuntimeError(f'Local receiver failed ({receiver.returncode}): ' + + receiver_error.decode(errors='replace').strip()) + return json.loads(out) + finally: + for process in processes: + if process.poll() is None: + process.terminate() + for process in processes: + if process not in communicated: + process.communicate() + with self.lock: + self.processes.discard(process) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('source', type=workspace_path) + parser.add_argument('destination', type=workspace_path) + parser.add_argument('--host', required=True, help='user@hostname') + parser.add_argument('--port', type=int, default=22) + parser.add_argument('--key', type=workspace_path, required=True) + parser.add_argument('--known-hosts', type=workspace_path, required=True, + help='Existing verified host-key file inside the workspace') + parser.add_argument('--connections', type=int, default=8) + parser.add_argument('--chunk-mib', type=int, default=256) + parser.add_argument('--retries', type=int, default=3) + parser.add_argument('--compress', action='store_true') + parser.add_argument('--pull', action='store_true', + help='Download source from SSH host to local destination') + parser.add_argument('--expected-sha256', + help='Reject a source that does not match this known SHA256') + parser.add_argument('--manifest-dir', type=workspace_path, + default=ROOT / 'tmp' / 'artifact-transfer') + args = parser.parse_args() + if args.connections < 1 or args.chunk_mib < 1 or args.retries < 1: + parser.error('Connections, chunk size, and retries must be positive') + if args.host.startswith('-') or not 1 <= args.port <= 65535: + parser.error('Invalid SSH host or port') + if args.expected_sha256 is not None: + if re.fullmatch(r'[0-9a-fA-F]{64}', args.expected_sha256) is None: + parser.error('--expected-sha256 must contain exactly 64 hexadecimal digits') + args.expected_sha256 = args.expected_sha256.lower() + for path in (args.key, args.known_hosts) + (() if args.pull else (args.source,)): + identity(path) + args.manifest_dir.mkdir(parents=True, exist_ok=True) + workspace_path(args.manifest_dir) + print(f'Computing source SHA256 and {args.chunk_mib} MiB chunk hashes: {args.source}', + flush=True) + remote_source = SSHSource(args, args.source) if args.pull else None + if args.pull: + manifest, initial_identity = remote_source.manifest(args.chunk_mib * BLOCK) + else: + manifest, initial_identity = source_manifest(args.source, args.chunk_mib * BLOCK) + if args.expected_sha256 is not None and manifest['sha256'] != args.expected_sha256: + raise ValueError('Source SHA256 does not match --expected-sha256') + manifest_key = (f'{args.host}:{args.port}:{args.source}' if args.pull else str(args.source)) + manifest_name = hashlib.sha256(manifest_key.encode()).hexdigest()[:20] + '.json' + manifest_path = workspace_path(args.manifest_dir / manifest_name) + # This local manifest is informational; each invocation hashes its source + # afresh and validates remote receipts rather than trusting cached claims. + temporary = workspace_path(str(manifest_path) + f'.{os.getpid()}.tmp') + with temporary.open('x') as stream: + json.dump({'source': str(args.source), **manifest}, stream, indent=2) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, manifest_path) + print(f'Source: {manifest["size"]:,} bytes; SHA256 {manifest["sha256"]}', flush=True) + receiver = (PullReceiver(args, args.destination, manifest, remote_source, initial_identity) + if args.pull else SSHReceiver(args, args.destination, manifest)) + prepared = receiver.call('prepare', json.dumps(manifest).encode()) + if prepared.get('complete'): + print('Destination already exists and its full SHA256 matches.', flush=True) + return + completed = set(prepared['completed']) + pending = [i for i in range(len(manifest['chunk_sha256'])) if i not in completed] + done_bytes = sum(min(manifest['chunk_size'], manifest['size'] - i * manifest['chunk_size']) + for i in completed) + initial_bytes = done_bytes + start = time.monotonic() + print(f'Resuming: {len(completed)} verified chunks; {len(pending)} remaining; ' + f'{args.connections} SSH connections.', flush=True) + + def send_chunk(index): + for attempt in range(args.retries): + if not args.pull and identity(args.source) != initial_identity: + raise ValueError('Source changed during transfer') + try: + result = receiver.call('chunk', source=args.source, index=index) + if (result['index'] != index or + result['sha256'] != manifest['chunk_sha256'][index]): + raise ValueError('Receiver returned an invalid chunk acknowledgment') + return result + except Exception: + if receiver.cancelled.is_set() or attempt + 1 == args.retries: + raise + receiver.cancelled.wait(min(2 ** attempt, 10)) + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=args.connections) + try: + futures = [executor.submit(send_chunk, index) for index in pending] + for future in concurrent.futures.as_completed(futures): + result = future.result() + completed.add(result['index']) + done_bytes += result['bytes'] + elapsed = max(time.monotonic() - start, 0.001) + speed = (done_bytes - initial_bytes) / elapsed / BLOCK + percent = 100 * done_bytes / max(manifest['size'], 1) + print(f'{percent:6.2f}% {done_bytes:,}/{manifest["size"]:,} bytes ' + f'{len(completed)}/{len(manifest["chunk_sha256"])} chunks ' + f'{speed:.1f} MiB/s', flush=True) + except BaseException: + receiver.cancel() + executor.shutdown(wait=True, cancel_futures=True) + raise + executor.shutdown(wait=True) + if args.pull: + remote_source.check_identity(initial_identity) + elif identity(args.source) != initial_identity: + raise ValueError('Source changed during transfer; refusing final publication') + print('Verifying full destination SHA256 before atomic publication...', flush=True) + result = receiver.call('finalize') + if result['sha256'] != manifest['sha256'] or not result['complete']: + raise ValueError('Invalid final verification response') + print(f'Complete: {args.destination}; SHA256 {result["sha256"]}', flush=True) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('Interrupted; verified chunks are retained for resume.', file=sys.stderr) + raise SystemExit(130) diff --git a/ops/tune_matmul.py b/ops/tune_matmul.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ca5e292895c67763216a48e18b6c2d64767a82 --- /dev/null +++ b/ops/tune_matmul.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Screen BF16 FFN GEMM/epilogue tiles with alternating warmed ATen controls. + +No model/checkpoint/token files are opened. Generated policy entries are +isolated-kernel candidates and require full-model validation before promotion. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import subprocess +import sys +import time + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +sys.path.insert(0, str(PACKAGE)) + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kind", choices=["mm", "swiglu"], default="mm") + parser.add_argument("--operations", nargs="+", choices=["fwd", "dw"]) + parser.add_argument("--algorithms", nargs="+", choices=["tiled", "persistent", "persistent_tma"], + help="defaults to all supported algorithms for the selected kind") + parser.add_argument("--tile", action="append", metavar="BM,BN,BK,WARPS,STAGES", + help="repeat to select explicit tiles instead of the bounded default sweep") + parser.add_argument("--rows", type=int, default=8192) + parser.add_argument("--repeats", type=int, default=30) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--warmup", type=int, default=10) + parser.add_argument("--burnin-seconds", type=float, default=1.0) + parser.add_argument("--minimum-gain", type=float, default=0.01) + parser.add_argument("--maximum-relative-rms", type=float, default=0.003) + parser.add_argument("--seed", type=int, default=1337) + parser.add_argument("--input-scale", type=float, default=1.0, + help="multiply both BF16 random operands by this positive scalar") + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--list-cases", action="store_true") + parser.add_argument("--output-dir", type=Path, default=PACKAGE / "work" / + ("benchmark-matmul-tiles-" + dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%S"))) + args = parser.parse_args(argv) + if args.operations is None: + args.operations = ["fwd"] if args.kind == "swiglu" else ["fwd", "dw"] + if args.kind == "swiglu" and args.operations != ["fwd"]: + parser.error("swiglu supports only --operations fwd") + if args.algorithms is None: + args.algorithms = ["tiled", "persistent"] + if args.kind == "mm": + args.algorithms.append("persistent_tma") + if args.kind == "swiglu" and "persistent_tma" in args.algorithms: + parser.error("persistent_tma is implemented for mm only") + for key in ("operations", "algorithms"): + if len(getattr(args, key)) != len(set(getattr(args, key))): + parser.error(key + " must not contain duplicate selections") + if args.tile: + from ops.matmul_kernels import normalize_tile + try: + args.tile = [normalize_tile(tuple(map(int, tile.split(",")))) for tile in args.tile] + if len(args.tile) != len(set(args.tile)): + raise ValueError("tile must not contain duplicate selections; use --trials for repeats") + if args.kind == "swiglu" and any((bm + 2 * bn) * bk * 2 * stages > 96 * 1024 + for bm, bn, bk, _, stages in args.tile): + raise ValueError("Paired SwiGLU pipeline exceeds the conservative 96 KiB budget") + except ValueError as error: + parser.error(str(error)) + for key in ["rows", "repeats", "trials", "warmup"]: + if getattr(args, key) < 1: + parser.error(key + " must be positive") + for key in ["burnin_seconds", "minimum_gain", "maximum_relative_rms"]: + if not math.isfinite(getattr(args, key)) or getattr(args, key) < 0: + parser.error(key + " must be finite and nonnegative") + if args.device < 0: + parser.error("device must be nonnegative") + if not math.isfinite(args.input_scale) or args.input_scale <= 0: + parser.error("input-scale must be finite and positive") + args.output_dir = args.output_dir.expanduser().resolve() + work = PACKAGE / "work" + if (not args.output_dir.is_relative_to(work) or args.output_dir == work + or not args.output_dir.relative_to(work).parts[0].startswith("benchmark-")): + parser.error("output-dir must be below rtx6000_40b/work/benchmark-*") + if not args.list_cases: + for key in ["TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR", "TORCHINDUCTOR_CACHE_DIR"]: + if not os.environ.get(key) or not Path(os.environ[key]).resolve().is_relative_to(WORKSPACE): + parser.error("Source ops/training-env.sh; " + key + " must stay inside the workspace") + return args + + +def cases(rows, operations): + result = [] + for operation in operations: + m, n, k, at, bt = ((rows, 14336, 2048, False, True) if operation == "fwd" + else (14336, 2048, rows, True, False)) + result.append({"operation": "ffn_up_gate." + operation, "m": m, "n": n, "k": k, + "a_transposed": at, "b_transposed": bt, + "a_stride": [1, m] if at else [k, 1], + "b_stride": [1, k] if bt else [n, 1], "dtype": "bfloat16"}) + return result + + +def telemetry(): + try: + result = subprocess.run([ + "nvidia-smi", "--query-gpu=index,uuid,name,power.draw,power.limit,clocks.sm," + "clocks.mem,temperature.gpu,utilization.gpu", "--format=csv,noheader"], + capture_output=True, text=True, timeout=10) + return {"timestamp": time.time(), "csv": result.stdout.strip(), "returncode": result.returncode} + except (OSError, subprocess.TimeoutExpired) as error: + return {"error": str(error)} + + +def output_metrics(output, reference): + """Validate every returned BF16 output; one bad epilogue cannot hide in RMS.""" + import torch + + outputs = output if isinstance(output, tuple) else (output,) + references = reference if isinstance(reference, tuple) else (reference,) + if len(outputs) != len(references): + raise ValueError("Candidate and reference must return the same number of tensors") + metrics = [] + for result, expected in zip(outputs, references, strict=True): + if result.shape != expected.shape or result.dtype != torch.bfloat16: + raise ValueError("Candidate output must preserve the reference shape and BF16 dtype") + difference = result.float() - expected.float() + relative_rms = (difference.square().mean().sqrt() + / expected.float().square().mean().sqrt().clamp_min(1e-30)).item() + if not bool(torch.isfinite(result).all()) or not math.isfinite(relative_rms): + raise ValueError("Nonfinite candidate output or error metric") + metrics.append({"max_abs_error": difference.abs().max().item(), + "relative_rms_error": relative_rms}) + return {"output_metrics": metrics, "finite": True, + "max_abs_error": max(metric["max_abs_error"] for metric in metrics), + "relative_rms_error": max(metric["relative_rms_error"] for metric in metrics)} + + +def paired_timing_metrics(trials, minimum_gain): + """Compare each candidate with its adjacent controls, retaining the worst trial.""" + if not trials or any(not math.isfinite(value) or value <= 0 + for trial in trials for value in trial.values()): + raise ValueError("Paired CUDA timings must be finite and positive") + controls = [(trial["aten_before_ms"] + trial["aten_after_ms"]) / 2 for trial in trials] + gains = [control / trial["candidate_ms"] - 1 + for control, trial in zip(controls, trials, strict=True)] + return {"median_ms": statistics.median(trial["candidate_ms"] for trial in trials), + "median_control_ms": statistics.median(controls), + "paired_gain": statistics.median(gains), "minimum_trial_gain": min(gains), + "timing_eligible": min(gains) >= minimum_gain, + "maximum_control_drift_fraction": max( + abs(trial["aten_after_ms"] - trial["aten_before_ms"]) / control + for control, trial in zip(controls, trials, strict=True))} + + +def main(argv=None): + args = arguments(argv) + selected_cases = cases(args.rows, args.operations) + if args.list_cases: + print(json.dumps(selected_cases, indent=2)) + return + import torch + from ops.matmul_kernels import TUNING_TILES, launch_mm, matrix_signature, policy_key, tile_dict + + tiles = args.tile or TUNING_TILES + source_names = ["ops/tune_matmul.py", "ops/matmul_kernels.py"] + if args.kind == "swiglu": + from ops.matmul_epilogue import SUPPORTED_TILES, launch_swiglu_mm + tiles = args.tile or SUPPORTED_TILES + source_names.append("ops/matmul_epilogue.py") + + def swiglu_reference(a, b): + packed = torch.mm(a, b) + up, gate = packed.chunk(2, dim=-1) + # Matches the existing compiled FFN: the packed projection is + # rounded to BF16; the SiLU/multiply has no intermediate BF16 store. + output = (torch.nn.functional.silu(gate.float()) * up.float()).to(torch.bfloat16) + return packed, output + + compiled_swiglu_reference = torch.compile(swiglu_reference, fullgraph=True) + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required; --list-cases is available without CUDA") + torch.cuda.set_device(args.device) + device = torch.device("cuda", args.device) + if torch.cuda.get_device_capability(device) != (12, 0): + raise RuntimeError("This bounded policy tuning run targets SM120") + # A process-local reference setting: all tested dot accumulators stay FP32. + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False + torch.backends.cuda.matmul.allow_tf32 = False + args.output_dir.mkdir(parents=True, exist_ok=False) + properties = torch.cuda.get_device_properties(device) + report = { + "format": "warpstate-matmul-tuning-v1", "started_utc": dt.datetime.now(dt.UTC).isoformat(), + "arguments": {k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()}, + "torch_version": torch.__version__, "cuda_version": torch.version.cuda, + "device": {"name": properties.name, "capability": [12, 0], + "multiprocessors": properties.multi_processor_count, + "cuda_visible_devices": os.environ.get("CUDA_VISIBLE_DEVICES")}, + "input_dtype": "bfloat16", "output_dtype": "bfloat16", "accumulation_dtype": "float32", + "allow_bf16_reduced_precision_reduction": torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction, + "allow_tf32": torch.backends.cuda.matmul.allow_tf32, + "source_sha256": {name: hashlib.sha256((PACKAGE / name).read_bytes()).hexdigest() + for name in source_names}, + "control": "torch.mm" if args.kind == "mm" else "compiled torch.mm + SwiGLU", + "output_names": ["matrix"] if args.kind == "mm" else ["packed_projection", "activation"], + "results": [], + "limitations": ["Isolated kernels, not measured training speedups.", + "Inputs are reused; timing includes output allocation and launch scheduling.", + "Every candidate trial has a preceding and following warmed ATen control.", + "Correctness metrics compare BF16 outputs, not exact real arithmetic.", + "Pipeline footprint checks cannot predict every compiler resource requirement."], + } + policy = {"format": "warpstate-matmul-plan-v1", + "description": "Isolated SM120 BF16 winners; require full-model validation before promotion. " + "Unmatched shapes retain ATen.", "entries": []} + + def save(): + for name, value in [("results.json", report), ("candidate-policy.json", policy)]: + temporary = args.output_dir / (name + ".partial") + temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") + temporary.replace(args.output_dir / name) + + def elapsed_ms(operation): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(args.repeats): + output = operation() + end.record() + end.synchronize() + del output + return start.elapsed_time(end) / args.repeats + + def warm(operation): + for _ in range(args.warmup): + output = operation() + torch.cuda.synchronize(device) + start = time.monotonic() + while time.monotonic() - start < args.burnin_seconds: + for _ in range(10): + output = operation() + torch.cuda.synchronize(device) + del output + + save() + with torch.inference_mode(): + for case_index, case in enumerate(selected_cases): + generator = torch.Generator(device=device).manual_seed(args.seed + case_index) + + def operand(rows, cols, transposed): + value = torch.randn((cols, rows) if transposed else (rows, cols), + device=device, dtype=torch.bfloat16, generator=generator) + if args.input_scale != 1.0: + value.mul_(args.input_scale) + return value.T if transposed else value + + a = operand(case["m"], case["k"], case["a_transposed"]) + b = operand(case["k"], case["n"], case["b_transposed"]) + control = (lambda: launch_mm(a, b, "aten")) if args.kind == "mm" else ( + lambda: compiled_swiglu_reference(a, b)) + reference = control() + candidates = [] + for tile in tiles: + for algorithm in args.algorithms: + row = {"case": case, "signature": matrix_signature(a, b), "key": policy_key(a, b), + "algorithm": algorithm, "tile": tile_dict(tile), "status": "running"} + print(json.dumps({"operation": case["operation"], "algorithm": algorithm, + "tile": tile, "status": "starting"}), flush=True) + operation = (lambda: launch_mm(a, b, algorithm, tile)) if args.kind == "mm" else ( + lambda: launch_swiglu_mm(a, b, algorithm, tile, eager_rounding=False)) + try: + start = time.monotonic() + output = operation() + torch.cuda.synchronize(device) + row["first_call_seconds"] = time.monotonic() - start + row.update(output_metrics(output, reference)) + del output + warm(control) + warm(operation) + row["telemetry_before"] = telemetry() + triples = [] + for _ in range(args.trials): + triples.append({"aten_before_ms": elapsed_ms(control), + "candidate_ms": elapsed_ms(operation), + "aten_after_ms": elapsed_ms(control)}) + row["trials"] = triples + row["telemetry_after"] = telemetry() + row.update(paired_timing_metrics(triples, args.minimum_gain)) + row["tflops"] = 2 * case["m"] * case["n"] * case["k"] / (row["median_ms"] * 1e9) + row["status"] = "ok" + row["eligible"] = (row["relative_rms_error"] <= args.maximum_relative_rms + and row["timing_eligible"]) + if row["eligible"]: + candidates.append(row) + except Exception as error: + row["status"] = "error" + row["error"] = type(error).__name__ + ": " + str(error)[:4000] + report["results"].append(row) + save() + print(json.dumps({k: row[k] for k in ["algorithm", "tile", "status", "median_ms", + "paired_gain", "relative_rms_error", "error"] if k in row}), flush=True) + if candidates: + winner = max(candidates, key=lambda row: row["paired_gain"]) + policy["entries"].append({"kind": args.kind, "name": case["operation"], + **winner["signature"], "algorithm": winner["algorithm"], "tile": winner["tile"]}) + save() + del a, b, reference + report["finished_utc"] = dt.datetime.now(dt.UTC).isoformat() + save() + + +if __name__ == "__main__": + main() diff --git a/ops/validate_backward_layout_training.py b/ops/validate_backward_layout_training.py new file mode 100644 index 0000000000000000000000000000000000000000..5e74d8fabdfe2a49a9396e31117d073f4e8c9f79 --- /dev/null +++ b/ops/validate_backward_layout_training.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""Compare the opt-in backward graph replacement in real WarpState training. + +The fixture retains production attention and FFN dimensions, with one physical +core and one logical layer by default; --cores selects additional distinct cores +and layers to exercise cross-core graph optimization. Its default 512-token vocabulary exercises repeated +embedding indices. --unique-inputs uses an 8192-token vocabulary and each input +index once per microbatch, removing the existing embedding-atomic nondeterminism +for strict continuation checks. It is a correctness test, not a throughput +benchmark or a claim about all 28 trained layers. Separate +processes compare compiled baseline/candidate execution and candidate checkpoint +continuation. Every artifact stays in a fresh directory below work/validation. +""" +from __future__ import annotations + +import argparse +import contextlib +from dataclasses import asdict +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +import torch + +PACKAGE = Path(__file__).resolve().parents[1] +WORKSPACE = PACKAGE.parent +sys.path.insert(0, str(PACKAGE)) + +# Declared independently of measured results. The graph pass preserves BF16 +# arithmetic, but the existing full graph contains relaxed embedding atomics. +# Byte identity is reported separately and is mandatory for resumed full state. +TOLERANCES = { + "gradient_relative_l2": 0.005, + "gradient_relative_max": 0.01, + "gradient_absolute": 1e-8, + "parameter_absolute": 2e-5, + "moment_relative_l2": 0.005, + "moment_relative_max": 0.01, + "moment_absolute": 1e-8, + "loss_absolute": 2e-4, +} +MICRO_BATCH = 4 +SEQ_LEN = 2048 +MICRO_TOKENS = MICRO_BATCH * SEQ_LEN +# Three exercises three distinct CA graphs: first no-sync with None gradients, +# later no-sync accumulation, and final accumulation with DDP synchronization. +ACCUMULATION = 3 + + +def tensor_comparison(actual, expected): + """Finite, byte-sensitive error metrics with bounded CPU scratch space.""" + if actual.shape != expected.shape or actual.dtype != expected.dtype: + raise ValueError("Tensor shape/dtype mismatch") + shape = list(actual.shape) + actual = actual.detach().cpu().contiguous().reshape(-1) + expected = expected.detach().cpu().contiguous().reshape(-1) + squared_error = squared_reference = maximum_error = maximum_reference = 0.0 + unequal = bitwise_unequal = 0 + finite = True + for offset in range(0, actual.numel(), 1 << 20): + observed = actual[offset:offset + (1 << 20)] + wanted = expected[offset:offset + (1 << 20)] + bitwise_unequal += int((observed.view(torch.uint8).reshape(-1, observed.element_size()) + != wanted.view(torch.uint8).reshape(-1, wanted.element_size())) + .any(dim=1).sum()) + unequal += int((observed != wanted).sum()) + observed, wanted = observed.double(), wanted.double() + if not (torch.isfinite(observed).all() and torch.isfinite(wanted).all()): + finite = False + continue + difference = observed - wanted + squared_error += float(difference.square().sum()) + squared_reference += float(wanted.square().sum()) + maximum_error = max(maximum_error, float(difference.abs().max())) + maximum_reference = max(maximum_reference, float(wanted.abs().max())) + return { + "shape": shape, "dtype": str(actual.dtype), + "elements": actual.numel(), "finite": bool(finite), + "unequal": unequal, "bitwise_unequal": bitwise_unequal, + "error_l2": squared_error ** 0.5 if finite else None, + "reference_l2": squared_reference ** 0.5 if finite else None, + "relative_l2": ((squared_error / squared_reference) ** 0.5 + if finite and squared_reference else (0.0 if finite and not squared_error else None)), + "max_absolute": maximum_error if finite else None, + "reference_max_absolute": maximum_reference if finite else None, + } + + +def within_tolerance(metrics, kind): + if not metrics["finite"]: + return False + if kind == "exact": + return metrics["bitwise_unequal"] == 0 + if kind == "parameter": + return metrics["max_absolute"] <= TOLERANCES["parameter_absolute"] + if kind not in {"gradient", "moment"}: + raise ValueError("Unknown comparison kind: " + kind) + absolute = TOLERANCES[kind + "_absolute"] + return (metrics["error_l2"] <= metrics["reference_l2"] * TOLERANCES[kind + "_relative_l2"] + absolute + and metrics["max_absolute"] <= metrics["reference_max_absolute"] + * TOLERANCES[kind + "_relative_max"] + absolute) + + +def compare_named_tensors(actual, expected, kind): + if actual.keys() != expected.keys(): + raise ValueError("Tensor names differ") + results = {name: tensor_comparison(value, expected[name]) for name, value in actual.items()} + for name, metrics in results.items(): + # Adam step counters have discrete semantics despite their FP32 dtype. + selected_kind = "exact" if kind == "moment" and name.endswith("/step") else kind + metrics["accepted"] = within_tolerance(metrics, selected_kind) + return {"accepted": all(value["accepted"] for value in results.values()), + "bitwise_equal": all(value["bitwise_unequal"] == 0 for value in results.values()), + "tensors": results} + + +def exact_tree_equal(actual, expected): + if isinstance(expected, torch.Tensor): + return (isinstance(actual, torch.Tensor) and actual.shape == expected.shape + and actual.dtype == expected.dtype + and torch.equal(actual.detach().cpu().contiguous().reshape(-1).view(torch.uint8), + expected.detach().cpu().contiguous().reshape(-1).view(torch.uint8))) + if isinstance(expected, dict): + return (isinstance(actual, dict) and actual.keys() == expected.keys() + and all(exact_tree_equal(actual[key], value) for key, value in expected.items())) + if isinstance(expected, (list, tuple)): + return (type(actual) is type(expected) and len(actual) == len(expected) + and all(exact_tree_equal(a, b) for a, b in zip(actual, expected, strict=True))) + return type(actual) is type(expected) and actual == expected + + +def cpu_tree(value): + if isinstance(value, torch.Tensor): + return value.detach().to(device="cpu", copy=True) + if isinstance(value, dict): + return {key: cpu_tree(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return type(value)(cpu_tree(item) for item in value) + return value + + +def update_plan(world_size, update): + """Two full accumulated updates, then nine valid global tokens.""" + if world_size not in {1, 2} or update not in {0, 1, 2}: + raise ValueError("The fixture supports one/two ranks and three updates") + return MICRO_TOKENS * ACCUMULATION * world_size if update < 2 else 9 + + +def fixture_config(unique_inputs=False, cores=1): + from warpstate import WarpStateConfig + if isinstance(cores, bool) or not isinstance(cores, int) or cores < 1: + raise ValueError("cores must be a positive integer") + return WarpStateConfig(vocab_size=MICRO_TOKENS if unique_inputs else 512, d_model=2048, n_heads=32, + n_cores=cores, logical_depth=cores, ffn_hidden=7168, + chunk_size=128, gradient_checkpointing=False, + parallel_memory=True, loss_chunk_size=512) + + +def fixture_batch_cpu(seed, rank, update, micro, unique_inputs=False): + generator = torch.Generator().manual_seed(seed + 100000 * rank + 100 * update + micro) + vocab_size = MICRO_TOKENS if unique_inputs else 512 + if unique_inputs: + inputs = torch.randperm(MICRO_TOKENS, generator=generator).reshape(MICRO_BATCH, SEQ_LEN) + else: + inputs = torch.randint(vocab_size, (MICRO_BATCH, SEQ_LEN), generator=generator) + targets = torch.randint(vocab_size, inputs.shape, generator=generator) + return inputs, targets + + +def save_json(path, document): + with path.open("x") as handle: + json.dump(document, handle, indent=2, sort_keys=True, allow_nan=False) + handle.write("\n") + + +def source_identity(): + names = ["ops/validate_backward_layout_training.py", "ops/gradient_execution.py", + "ops/distributed_execution.py", "ops/loss_experiment.py", "ops/residual_experiment.py", + "ops/backward_layout_execution.py", "ops/backward_layout.py", + "warpstate/model.py", "warpstate/config.py"] + return {name: hashlib.sha256((PACKAGE / name).read_bytes()).hexdigest() for name in names} + + +def check_workspace_environment(): + for name in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR", + "TORCHINDUCTOR_CACHE_DIR"): + if not os.environ.get(name) or not Path(os.environ[name]).resolve().is_relative_to(WORKSPACE): + raise ValueError("Source ops/training-env.sh first: " + name) + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--world-size", type=int, choices=(1, 2), default=2) + parser.add_argument("--timeout", type=int, default=1800, help="Seconds per subprocess") + parser.add_argument("--seed", type=int, default=41619) + parser.add_argument("--cores", type=int, default=1, + help="Number of distinct physical cores and logical layers") + parser.add_argument("--unique-inputs", action="store_true", + help="Use each input embedding row once per microbatch for strict resume checks") + parser.add_argument("--worker", choices=("baseline", "candidate", "partial", "resumed"), + help=argparse.SUPPRESS) + args = parser.parse_args(argv) + args.output_dir = args.output_dir.expanduser().resolve() + if not args.output_dir.is_relative_to(PACKAGE / "work/validation"): + parser.error("--output-dir must be below rtx6000_40b/work/validation") + if args.output_dir == PACKAGE / "work/validation": + parser.error("--output-dir must name a new child directory") + if args.timeout <= 0: + parser.error("--timeout must be positive") + if args.cores < 1: + parser.error("--cores must be positive") + return args + + +def worker(args): + from torch.nn.parallel import DistributedDataParallel + import torch.distributed as dist + from ops.backward_layout_execution import (backward_layout_stats, + configure_backward_layout, require_backward_layout_dispatch) + from ops.distributed_execution import ddp_kwargs, ddp_runtime_settings + from ops.gradient_execution import make_backward, scale_loss + from ops.loss_experiment import apply_loss_execution + from ops.residual_experiment import apply_bf16_residuals + from warpstate import WarpStateLM + + rank = int(os.environ.get("RANK", 0)) + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + world_size = int(os.environ.get("WORLD_SIZE", 1)) + if world_size != args.world_size: + raise ValueError("Worker world size differs from its fixture") + torch.set_num_threads(4) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + torch.manual_seed(args.seed) + torch.set_float32_matmul_precision("high") + if world_size > 1: + dist.init_process_group("nccl", device_id=device) + selected = "baseline" if args.worker == "baseline" else "paired" + layout_metadata = configure_backward_layout(selected) + backward = make_backward("compiled-autograd-overlap", distributed=world_size > 1) + initial = torch.load(args.output_dir / "initial.pt", map_location="cpu", weights_only=False) + model = WarpStateLM(fixture_config(args.unique_inputs, args.cores)).to(device=device, dtype=torch.float32) + model.load_state_dict(initial["model"]) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, betas=(0.9, 0.95), + eps=1e-8, weight_decay=0.1, fused=True) + # Nonzero FP32 Adam state avoids first-update sign amplification near zero; + # this is a deterministic synthetic fixture, not imported production state. + for parameter in model.parameters(): + optimizer.state[parameter] = { + "step": torch.tensor(40081.0, device=device), + "exp_avg": torch.full_like(parameter, 0.001), + "exp_avg_sq": torch.full_like(parameter, 0.0001), + } + del initial + start_update = 0 + if args.worker == "resumed": + saved = torch.load(args.output_dir / f"partial/rank-{rank}-state.pt", + map_location="cpu", weights_only=False) + if (saved["layout_execution"] != layout_metadata + or saved["source_sha256"] != source_identity() + or saved["config"] != asdict(model.cfg) + or saved["world_size"] != world_size): + raise ValueError("Checkpoint execution identity/configuration changed") + model.load_state_dict(saved["model"]) + optimizer.load_state_dict(saved["optimizer"]) + start_update = saved["update"] + resume_rng = saved["rng"] + del saved + for core in model.cores: + core.forward = torch.compile(core.forward, fullgraph=True, dynamic=False, mode="default") + apply_loss_execution(model, "compiled-full") + apply_bf16_residuals(model) + wrapped = model + if world_size > 1: + wrapped = DistributedDataParallel(model, device_ids=[local_rank], output_device=local_rank, + find_unused_parameters=False, **ddp_kwargs(ddp_runtime_settings(bucket_cap_mb=128))) + model.train() + directory = args.output_dir / args.worker + directory.mkdir(exist_ok=True) + records = [] + accepted = True + + def fixture_batch(update, micro): + inputs, targets = fixture_batch_cpu(args.seed, rank, update, micro, args.unique_inputs) + return inputs.to(device), targets.to(device) + + if args.worker != "resumed": + inputs, _ = fixture_batch(0, 0) + with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16): + logits = wrapped(inputs) + logits = logits.detach().cpu() + if args.worker == "baseline": + torch.save(logits, directory / f"rank-{rank}-forward.pt") + else: + expected = torch.load(args.output_dir / f"baseline/rank-{rank}-forward.pt", + map_location="cpu", weights_only=True) + metrics = tensor_comparison(logits, expected) + metrics["accepted"] = within_tolerance(metrics, "exact") + accepted &= metrics["accepted"] + records.append({"stage": "initial_forward_logits", **metrics}) + del logits, inputs + # Startup/cold compilation need not consume the same RNG as a warm process. + # Set the training RNG after initialization, or restore it after resume setup. + if args.worker == "resumed": + torch.set_rng_state(resume_rng["cpu"]) + torch.cuda.set_rng_state(resume_rng["cuda"], device) + else: + torch.manual_seed(args.seed + rank + 81) + stop_update = 1 if args.worker == "partial" else 3 + for update in range(start_update, stop_update): + optimizer.zero_grad(set_to_none=True) + target_tokens = update_plan(world_size, update) + microbatches = (target_tokens + MICRO_TOKENS * world_size - 1) // (MICRO_TOKENS * world_size) + for micro in range(microbatches): + offset = (micro * world_size + rank) * MICRO_TOKENS + valid = max(0, min(MICRO_TOKENS, target_tokens - offset)) + inputs, targets = fixture_batch(update, micro) + targets.reshape(-1)[valid:] = -100 + synchronize = micro == microbatches - 1 + synchronization = contextlib.nullcontext() if synchronize or world_size == 1 else wrapped.no_sync() + previous_calls = backward_layout_stats()["runtime_calls"] + with synchronization: + with torch.autocast("cuda", dtype=torch.bfloat16, cache_enabled=True): + _, loss = wrapped(inputs, targets, return_logits=False) + backward(scale_loss(loss, world_size * valid / target_tokens, "compiled-autograd-overlap")) + torch.cuda.synchronize(device) + if selected == "paired": + # Every backward variant must dispatch, including later + # accumulation and the masked final rank; an earlier match + # must not conceal fallback in a later graph. + require_backward_layout_dispatch(min_calls=previous_calls + model.cfg.logical_depth) + gradients = {} + for name, parameter in model.named_parameters(): + if parameter.dtype != torch.float32 or parameter.grad is None or parameter.grad.dtype != torch.float32: + raise RuntimeError("Missing FP32 master/gradient: " + name) + gradients[name] = parameter.grad.detach().cpu() + reference_path = args.output_dir / f"baseline/rank-{rank}-u{update}-m{micro}.pt" + record = {"stage": "gradient", "update": update, "microbatch": micro, + "synchronize": synchronize, "valid_tokens": valid, + "loss": float(loss.detach()), "layout_stats": backward_layout_stats(), + "runtime_calls_this_backward": backward_layout_stats()["runtime_calls"] - previous_calls} + if args.worker == "baseline": + if not all(torch.isfinite(value).all() for value in gradients.values()): + raise RuntimeError("Baseline produced nonfinite gradients") + torch.save({"gradients": gradients, "loss": record["loss"]}, reference_path) + else: + expected = torch.load(reference_path, map_location="cpu", weights_only=True, mmap=True) + record["comparison"] = compare_named_tensors(gradients, expected["gradients"], "gradient") + record["loss_difference"] = abs(record["loss"] - expected["loss"]) + record["accepted"] = (record["comparison"]["accepted"] + and record["loss_difference"] <= TOLERANCES["loss_absolute"]) + accepted &= record["accepted"] + del expected + records.append(record) + del gradients, inputs, targets, loss + norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) + optimizer.step() + torch.cuda.synchronize(device) + actual_parameters = cpu_tree(model.state_dict()) + actual_moments = {f"{name}/{key}": cpu_tree(value) + for name, parameter in model.named_parameters() + for key, value in optimizer.state[parameter].items()} + reference_path = args.output_dir / f"baseline/rank-{rank}-update-{update}.pt" + record = {"stage": "adam_update", "update": update, "grad_norm": float(norm)} + if args.worker == "baseline": + torch.save({"model": actual_parameters, "moments": actual_moments}, reference_path) + else: + expected = torch.load(reference_path, map_location="cpu", weights_only=True, mmap=True) + record["parameters"] = compare_named_tensors(actual_parameters, expected["model"], "parameter") + record["moments"] = compare_named_tensors(actual_moments, expected["moments"], "moment") + record["accepted"] = record["parameters"]["accepted"] and record["moments"]["accepted"] + accepted &= record["accepted"] + del expected + records.append(record) + del actual_parameters, actual_moments + state = {"config": asdict(model.cfg), "world_size": world_size, "update": stop_update, + "tokens": sum(update_plan(world_size, update) for update in range(stop_update)), + "model": cpu_tree(model.state_dict()), "optimizer": cpu_tree(optimizer.state_dict()), + "rng": {"cpu": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state(device)}, + "layout_execution": layout_metadata, "source_sha256": source_identity()} + expected_dispatches = model.cfg.logical_depth * sum( + (update_plan(world_size, update) + MICRO_TOKENS * world_size - 1) + // (MICRO_TOKENS * world_size) + for update in range(start_update, stop_update)) if selected == "paired" else 0 + dispatch_exact = backward_layout_stats()["runtime_calls"] == expected_dispatches + accepted &= dispatch_exact + records.append({"stage": "dispatch_count", "expected": expected_dispatches, + "actual": backward_layout_stats()["runtime_calls"], "accepted": dispatch_exact}) + torch.save(state, directory / f"rank-{rank}-state.pt") + if args.worker == "resumed": + expected = torch.load(args.output_dir / f"candidate/rank-{rank}-state.pt", + map_location="cpu", weights_only=False, mmap=True) + exact = {key: exact_tree_equal(state[key], expected[key]) for key in state} + accepted &= all(exact.values()) + records.append({"stage": "checkpoint_continuation", "accepted": all(exact.values()), + "exact_fields": exact}) + save_json(directory / f"rank-{rank}.json", { + "rank": rank, "worker": args.worker, "accepted": accepted, + "tolerances": TOLERANCES, "records": records, "layout_execution": layout_metadata, + "layout_stats": backward_layout_stats(), "source_sha256": source_identity(), + "max_allocated_bytes": torch.cuda.max_memory_allocated(device), + "max_reserved_bytes": torch.cuda.max_memory_reserved(device), + }) + if world_size > 1: + dist.barrier() + dist.destroy_process_group() + print(json.dumps({"worker": args.worker, "rank": rank, "accepted": accepted}), flush=True) + + +def main(argv=None): + args = arguments(argv) + check_workspace_environment() + if args.worker: + worker(args) + return + from warpstate import WarpStateLM + args.output_dir.mkdir(parents=True, exist_ok=False) + torch.set_num_threads(4) + torch.manual_seed(args.seed) + cfg = fixture_config(args.unique_inputs, args.cores) + initial = WarpStateLM(cfg) + torch.save({"config": asdict(cfg), "model": initial.state_dict()}, args.output_dir / "initial.pt") + del initial + source_hashes = source_identity() + save_json(args.output_dir / "fixture.json", { + "config": asdict(cfg), "seed": args.seed, "world_size": args.world_size, + "micro_batch": MICRO_BATCH, "seq_len": SEQ_LEN, "accumulation": ACCUMULATION, + "unique_inputs": args.unique_inputs, + "global_tokens": [update_plan(args.world_size, update) for update in range(3)], + "tolerances": TOLERANCES, "source_sha256": source_hashes, + "scope": f"{args.cores} production-shaped cores/layers; synthetic weights/tokens and nonzero FP32 Adam moments", + }) + environment = os.environ.copy() + for name in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "GROUP_RANK", + "ROLE_RANK", "ROLE_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT"): + environment.pop(name, None) + reports = [] + for mode in ("baseline", "candidate", "partial", "resumed"): + command = [sys.executable] + if args.world_size > 1: + command += ["-m", "torch.distributed.run", "--standalone", "--nnodes", "1", + "--nproc-per-node", str(args.world_size)] + command += [str(Path(__file__).resolve()), "--output-dir", str(args.output_dir), + "--world-size", str(args.world_size), "--seed", str(args.seed), + "--cores", str(args.cores), "--worker", mode] + if args.unique_inputs: + command.append("--unique-inputs") + with (args.output_dir / f"{mode}.log").open("x") as log: + print("Running " + mode, flush=True) + result = subprocess.run(command, cwd=PACKAGE, env=environment, stdout=log, + stderr=subprocess.STDOUT, timeout=args.timeout) + if result.returncode: + raise RuntimeError(f"{mode} exited {result.returncode}; inspect {mode}.log") + if source_identity() != source_hashes: + raise RuntimeError("Validation source changed during the experiment") + reports.extend(json.loads((args.output_dir / f"{mode}/rank-{rank}.json").read_text()) + for rank in range(args.world_size)) + rank_consistency = {} + if args.world_size == 2: + for mode in ("baseline", "candidate", "partial", "resumed"): + states = [torch.load(args.output_dir / f"{mode}/rank-{rank}-state.pt", map_location="cpu", + weights_only=False, mmap=True) for rank in range(args.world_size)] + rank_consistency[mode] = {key: exact_tree_equal(states[0][key], states[1][key]) + for key in ("model", "optimizer", "update", "tokens")} + del states + summary = {"accepted": (all(report["accepted"] for report in reports) + and all(all(fields.values()) for fields in rank_consistency.values())), + "rank_consistency": rank_consistency, "runs": reports, + "source_sha256": source_hashes, "tolerances": TOLERANCES} + save_json(args.output_dir / "results.json", summary) + print(json.dumps({"accepted": summary["accepted"], "report": str(args.output_dir / "results.json")}), flush=True) + if not summary["accepted"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/ops/weight_cast_execution.py b/ops/weight_cast_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..c6f9db7bb41a2b794614c11d04d313949db4a1c2 --- /dev/null +++ b/ops/weight_cast_execution.py @@ -0,0 +1,126 @@ +"""Experimental per-update BF16 core weight views with FP32 master gradients.""" + +from collections import Counter +from contextlib import contextmanager + +import torch +from torch import nn +from torch.func import functional_call + + +class _OriginalForward(nn.Module): + def __init__(self, model): + super().__init__() + self.wrapped = model + self.original_forward = model.forward + + def forward(self, *args, **kwargs): + return self.original_forward(*args, **kwargs) + + +class CoreWeightCastCache: + """Own differentiable casts for one complete accumulation update only. + + Install after selecting the model's forward/loss/residual execution and + before DDP construction. Enclose all accumulation forwards and backwards in + ``with cache.update():`` and exit before clipping, optimizer, evaluation, or + checkpoint operations. Ordinary execution outside that scope is unchanged. + + ``ToCopyBackward`` returns each BF16 gradient to its original FP32 leaf on + every backward. Only that stateless conversion node is reused across + microsteps; completed forward graphs are never retained. FP32 parameters, + optimizer ownership, DDP hooks, and state dictionary names remain unchanged. + """ + + def __init__(self, model): + if getattr(model, "_core_weight_cast_cache", None) is not None: + raise ValueError("Core weight cast caching has already been applied") + cfg = getattr(model, "cfg", None) + if cfg is None or cfg.logical_depth != cfg.n_cores: + raise ValueError("Core weight cast caching requires one use of each core per forward") + if cfg.gradient_checkpointing: + raise ValueError("Core weight cast caching requires core activation checkpointing off") + self._model = model + self._masters = {} + for core_name, core in model.cores.named_children(): + for name, module in core.named_modules(): + if isinstance(module, nn.Linear): + parameter_name = f"cores.{core_name}.{name}.weight" + parameter = module.weight + if parameter.dtype != torch.float32 or not parameter.requires_grad: + raise ValueError("Core weight cast caching requires trainable FP32 linear weights") + self._masters[parameter_name] = parameter + if not self._masters: + raise ValueError("No core linear weights were found") + references = Counter(id(parameter) for _, parameter in + model.named_parameters(remove_duplicate=False)) + if any(references[id(parameter)] != 1 for parameter in self._masters.values()): + raise ValueError("Core weight cast caching does not support aliased linear weights") + self._views = None + self._versions = None + self._wrapper = _OriginalForward(model) + model.forward = self._forward + model._core_weight_cast_cache = self + + @property + def active(self): + return self._views is not None + + @property + def cached_tensor_count(self): + return len(self._views) if self._views is not None else 0 + + @property + def cached_bytes(self): + return sum(tensor.numel() * tensor.element_size() + for tensor in (self._views or {}).values()) + + def _validate_masters(self, *, check_versions): + current = dict(self._model.named_parameters()) + for name, parameter in self._masters.items(): + if current.get(name) is not parameter: + raise RuntimeError("Core master parameter identity changed; recreate the model and cast cache") + if parameter.dtype != torch.float32 or not parameter.requires_grad: + raise RuntimeError("Core master weights must remain trainable FP32 parameters") + if check_versions and parameter._version != self._versions[name]: + raise RuntimeError("Core weights changed inside an accumulation scope; " + "finish the cast-cache scope before optimizer or checkpoint loading") + cfg = self._model.cfg + if cfg.logical_depth != cfg.n_cores or cfg.gradient_checkpointing: + raise RuntimeError("Core reuse or activation checkpointing changed after cast-cache setup") + + @contextmanager + def update(self): + """Create fresh casts, then release them even when an update fails.""" + if self.active: + raise RuntimeError("Core weight cast accumulation scopes cannot be nested") + if not self._model.training or not torch.is_grad_enabled(): + raise RuntimeError("Core weight cast accumulation requires training with gradients enabled") + self._validate_masters(check_versions=False) + self._versions = {name: parameter._version for name, parameter in self._masters.items()} + try: + self._views = {f"wrapped.{name}": parameter.to(torch.bfloat16) + for name, parameter in self._masters.items()} + yield self + self._validate_masters(check_versions=True) + finally: + self._views = None + self._versions = None + + def _forward(self, input_ids, *args, **kwargs): + if not self.active: + return self._wrapper.original_forward(input_ids, *args, **kwargs) + if not self._model.training or not torch.is_grad_enabled(): + raise RuntimeError("Exit the cast-cache accumulation scope before inference or evaluation") + device_type = input_ids.device.type + if (not torch.is_autocast_enabled(device_type) + or torch.get_autocast_dtype(device_type) != torch.bfloat16): + raise RuntimeError("An active core weight cast cache requires BF16 autocast") + self._validate_masters(check_versions=True) + return functional_call(self._wrapper, self._views, (input_ids, *args), kwargs, + strict=False) + + +def apply_weight_cast_cache(model): + """Install the opt-in forward adapter and return its accumulation controller.""" + return CoreWeightCastCache(model) diff --git a/ops/workspace-tools.sh b/ops/workspace-tools.sh new file mode 100644 index 0000000000000000000000000000000000000000..40bb777dd03e7bdf5d21d3c0795a6ade9eb73b0a --- /dev/null +++ b/ops/workspace-tools.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Source this after training-env.sh when using workspace-local Git packages. +if [[ -x /workspace/WarpState/.tools/usr/bin/git ]]; then + export PATH="/workspace/WarpState/.tools/usr/bin:$PATH" + export GIT_EXEC_PATH=/workspace/WarpState/.tools/usr/lib/git-core + export GIT_TEMPLATE_DIR=/workspace/WarpState/.tools/usr/share/git-core/templates +fi diff --git a/posttrain.py b/posttrain.py new file mode 100644 index 0000000000000000000000000000000000000000..ab6a99d0f9df97f4467b695d41c192593cb12df0 --- /dev/null +++ b/posttrain.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Independent full-parameter assistant-only SFT from a frozen model export. + +The output directory owns a fresh AdamW optimizer, token-weighted SFT metrics, +and resumable checkpoints. No pretraining checkpoint or source export is edited. +Run with torchrun for DDP; --device cpu exists only for small correctness fixtures. +""" + +from __future__ import annotations + +import argparse +import contextlib +import fcntl +from functools import lru_cache +import hashlib +import json +import math +import os +from pathlib import Path +import random +import signal +import time + +import numpy as np +import torch +import torch.distributed as dist + +from ops.posttrain_data import encode_messages, renderer_sha256, RENDER_FORMAT + + +FORMAT = 'warpstate-posttrain-v1' +WORKSPACE = Path('/workspace/WarpState') +STOP = False +DEFAULTS = dict(sequence_length=2048, micro_batch=4, grad_accum=4, lr=5e-6, + min_lr=0.0, warmup_ratio=0.03, epochs=1, max_steps=None, + weight_decay=0.0, seed=42, eval_every=100, save_every=100, + eval_batches=32, compile_core=True, loss_mode='compiled-full', + bf16_residuals=True, gradient_checkpointing=False, workers=4, + deterministic=True, clip_grad_norm=1.0) + + +def digest(path): + result = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(8 << 20), b''): + result.update(block) + return result.hexdigest() + + +def atomic_json(path, report): + path = Path(path) + temporary = path.with_suffix(path.suffix + '.tmp') + with temporary.open('w') as stream: + stream.write(json.dumps(report, indent=2, allow_nan=False) + '\n') + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def load_plan(path): + plan = json.loads(Path(path).read_text()) + if plan.get('format') != 'warpstate-posttrain-plan-v1' or plan.get('stage') != 'sft': + raise ValueError('Expected warpstate-posttrain-plan-v1 stage sft') + supplied = plan.get('training', {}) + unknown = set(supplied) - set(DEFAULTS) + if unknown: + raise ValueError(f'Unknown training settings: {sorted(unknown)}') + settings = DEFAULTS | supplied + for name in ('sequence_length', 'micro_batch', 'grad_accum', 'epochs', + 'eval_every', 'save_every', 'eval_batches', 'workers'): + if type(settings[name]) is not int or settings[name] < 1: + raise ValueError(f'{name} must be a positive integer') + ratio = settings['warmup_ratio'] + if isinstance(ratio, bool) or not isinstance(ratio, (int, float)) or not 0 <= ratio < 1: + raise ValueError('warmup_ratio must be in [0, 1)') + if settings['max_steps'] is not None and ( + type(settings['max_steps']) is not int or settings['max_steps'] < 1): + raise ValueError('max_steps must be null or a positive integer') + for name in ('lr', 'min_lr', 'weight_decay', 'clip_grad_norm'): + value = settings[name] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise ValueError(f'{name} must be finite and nonnegative') + if not 0 < settings['lr'] or settings['min_lr'] > settings['lr'] or not settings['clip_grad_norm']: + raise ValueError('Require 0 <= min_lr <= lr and positive lr/clip_grad_norm') + if type(settings['seed']) is not int or settings['seed'] < 0: + raise ValueError('seed must be a nonnegative integer') + for name in ('compile_core', 'bf16_residuals', 'gradient_checkpointing', 'deterministic'): + if type(settings[name]) is not bool: + raise ValueError(f'{name} must be a boolean') + if settings['loss_mode'] not in ('full', 'compiled-full', 'checkpointed'): + raise ValueError('Unsupported loss_mode') + return plan, settings + + +def validate_paths(work_dir, data_dir, weights, tokenizer, config): + paths = [Path(p).resolve() for p in (work_dir, data_dir, weights, tokenizer, config)] + if any(not path.is_relative_to(WORKSPACE) for path in paths): + raise ValueError('All inputs and outputs must stay inside /workspace/WarpState') + work, data, source, tok, plan = paths + if work == WORKSPACE or any(path == work or path.is_relative_to(work) for path in (data, source, tok, plan)): + raise ValueError('Training outputs must be separate from every input') + if work.is_relative_to(data) or work.is_relative_to(source.parent): + raise ValueError('Training outputs must not be inside data or source-weight directories') + for path in (source, tok, plan): + if not path.is_file(): + raise ValueError(f'Missing required input: {path}') + if not data.is_dir(): + raise ValueError(f'Missing prepared data directory: {data}') + return paths + + +def load_data(data_dir, tokenizer, tokenizer_hash, sequence_length): + """Refuse partial/mismatched data; re-encode exactly before model allocation.""" + directory = Path(data_dir) + manifest_path = directory / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + if manifest.get('format') != 'warpstate-posttrain-data-v1' or manifest.get('complete') is not True: + raise ValueError('Posttraining data is not complete') + if manifest.get('tokenizer_sha256') != tokenizer_hash: + raise ValueError('Prepared data tokenizer hash differs') + if manifest.get('renderer_sha256') != renderer_sha256(): + raise ValueError('Prepared data renderer hash differs') + if manifest.get('max_length') != sequence_length: + raise ValueError('Prepared data max_length differs from sequence_length') + result = [] + for filename in ('sft_train.jsonl', 'sft_val.jsonl'): + info = manifest.get('outputs', {}).get(filename) + path = directory / filename + if not info or not path.is_file() or digest(path) != info.get('sha256'): + raise ValueError(f'Missing or changed prepared file: {filename}') + records = [] + token_count = target_count = 0 + with path.open() as stream: + for number, line in enumerate(stream, 1): + row = json.loads(line) + if row.get('tools'): + raise ValueError(f'{filename}:{number}: tools must be injected into messages') + encoded = encode_messages(row['messages'], tokenizer, max_length=sequence_length) + tokens = len(encoded['input_ids']) + targets = sum(label != -100 for label in encoded['labels'][1:]) + if row.get('tokens') != tokens or row.get('supervised_tokens') != targets: + raise ValueError(f'{filename}:{number}: token metadata differs from exact encoding') + records.append(encoded) + token_count += tokens + target_count += targets + if not records or len(records) != info.get('rows'): + raise ValueError(f'{filename}: empty data or row count mismatch') + if token_count != info.get('tokens') or target_count != info.get('supervised_tokens'): + raise ValueError(f'{filename}: manifest token totals differ') + result.append(records) + return *result, manifest, digest(manifest_path) + + +def collate(records, sequence_length, pad_id): + """Right-pad independent causal examples; mask padding and all prompt labels.""" + x = torch.full((len(records), sequence_length), pad_id, dtype=torch.long) + y = torch.full_like(x, -100) + for index, record in enumerate(records): + if record is None: + continue + ids, labels = record['input_ids'], record['labels'] + if len(ids) != len(labels) or not 2 <= len(ids) <= sequence_length: + raise ValueError('Invalid encoded record length') + length = len(ids) - 1 + x[index, :length] = torch.tensor(ids[:-1]) + y[index, :length] = torch.tensor(labels[1:]) + return x, y + + +@lru_cache(maxsize=2) +def epoch_order(size, seed, epoch): + order = list(range(size)) + random.Random(seed + epoch).shuffle(order) + return tuple(order) + + +def update_indices(size, seed, cursor, epochs, global_batch): + """One deterministic batch, never crossing an epoch or repeating tail data.""" + epoch, offset = divmod(cursor, size) + if epoch >= epochs: + return [], cursor + count = min(global_batch, size - offset) + order = epoch_order(size, seed, epoch) + return list(order[offset:offset + count]), cursor + count + + +def learning_rate(step, total_steps, settings): + warmup = min(math.ceil(settings['warmup_ratio'] * total_steps), max(0, total_steps - 1)) + if step < warmup: + return settings['lr'] * (step + 1) / warmup + progress = min(1., (step - warmup) / max(1, total_steps - warmup - 1)) + return settings['min_lr'] + .5 * (settings['lr'] - settings['min_lr']) * (1 + math.cos(math.pi * progress)) + + +def token_weighted_loss(mean_loss, local_targets, global_targets, world_size): + """DDP averages ranks, so multiply each local token sum by world/global sum.""" + if global_targets < 1: + raise ValueError('Update has no supervised tokens') + return mean_loss * (local_targets * world_size / global_targets) + + +def rng_state(device): + return {'torch': torch.get_rng_state(), 'cuda': torch.cuda.get_rng_state(device) + if device.type == 'cuda' else None, 'numpy': np.random.get_state(), 'python': random.getstate()} + + +def restore_rng(state, device): + torch.set_rng_state(state['torch']) + if device.type == 'cuda': + torch.cuda.set_rng_state(state['cuda'], device) + np.random.set_state(state['numpy']) + random.setstate(state['python']) + + +def validate_resume(checkpoint, signature, world_size): + if checkpoint.get('format') != FORMAT or checkpoint.get('signature') != signature: + raise ValueError('Posttraining resume signature differs; use a new output directory') + if len(checkpoint.get('rng_by_rank', [])) != world_size: + raise ValueError('Checkpoint lacks exact per-rank RNG state') + + +@contextlib.contextmanager +def shared_pipeline_lock(path): + """Every rank excludes pretraining, even if its launcher dies unexpectedly.""" + path = Path(path).resolve() + if not path.is_relative_to(WORKSPACE): + raise ValueError('Pipeline lock must stay inside the workspace') + with path.open('a+') as handle: + try: + fcntl.flock(handle, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError as error: + raise RuntimeError('Pretraining or its handoff currently owns pipeline.lock') from error + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +@contextlib.contextmanager +def base_pipeline_lock(plan): + base = plan.get('handoff', {}).get('base_work_dir') + if base is None: + yield + return + path = (WORKSPACE / base).resolve() + if not path.is_relative_to(WORKSPACE) or path == WORKSPACE or not path.is_dir(): + raise ValueError('handoff.base_work_dir must be an existing workspace directory') + with shared_pipeline_lock(path / 'pipeline.lock'): + yield + + +def training_update(model, optimizer, microbatches, global_targets, world_size, device, clip_norm): + optimizer.zero_grad(set_to_none=True) + loss_sum = torch.zeros((), device=device, dtype=torch.float64) + target_count = 0 + for index, (x, y) in enumerate(microbatches): + targets = int(y.ne(-100).sum()) + target_count += targets + sync = model.no_sync() if world_size > 1 and index < len(microbatches) - 1 else contextlib.nullcontext() + with sync: + with torch.autocast(device.type, dtype=torch.bfloat16, enabled=True, cache_enabled=False): + _, loss = model(x.to(device), y.to(device), return_logits=False) + if not torch.isfinite(loss): + raise RuntimeError('Non-finite SFT training loss') + token_weighted_loss(loss, targets, global_targets, world_size).backward() + loss_sum += loss.detach().double() * targets + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm, error_if_nonfinite=True) + optimizer.step() + totals = torch.stack((loss_sum, torch.tensor(target_count, device=device, dtype=torch.float64))) + if world_size > 1: + dist.all_reduce(totals) + return float(totals[0] / totals[1]), float(grad_norm) + + +def evaluate(model, records, settings, device, pad_id, *, full=False): + was_training = model.training + state = rng_state(device) + model.eval() + total_loss, total_targets = 0., 0 + try: + with torch.no_grad(): + limit = len(records) if full else settings['eval_batches'] * settings['micro_batch'] + for start in range(0, min(len(records), limit), settings['micro_batch']): + batch = records[start:min(start + settings['micro_batch'], limit)] + x, y = collate(batch, settings['sequence_length'], pad_id) + targets = int(y.ne(-100).sum()) + with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): + _, loss = model(x.to(device), y.to(device), return_logits=False) + total_loss += float(loss) * targets + total_targets += targets + finally: + model.train(was_training) + restore_rng(state, device) + value = total_loss / total_targets + if not math.isfinite(value): + raise RuntimeError('Non-finite SFT validation loss') + return value, total_targets + + +def source_identity(): + root = Path(__file__).parent + files = [Path(__file__), root / 'ops/posttrain_data.py', root / 'ops/loss_experiment.py', + root / 'ops/residual_experiment.py', root / 'ops/checkpoint_writer.py', root / 'run_40b.py'] + files += sorted((root / 'warpstate').glob('*.py')) + return {str(path.relative_to(root)): digest(path) for path in files} + + +def train(args): + import sentencepiece as spm + from warpstate import WarpStateConfig, WarpStateLM + from run_40b import compile_training_cores + from ops.checkpoint_writer import DistributedCheckpointWriter + from ops.loss_experiment import apply_loss_execution + from ops.residual_experiment import apply_bf16_residuals + + work, data_dir, weights, tokenizer_path, config = validate_paths( + args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) + plan, settings = load_plan(config) + rank = int(os.environ.get('RANK', '0')) + world = int(os.environ.get('WORLD_SIZE', '1')) + local_rank = int(os.environ.get('LOCAL_RANK', '0')) + device = torch.device('cuda', local_rank) if args.device == 'cuda' else torch.device('cpu') + if device.type == 'cuda': + torch.cuda.set_device(device) + if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): + raise RuntimeError('BF16-capable CUDA GPU required') + torch.set_num_threads(settings['workers']) + torch.use_deterministic_algorithms(settings['deterministic']) + random.seed(settings['seed'] + rank) + np.random.seed(settings['seed'] + rank) + torch.manual_seed(settings['seed'] + rank) + tokenizer_hash = digest(tokenizer_path) + sp = spm.SentencePieceProcessor(model_file=str(tokenizer_path)) + train_data, val_data, manifest, manifest_hash = load_data(data_dir, sp, tokenizer_hash, settings['sequence_length']) + base_hash = digest(weights) + signature = dict(plan=plan, settings=settings, data_sha256=manifest_hash, tokenizer_sha256=tokenizer_hash, + renderer=RENDER_FORMAT, base_sha256=base_hash, world_size=world, + device_type=device.type, torch_version=torch.__version__, sources=source_identity()) + latest = work / 'latest.pt' + resuming = latest.exists() + checkpoint = torch.load(latest if resuming else weights, map_location='cpu', weights_only=False, mmap=True) + if resuming: + validate_resume(checkpoint, signature, world) + origin = checkpoint['origin'] + else: + if checkpoint.get('format') != 'warpstate-chat-v1' or checkpoint.get('inference_only') is not True: + raise ValueError('Initialize SFT from an independent warpstate-chat-v1 model-only export') + if checkpoint.get('tokenizer_sha256') != tokenizer_hash: + raise ValueError('Base model export tokenizer differs') + for key in ('step', 'tokens_seen', 'tokenizer_sha256'): + if key in plan.get('base', {}) and plan['base'][key] != checkpoint.get(key): + raise ValueError(f'Base model {key} differs from plan') + origin = {key: value for key, value in checkpoint.items() if key != 'model'} + origin.update(weights_sha256=base_hash, weights_path=str(weights)) + cfg = WarpStateConfig(**checkpoint['config']) + cfg.gradient_checkpointing = settings['gradient_checkpointing'] + cfg.validate() + if cfg.vocab_size != sp.vocab_size() or settings['sequence_length'] % cfg.chunk_size: + raise ValueError('Model/tokenizer vocabulary or sequence/chunk mismatch') + with torch.device('meta'): + raw = WarpStateLM(cfg) + raw.load_state_dict(checkpoint['model'], strict=True, assign=True) + raw.to(device=device, dtype=torch.float32).requires_grad_(True) + optimizer = torch.optim.AdamW(raw.parameters(), lr=settings['lr'], betas=(.9, .95), + eps=1e-8, weight_decay=settings['weight_decay'], fused=device.type == 'cuda') + step = cursor = tokens_seen = assistant_tokens_seen = 0 + if resuming: + optimizer.load_state_dict(checkpoint['optimizer']) + step, cursor = checkpoint['step'], checkpoint['sample_cursor'] + tokens_seen = checkpoint['tokens_seen'] + assistant_tokens_seen = checkpoint['assistant_tokens_seen'] + resume_rng = checkpoint['rng_by_rank'][rank] if resuming else None + del checkpoint + if settings['compile_core']: + compile_training_cores(raw) + apply_loss_execution(raw, settings['loss_mode']) + if settings['bf16_residuals']: + apply_bf16_residuals(raw) + if world > 1: + dist.init_process_group(backend='nccl' if device.type == 'cuda' else 'gloo') + model = torch.nn.parallel.DistributedDataParallel(raw, + device_ids=[local_rank] if device.type == 'cuda' else None, + broadcast_buffers=False, gradient_as_bucket_view=True, bucket_cap_mb=128) + else: + model = raw + if resume_rng is not None: + restore_rng(resume_rng, device) + global_batch = settings['micro_batch'] * settings['grad_accum'] * world + total_steps = math.ceil(len(train_data) / global_batch) * settings['epochs'] + if settings['max_steps'] is not None: + total_steps = min(total_steps, settings['max_steps']) + end_step = min(total_steps, step + args.max_steps) if args.max_steps is not None else total_steps + work.mkdir(parents=True, exist_ok=True) + if rank == 0: + (work / 'settings.json').write_text(json.dumps(dict(signature=signature, origin=origin, + total_steps=total_steps, train_rows=len(train_data), val_rows=len(val_data)), indent=2) + '\n') + if step < total_steps: + (work / 'completion.json').unlink(missing_ok=True) + pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id() + last_validation = None + last_validation_step = None + + def validation(*, full=False): + values = [evaluate(raw, val_data, settings, device, pad_id, full=full) if rank == 0 else None] + if world > 1: + dist.broadcast_object_list(values, src=0, device=device) + return values[0] + + def metric(values): + if rank == 0: + values.update(timestamp=time.time(), stage='sft', step=step, tokens_seen=tokens_seen, + assistant_tokens_seen=assistant_tokens_seen, sample_cursor=cursor) + with (work / 'metrics.jsonl').open('a') as stream: + stream.write(json.dumps(values, allow_nan=False) + '\n') + if values.get('event') == 'validation': + count = len(val_data) if values.get('full_holdout') else min(len(val_data), settings['eval_batches'] * settings['micro_batch']) + report = dict(values, evaluated_samples=count, + available_validation_samples=len(val_data), data_sha256=manifest_hash, + base_sha256=base_hash, base_step=origin['step'], objective='assistant-token-mean', + tokenizer_sha256=tokenizer_hash, renderer_format=RENDER_FORMAT) + atomic_json(work / 'evaluation.json', report) + print(json.dumps(values, allow_nan=False), flush=True) + + def save(writer): + state = rng_state(device) + rank_states = [None] * world + if world > 1: + dist.all_gather_object(rank_states, state) + else: + rank_states = [state] + writer.submit(state_factory=lambda: dict(format=FORMAT, signature=signature, origin=origin, + config=vars(cfg), model=raw.state_dict(), optimizer=optimizer.state_dict(), step=step, + sample_cursor=cursor, tokens_seen=tokens_seen, assistant_tokens_seen=assistant_tokens_seen, + val_loss=last_validation, val_step=last_validation_step, rng_by_rank=rank_states, + completed=step >= total_steps, total_steps=total_steps)) + + def stop_requested(): + flag = STOP or bool(args.stop_file and args.stop_file.exists()) + if world > 1: + value = torch.tensor(int(flag), device=device) + dist.all_reduce(value, op=dist.ReduceOp.MAX) + return bool(value.item()) + return flag + + with DistributedCheckpointWriter(latest, rank=rank, device=device, distributed=world > 1) as writer: + last_validation, eval_tokens = validation(full=True) + last_validation_step = step + metric(dict(event='validation', val_loss=last_validation, val_perplexity=math.exp(last_validation), + validation_assistant_tokens=eval_tokens, full_holdout=True)) + last_saved_step = step if resuming else -1 + model.train() + while step < end_step and not stop_requested(): + writer.check() + indices, next_cursor = update_indices(len(train_data), settings['seed'], cursor, + settings['epochs'], global_batch) + if not indices: + break + global_targets = sum(sum(label != -100 for label in train_data[i]['labels'][1:]) for i in indices) + microbatches = [] + for micro in range(settings['grad_accum']): + offset = (micro * world + rank) * settings['micro_batch'] + selected = [train_data[i] for i in indices[offset:offset + settings['micro_batch']]] + selected += [None] * (settings['micro_batch'] - len(selected)) + microbatches.append(collate(selected, settings['sequence_length'], pad_id)) + lr = learning_rate(step, total_steps, settings) + for group in optimizer.param_groups: + group['lr'] = lr + started = time.perf_counter() + loss, grad_norm = training_update(model, optimizer, microbatches, global_targets, world, + device, settings['clip_grad_norm']) + if device.type == 'cuda': + torch.cuda.synchronize(device) + seconds = time.perf_counter() - started + step += 1 + cursor = next_cursor + actual_tokens = sum(len(train_data[i]['input_ids']) - 1 for i in indices) + tokens_seen += actual_tokens + assistant_tokens_seen += global_targets + metric(dict(event='train', loss=loss, lr=lr, grad_norm=grad_norm, update_seconds=seconds, + tokens_per_second=actual_tokens / seconds, + assistant_tokens_per_second=global_targets / seconds, update_samples=len(indices))) + if step % settings['eval_every'] == 0 or step == end_step: + full = step >= total_steps + last_validation, eval_tokens = validation(full=full) + last_validation_step = step + metric(dict(event='validation', val_loss=last_validation, val_perplexity=math.exp(last_validation), + validation_assistant_tokens=eval_tokens, full_holdout=full)) + if step % settings['save_every'] == 0 or step == end_step: + save(writer) + last_saved_step = step + if last_saved_step != step: + save(writer) + writer.flush() + if rank == 0 and step >= total_steps: + atomic_json(work / 'completion.json', dict(format='warpstate-posttrain-completion-v1', + stage='sft', completed=True, paused_for_evaluation=True, step=step, + total_steps=total_steps, base_step=origin['step'], tokens_seen=tokens_seen, + assistant_tokens_seen=assistant_tokens_seen, val_loss=last_validation, + evaluation_file=str(work / 'evaluation.json'), checkpoint=str(latest), + timestamp=time.time())) + metric(dict(event='complete' if step >= total_steps else 'paused', total_steps=total_steps, + checkpoint=str(latest))) + if world > 1: + dist.destroy_process_group() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('config', 'work-dir', 'data-dir', 'weights', 'tokenizer'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--max-steps', type=int, help='Pause after this many additional updates; schedule is unchanged') + parser.add_argument('--stop-file', type=Path, help='Defaults to work-dir/control/training-stop-request') + parser.add_argument('--device', choices=('cuda', 'cpu'), default='cuda', help='CPU is only for tiny correctness fixtures') + args = parser.parse_args() + if args.stop_file is None: + args.stop_file = args.work_dir / 'control/training-stop-request' + if args.max_steps is not None and args.max_steps < 1: + parser.error('--max-steps must be positive') + if args.stop_file is not None and not args.stop_file.resolve().is_relative_to(WORKSPACE): + parser.error('--stop-file must stay inside the workspace') + def stop(signum, frame): + global STOP + STOP = True + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + from run_40b import process_lock + rank = int(os.environ.get('RANK', '0')) + # Separate lock files avoid duplicate rank launches without serializing DDP. + lock_dir = args.work_dir.resolve() + validate_paths(args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) + lock_dir.mkdir(parents=True, exist_ok=True) + plan, _ = load_plan(args.config) + with process_lock(lock_dir / f'rank-{rank}.lock'), base_pipeline_lock(plan): + train(args) + + +if __name__ == '__main__': + main() diff --git a/run_40b.py b/run_40b.py new file mode 100644 index 0000000000000000000000000000000000000000..f3b35f52a15f0038526ef2a218ebe5fc6d970603 --- /dev/null +++ b/run_40b.py @@ -0,0 +1,885 @@ +#!/usr/bin/env python3 +"""One entry point: isolated dependencies, resumable 40B corpus, full GPU training.""" +from __future__ import annotations +import argparse +import contextlib +import hashlib +import itertools +import json +import math +import os +from pathlib import Path +import pickle +import random +import shutil +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent +STOP = False +RECIPE = 1 + + +def arguments(): + p = argparse.ArgumentParser(description=__doc__) + initialization = p.add_mutually_exclusive_group() + initialization.add_argument('--weights', type=Path, help='Your existing trusted WarpState checkpoint; used only to initialize a new run') + initialization.add_argument('--from-scratch', action='store_true', help='Initialize a new model from --config and --seed; existing runs still resume') + p.add_argument('--work-dir', type=Path, default=ROOT / 'work') + p.add_argument('--tokenizer', type=Path, default=ROOT / 'tokenizer.model') + p.add_argument('--config', type=Path, default=ROOT / 'configs/warpstate_2b.json') + p.add_argument('--stage', choices=['all', 'prepare', 'train'], default='all') + p.add_argument('--verify-data', action='store_true', help='Read the entire corpus and check SHA256 before training') + p.add_argument('--dataset', default='HuggingFaceFW/fineweb-edu') + p.add_argument('--dataset-config', default='sample-100BT') + p.add_argument('--revision', default='main', help='Resolved to an immutable Hub commit on first preparation') + p.add_argument('--train-tokens', type=int, default=40_000_000_000) + p.add_argument('--val-tokens', type=int, default=1_048_576) + p.add_argument('--workers', type=int, default=min(16, os.cpu_count() or 4)) + p.add_argument('--documents-per-batch', type=int, default=256) + p.add_argument('--seq-len', type=int, default=2048) + p.add_argument('--micro-batch', type=int, default=8) + p.add_argument('--grad-accum', type=int, default=8) + p.add_argument('--lr', type=float, default=1.5e-4) + p.add_argument('--min-lr', type=float, default=1.5e-5) + p.add_argument('--warmup-steps', type=int, default=1000) + p.add_argument('--weight-decay', type=float, default=0.1) + p.add_argument('--activation-checkpointing', action=argparse.BooleanOptionalAction, default=True) + p.add_argument('--autocast-cache', action=argparse.BooleanOptionalAction, default=True, + help='Disable on a shared GPU to avoid retaining BF16 copies of all weights') + p.add_argument('--vram-limit-gib', type=float, help='Cap the PyTorch CUDA allocator, excluding CUDA context/library memory') + p.add_argument('--vram-headroom-gib', type=float, default=3.0, + help='Require this much additional free device memory before applying a VRAM limit') + p.add_argument('--loss-chunk-size', type=int, default=1024) + p.add_argument('--loss-mode', choices=['checkpointed', 'full', 'compiled-full'], default='checkpointed', + help='Full loss retains vocabulary logits to avoid head recomputation; requires additional VRAM') + p.add_argument('--bf16-residuals', action=argparse.BooleanOptionalAction, default=False, + help='Use BF16 residual activations while retaining FP32 parameters, gradients and optimizer state') + p.add_argument('--activation-mode', choices=['baseline', 'swiglu', 'rmsnorm', 'all'], default='baseline', + help='Optional BF16 activation/normalization kernels; requires explicit reconfiguration on resume') + p.add_argument('--matmul-plan', type=Path, + help='Explicit BF16 shape/tile recipe; missing shapes retain ATen') + p.add_argument('--gradient-mode', choices=['eager', 'compiled-autograd', 'compiled-autograd-overlap'], default='eager', + help='Optional compiled backward and FP32 gradient accumulation') + p.add_argument('--backward-layout', choices=['baseline', 'inspect', 'paired'], default='baseline', + help='Opt-in production-shape packed attention backward graph experiment') + p.add_argument('--overlap-swap-collectives', action='store_true', + help='Allow wait/copy groups past later collective launches, preserving launch order') + p.add_argument('--overlap-runtime-estimates', action='store_true', + help='Use measured, rank-aligned GEMM estimates for communication scheduling') + p.add_argument('--overlap-producer-schedule', action='store_true', + help='Move ready gradient producers with collectives under zero estimated memory growth') + p.add_argument('--overlap-memory-budget', type=float, default=0.0, + help='Allowed fractional increase in estimated scheduler peak memory (0 to 0.25)') + p.add_argument('--cache-core-weight-casts', action=argparse.BooleanOptionalAction, default=False, + help='Reuse differentiable BF16 core weight views across one accumulation update') + p.add_argument('--ddp-bucket-cap-mb', type=float, default=128, + help='Gradient communication bucket capacity in MiB') + p.add_argument('--ddp-batched-grad-copy', action=argparse.BooleanOptionalAction, default=False, + help='Batch FP32 gradient-to-bucket copying and scaling') + p.add_argument('--compile', action='store_true', help='Optional Linux Inductor; initial compilation can take minutes') + p.add_argument('--compile-core', action='store_true', help='Compile individual training blocks while preserving canonical checkpoint keys') + p.add_argument('--compile-mode', choices=['default', 'reduce-overhead', 'max-autotune-no-cudagraphs'], + default='default', help='Compilation mode for --compile-core or --compile') + p.add_argument('--reconfigure', action='store_true', help='Allow execution changes on resume while preserving context, global batch, data and optimizer schedule') + p.add_argument('--eval-every', type=int, default=500) + p.add_argument('--eval-batches', type=int, default=8) + p.add_argument('--eval-micro-batch', type=int, default=1, help='Validation batch size independent of training microbatch') + p.add_argument('--save-every', type=int, default=1000) + p.add_argument('--async-checkpoint', action=argparse.BooleanOptionalAction, default=False, + help='Stage an immutable full CPU snapshot on rank zero, then overlap disk writing with training') + p.add_argument('--max-steps', type=int, help='Stop and save after N additional steps, keeping the 40B schedule') + p.add_argument('--stop-at-step', type=int, + help='Evaluate, save and stop at this absolute optimizer step, keeping the 40B schedule') + p.add_argument('--seed', type=int, default=1337) + p.add_argument('--no-bootstrap', action='store_true', help='Use an already provisioned environment') + p.add_argument('--test-small-gpu', action='store_true', help='Tests only: bypass the 80 GiB device requirement') + a = p.parse_args() + for key in ('train_tokens','val_tokens','workers','documents_per_batch','seq_len','micro_batch','grad_accum', + 'loss_chunk_size','eval_every','eval_batches','eval_micro_batch','save_every','warmup_steps'): + if getattr(a, key) < 1: + p.error(key + ' must be positive') + if a.max_steps is not None and a.max_steps < 1: + p.error('--max-steps must be positive') + if a.stop_at_step is not None and a.stop_at_step < 1: + p.error('--stop-at-step must be positive') + if a.vram_limit_gib is not None and (not math.isfinite(a.vram_limit_gib) or a.vram_limit_gib <= 0): + p.error('--vram-limit-gib must be finite and positive') + if not math.isfinite(a.vram_headroom_gib) or a.vram_headroom_gib < 0: + p.error('--vram-headroom-gib must be finite and nonnegative') + if not 0 < a.min_lr <= a.lr or a.weight_decay < 0: + p.error('Invalid learning rate or weight decay') + if a.train_tokens < 2 or a.val_tokens <= a.seq_len: + p.error('Need at least 2 training tokens and more validation tokens than seq-len') + if a.compile and a.compile_core: + p.error('Choose --compile or --compile-core') + if a.compile_mode != 'default' and not (a.compile or a.compile_core): + p.error('A nondefault --compile-mode requires --compile-core or --compile') + if not math.isfinite(a.ddp_bucket_cap_mb) or a.ddp_bucket_cap_mb * 1024**2 < 1: + p.error('--ddp-bucket-cap-mb must be finite and at least one byte in MiB') + if not math.isfinite(a.overlap_memory_budget) or not 0 <= a.overlap_memory_budget <= 0.25: + p.error('--overlap-memory-budget must be finite and between 0 and 0.25') + if a.gradient_mode != 'compiled-autograd-overlap' and ( + a.overlap_swap_collectives or a.overlap_runtime_estimates or a.overlap_memory_budget + or a.overlap_producer_schedule): + p.error('Overlap scheduling options require --gradient-mode compiled-autograd-overlap') + if a.overlap_producer_schedule and ( + a.overlap_swap_collectives or a.overlap_runtime_estimates or a.overlap_memory_budget): + p.error('--overlap-producer-schedule must be tested separately from other overlap scheduling options') + if a.gradient_mode != 'eager' and a.ddp_batched_grad_copy: + p.error('--ddp-batched-grad-copy uses the C++ reducer and cannot be combined with compiled autograd') + if a.backward_layout != 'baseline': + if (a.gradient_mode != 'compiled-autograd-overlap' or not a.compile_core + or a.activation_checkpointing or not a.bf16_residuals + or a.micro_batch != 4 or a.seq_len != 2048 + or a.activation_mode != 'baseline' or a.matmul_plan is not None + or a.compile_mode != 'default'): + p.error('--backward-layout requires compiled-autograd-overlap, --compile-core, ' + '--no-activation-checkpointing, --bf16-residuals, microbatch 4, sequence 2048, ' + 'default compile mode, baseline activations and no matmul plan') + if a.gradient_mode != 'eager' and a.activation_mode != 'baseline': + p.error('Standalone activation modes require eager gradients; use baseline activations ' + 'for compiled autograd, including matmul-plan epilogues') + if a.cache_core_weight_casts and a.activation_checkpointing: + p.error('--cache-core-weight-casts requires --no-activation-checkpointing') + if a.cache_core_weight_casts and a.gradient_mode != 'eager': + p.error('Core weight cast caching and compiled autograd must be tested separately') + if a.compile_mode == 'reduce-overhead' and (a.cache_core_weight_casts or a.gradient_mode != 'eager'): + p.error('Weight cast caching and compiled autograd require a compile mode without CUDA graphs') + if a.matmul_plan is not None and a.cache_core_weight_casts: + p.error('--matmul-plan cannot be combined with external core weight cast caching') + if a.matmul_plan is not None and a.compile_mode == 'reduce-overhead': + p.error('--matmul-plan currently requires a compile mode without CUDA graphs') + if a.matmul_plan is not None and a.gradient_mode != 'eager' and a.activation_checkpointing: + p.error('--matmul-plan with compiled autograd requires --no-activation-checkpointing') + a.rank = int(os.environ.get('RANK', '0')) + a.world_size = int(os.environ.get('WORLD_SIZE', '1')) + a.local_rank = int(os.environ.get('LOCAL_RANK', '0')) + if a.world_size > 1 and a.stage != 'train': + p.error('torchrun supports --stage train only; prepare the corpus once beforehand') + for name in ('work_dir','tokenizer','config','weights','matmul_plan'): + if getattr(a,name) is not None: + setattr(a,name,getattr(a,name).expanduser().resolve()) + return a + + +def bootstrap(a): + if a.no_bootstrap: + return + if sys.platform != 'linux': + raise RuntimeError('Automatic installation targets Linux. Tests may use --no-bootstrap.') + if not (3, 12) <= sys.version_info[:2] <= (3, 14): + raise RuntimeError('Use Python 3.12–3.14; Python 3.12 is recommended.') + env = ROOT / '.server-venv' + python = env / 'bin/python' + if not python.exists(): + subprocess.run([sys.executable, '-m', 'venv', str(env)], check=True) + marker = env / 'warpstate-deps-v1.json' + if not marker.exists(): + subprocess.run([str(python), '-m', 'pip', 'install', '--upgrade', 'pip'], check=True) + subprocess.run([str(python), '-m', 'pip', 'install', 'torch==2.14.0', + '--index-url', 'https://download.pytorch.org/whl/cu130'], check=True) + subprocess.run([str(python), '-m', 'pip', 'install', 'numpy==2.5.3', 'sentencepiece==0.2.2', + 'datasets==5.0.1', 'huggingface-hub==1.30.0'], check=True) + subprocess.run([str(python), '-m', 'pip', 'check'], check=True) + freeze = subprocess.check_output([str(python), '-m', 'pip', 'freeze'], text=True) + (env / 'requirements-resolved.txt').write_text(freeze) + marker.write_text(json.dumps({'installed': time.time()})) + os.execv(str(python), [str(python), '-u', str(Path(__file__).resolve()), *sys.argv[1:], '--no-bootstrap']) + + +def digest(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for block in iter(lambda: f.read(1024*1024), b''): + h.update(block) + return h.hexdigest() + + +def atomic(path, writer): + path = Path(path) + temporary = path.with_suffix(path.suffix + '.tmp') + with temporary.open('wb') as f: + writer(f) + f.flush() + os.fsync(f.fileno()) + os.replace(temporary, path) + + +@contextlib.contextmanager +def process_lock(path): + with Path(path).open('a+b') as f: + if os.name == 'nt': + import msvcrt + f.seek(0) + f.write(b'0') + f.flush() + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + + +def data_signature(a): + return {'recipe': RECIPE, 'dataset': a.dataset, 'config': a.dataset_config, 'requested_revision': a.revision, + 'train_tokens': a.train_tokens, 'val_tokens': a.val_tokens, 'tokenizer_sha256': digest(a.tokenizer), + 'dtype': ' 65536 or sp.eos_id() < 0: + raise ValueError('Need a uint16-compatible tokenizer with EOS') + progress = directory / 'prepare-state.pkl' + if progress.exists(): + with progress.open('rb') as f: + state = pickle.load(f) + if state['signature'] != signature or state['datasets_version'] != datasets.__version__: + raise ValueError('Preparation settings/version changed; use the original environment or a different work directory') + else: + if any((directory / (s+'.bin')).exists() for s in ('train','val')): + raise RuntimeError('Token files without preparation state: refusing to overwrite') + state = {'signature': signature, 'datasets_version': datasets.__version__, 'train': 0, 'val': 0, + 'revision': HfApi().dataset_info(a.dataset, revision=a.revision).sha, 'iterator': None, 'documents': 0} + atomic(progress, lambda f: pickle.dump(state, f)) + remaining_bytes = 2*(a.train_tokens+a.val_tokens-state['train']-state['val']) + reserve = 70*1024**3 if not a.test_small_gpu else 0 + if shutil.disk_usage(directory).free < remaining_bytes + reserve: + raise RuntimeError(f'Need {(remaining_bytes+reserve)/1024**3:.1f} GiB free for remaining data and checkpoints') + ds = datasets.load_dataset(a.dataset, name=a.dataset_config, split='train', streaming=True, revision=state['revision']) + print(f"Preparing {a.dataset}/{a.dataset_config} at {state['revision']}; " + f"resuming train={state['train']:,}, val={state['val']:,}", flush=True) + if state['iterator'] is not None: + ds.load_state_dict(state['iterator']) + iterator = iter(ds) + with contextlib.ExitStack() as stack: + stack.callback(lambda: getattr(iterator, 'close', lambda: None)()) + files = {} + for split in ('train', 'val'): + path = directory / (split+'.bin') + if not path.exists() and state[split]: + raise RuntimeError('Missing committed data: '+split) + f = stack.enter_context(path.open('r+b' if path.exists() else 'w+b')) + if path.stat().st_size < state[split]*2: + raise RuntimeError('Truncated committed data: '+split) + f.truncate(state[split]*2) # Discard only uncommitted output after interruption. + f.seek(state[split]*2) + files[split] = f + def commit(): + for f in files.values(): + f.flush() + os.fsync(f.fileno()) + state['iterator'] = ds.state_dict() + atomic(progress, lambda f: pickle.dump(state, f, protocol=pickle.HIGHEST_PROTOCOL)) + last_commit = state['train']+state['val'] + started = time.monotonic() + start_count = last_commit + while state['train'] < a.train_tokens or state['val'] < a.val_tokens: + rows = list(itertools.islice(iterator, a.documents_per_batch)) + if not rows: + commit() + raise RuntimeError('Source exhausted before the requested token budget; do not repeat it silently') + texts = [row['text'] for row in rows] + encoded = sp.encode(texts, out_type=int, num_threads=a.workers) + for text, ids in zip(texts, encoded): + state['documents'] += 1 + if not text.strip(): + continue + split = 'val' if is_holdout(text) else 'train' + target = a.val_tokens if split == 'val' else a.train_tokens + room = target - state[split] + if room <= 0: + continue + tokens = np.asarray((ids+[sp.eos_id()])[:room], dtype='= 8_000_000: + commit() + last_commit = count + print(f"prepared train={state['train']:,}/{a.train_tokens:,}, val={state['val']:,}; " + f"{(count-start_count)/max(time.monotonic()-started,1):,.0f} tokens/s", flush=True) + if STOP: + return False + commit() + manifest = {'signature': signature, 'source_revision': state['revision'], 'complete': True, + 'documents_read': state['documents'], 'datasets_version': datasets.__version__, + 'sha256': {s:digest(directory/(s+'.bin')) for s in ('train','val')}, + 'source_url': 'https://huggingface.co/datasets/'+a.dataset} + atomic(directory / 'manifest.json', lambda f: f.write(json.dumps(manifest, indent=2).encode())) + check_manifest(a) + print('Prepared the exact requested token counts.', flush=True) + return True + + +def batches(data, cursor, batch, length, valid, device): + import numpy as np + import torch + # A cyclic contiguous pass: no random sampling with replacement over 40B tokens. + indexes = (np.arange(batch*length+1, dtype=np.int64)+cursor) % len(data) + tokens = np.asarray(data[indexes], dtype=np.int64) + x = torch.from_numpy(tokens[:-1].reshape(batch, length).copy()) + y = torch.from_numpy(tokens[1:].reshape(batch, length).copy()) + y.view(-1)[valid:] = -100 + if device.type == 'cuda': + x, y = x.pin_memory(), y.pin_memory() + return x.to(device, non_blocking=True), y.to(device, non_blocking=True) + + +def train_signature(a, manifest): + keys = ('seq_len','micro_batch','grad_accum','train_tokens','lr','min_lr','warmup_steps','weight_decay', + 'seed','activation_checkpointing','loss_chunk_size','compile') + settings = {k: getattr(a,k) for k in keys} + settings.update(world_size=getattr(a, 'world_size', 1), compile_core=getattr(a, 'compile_core', False), + loss_mode=getattr(a, 'loss_mode', 'checkpointed'), + compile_mode=getattr(a, 'compile_mode', 'default'), + bf16_residuals=getattr(a, 'bf16_residuals', False), + activation_mode=getattr(a, 'activation_mode', 'baseline'), + gradient_mode=getattr(a, 'gradient_mode', 'eager'), + overlap_swap_collectives=getattr(a, 'overlap_swap_collectives', False), + overlap_runtime_estimates=getattr(a, 'overlap_runtime_estimates', False), + overlap_memory_budget=getattr(a, 'overlap_memory_budget', 0.0), + overlap_producer_schedule=getattr(a, 'overlap_producer_schedule', False), + matmul_execution=getattr(a, 'matmul_metadata', None), + backward_layout_execution=getattr(a, 'backward_layout_metadata', None), + cache_core_weight_casts=getattr(a, 'cache_core_weight_casts', False), + ddp_bucket_cap_mb=getattr(a, 'ddp_bucket_cap_mb', 128), + ddp_batched_grad_copy=getattr(a, 'ddp_batched_grad_copy', False)) + return {'settings': settings, 'data': manifest} + + +def check_resume_signature(old, new, reconfigure=False): + """Execution can change explicitly; the objective and global schedule cannot.""" + def normalized(signature): + settings = dict(signature['settings']) + settings.setdefault('world_size', 1) + settings.setdefault('compile_core', False) + settings.setdefault('loss_mode', 'checkpointed') + settings.setdefault('compile_mode', 'default') + settings.setdefault('bf16_residuals', False) + settings.setdefault('activation_mode', 'baseline') + settings.setdefault('gradient_mode', 'eager') + settings.setdefault('overlap_swap_collectives', False) + settings.setdefault('overlap_runtime_estimates', False) + settings.setdefault('overlap_memory_budget', 0.0) + settings.setdefault('overlap_producer_schedule', False) + settings.setdefault('matmul_execution', None) + settings.setdefault('backward_layout_execution', None) + settings.setdefault('cache_core_weight_casts', False) + settings.setdefault('ddp_bucket_cap_mb', 128) + settings.setdefault('ddp_batched_grad_copy', False) + return {'settings': settings, 'data': signature['data']} + old, new = normalized(old), normalized(new) + if old == new: + return False + message = 'Resume settings/data differ; keep original settings or use --reconfigure for execution-only changes' + if not reconfigure or old['data'] != new['data']: + raise ValueError(message) + execution = {'micro_batch', 'grad_accum', 'world_size', 'activation_checkpointing', + 'loss_chunk_size', 'loss_mode', 'compile', 'compile_core', 'compile_mode', + 'bf16_residuals', 'activation_mode', 'gradient_mode', 'cache_core_weight_casts', 'matmul_execution', + 'overlap_swap_collectives', 'overlap_runtime_estimates', 'overlap_memory_budget', + 'overlap_producer_schedule', + 'ddp_bucket_cap_mb', 'ddp_batched_grad_copy', 'backward_layout_execution'} + objective = lambda s: {k: v for k, v in s.items() if k not in execution} + batch = lambda s: s['seq_len']*s['micro_batch']*s['grad_accum']*s['world_size'] + if objective(old['settings']) != objective(new['settings']) or batch(old['settings']) != batch(new['settings']): + raise ValueError(message + '; objective and global tokens per update must stay fixed') + return True + + +def update_slices(target, micro_tokens, world_size, rank): + """Disjoint contiguous slices; empty ranks still participate in the final backward.""" + for iteration in range(math.ceil(target / (world_size*micro_tokens))): + offset = (iteration*world_size + rank)*micro_tokens + yield offset, max(0, min(micro_tokens, target-offset)) + + +def compile_training_cores(model, mode='default'): + import torch + for core in model.cores: + eager = core.forward + compiled = torch.compile(eager, fullgraph=True, dynamic=False, mode=mode) + def forward(*args, _core=core, _eager=eager, _compiled=compiled, **kwargs): + # Validation does not need another compiled graph for a different batch size. + fn = _compiled if _core.training and torch.is_grad_enabled() else _eager + return fn(*args, **kwargs) + core.forward = forward + + +def configure_memory(a): + """Apply the cap before model allocation; this does not reserve another job's memory.""" + if a.vram_limit_gib is None: + return + import torch + device = torch.device('cuda', getattr(a, 'local_rank', 0)) + free, total = torch.cuda.mem_get_info(device) + limit = int(a.vram_limit_gib * 1024**3) + required = limit + int(a.vram_headroom_gib * 1024**3) + if limit > total or free < required: + raise RuntimeError(f'Only {free/1024**3:.2f} GiB free; need {required/1024**3:.2f} GiB ' + 'for the requested allocator cap and headroom') + torch.cuda.set_per_process_memory_fraction(limit / total, device) + print(f'CUDA allocator cap {a.vram_limit_gib:.2f} GiB; free before model ' + f'{free/1024**3:.2f} GiB; autocast cache={a.autocast_cache}', flush=True) + + +def train(a): + import numpy as np + import sentencepiece as spm + import torch + import torch.distributed as dist + from warpstate import WarpStateConfig, WarpStateLM + manifest = check_manifest(a) + matmul_plan = None + if a.matmul_plan is not None: + from ops.matmul_execution import load_matmul_plan + matmul_plan = load_matmul_plan(a.matmul_plan) + a.matmul_metadata = matmul_plan.metadata if matmul_plan else None + a.backward_layout_metadata = None + if a.backward_layout != 'baseline': + from ops.backward_layout_execution import configure_backward_layout + a.backward_layout_metadata = configure_backward_layout(a.backward_layout) + if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): + raise RuntimeError('BF16 CUDA GPU required') + device = torch.device('cuda', a.local_rank) + distributed = a.world_size > 1 + props = torch.cuda.get_device_properties(device) + if not a.test_small_gpu and props.total_memory < 80*1024**3: + raise RuntimeError('This preset requires the 96 GB RTX PRO 6000; use --test-small-gpu only for tiny tests') + torch.set_num_threads(min(a.workers, 8)) + torch.set_float32_matmul_precision('high') + torch.backends.cuda.matmul.allow_tf32 = True + random.seed(a.seed) + np.random.seed(a.seed) + torch.manual_seed(a.seed) + torch.cuda.manual_seed(a.seed + a.rank) + run = a.work_dir / 'run' + run.mkdir(exist_ok=True) + latest = run / 'latest.pt' + resuming = latest.exists() + source = latest if resuming else a.weights + if not resuming and a.from_scratch: + checkpoint = {} + elif source is None or not source.is_file(): + raise ValueError('First launch requires --weights /path/to/your/checkpoint.pt') + else: + checkpoint = torch.load(source, map_location='cpu', mmap=True, weights_only=False) + cfg = WarpStateConfig(**checkpoint.get('config', json.loads(a.config.read_text()))) + cfg.validate() + if a.backward_layout != 'baseline': + from ops.backward_layout_execution import validate_backward_layout_config + validate_backward_layout_config(cfg) + if a.seq_len % cfg.chunk_size: + raise ValueError('seq-len must be divisible by model chunk_size') + sp = spm.SentencePieceProcessor(model_file=str(a.tokenizer)) + if cfg.vocab_size != sp.vocab_size(): + raise ValueError('Tokenizer/model vocabulary mismatch; use the exact original tokenizer') + signature = train_signature(a, manifest) + changed = False + if resuming: + if checkpoint.get('format') != 'warpstate-server-v1': + raise ValueError('Resume settings/data differ; unsupported checkpoint format') + changed = check_resume_signature(checkpoint['signature'], signature, a.reconfigure) + if not resuming and a.from_scratch: + print(f'Initializing fresh FP32 weights on CPU with seed {a.seed}.', flush=True) + model = WarpStateLM(cfg) + else: + with torch.device('meta'): + model = WarpStateLM(cfg) + model.load_state_dict(checkpoint['model'], assign=True) + model.to(device=device, dtype=torch.float32).requires_grad_(True) + # Prefer unrounded FP32 masters when importing the laptop's CPU-offload checkpoint. + old_opt = checkpoint.get('optimizer', {}) + if not resuming and old_opt.get('kind') == 'cpu_offload_adamw': + with torch.no_grad(): + for parameter, master in zip(model.parameters(), old_opt['masters'], strict=True): + if parameter.shape != master.shape: + raise ValueError('Master weight shape mismatch') + parameter.copy_(master) + cfg.gradient_checkpointing = a.activation_checkpointing + model.loss_chunk_size = a.loss_chunk_size + optimizer = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9,0.95), eps=1e-8, + weight_decay=a.weight_decay, fused=True) + seen, step = 0, 0 + if resuming: + optimizer.load_state_dict(checkpoint['optimizer']) + seen, step = checkpoint['tokens_seen'], checkpoint['step'] + rank_states = checkpoint.get('rng_by_rank') + if rank_states is None: + rank_states = [{'torch': checkpoint['rng_torch'], 'cuda': checkpoint['rng_cuda'][0], + 'numpy': checkpoint['rng_numpy'], 'python': checkpoint['rng_python']}] + if a.rank < len(rank_states): + state = rank_states[a.rank] + torch.set_rng_state(state['torch']) + torch.cuda.set_rng_state(state['cuda'], device) + np.random.set_state(state['numpy']) + random.setstate(state['python']) + else: + # A newly added rank gets a reproducible independent stream. + random.seed(a.seed + a.rank) + np.random.seed(a.seed + a.rank) + torch.manual_seed(a.seed + a.rank) + torch.cuda.manual_seed(a.seed + a.rank) + origin = checkpoint.get('origin', + {'initialization': 'random', 'seed': a.seed, 'previous_training_tokens': 0} + if not resuming and a.from_scratch else + {'path': str(source), 'previous_training_tokens': checkpoint.get('tokens_seen')}) + history = list(checkpoint.get('execution_history', [])) + if changed: + history.append({'step': step, 'tokens': seen, 'timestamp': time.time(), + 'from': checkpoint['signature']['settings'], 'to': signature['settings']}) + if a.rank == 0: + print(f'Reconfiguring execution at step {step}, tokens={seen:,}; preserving model, optimizer and schedule.', flush=True) + del old_opt, checkpoint + count = model.num_parameters() + if shutil.disk_usage(run).free < count*28: + raise RuntimeError('Need free space for checkpoint plus atomic replacement (about 53 GiB for 2B)') + raw = model + from ops.gradient_execution import OverlapSettings, make_backward, scale_loss + from ops.distributed_execution import ddp_kwargs, ddp_runtime_settings + overlap_settings = (OverlapSettings(swap_with_collectives=a.overlap_swap_collectives, + use_runtime_estimations=a.overlap_runtime_estimates, + producer_schedule=a.overlap_producer_schedule, + peak_memory_budget=a.overlap_memory_budget) + if a.gradient_mode == 'compiled-autograd-overlap' else None) + backward = make_backward(a.gradient_mode, distributed=distributed, overlap_settings=overlap_settings) + ddp_settings = ddp_runtime_settings(bucket_cap_mb=a.ddp_bucket_cap_mb, + batched_grad_copy=a.ddp_batched_grad_copy) + if a.activation_mode != 'baseline': + from ops.activation_execution import apply_activation_execution + apply_activation_execution(raw, a.activation_mode) + if matmul_plan is not None: + from ops.matmul_execution import apply_matmul_execution + apply_matmul_execution(raw, matmul_plan) + if a.compile_core: + compile_training_cores(raw, mode=a.compile_mode) + if a.loss_mode != 'checkpointed': + from ops.loss_experiment import apply_loss_execution + apply_loss_execution(raw, a.loss_mode) + if a.bf16_residuals: + from ops.residual_experiment import apply_bf16_residuals + apply_bf16_residuals(raw) + weight_cast_cache = None + if a.cache_core_weight_casts: + from ops.weight_cast_execution import apply_weight_cast_cache + weight_cast_cache = apply_weight_cast_cache(raw) + if a.compile: + model = torch.compile(model, mode=a.compile_mode) # Optional: GPU-only optimizer, no CPU offload hooks. + if distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[a.local_rank], + output_device=a.local_rank, **ddp_kwargs(ddp_settings)) + graph_gradients = (a.compile or a.compile_core) and a.compile_mode == 'reduce-overhead' + if graph_gradients: + # Replay must accumulate into stable buffers outside the graph's private pool. + for parameter in model.parameters(): + if parameter.requires_grad: + parameter.grad = torch.zeros_like(parameter) + train_data = np.memmap(a.work_dir/'data/train.bin', dtype='= end_step: + print('Pause boundary already reached; no further optimizer updates.', flush=True) + (run/'settings.json').write_text(json.dumps({'signature':signature, 'gpu':props.name, + 'torch':torch.__version__, 'cuda':torch.version.cuda, 'parameters':count, 'origin':origin, + 'execution_history': history, + 'runtime': {'stop_at_step': a.stop_at_step, 'max_steps': a.max_steps, 'end_step': end_step, + 'vram_limit_gib': a.vram_limit_gib, 'vram_headroom_gib': a.vram_headroom_gib, + 'autocast_cache': a.autocast_cache, 'eval_micro_batch': a.eval_micro_batch, + 'eval_every': a.eval_every, 'save_every': a.save_every, + 'async_checkpoint': a.async_checkpoint, + 'bf16_residuals': a.bf16_residuals, + 'activation_mode': a.activation_mode, 'gradient_mode': a.gradient_mode, + 'overlap_swap_collectives': a.overlap_swap_collectives, + 'overlap_runtime_estimates': a.overlap_runtime_estimates, + 'overlap_memory_budget': a.overlap_memory_budget, + 'overlap_producer_schedule': a.overlap_producer_schedule, + 'matmul_execution': a.matmul_metadata, + 'backward_layout_execution': a.backward_layout_metadata, + 'cache_core_weight_casts': a.cache_core_weight_casts, + **ddp_settings, + 'blas_preference': str(torch.backends.cuda.preferred_blas_library()), + 'cublaslt_workspace_size_env': os.getenv('CUBLASLT_WORKSPACE_SIZE'), + 'cuda_visible_devices': os.getenv('CUDA_VISIBLE_DEVICES')}}, indent=2)) + def evaluate(): + model.eval() + values = [] + if a.rank == 0: + validation_tokens = a.eval_micro_batch*a.seq_len + with torch.no_grad(): + for cursor in np.linspace(0, max(0,len(val_data)-validation_tokens-1), a.eval_batches, dtype=np.int64): + x,y = batches(val_data,int(cursor),a.eval_micro_batch,a.seq_len,validation_tokens,device) + with torch.autocast('cuda', dtype=torch.bfloat16, cache_enabled=a.autocast_cache): + _, loss = raw(x,y,return_logits=False) + values.append(float(loss)) + model.train() + value = sum(values)/len(values) if values else 0.0 + if distributed: + tensor = torch.tensor(value, dtype=torch.float64, device=device) + dist.broadcast(tensor, src=0) + value = float(tensor) + if not math.isfinite(value): + raise RuntimeError('Non-finite validation loss') + return value + def save(validation): + state = {'torch': torch.get_rng_state(), 'cuda': torch.cuda.get_rng_state(device), + 'numpy': np.random.get_state(), 'python': random.getstate()} + rank_states = [state] + if distributed: + rank_states = [None]*a.world_size + dist.all_gather_object(rank_states, state) + def build_state(): + print('Staging full CPU checkpoint...' if a.async_checkpoint else + 'Saving full GPU-optimizer checkpoint...', flush=True) + return {'format':'warpstate-server-v1', 'signature':signature, 'origin':origin, + 'execution_history': history, + 'config':vars(cfg), 'model':raw.state_dict(), 'optimizer':optimizer.state_dict(), + 'step':step, 'tokens_seen':seen, 'val_loss':validation, + 'val_step':last_val_step, 'val_tokens':last_val_tokens, 'rng_by_rank': rank_states, + 'rng_torch':state['torch'], 'rng_cuda':[state['cuda']], + 'rng_numpy':state['numpy'], 'rng_python':state['python']} + if checkpoint_writer is not None: + # Every rank waits for the frozen CPU snapshot or a coordinated error. + # Only rank zero writes; subsequent updates overlap its CPU thread. + staging_seconds = checkpoint_writer.submit(state_factory=build_state) + if a.rank == 0: + print(f'Staged step {step}, tokens={seen:,} for asynchronous checkpoint; ' + f'synchronous CPU staging {staging_seconds:.2f}s.', flush=True) + else: + if a.rank == 0: + obj = build_state() + atomic(latest, lambda f: torch.save(obj,f)) + print('Saved '+str(latest), flush=True) + if distributed: + dist.barrier() + def stop_requested(): + stop_file = os.environ.get('WARPSTATE_STOP_FILE') + requested = STOP or bool(stop_file and Path(stop_file).is_file()) + if not distributed: + return requested + flag = torch.tensor(int(requested), device=device) + dist.all_reduce(flag, op=dist.ReduceOp.MAX) + return bool(flag.item()) + checkpoint_writer = None + if a.async_checkpoint: + from ops.checkpoint_writer import DistributedCheckpointWriter + checkpoint_writer = DistributedCheckpointWriter(latest, rank=a.rank, + device=device, distributed=distributed) + with checkpoint_writer if checkpoint_writer is not None else contextlib.nullcontext(): + last_val = evaluate() + last_val_step, last_val_tokens = step, seen + if a.rank == 0: + print(f'initial validation loss: {last_val:.5f}', flush=True) + while seen < a.train_tokens and step < end_step: + if checkpoint_writer is not None: + checkpoint_writer.check() + if stop_requested(): + save(last_val) + break + torch.cuda.synchronize() + begin = time.perf_counter() + optimizer.zero_grad(set_to_none=not graph_gradients) + target = min(update_tokens,a.train_tokens-seen) + warm = min(a.warmup_steps,total_steps) + rate = (a.lr*(step+1)/warm if step < warm else a.min_lr+(a.lr-a.min_lr)*0.5* + (1+math.cos(math.pi*(step-warm)/max(1,total_steps-warm)))) + for group in optimizer.param_groups: + group['lr'] = rate + loss_sum = torch.zeros((),device=device) + slices = list(update_slices(target, batch_tokens, a.world_size, a.rank)) + if a.backward_layout == 'paired': + from ops.backward_layout_execution import backward_layout_stats, require_backward_layout_dispatch + layout_calls_before = backward_layout_stats()['runtime_calls'] + with weight_cast_cache.update() if weight_cast_cache else contextlib.nullcontext(): + for index, (offset, valid) in enumerate(slices): + x,y = batches(train_data,seen+offset,a.micro_batch,a.seq_len,valid,device) + sync = model.no_sync() if distributed and index < len(slices)-1 else contextlib.nullcontext() + with sync: + with torch.autocast('cuda',dtype=torch.bfloat16,cache_enabled=a.autocast_cache): + _, loss = model(x,y,return_logits=False) + scaled = scale_loss(loss, valid*a.world_size/target, a.gradient_mode) + backward(scaled) + loss_sum += loss.detach()*(valid/target) + if a.backward_layout == 'paired': + require_backward_layout_dispatch(min_calls=layout_calls_before + cfg.logical_depth * len(slices)) + norm = torch.nn.utils.clip_grad_norm_(model.parameters(),1.0,error_if_nonfinite=True) + optimizer.step() + if distributed: + dist.all_reduce(loss_sum, op=dist.ReduceOp.SUM) + torch.cuda.synchronize() + elapsed = time.perf_counter()-begin + gpu_stats = [{'allocated': torch.cuda.max_memory_allocated(device)/1024**3, + 'reserved': torch.cuda.max_memory_reserved(device)/1024**3, + 'free': torch.cuda.mem_get_info(device)[0]/1024**3, 'seconds': elapsed}] + if distributed: + local_stats = gpu_stats[0] + gpu_stats = [None]*a.world_size + dist.all_gather_object(gpu_stats, local_stats) + elapsed = max(item['seconds'] for item in gpu_stats) + seen += target + step += 1 + record = {'step':step,'tokens':seen,'loss':float(loss_sum),'lr':rate,'grad_norm':float(norm), + 'tokens_per_second':target/elapsed,'seconds':elapsed,'timestamp':time.time(), + 'world_size': a.world_size, 'gpu_memory': gpu_stats, + 'peak_vram_gib':max(item['allocated'] for item in gpu_stats), + 'peak_reserved_vram_gib':max(item['reserved'] for item in gpu_stats), + 'device_free_vram_gib':min(item['free'] for item in gpu_stats)} + torch.cuda.reset_peak_memory_stats() + ending = stop_requested() or seen == a.train_tokens or step >= end_step + if step % a.eval_every == 0 or ending: + last_val = evaluate() + last_val_step, last_val_tokens = step, seen + if a.rank == 0: + with (run/'validation.jsonl').open('a') as f: + f.write(json.dumps({'step':step,'tokens':seen,'loss':last_val})+'\n') + record.update(val_loss=last_val, val_step=last_val_step, val_tokens=last_val_tokens) + if a.backward_layout != 'baseline': + from ops.backward_layout_execution import backward_layout_stats + record['backward_layout'] = backward_layout_stats() + if a.rank == 0: + validation_log = f'val {last_val:.5f} (step {last_val_step}) | ' if last_val_step == step else '' + print(f"step {step}/{total_steps} | {seen:,}/{a.train_tokens:,} tokens | loss {record['loss']:.4f} | " + f"{validation_log}" + f"{record['tokens_per_second']:,.0f} tok/s | peak {record['peak_vram_gib']:.2f} GiB allocated / " + f"{record['peak_reserved_vram_gib']:.2f} GiB reserved | free {record['device_free_vram_gib']:.2f} GiB",flush=True) + with (run/'metrics.jsonl').open('a') as f: + f.write(json.dumps(record)+'\n') + if step % a.save_every == 0 or ending: + save(last_val) + if ending: + break + + +def request_stop(signum, frame): + global STOP + STOP = True + print('Stop requested; committing preparation or finishing the current optimizer update.', flush=True) + + +def preflight(a): + """Reject incompatible weights/hardware before a multi-day corpus preparation.""" + import torch + import sentencepiece as spm + from warpstate import WarpStateConfig, WarpStateLM + if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): + raise RuntimeError('Install a CUDA 13.0-compatible NVIDIA driver; BF16 CUDA is required') + if not a.test_small_gpu and torch.cuda.get_device_properties(a.local_rank).total_memory < 80*1024**3: + raise RuntimeError('The normal preset requires the 96 GB GPU') + latest = a.work_dir/'run/latest.pt' + source = latest if latest.exists() else a.weights + fresh = not latest.exists() and a.from_scratch + checkpoint = {} if fresh else torch.load(source, map_location='cpu', mmap=True, weights_only=False) + if not fresh and 'model' not in checkpoint: + raise ValueError('Expected a WarpState checkpoint with a model state dictionary, not arbitrary model weights') + cfg = WarpStateConfig(**checkpoint.get('config',json.loads(a.config.read_text()))) + cfg.validate() + if cfg.vocab_size != spm.SentencePieceProcessor(model_file=str(a.tokenizer)).vocab_size(): + raise ValueError('Tokenizer/model vocabulary mismatch') + if a.seq_len % cfg.chunk_size: + raise ValueError('Sequence length must be divisible by the local tile size') + with torch.device('meta'): + probe = WarpStateLM(cfg) + if not fresh: + probe.load_state_dict(checkpoint['model'], assign=True, strict=True) + if not a.test_small_gpu and not 2_000_000_000 <= probe.num_parameters() <= 2_020_000_000: + raise ValueError('This package expects the approximately 2B WarpState architecture') + optimizer_state = checkpoint.get('optimizer', {}) + if optimizer_state.get('kind') == 'cpu_offload_adamw': + for parameter, master in zip(probe.parameters(), optimizer_state['masters'], strict=True): + if parameter.shape != master.shape: + raise ValueError('FP32 master weight shape mismatch') + if latest.exists() and checkpoint.get('format') != 'warpstate-server-v1': + raise ValueError('Use --weights to initialize a new server run; do not place a laptop checkpoint in work/run') + print(f'Preflight: compatible WarpState {"configuration for fresh initialization" if fresh else "weights"} ' + f'({probe.num_parameters():,} parameters) and CUDA GPU.', flush=True) + + +def main(): + a = arguments() + if not a.tokenizer.is_file(): + raise FileNotFoundError(a.tokenizer) + if (a.stage != 'prepare' and not (a.work_dir/'run/latest.pt').exists() and not a.from_scratch + and (a.weights is None or not a.weights.is_file())): + raise ValueError('Supply --weights or explicitly choose --from-scratch before training') + bootstrap(a) + a.work_dir.mkdir(parents=True,exist_ok=True) + os.environ.setdefault('HF_HOME',str(a.work_dir/'hf-cache')) + os.environ.setdefault('TORCHINDUCTOR_CACHE_DIR',str(a.work_dir/'compile-cache')) + signals = [signal.SIGINT, signal.SIGTERM] + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for sig in signals: + signal.signal(sig,request_stop) + distributed = a.world_size > 1 + if a.stage != 'prepare': + import torch + torch.cuda.set_device(a.local_rank) + configure_memory(a) + if distributed: + import torch.distributed as dist + dist.init_process_group('nccl', device_id=torch.device('cuda', a.local_rank)) + try: + with contextlib.ExitStack() as stack: + if a.rank == 0: + stack.enter_context(process_lock(a.work_dir/'pipeline.lock')) + if distributed: + dist.barrier() + if a.stage != 'prepare': + preflight(a) + if a.stage in ('all','prepare') and not prepare(a): + return + if a.stage in ('all','train'): + train(a) + finally: + if distributed: + dist.destroy_process_group() + + +if __name__ == '__main__': + main() diff --git a/test_activation_execution.py b/test_activation_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..83bc47f038ab6ddee55972472482667383804122 --- /dev/null +++ b/test_activation_execution.py @@ -0,0 +1,169 @@ +"""Optional activation kernels preserve state, rounding and usable gradients.""" + +from __future__ import annotations + +import copy +import os + +import pytest +import torch +import torch.nn.functional as F + +from ops.activation_execution import apply_activation_execution, bf16_rmsnorm, bf16_swiglu +from warpstate import WarpStateConfig, WarpStateLM +from warpstate.model import RMSNorm + + +def _model(): + torch.manual_seed(943) + return WarpStateLM(WarpStateConfig(vocab_size=48, d_model=32, n_heads=2, + n_cores=1, logical_depth=2, ffn_hidden=56, chunk_size=4, + gradient_checkpointing=False, parallel_memory=True)).train() + + +@pytest.mark.parametrize("mode", ["baseline", "swiglu", "rmsnorm", "all"]) +@pytest.mark.parametrize("autocast", [False, True]) +def test_cpu_fallback_retains_parameters_state_logits_and_gradients(mode, autocast): + reference = _model() + actual = copy.deepcopy(reference) + parameters = tuple(actual.parameters()) + keys = tuple(actual.state_dict()) + apply_activation_execution(actual, mode) + tokens = torch.randint(48, (2, 8)) + with torch.autocast("cpu", dtype=torch.bfloat16, enabled=autocast): + expected = reference(tokens) + result = actual(tokens) + torch.testing.assert_close(result, expected, rtol=0, atol=0) + expected.float().square().mean().backward() + result.float().square().mean().backward() + assert tuple(actual.state_dict()) == keys + assert all(a is b for a, b in zip(actual.parameters(), parameters, strict=True)) + for p, q in zip(actual.parameters(), reference.parameters(), strict=True): + assert p.dtype == p.grad.dtype == torch.float32 + torch.testing.assert_close(p.grad, q.grad, rtol=0, atol=0) + reference.eval() + actual.eval() + with torch.no_grad(): + torch.testing.assert_close(actual(tokens), reference(tokens), rtol=0, atol=0) + + +def test_mode_validation_and_noop(): + model = _model() + forward = model.cores[0].ffn.forward + apply_activation_execution(model, "baseline") + assert model.cores[0].ffn.forward == forward + with pytest.raises(ValueError, match="Unknown activation"): + apply_activation_execution(model, "invalid") + apply_activation_execution(model, "all") + with pytest.raises(ValueError, match="already been applied"): + apply_activation_execution(model, "all") + with pytest.raises(ValueError, match="FP32 master"): + apply_activation_execution(_model().bfloat16(), "all") + + +# CUDA tests require an explicit maintenance window. Normal CPU CI never starts +# CUDA or competes with the running training process merely on test collection. +cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1", + reason="set WARPSTATE_TEST_CUDA=1 in an exclusive GPU window") + + +def _check_error(actual, expected, *, relative_rms=0.003, atol=0.0625): + assert actual.dtype == expected.dtype + assert torch.isfinite(actual).all() + difference = actual.float() - expected.float() + relative = difference.square().mean().sqrt() / expected.float().square().mean().sqrt().clamp_min(1e-20) + assert relative.item() <= relative_rms, relative.item() + assert difference.abs().max().item() <= atol, difference.abs().max().item() + + +@cuda_window +@pytest.mark.parametrize("shape", [(3, 224), (2, 32, 256), (257, 14336)]) +@pytest.mark.parametrize("compiled", [False, True]) +def test_cuda_swiglu_forward_and_backward(shape, compiled): + torch.manual_seed(947) + source = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + reference = source.clone().requires_grad_() + actual = source.clone().requires_grad_() + def reference_fn(x): + up, gate = x.chunk(2, dim=-1) + return F.silu(gate) * up + fn = lambda x: bf16_swiglu(x, eager_rounding=not compiled) + if compiled: + reference_fn = torch.compile(reference_fn, fullgraph=True) + fn = torch.compile(fn, fullgraph=True) + expected = reference_fn(reference) + result = fn(actual) + grad = torch.randn_like(expected) + expected.backward(grad) + result.backward(grad) + _check_error(result, expected, relative_rms=0.0002) + _check_error(actual.grad, reference.grad, relative_rms=0.001) + + +@cuda_window +@pytest.mark.parametrize("shape", [(3, 24), (2, 16, 128), (257, 2048)]) +@pytest.mark.parametrize("compiled", [False, True]) +def test_cuda_rmsnorm_rounding_and_master_gradient(shape, compiled): + torch.manual_seed(949) + reference = RMSNorm(shape[-1]).cuda() + with torch.no_grad(): + reference.weight.uniform_(0.1, 2.0) + weight = reference.weight.detach().clone().requires_grad_() + source = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + x_ref = source.clone().requires_grad_() + x_actual = source.clone().requires_grad_() + fn = lambda x, w: bf16_rmsnorm(x, w, reference.eps, eager_rounding=not compiled)[0] + ref_fn = reference + if compiled: + ref_fn = torch.compile(reference, fullgraph=True) + fn = torch.compile(fn, fullgraph=True) + expected = ref_fn(x_ref) + result = fn(x_actual, weight) + # A strided upstream gradient verifies backward's contiguous handling. + grad = torch.randn((*shape, 2), device="cuda", dtype=torch.bfloat16)[..., 0] + expected.backward(grad) + result.backward(grad) + _check_error(result, expected, relative_rms=0.0002) + _check_error(x_actual.grad, x_ref.grad, relative_rms=0.001) + _check_error(weight.grad, reference.weight.grad, + relative_rms=0.003, atol=0.5) + assert weight.grad.dtype == torch.float32 + + +@cuda_window +def test_cuda_compiled_model_with_bf16_residuals_and_optimizer(): + from ops.residual_experiment import apply_bf16_residuals + from run_40b import compile_training_cores + reference = _model().cuda() + actual = copy.deepcopy(reference) + apply_bf16_residuals(reference) + apply_bf16_residuals(actual) + parameters = tuple(actual.parameters()) + keys = tuple(actual.state_dict()) + apply_activation_execution(actual, "all") + compile_training_cores(reference) + compile_training_cores(actual) + optimizer = torch.optim.AdamW(actual.parameters(), lr=1e-4) + tokens = torch.randint(48, (2, 8), device="cuda") + with torch.autocast("cuda", dtype=torch.bfloat16): + ref_loss = reference(tokens, tokens, return_logits=False)[1] + loss = actual(tokens, tokens, return_logits=False)[1] + ref_loss.backward() + loss.backward() + torch.testing.assert_close(loss, ref_loss, rtol=0.001, atol=0.001) + grads = torch.cat([p.grad.flatten() for p in parameters]) + ref_grads = torch.cat([p.grad.flatten() for p in reference.parameters()]) + assert F.cosine_similarity(grads, ref_grads, dim=0).item() > 0.999 + assert (grads - ref_grads).norm().item() / ref_grads.norm().item() < 0.02 + optimizer.step() + assert tuple(actual.state_dict()) == keys + assert all(a is b for a, b in zip(actual.parameters(), parameters, strict=True)) + assert all(p.dtype == p.grad.dtype == optimizer.state[p]["exp_avg"].dtype + == optimizer.state[p]["exp_avg_sq"].dtype == torch.float32 for p in parameters) + restored = _model().cuda() + restored.load_state_dict(actual.state_dict(), strict=True) + actual.eval() + with torch.no_grad(): + # Outside autocast the BF16 residual wrapper and activation patches both + # retain exactly the archived model's original FP32 inference behavior. + torch.testing.assert_close(actual(tokens), restored.eval()(tokens), rtol=0, atol=0) diff --git a/test_analyze_communication_profile.py b/test_analyze_communication_profile.py new file mode 100644 index 0000000000000000000000000000000000000000..dccb10dda0de7d8a87587ad94e98d6776132b938 --- /dev/null +++ b/test_analyze_communication_profile.py @@ -0,0 +1,102 @@ +"""CPU fixtures for stream unions, exposed communication, and collective metadata.""" + +import copy + +import pytest + +from ops.analyze_communication_profile import merge_intervals, summarize_trace + + +def activity(name, begin, duration, *, device=0, category="kernel", **metadata): + return {"ph": "X", "cat": category, "name": name, "ts": begin, "dur": duration, + "pid": device, "args": {"device": device, **metadata}} + + +def nccl(begin, duration, sequence=1, **metadata): + return activity("nccl_AllReduce", begin, duration, + **{"Process Group Name": "0", "Seq": sequence, "Collective name": "allreduce", + "In msg nelems": 32, "dtype": "Float", **metadata}) + + +def test_interval_union_excludes_duplicate_streams_and_gpu_annotations(): + events = [activity("gemm", 0, 10), activity("pointwise", 5, 10), + nccl(8, 12), nccl(9, 2), activity("copy", 25, 5, category="gpu_memcpy"), + activity("nccl:annotation", 0, 100, category="gpu_user_annotation")] + result = summarize_trace({"traceEvents": events, "distributedInfo": {"rank": 1}}) + device = result["devices"][0] + assert result["rank"] == 1 and result["excluded_gpu_annotation_ranges"] == 1 + assert device["noncommunication_union_us"] == 20 + assert device["communication_union_us"] == 12 + assert device["compute_communication_overlap_us"] == 7 + assert device["compute_only_us"] == 13 + assert device["communication_only_us"] == 5 + assert device["active_union_us"] == 25 + assert device["activity_span_us"] == 30 + assert device["idle_within_activity_span_us"] == 5 + assert device["summed_communication_us"] == 14 + assert device["collective_call_count"] == 1 + assert device["nccl_kernel_count"] == 2 + assert device["logical_input_payload_bytes"] == 128 + + +def test_devices_are_separate_and_unknown_transport_bytes_remain_unknown(): + trace = {"traceEvents": [activity("gemm", 0, 10, device=0), + activity("gemm", 0, 10, device=1), + activity("nccl_kernel", 8, 5, device=1)]} + first, second = summarize_trace(trace)["devices"] + assert first["active_union_us"] == 10 + assert second["active_union_us"] == 13 + assert second["compute_communication_overlap_us"] == 2 + assert second["collective_call_count"] is None + assert second["logical_input_payload_bytes"] is None + assert second["nccl_kernel_count"] == 1 + + +@pytest.mark.parametrize("metadata", [{"dtype": "unknown"}, {"In msg nelems": "32"}]) +def test_unknown_payload_metadata_cannot_generate_plausible_byte_totals(metadata): + device = summarize_trace({"traceEvents": [nccl(0, 1, **metadata)]})["devices"][0] + assert device["collective_call_count"] == 1 + assert device["logical_input_payload_bytes"] is None + + +@pytest.mark.parametrize("sequence", [-1, None, True]) +def test_unavailable_sequence_numbers_are_not_deduplicated_into_fake_calls(sequence): + device = summarize_trace({"traceEvents": [nccl(0, 1, sequence), nccl(2, 1, sequence)]})["devices"][0] + assert device["nccl_kernel_count"] == 2 + assert device["collective_call_count"] is None + assert device["logical_input_payload_bytes"] is None + + +def test_inconsistent_duplicate_collective_metadata_is_reported(): + device = summarize_trace({"traceEvents": [nccl(0, 1), nccl(1, 1, **{"In msg nelems": 64})]})["devices"][0] + assert device["collective_call_count"] == 1 + assert device["logical_input_payload_bytes"] is None + assert device["collectives"][0]["metadata_consistent"] is False + + +def test_cpu_fingerprint_uses_submission_order_not_annotations_or_timestamps(): + def collective(begin, size): + return {"ph": "X", "cat": "cpu_op", "name": "c10d::allreduce_", "ts": begin, "dur": 2, + "args": {"Input Dims": [[[size]]], "Input type": ["TensorList"]}} + + trace = {"traceEvents": [collective(3, 64), collective(1, 32), + activity("nccl:all_reduce", 2, 1, category="user_annotation")]} + expected = summarize_trace(trace)["cpu_collective_fingerprint"] + assert expected["call_count"] == 2 + shifted = copy.deepcopy(trace) + for event in shifted["traceEvents"]: + event["ts"] += 100 + assert summarize_trace(shifted)["cpu_collective_fingerprint"]["sha256"] == expected["sha256"] + shifted["traceEvents"][0]["ts"] = 0 + assert summarize_trace(shifted)["cpu_collective_fingerprint"]["sha256"] != expected["sha256"] + + +def test_empty_trace_and_touching_intervals(): + assert merge_intervals([(2, 3), (0, 2), (1, 1)]) == [(0, 3)] + assert summarize_trace({"traceEvents": []})["devices"] == [] + + +@pytest.mark.parametrize("interval", [(1, 0), (0, float("inf")), (float("nan"), 2)]) +def test_invalid_intervals_fail_explicitly(interval): + with pytest.raises(ValueError): + merge_intervals([interval]) diff --git a/test_archive_dpo.py b/test_archive_dpo.py new file mode 100644 index 0000000000000000000000000000000000000000..d1bb9ce87a933d6e16564967b107fa8899b7cccf --- /dev/null +++ b/test_archive_dpo.py @@ -0,0 +1,177 @@ +"""CPU-only checks for final DPO archive integrity and immutable publication.""" + +import copy +import json +from pathlib import Path + +import numpy as np +import pytest +import torch + +import dpo +from ops import archive_dpo as archive + + +@pytest.fixture +def fixture(tmp_path): + base = tmp_path / 'base' + work = base / 'posttrain/dpo' + run = work / 'run' + reference_dir = work / 'reference-cache' + data_dir = base / 'posttrain/data' + for path in (run, reference_dir, data_dir, work / 'control', work / 'preflight'): + path.mkdir(parents=True) + tokenizer = tmp_path / 'tokenizer.model' + tokenizer.write_bytes(b'original tokenizer fixture') + tokenizer_hash = archive.sha256(tokenizer) + sources = dpo.source_identity() + renderer_hash = sources['ops/posttrain_data.py'] + data = dict(format='warpstate-posttrain-data-v1', complete=True, max_length=2048, + tokenizer_sha256=tokenizer_hash, renderer_sha256=renderer_hash, + outputs={'dpo_train.jsonl': dict(rows=2, tokens=20, supervised_tokens=8), + 'dpo_val.jsonl': dict(rows=1, tokens=10, supervised_tokens=4)}) + archive.atomic_json(data_dir / 'manifest.json', data) + data_hash = archive.sha256(data_dir / 'manifest.json') + plan = dict(format='warpstate-dpo-plan-v1', stage='dpo', after_completion='pause_for_evaluation', + training={}, sft=dict(step=10312, tokens_seen=212321372, base_step=76294, + base_tokens_seen=10000007168, tokenizer_sha256=tokenizer_hash), + handoff=dict(processes=2, data_dir=str(data_dir), data_manifest_sha256=data_hash)) + reference_identity = dpo.reference_identity('sft-model-hash', data_hash, tokenizer_hash, + dpo.DEFAULTS, torch.device('cuda'), 2) + arrays = {} + for split, count in (('train', 2), ('val', 1)): + path = reference_dir / f'{split}.npy' + np.save(path, np.asarray([[-2., -3.]] * count, dtype=np.float64), allow_pickle=False) + arrays[path.name] = dict(sha256=archive.sha256(path), rows=count) + reference = dict(format='warpstate-dpo-reference-v1', complete=True, identity=reference_identity, + identity_sha256=dpo.identity_digest(reference_identity), files=arrays) + archive.atomic_json(reference_dir / 'manifest.json', reference) + reference_hash = archive.sha256(reference_dir / 'manifest.json') + signature = dict(plan=plan, settings=dpo.DEFAULTS, base_sha256='sft-model-hash', + data_sha256=data_hash, tokenizer_sha256=tokenizer_hash, reference_sha256=reference_hash, + world_size=2, sources=sources, device_type='cuda', torch_version=torch.__version__, + mask='warpstate-final-assistant-response-v1') + evaluation = dict(stage='dpo', step=1, full_holdout=True, evaluated_samples=1, + available_validation_samples=1, val_loss=.5, reward_margin=.4, reward_accuracy=1., chosen_nll=2., + objective='length-normalized-dpo', base_sha256='sft-model-hash', data_sha256=data_hash, + tokenizer_sha256=tokenizer_hash, reference_sha256=reference_hash) + checkpoint = dict(format='warpstate-dpo-v1', completed=True, step=1, total_steps=1, + sample_cursor=2, tokens_seen=16, response_tokens_seen=8, val_step=1, val_loss=.5, + config={'d_model': 2}, model={'weight': torch.arange(6, dtype=torch.float32).reshape(2, 3)}, + optimizer={'state': {0: dict(step=torch.tensor(1.), exp_avg=torch.arange(6.), exp_avg_sq=torch.arange(6.))}}, + rng_by_rank=[dict(torch=torch.tensor([1, 2], dtype=torch.uint8), cuda=torch.tensor([3], dtype=torch.uint8))] * 2, + signature=signature, validation={key: evaluation[key] for key in + ('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll', 'evaluated_samples')}, + origin=dict(stage='sft', step=10312, tokens_seen=212321372, weights_sha256='sft-model-hash', + origin=dict(step=76294, tokens_seen=10000007168))) + completion = dict(format='warpstate-dpo-completion-v1', stage='dpo', completed=True, + paused_for_evaluation=True, reference_sha256=reference_hash, + **{key: checkpoint[key] for key in ('step', 'total_steps', 'sample_cursor', 'tokens_seen', 'response_tokens_seen', 'val_loss')}) + settings_report = dict(signature=signature, total_steps=1, train_rows=2, val_rows=1) + torch.save(checkpoint, run / 'latest.pt') + for name, value in (('evaluation.json', evaluation), ('completion.json', completion), + ('settings.json', settings_report)): + archive.atomic_json(run / name, value) + (run / 'metrics.jsonl').write_text(json.dumps(evaluation) + '\n') + archive.atomic_json(work / 'control/execution.json', {'plan': plan}) + archive.atomic_json(work / 'control/handoff.json', {'phase': 'complete'}) + archive.atomic_json(work / 'preflight/report.json', {'accepted': True}) + return dict(work=work, run=run, reference_dir=reference_dir, tokenizer=tokenizer, + destination=base / 'archives/dpo-final', plan=plan, checkpoint=checkpoint, + evaluation=evaluation, completion=completion, settings=settings_report, + data=data, reference=reference) + + +def create(fixture): + return archive.archive_dpo(fixture['work'], fixture['tokenizer'], fixture['destination'], + fixture['plan'], 'test-revision') + + +def test_full_dpo_archive_exact_model_optimizer_rng_and_idempotent_reuse(fixture): + source = fixture['run'] / 'latest.pt' + before = source.read_bytes() + metadata, entries = create(fixture) + folder = fixture['destination'] + assert (folder / 'checkpoint.pt').read_bytes() == before == source.read_bytes() + assert (folder / 'checkpoint.pt').stat().st_ino != source.stat().st_ino + full = torch.load(folder / 'checkpoint.pt', weights_only=False, mmap=True) + model = torch.load(folder / 'model.pt', weights_only=False, mmap=True) + assert model['stage'] == 'dpo' and 'optimizer' not in model + assert torch.equal(full['model']['weight'], model['model']['weight']) + assert full['optimizer']['state'][0]['step'].item() == 1 + assert len(full['rng_by_rank']) == 2 + assert model['origin']['step'] == 10312 and model['origin']['origin']['step'] == 76294 + assert entries['checkpoint.pt'] == archive.sha256(source) + assert all(path.stat().st_mode & 0o222 == 0 for path in folder.iterdir()) + original_inodes = {path.name: path.stat().st_ino for path in folder.iterdir()} + assert create(fixture) == (metadata, entries) + assert {path.name: path.stat().st_ino for path in folder.iterdir()} == original_inodes + assert archive.verify_dpo_archive(folder) == (metadata, entries) + + +@pytest.mark.parametrize('field,value', [('completed', False), ('step', 0), ('sample_cursor', 1), + ('tokens_seen', 15), ('response_tokens_seen', 7), ('val_step', 0), ('val_loss', float('nan'))]) +def test_incomplete_or_wrong_final_state_is_rejected(fixture, field, value): + fixture['checkpoint'][field] = value + torch.save(fixture['checkpoint'], fixture['run'] / 'latest.pt') + with pytest.raises(ValueError): + create(fixture) + assert not fixture['destination'].exists() + + +def test_changed_reference_or_subset_evaluation_is_rejected(fixture): + fixture['evaluation']['full_holdout'] = False + with pytest.raises(ValueError, match='full-holdout'): + archive.validate_state(fixture['checkpoint'], fixture['evaluation'], fixture['completion'], + fixture['settings'], fixture['data'], fixture['reference'], fixture['plan']) + fixture['evaluation']['full_holdout'] = True + fixture['reference']['identity']['weights_sha256'] = 'changed-reference' + with pytest.raises(ValueError, match='reference'): + archive.validate_state(fixture['checkpoint'], fixture['evaluation'], fixture['completion'], + fixture['settings'], fixture['data'], fixture['reference'], fixture['plan']) + + +def test_existing_unrecognized_archive_is_never_changed(fixture): + fixture['destination'].mkdir(parents=True) + marker = fixture['destination'] / 'keep.txt' + marker.write_text('user data') + with pytest.raises(ValueError, match='checksum'): + create(fixture) + assert list(fixture['destination'].iterdir()) == [marker] + assert marker.read_text() == 'user data' + + +def test_corrupt_sealed_reference_copy_is_detected(fixture): + create(fixture) + array = fixture['destination'] / 'reference-train.npy' + array.chmod(0o644) + array.write_bytes(array.read_bytes() + b'changed') + array.chmod(0o444) + with pytest.raises(ValueError, match='checksum'): + archive.verify_dpo_archive(fixture['destination']) + + +def test_source_replacement_during_archive_blocks_publication(fixture, monkeypatch): + real_copy = archive.copy_regular + def changing_copy(source, destination): + real_copy(source, destination) + if destination.name == 'tokenizer.model': + checkpoint = copy.deepcopy(fixture['checkpoint']) + replacement = fixture['run'] / 'replacement.pt' + torch.save(checkpoint, replacement) + replacement.replace(fixture['run'] / 'latest.pt') + monkeypatch.setattr(archive, 'copy_regular', changing_copy) + with pytest.raises(ValueError, match='changed during archival'): + create(fixture) + assert not fixture['destination'].exists() + + +def test_running_guardian_blocks_before_archival(fixture, monkeypatch): + paths = dict(work_dir=fixture['work'], base_work_dir=fixture['work'].parents[1], + sft_work_dir=fixture['work'].parent / 'sft', archive_dir=fixture['destination'].parent / 'sft-final', + supervisor_socket=fixture['work'] / 'control/socket', tokenizer=fixture['tokenizer']) + monkeypatch.setattr(archive, 'load_plan', lambda p: (fixture['plan'], paths)) + monkeypatch.setattr(archive, 'process_status', lambda *a: {'statename': 'RUNNING', 'exitstatus': 0}) + monkeypatch.setattr(archive, 'archive_dpo', lambda *a: pytest.fail('must not archive an active job')) + with pytest.raises(ValueError, match='exited successfully'): + archive.run(fixture['tokenizer'], fixture['destination']) diff --git a/test_artifact_transfer.py b/test_artifact_transfer.py new file mode 100644 index 0000000000000000000000000000000000000000..7df19b722a5f8ee7fb7d3c38d875715d92950880 --- /dev/null +++ b/test_artifact_transfer.py @@ -0,0 +1,278 @@ +"""CPU-only checks for transfer integrity, interruption, and safe publication.""" + +import concurrent.futures +import hashlib +import importlib.util +import inspect +import json +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +import pytest + + +SPEC = importlib.util.spec_from_file_location( + 'transfer_artifacts', Path(__file__).parent / 'ops' / 'transfer_artifacts.py') +transfer = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(transfer) +RECEIVER = inspect.getsource(transfer.remote_main) + '\nremote_main()\n' +LOCAL_SSH = [sys.executable, '-c', + 'import os, shlex, sys; command = shlex.split(sys.argv[1]); ' + 'os.execvp(command[0], command)'] + + +def receive(action, target, manifest, data=b'', index=None, success=True): + command = [sys.executable, '-c', RECEIVER, action, str(target), manifest['sha256']] + if index is not None: + command.append(str(index)) + result = subprocess.run(command, input=data, capture_output=True) + if not success: + assert result.returncode != 0 + return result.stderr.decode() + assert result.returncode == 0, result.stderr.decode() + return json.loads(result.stdout) + + +@pytest.fixture +def artifact(tmp_path): + # Tests must be invoked with --basetemp below the workspace. + transfer.workspace_path(tmp_path) + data = bytes(range(251)) * 100 + source = tmp_path / 'source.bin' + source.write_bytes(data) + manifest, _ = transfer.source_manifest(source, 8192) + return source, tmp_path / 'remote' / 'target.bin', data, manifest + + +def prepare(target, manifest): + return receive('prepare', target, manifest, json.dumps(manifest).encode()) + + +def upload(index, target, data, manifest): + size = manifest['chunk_size'] + return receive('chunk', target, manifest, data[index * size:(index + 1) * size], index) + + +def test_parallel_upload_verifies_before_atomic_publication(artifact): + source, target, data, manifest = artifact + assert manifest['sha256'] == hashlib.sha256(data).hexdigest() + assert prepare(target, manifest)['completed'] == [] + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + results = list(pool.map(lambda i: upload(i, target, data, manifest), + range(len(manifest['chunk_sha256'])))) + assert sum(row['bytes'] for row in results) == len(data) + assert not target.exists() + result = receive('finalize', target, manifest) + assert result['complete'] and result['sha256'] == manifest['sha256'] + assert target.read_bytes() == source.read_bytes() + assert not Path(str(target) + '.uploading').exists() + assert prepare(target, manifest)['complete'] + + +def test_resume_rechecks_receipts_and_retransmits_corrupt_chunks(artifact): + _, target, data, manifest = artifact + prepare(target, manifest) + upload(0, target, data, manifest) + assert prepare(target, manifest)['completed'] == [0] + partial = Path(str(target) + '.uploading') + with partial.open('r+b') as stream: + stream.write(b'corrupt') + assert prepare(target, manifest)['completed'] == [] + error = receive('chunk', target, manifest, b'interrupted', 0, success=False) + assert 'ended early' in error + assert prepare(target, manifest)['completed'] == [] + for i in range(len(manifest['chunk_sha256'])): + upload(i, target, data, manifest) + receive('finalize', target, manifest) + assert target.read_bytes() == data + + +def test_wrong_chunk_hash_and_final_corruption_never_publish(artifact): + _, target, data, manifest = artifact + prepare(target, manifest) + error = receive('chunk', target, manifest, b'x' * manifest['chunk_size'], 0, success=False) + assert 'Chunk SHA256 mismatch' in error + for i in range(len(manifest['chunk_sha256'])): + upload(i, target, data, manifest) + with Path(str(target) + '.uploading').open('r+b') as stream: + stream.write(b'corrupt after receiving') + error = receive('finalize', target, manifest, success=False) + assert 'Full destination SHA256 mismatch' in error + assert not target.exists() + + +def test_existing_destination_and_foreign_partial_are_protected(artifact): + _, target, _, manifest = artifact + target.parent.mkdir() + target.write_bytes(b'existing content') + error = receive('prepare', target, manifest, json.dumps(manifest).encode(), success=False) + assert 'refusing overwrite' in error + assert target.read_bytes() == b'existing content' + target.unlink() + partial = Path(str(target) + '.uploading') + partial.write_bytes(b'foreign partial') + error = receive('prepare', target, manifest, json.dumps(manifest).encode(), success=False) + assert 'Unclaimed upload sidecar' in error + assert partial.read_bytes() == b'foreign partial' + + +def test_paths_and_symlinks_are_rejected(artifact): + source, target, _, manifest = artifact + with pytest.raises(ValueError, match='below'): + transfer.workspace_path('/tmp/outside.bin') + target.parent.symlink_to(source.parent, target_is_directory=True) + with pytest.raises(ValueError, match='Symlink'): + transfer.workspace_path(target) + error = receive('prepare', target, manifest, json.dumps(manifest).encode(), success=False) + assert 'Symlink is not allowed' in error + + +def test_destination_created_before_finalize_is_preserved(artifact): + _, target, data, manifest = artifact + prepare(target, manifest) + for i in range(len(manifest['chunk_sha256'])): + upload(i, target, data, manifest) + target.write_bytes(b'created independently') + error = receive('finalize', target, manifest, success=False) + assert 'refusing overwrite' in error + assert target.read_bytes() == b'created independently' + + +def test_client_streaming_without_an_ssh_server(artifact): + source, target, data, manifest = artifact + args = SimpleNamespace(port=22, key=source, known_hosts=source, + host='unused', compress=False) + receiver = transfer.SSHReceiver(args, target, manifest) + # Replace just the SSH transport with a local command dispatcher. The client + # still serializes the remote command and streams the actual binary input. + receiver.ssh = [sys.executable, '-c', + 'import os, shlex, sys; command = shlex.split(sys.argv[1]); ' + 'os.execvp(command[0], command)'] + assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == [] + for index in range(len(manifest['chunk_sha256'])): + assert receiver.call('chunk', source=source, index=index)['index'] == index + assert receiver.call('finalize')['sha256'] == hashlib.sha256(data).hexdigest() + assert target.read_bytes() == data + + +def test_expected_source_hash_rejected_before_ssh(artifact): + source, target, _, _ = artifact + command = [sys.executable, str(Path(transfer.__file__)), str(source), str(target), + '--host', 'unused', '--key', str(source), '--known-hosts', str(source), + '--expected-sha256', '0' * 64] + result = subprocess.run(command, capture_output=True) + assert result.returncode != 0 + assert b'Source SHA256 does not match --expected-sha256' in result.stderr + assert not target.exists() + + +def pull_client(source, target, chunk_size=8192): + args = SimpleNamespace(port=22, key=source, known_hosts=source, + host='unused', compress=False) + remote = transfer.SSHSource(args, source) + remote.ssh = LOCAL_SSH + manifest, initial = remote.manifest(chunk_size) + receiver = transfer.PullReceiver(args, target, manifest, remote, initial) + return remote, receiver, manifest, initial + + +def test_parallel_pull_resumes_with_verified_receipts(artifact): + source, target, data, expected = artifact + remote, receiver, manifest, initial = pull_client(source, target) + assert manifest == expected + assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == [] + receiver.call('chunk', index=0) + assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == [0] + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + results = list(pool.map(lambda i: receiver.call('chunk', index=i), + range(1, len(manifest['chunk_sha256'])))) + assert len(results) == len(manifest['chunk_sha256']) - 1 + assert not target.exists() + remote.check_identity(initial) + assert receiver.call('finalize')['sha256'] == manifest['sha256'] + assert target.read_bytes() == data + assert receiver.processes == set() + + +def test_pull_changed_source_is_rejected_before_streaming(artifact): + source, target, data, _ = artifact + remote, receiver, manifest, initial = pull_client(source, target) + receiver.call('prepare', json.dumps(manifest).encode()) + source.write_bytes(data + b'changed') + with pytest.raises(RuntimeError, match='Source changed during transfer'): + receiver.call('chunk', index=0) + with pytest.raises(RuntimeError, match='Source changed during transfer'): + remote.check_identity(initial) + assert not target.exists() + assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == [] + assert receiver.processes == set() + + +@pytest.mark.parametrize('failure', ['corrupt', 'exit']) +def test_pull_transport_corruption_or_failure_never_publishes(artifact, failure): + source, target, _, _ = artifact + remote, receiver, manifest, _ = pull_client(source, target) + receiver.call('prepare', json.dumps(manifest).encode()) + # The fake transport either corrupts a byte or reports failure after sending + # all bytes. Both endpoints' exit status must succeed before acknowledgment. + code = ('import shlex, subprocess, sys; ' + 'result = subprocess.run(shlex.split(sys.argv[1]), capture_output=True); ' + 'data = result.stdout; ' + + ('data = bytes([data[0] ^ 1]) + data[1:]; ' if failure == 'corrupt' else '') + + 'sys.stdout.buffer.write(data); sys.stderr.buffer.write(result.stderr); ' + + ('sys.exit(result.returncode)' if failure == 'corrupt' else 'sys.exit(42)')) + remote.ssh = [sys.executable, '-c', code] + expected = 'Chunk SHA256 mismatch' if failure == 'corrupt' else 'SSH source failed' + with pytest.raises(RuntimeError, match=expected): + receiver.call('chunk', index=0) + assert not target.exists() + assert receiver.processes == set() + + +def test_pull_source_symlinks_are_rejected_remotely(artifact): + source, target, _, _ = artifact + link = source.parent / 'linked.bin' + link.symlink_to(source) + args = SimpleNamespace(port=22, key=source, known_hosts=source, + host='unused', compress=False) + remote = transfer.SSHSource(args, link) + remote.ssh = LOCAL_SSH + with pytest.raises(RuntimeError, match='Symlink is not allowed'): + remote.manifest(8192) + assert not target.exists() + + +@pytest.mark.parametrize('failure', [None, 'expected_hash', 'changed_before_finalize']) +def test_pull_cli_verifies_remote_source_and_final_publication(artifact, monkeypatch, failure): + source, target, data, _ = artifact + digest = hashlib.sha256(data).hexdigest() + monkeypatch.setattr(transfer, 'ssh_command', lambda args: LOCAL_SSH) + argv = [str(Path(transfer.__file__)), str(source), str(target), '--pull', + '--host', 'unused', '--key', str(source), '--known-hosts', str(source), + '--chunk-mib', '1', '--connections', '3', '--retries', '1', + '--manifest-dir', str(source.parent / 'manifests'), + '--expected-sha256', '0' * 64 if failure == 'expected_hash' else digest] + monkeypatch.setattr(sys, 'argv', argv) + if failure == 'changed_before_finalize': + original = transfer.SSHSource.check_identity + + def change_then_check(remote, expected): + source.write_bytes(data + b'changed after the last chunk') + original(remote, expected) + + monkeypatch.setattr(transfer.SSHSource, 'check_identity', change_then_check) + if failure: + error = ValueError if failure == 'expected_hash' else RuntimeError + expected = '--expected-sha256' if failure == 'expected_hash' else 'Source changed' + with pytest.raises(error, match=expected): + transfer.main() + assert not target.exists() + if failure == 'expected_hash': + assert not target.parent.exists() + else: + transfer.main() + assert target.read_bytes() == data + # An already complete destination is independently hashed on a rerun. + transfer.main() diff --git a/test_backward_layout.py b/test_backward_layout.py new file mode 100644 index 0000000000000000000000000000000000000000..39f9ab627df1569f20d9ca09fce74365b8b63667 --- /dev/null +++ b/test_backward_layout.py @@ -0,0 +1,144 @@ +"""Independent derivative/index checks for the isolated backward layout prototype.""" + +import os + +import pytest +import torch + +from ops.backward_layout import (PROFILE_SHAPE, launch_layout, make_inputs, + prepare_layout, reference_layout, validate_inputs) + + +def token_major(value): + b, n, h, c, d = value.shape + return value.permute(0, 1, 3, 2, 4).reshape(b, n * c, h * d) + + +def test_pair_direction_and_projection_packing(): + inputs = make_inputs((1, 2, 2, 3, 4)) + value = torch.arange(1, 49, dtype=torch.bfloat16).reshape(inputs.q_memory.shape) + inputs = inputs._replace(q_memory=value, q_local=torch.zeros_like(value), + k_write=torch.zeros_like(inputs.k_write), k_local=value, v_local=value + 1, + sin=torch.ones_like(inputs.sin), cos=torch.zeros_like(inputs.cos)) + q, k, v, _, _ = reference_layout(inputs).chunk(5, -1) + wanted = token_major(torch.stack((value[..., 1::2], -value[..., ::2]), -1).flatten(-2)) + assert torch.equal(q, wanted) and torch.equal(k, wanted) + assert torch.equal(v, token_major(value + 1)) + + +def test_q_matches_model_rope_vjp_with_unequal_pair_coefficients(): + from warpstate.model import apply_rope + inputs = make_inputs((2, 3, 2, 7, 6), seed=921) + generator = torch.Generator().manual_seed(799) + inputs = inputs._replace(sin=torch.randn(inputs.sin.shape, generator=generator), + cos=torch.randn(inputs.cos.shape, generator=generator)) + original = torch.zeros_like(inputs.q_memory, dtype=torch.float32, requires_grad=True) + output = apply_rope(original, inputs.cos, inputs.sin) + gradient, = torch.autograd.grad(output, original, + grad_outputs=inputs.q_memory.float() + inputs.q_local.float()) + assert torch.equal(reference_layout(inputs).chunk(5, -1)[0], token_major(gradient.bfloat16())) + + +def test_k_intermediate_bf16_rounding_is_observable(): + inputs = make_inputs((1, 2, 2, 7, 16), seed=19) + k = (inputs.k_write.transpose(-2, -1).float() + * (1 - inputs.tanh_k.float().square()) + inputs.k_local.float()) + sine = k * inputs.sin + unrounded_rotary = torch.stack((sine[..., 1::2], -sine[..., ::2]), -1).flatten(-2) + wrong = token_major((unrounded_rotary + k * inputs.cos).bfloat16()) + assert torch.any(reference_layout(inputs).chunk(5, -1)[1] != wrong) + + +def test_gate_matches_independent_autograd_and_u_packing(): + inputs = make_inputs((1, 2, 2, 3, 4), seed=987) + inputs = inputs._replace(tanh_u=torch.zeros_like(inputs.tanh_u)) + gate = inputs.gate.float().requires_grad_() + blended = gate * inputs.local.float() + (1 - gate) * inputs.memory.float() / 2 + dy = inputs.grad_y.reshape(1, 2, 3, 2, 4).permute(0, 1, 3, 2, 4).float() + derivative, = torch.autograd.grad(blended, gate, grad_outputs=dy) + expected = token_major((derivative * gate * (1 - gate)).bfloat16()) + _, _, _, actual, u = reference_layout(inputs).chunk(5, -1) + torch.testing.assert_close(actual, expected, rtol=0.01, atol=1e-6) + assert torch.equal(u, token_major(inputs.u_write)) + + +@pytest.mark.parametrize("case", ["dtype", "k_shape", "rope_dtype", "strides", "device", "odd_head"]) +def test_contract_rejects_unsupported_inputs(case): + inputs = make_inputs((1, 2, 2, 3, 4)) + if case == "dtype": + inputs = inputs._replace(q_memory=inputs.q_memory.float()) + elif case == "k_shape": + inputs = inputs._replace(k_write=inputs.k_write[..., :-1]) + elif case == "rope_dtype": + inputs = inputs._replace(sin=inputs.sin.bfloat16()) + elif case == "strides": + inputs = inputs._replace(v_local=inputs.v_local.transpose(1, 2)) + elif case == "device": + inputs = inputs._replace(gate=torch.empty(inputs.gate.shape, device="meta", dtype=torch.bfloat16)) + else: + inputs = inputs._replace(q_memory=inputs.q_memory[..., :-1]) + with pytest.raises(ValueError): + validate_inputs(inputs) + + +def test_cuda_launch_contract_rejects_cpu(): + with pytest.raises(ValueError, match="require CUDA"): + prepare_layout(make_inputs((1, 1, 1, 1, 2))) + + +def test_benchmark_distinguishes_signed_zero_and_serializes_nonfinite_outputs(): + import json + from ops.backward_layout_benchmark import output_errors + expected = torch.zeros((1, 1, 10), dtype=torch.bfloat16) + actual = expected.clone() + actual[0, 0, 0] = -0.0 + report = output_errors(actual, expected) + assert report["q"]["unequal"] == 0 + assert report["q"]["bitwise_unequal"] == 1 + actual[0, 0, 3] = float("nan") + report = output_errors(actual, expected) + assert not report["k"]["finite"] and report["k"]["relative_rms"] is None + json.dumps(report, allow_nan=False) + + +cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1", + reason="requires an explicitly reserved CUDA device") + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["paired", "single"]) +@pytest.mark.parametrize("shape", [(1, 2, 3, 7, 10), (2, 3, 2, 16, 64)]) +def test_cuda_ragged_and_production_head_arithmetic(algorithm, shape): + inputs = make_inputs(shape, device="cuda", seed=455) + # Distinct pair coefficients detect accidentally selecting the wrong RoPE lane. + inputs = inputs._replace(sin=torch.randn_like(inputs.sin), cos=torch.randn_like(inputs.cos)) + saved = [value.clone() for value in inputs] + actual = launch_layout(inputs, algorithm, block=256) + expected = reference_layout(inputs) + assert actual.dtype == torch.bfloat16 and actual.is_contiguous() + for value, wanted in zip(actual.chunk(5, -1), expected.chunk(5, -1), strict=True): + assert torch.isfinite(value).all() + error = (value.float() - wanted.float()).norm() / wanted.float().norm().clamp_min(1e-20) + assert float(error) < 0.002 + torch.testing.assert_close(value, wanted, rtol=0.015, atol=1e-4) + assert all(torch.equal(value, original) for value, original in zip(inputs, saved, strict=True)) + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["paired", "single"]) +def test_cuda_production_shape_matches_exact_profile_kernels(algorithm): + inputs = make_inputs(PROFILE_SHAPE, device="cuda", seed=817) + expected = launch_layout(inputs, "profile") + actual = launch_layout(inputs, algorithm, block=512) + for value, wanted in zip(actual.chunk(5, -1), expected.chunk(5, -1), strict=True): + assert torch.isfinite(value).all() + assert torch.equal(value.view(torch.int16), wanted.view(torch.int16)) + + +@cuda_window +def test_cuda_profile_scratch_cannot_alias(): + inputs = make_inputs(PROFILE_SHAPE, device="cuda") + scratch = torch.empty_like(inputs.q_memory) + for values in ((scratch, scratch), (inputs.q_local, scratch)): + with pytest.raises(ValueError, match="Scratch tensors may not alias"): + prepare_layout(inputs, "profile", scratch=values) diff --git a/test_backward_layout_cli.py b/test_backward_layout_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..b7381ef89b4e49da6c010bf38829ec80cd6b404f --- /dev/null +++ b/test_backward_layout_cli.py @@ -0,0 +1,154 @@ +"""Backward layout selection must be explicit, applicable, and resume-visible.""" + +from copy import deepcopy +from pathlib import Path +import sys + +import pytest + +import run_40b +from ops import benchmark_training + + +@pytest.fixture(params=[run_40b, benchmark_training], ids=["runner", "benchmark"]) +def cli(request, monkeypatch): + workspace = Path(__file__).resolve().parents[1] + for name in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR", + "TORCHINDUCTOR_CACHE_DIR"): + monkeypatch.setenv(name, str(workspace / "tmp")) + monkeypatch.delenv("WORLD_SIZE", raising=False) + compile_flag = "--compile-core" if request.param is run_40b else "--compile-cores" + + def parse(flags=(), *, supported=False, omit=()): + common = ["--micro-batch", "4"] + if supported: + common += ["--gradient-mode", "compiled-autograd-overlap", compile_flag, + "--no-activation-checkpointing", "--bf16-residuals"] + common = [flag for flag in common if flag not in omit] + monkeypatch.setattr(sys, "argv", [request.param.__file__, *common, *flags]) + return request.param.arguments() + + return parse, compile_flag + + +def test_baseline_keeps_existing_cli_usable(cli): + parse, _ = cli + args = parse() + assert args.backward_layout == "baseline" + assert args.gradient_mode == "eager" + + +@pytest.mark.parametrize("mode", ["inspect", "paired"]) +def test_supported_selection_requires_an_explicit_flag(cli, mode): + parse, _ = cli + args = parse(["--backward-layout", mode], supported=True) + assert args.backward_layout == mode + assert args.gradient_mode == "compiled-autograd-overlap" + assert args.bf16_residuals + assert not args.activation_checkpointing + + +@pytest.mark.parametrize("flags", [ + ["--gradient-mode", "eager"], + ["--gradient-mode", "compiled-autograd"], + ["--activation-checkpointing"], + ["--micro-batch", "2"], + ["--seq-len", "1024"], + ["--activation-mode", "swiglu"], + ["--matmul-plan", "/workspace/WarpState/rtx6000_40b/configs/not-loaded.json"], + ["--cache-core-weight-casts"], + ["--compile-mode", "reduce-overhead"], + ["--compile-mode", "max-autotune-no-cudagraphs"], + ["--ddp-batched-grad-copy"], +]) +@pytest.mark.parametrize("mode", ["inspect", "paired"]) +def test_reject_unsupported_selection_before_loading_cuda(cli, flags, mode): + parse, _ = cli + with pytest.raises(SystemExit) as error: + parse(["--backward-layout", mode, *flags], supported=True) + assert error.value.code == 2 + + +@pytest.mark.parametrize("missing", ["compile", "bf16"]) +@pytest.mark.parametrize("mode", ["inspect", "paired"]) +def test_reject_missing_required_execution_setting(cli, missing, mode): + parse, compile_flag = cli + omit = (compile_flag,) if missing == "compile" else ("--bf16-residuals",) + with pytest.raises(SystemExit) as error: + parse(["--backward-layout", mode], supported=True, omit=omit) + assert error.value.code == 2 + + +@pytest.mark.parametrize("mode", ["inspect", "paired"]) +def test_benchmark_rejects_unvalidated_fused_normalization(monkeypatch, capsys, mode): + monkeypatch.setattr(sys, "argv", [benchmark_training.__file__, "--micro-batch", "4", + "--gradient-mode", "compiled-autograd-overlap", "--compile-cores", + "--bf16-residuals", "--backward-layout", mode, "--fused-rms-norm"]) + with pytest.raises(SystemExit) as error: + benchmark_training.arguments() + assert error.value.code == 2 + assert "--backward-layout requires" in capsys.readouterr().err + + +def runner_signature(monkeypatch, metadata): + monkeypatch.delenv("WORLD_SIZE", raising=False) + monkeypatch.setattr(sys, "argv", [run_40b.__file__, "--micro-batch", "4"]) + args = run_40b.arguments() + args.backward_layout_metadata = metadata + return run_40b.train_signature(args, {"recipe": "fixed-corpus"}) + + +def test_signature_records_layout_implementation_identity(monkeypatch): + metadata = {"mode": "paired", "identity": "implementation-a", + "source_sha256": {"ops/backward_layout.py": "kernel-a"}, + "block": 256, "warps": 8} + expected = deepcopy(metadata) + signature = runner_signature(monkeypatch, metadata) + assert signature["settings"]["backward_layout_execution"] == expected + assert metadata == expected + + +def test_legacy_checkpoint_remains_compatible_with_baseline(monkeypatch): + current = runner_signature(monkeypatch, None) + legacy = deepcopy(current) + del legacy["settings"]["backward_layout_execution"] + before = deepcopy(legacy) + assert run_40b.check_resume_signature(legacy, current) is False + assert legacy == before + + +@pytest.mark.parametrize("change", [ + {"mode": "inspect", "identity": "implementation-a"}, + {"mode": "paired", "identity": "implementation-a"}, + {"mode": "paired", "identity": "implementation-b"}, + {"mode": "paired", "identity": "implementation-a", "source_sha256": {"kernel": "changed"}}, + None, +]) +def test_execution_identity_changes_require_reconfiguration(monkeypatch, change): + original = runner_signature(monkeypatch, {"mode": "paired", "identity": "implementation-a", + "source_sha256": {"kernel": "original"}}) + candidate = runner_signature(monkeypatch, change) + before = deepcopy((original, candidate)) + with pytest.raises(ValueError, match="Resume settings/data differ"): + run_40b.check_resume_signature(original, candidate) + assert run_40b.check_resume_signature(original, candidate, reconfigure=True) is True + assert (original, candidate) == before + + +def test_same_layout_identity_does_not_require_reconfiguration(monkeypatch): + original = runner_signature(monkeypatch, {"mode": "paired", "identity": "implementation-a"}) + assert run_40b.check_resume_signature(original, deepcopy(original)) is False + + +@pytest.mark.parametrize("change", ["data", "tokens", "batch"]) +def test_layout_reconfiguration_cannot_change_training_objective(monkeypatch, change): + original = runner_signature(monkeypatch, None) + candidate = runner_signature(monkeypatch, {"mode": "paired", "identity": "implementation-a"}) + if change == "data": + candidate["data"]["recipe"] = "another-corpus" + elif change == "tokens": + candidate["settings"]["train_tokens"] += 1 + else: + candidate["settings"]["grad_accum"] += 1 + with pytest.raises(ValueError): + run_40b.check_resume_signature(original, candidate, reconfigure=True) diff --git a/test_backward_layout_execution.py b/test_backward_layout_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..3c5714f61f3a727088013fa3a2d79575dd877b5f --- /dev/null +++ b/test_backward_layout_execution.py @@ -0,0 +1,262 @@ +"""Structural and configuration guards for the opt-in backward graph rewrite. + +The pattern fixtures isolate graph safety; actual compiled model numerical and +continuation checks live in the separate integration validation harness. +""" + +import copy +import hashlib +import operator +import os +from pathlib import Path + +import pytest +import torch +from torch._inductor import config + +from ops import backward_layout_execution as layout +from warpstate.config import WarpStateConfig + + +@pytest.fixture(autouse=True) +def restore_configuration(monkeypatch): + monkeypatch.setattr(layout, "_ACTIVE_PASS", None) + monkeypatch.setattr(layout, "_ACTIVE_MODE", "baseline") + monkeypatch.setattr(layout, "_COUNTS", layout.Counter()) + monkeypatch.setattr(layout, "_MISSES", layout.Counter()) + with config.patch({"post_grad_custom_post_pass": None, "emulate_precision_casts": False, + "triton.codegen_upcast_to_fp32": True, "eager_numerics.disable_ftz": False}): + yield + + +def candidate_graph(): + """Attach explicit native GEMM and ordered Flash VJP boundary producers.""" + gm = copy.deepcopy(layout._pattern()) + graph = gm.graph + placeholders = [node for node in graph.nodes if node.op == "placeholder"] + values = dict(zip(layout._NAMES, placeholders, strict=True)) + first = next(node for node in graph.nodes if node.op == "call_function") + with graph.inserting_before(first): + flash = graph.call_function(torch.ops.aten._scaled_dot_product_flash_attention_backward.default, + (None, None, None, None, values["local"], None, None, None, + 128, 128, 0.0, True, None, None), {"scale": 0.125}) + for name, index in (("q_local", 0), ("k_local", 1), ("v_local", 2)): + old = values[name] + new = graph.call_function(operator.getitem, (flash, index)) + new.meta = dict(old.meta) + old.replace_all_uses_with(new) + for name in ("q_memory", "k_write", "u_write", "grad_y"): + old = values[name] + users = list(old.users) + target = torch.ops.aten.mm.default if name == "grad_y" else torch.ops.aten.bmm.default + new = graph.call_function(target, (old, old)) + new.meta = dict(old.meta) + for user in users: + user.replace_input_with(old, new) + graph.lint() + root = next(node for node in graph.nodes if node.op == "output").args[0] + return gm, root, flash + + +def test_pattern_is_independent_of_node_names_and_preserves_native_inputs(): + gm, root, _ = candidate_graph() + for index, node in enumerate(gm.graph.nodes): + node.name = f"renamed_{index}" + inputs, interior = layout._match(root, layout._pattern()) + assert len(inputs) == 14 + assert len(interior) > 70 + assert tuple(inputs[4].meta["val"].shape) == (2048, 64, 128) + assert tuple(inputs[8].meta["val"].shape) == (8192, 2048) + + +def test_inspection_retains_graph_and_replacement_is_pure(): + gm, root, _ = candidate_graph() + original = str(gm.graph) + layout.BackwardLayoutPass("inspect", "inspect-test")(gm.graph) + assert str(gm.graph) == original + assert layout.backward_layout_stats()["matches"] == 1 + layout.BackwardLayoutPass("paired", "paired-test")(gm.graph) + gm.graph.lint() + calls = [node for node in gm.graph.nodes + if node.target == torch.ops.warpstate.paired_backward_layout.default] + assert len(calls) == 1 + assert root not in gm.graph.nodes + assert len(calls[0].args) == 14 + assert layout.backward_layout_stats()["replacements"] == 1 + assert layout.backward_layout_stats()["runtime_calls"] == 0 + + +@pytest.mark.parametrize("mutation", ["gate_scale", "projection_order", "rotary_lane", "rounding", + "dtype", "stride", "dropout", "flash_scale", "flash_order", + "input_producer", "missing_meta", "mixed_devices"]) +def test_unsupported_graph_is_left_unchanged(mutation): + gm, root, flash = candidate_graph() + nodes = list(gm.graph.nodes) + if mutation == "gate_scale": + node = next(n for n in nodes if n.target == torch.ops.aten.div.Tensor) + node.args = (node.args[0], 16.0) + elif mutation == "projection_order": + projections = list(root.args[0]); projections[0], projections[1] = projections[1], projections[0] + root.args = (projections, 2) + elif mutation == "rotary_lane": + node = next(n for n in nodes if n.target == torch.ops.aten.select.int) + node.args = (*node.args[:-1], 1 - node.args[-1]) + elif mutation == "rounding": + node = next(n for n in nodes if n.target == torch.ops.prims.convert_element_type.default) + node.args = (node.args[0], torch.float32) + elif mutation in ("dtype", "stride", "missing_meta"): + if mutation == "missing_meta": + root.meta.pop("val") + elif mutation == "dtype": + root.meta["val"] = root.meta["val"].float() + else: + root.meta["val"] = root.meta["val"].transpose(1, 2).contiguous().transpose(1, 2) + assert tuple(root.meta["val"].shape) == (4, 2048, 10240) + elif mutation == "mixed_devices": + from torch._subclasses.fake_tensor import FakeTensor + + node = next(n for n in nodes if n.target == torch.ops.aten.bmm.default) + value = node.meta["val"] + node.meta["val"] = FakeTensor(value.fake_mode, + torch.empty_strided(value.shape, value.stride(), + dtype=value.dtype, device="meta"), + torch.device("cuda:1")) + elif mutation == "dropout": + args = list(flash.args); args[10] = 0.1; flash.args = tuple(args) + elif mutation == "flash_scale": + flash.kwargs = {"scale": 0.25} + elif mutation == "flash_order": + node = next(n for n in nodes if n.target is operator.getitem) + node.args = (flash, 2) + else: + node = next(n for n in nodes if n.target == torch.ops.aten.bmm.default) + node.target = torch.ops.aten.mm.default + original = str(gm.graph) + layout.BackwardLayoutPass("paired", "test")(gm.graph) + assert str(gm.graph) == original + assert layout.backward_layout_stats()["replacements"] == 0 + assert sum(layout.backward_layout_stats()["miss_reasons"].values()) == 1 + + +def test_external_arithmetic_consumer_rejected(): + gm, root, _ = candidate_graph() + node = next(n for n in gm.graph.nodes if n.target == torch.ops.aten.add.Tensor) + with gm.graph.inserting_before(root): + gm.graph.call_function(torch.ops.aten.neg.default, (node,)) + with pytest.raises(layout._Mismatch, match="external intermediate consumer"): + layout._match(root, layout._pattern()) + + +def test_shared_gradient_view_retained_without_copy(): + gm, root, _ = candidate_graph() + node = next(n for n in gm.graph.nodes if n.target == torch.ops.aten.permute.default + and tuple(n.meta["val"].stride()) == (4194304, 262144, 64, 2048, 1)) + with gm.graph.inserting_before(root): + external = gm.graph.call_function(torch.ops.aten.neg.default, (node,)) + _, interior = layout._match(root, layout._pattern()) + assert node not in interior + layout.BackwardLayoutPass("paired", "test")(gm.graph) + assert external.args[0] is node + gm.graph.lint() + + +def test_cse_shared_exact_zero_across_two_regions(): + from torch._functorch.compile_utils import fx_graph_cse + + graph = torch.fx.Graph() + regions = [candidate_graph(), candidate_graph()] + mappings = [] + for index, (gm, _, _) in enumerate(regions): + mapping = {} + for node in gm.graph.nodes: + if node.op == "placeholder": + new = graph.placeholder(f"core_{index}_{node.name}") + new.meta = dict(node.meta) + mapping[node] = new + mappings.append(mapping) + roots = [] + for (gm, root, _), mapping in zip(regions, mappings, strict=True): + for node in gm.graph.nodes: + if node.op not in ("placeholder", "output"): + mapping[node] = graph.node_copy(node, lambda value: mapping[value]) + roots.append(mapping[root]) + graph.output(tuple(roots)) + graph = fx_graph_cse(graph) + zeros = [node for node in graph.nodes if node.target == torch.ops.aten.full.default] + assert len(zeros) == 1 and len(zeros[0].users) == 8 + layout.BackwardLayoutPass("paired", "test")(graph) + graph.lint() + assert sum(node.target == torch.ops.warpstate.paired_backward_layout.default + for node in graph.nodes) == 2 + assert not any(node.target == torch.ops.aten.full.default for node in graph.nodes) + assert layout.backward_layout_stats()["replacements"] == 2 + + +def test_configuration_identity_covers_sources(): + result = layout.configure_backward_layout("paired") + assert config.post_grad_custom_post_pass.uuid() == result["identity"] + for name, digest in result["source_sha256"].items(): + assert hashlib.sha256((Path(__file__).parent / name).read_bytes()).hexdigest() == digest + with pytest.raises(RuntimeError, match="already configured"): + layout.configure_backward_layout("paired") + + +def test_configuration_rejects_existing_pass_and_precision_changes(): + existing = lambda graph: None + with config.patch(post_grad_custom_post_pass=existing): + with pytest.raises(RuntimeError, match="already exists"): + layout.configure_backward_layout("paired") + assert config.post_grad_custom_post_pass is existing + for setting, value in (("emulate_precision_casts", True), ("triton.codegen_upcast_to_fp32", False), + ("eager_numerics.disable_ftz", True)): + with config.patch({setting: value}): + with pytest.raises(ValueError, match="FP32 pointwise"): + layout.configure_backward_layout("paired") + + +def test_baseline_has_no_compiler_hook_or_metadata(): + assert layout.configure_backward_layout() is None + assert config.post_grad_custom_post_pass is None + + +def test_dispatch_evidence_requires_actual_runtime_calls(): + layout.configure_backward_layout("paired") + layout._COUNTS["replacements"] = 100 + with pytest.raises(RuntimeError, match="runtime calls"): + layout.require_backward_layout_dispatch() + layout._COUNTS["runtime_calls"] = 2 + assert layout.require_backward_layout_dispatch(2)["runtime_calls"] == 2 + with pytest.raises(RuntimeError, match="at least 3"): + layout.require_backward_layout_dispatch(3) + with pytest.raises(ValueError): + layout.require_backward_layout_dispatch(True) + + +def test_native_entry_rejects_unsupported_inputs_before_launch(): + with pytest.raises(ValueError, match="q_memory"): + layout._validate_native_inputs([torch.empty(1)] * 14) + + +@pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1" or not torch.cuda.is_available(), + reason="requires an explicitly reserved CUDA device") +def test_opaque_boundary_schema_and_fake_registration(): + from ops.backward_layout import make_inputs + + saved = make_inputs(device="cuda", seed=9221) + native = tuple(value.view(shape) for value, shape in + zip(saved, layout._NATIVE_SHAPES, strict=True)) + # This is a backward-only internal operator; schema purity and accurate fake + # metadata apply, whereas registering another autograd VJP would be wrong. + result = torch.library.opcheck(torch.ops.warpstate.paired_backward_layout.default, native, + test_utils=("test_schema", "test_faketensor")) + assert set(result.values()) == {"SUCCESS"} + + +@pytest.mark.parametrize("field,value", [("d_model", 1024), ("n_heads", 16), ("chunk_size", 64), + ("parallel_memory", False), ("dropout", 0.1)]) +def test_model_configuration_guard(field, value): + cfg = WarpStateConfig(d_model=2048, n_heads=32, chunk_size=128, parallel_memory=True) + layout.validate_backward_layout_config(cfg) + setattr(cfg, field, value) + with pytest.raises(ValueError, match=field): + layout.validate_backward_layout_config(cfg) diff --git a/test_backward_layout_training.py b/test_backward_layout_training.py new file mode 100644 index 0000000000000000000000000000000000000000..253977997e958c976900cb19e02f9d9b426d4206 --- /dev/null +++ b/test_backward_layout_training.py @@ -0,0 +1,124 @@ +"""CPU checks of the full-training backward-layout validation oracle.""" +import json + +import pytest +import torch + +from ops.validate_backward_layout_training import ( + ACCUMULATION, MICRO_TOKENS, TOLERANCES, arguments, compare_named_tensors, exact_tree_equal, + fixture_batch_cpu, fixture_config, tensor_comparison, update_plan, within_tolerance, +) + + +def test_fixture_retains_production_attention_and_ffn_geometry(): + cfg = fixture_config() + assert (cfg.d_model, cfg.n_heads, cfg.head_dim, cfg.chunk_size) == (2048, 32, 64, 128) + assert cfg.ffn_hidden == 7168 and cfg.parallel_memory and cfg.dropout == 0 + assert cfg.n_cores == cfg.logical_depth == 1 + assert not cfg.gradient_checkpointing + + +def test_multicore_fixture_has_distinct_parameters_at_each_logical_layer(): + cfg = fixture_config(True, cores=2) + assert cfg.n_cores == cfg.logical_depth == 2 + assert cfg.d_model == 2048 and cfg.ffn_hidden == 7168 + + +@pytest.mark.parametrize("cores", [0, -1, True, 1.5]) +def test_fixture_rejects_invalid_core_counts(cores): + with pytest.raises(ValueError, match="positive integer"): + fixture_config(cores=cores) + + +def test_unique_fixture_removes_repeated_embedding_indices_reproducibly(): + assert fixture_config(True).vocab_size == MICRO_TOKENS + first = fixture_batch_cpu(191, 0, 0, 0, True) + again = fixture_batch_cpu(191, 0, 0, 0, True) + next_micro = fixture_batch_cpu(191, 0, 0, 1, True) + assert torch.equal(first[0], again[0]) and torch.equal(first[1], again[1]) + assert not torch.equal(first[0], next_micro[0]) + assert torch.equal(first[0].reshape(-1).sort().values, torch.arange(MICRO_TOKENS)) + assert first[1].min() >= 0 and first[1].max() < MICRO_TOKENS + repeated, _ = fixture_batch_cpu(191, 0, 0, 0, False) + assert repeated.unique().numel() <= 512 < repeated.numel() + + +@pytest.mark.parametrize("world_size", [1, 2]) +def test_update_plan_has_accumulation_then_masked_final_rank(world_size): + assert [update_plan(world_size, update) for update in range(3)] == [ + MICRO_TOKENS * ACCUMULATION * world_size, MICRO_TOKENS * ACCUMULATION * world_size, 9] + assert ACCUMULATION >= 3 + final_valid = [max(0, min(MICRO_TOKENS, 9 - rank * MICRO_TOKENS)) for rank in range(world_size)] + assert sum(final_valid) == 9 + if world_size == 2: + assert final_valid == [9, 0] + + +@pytest.mark.parametrize(("actual", "expected"), [ + (torch.ones(3), torch.ones(4)), + (torch.ones(3), torch.ones(3, dtype=torch.bfloat16)), +]) +def test_oracle_rejects_mismatched_tensor_contract(actual, expected): + with pytest.raises(ValueError, match="shape/dtype"): + tensor_comparison(actual, expected) + + +def test_oracle_distinguishes_signed_zero_and_nonfinite_values(): + expected = torch.zeros(3, dtype=torch.bfloat16) + observed = expected.clone() + observed[0] = -0.0 + metrics = tensor_comparison(observed, expected) + assert metrics["unequal"] == 0 and metrics["bitwise_unequal"] == 1 + assert not within_tolerance(metrics, "exact") + observed[1] = float("nan") + metrics = tensor_comparison(observed, expected) + assert not metrics["finite"] and not within_tolerance(metrics, "gradient") + json.dumps(metrics, allow_nan=False) + + +def test_oracle_checks_relative_norm_and_worst_element(): + expected = torch.ones(1000) + observed = expected + TOLERANCES["gradient_relative_l2"] / 2 + assert within_tolerance(tensor_comparison(observed, expected), "gradient") + observed = expected.clone() + observed[0] += TOLERANCES["gradient_relative_max"] * 2 + metrics = tensor_comparison(observed, expected) + assert metrics["relative_l2"] < TOLERANCES["gradient_relative_l2"] + assert not within_tolerance(metrics, "gradient") + assert not within_tolerance(tensor_comparison(expected * 1.02, expected), "gradient") + + +def test_oracle_uses_absolute_parameter_limit(): + expected = torch.zeros(3) + assert within_tolerance(tensor_comparison(expected + 1e-5, expected), "parameter") + assert not within_tolerance(tensor_comparison(expected + 3e-5, expected), "parameter") + + +def test_named_comparison_checks_every_parameter(): + expected = {"first": torch.ones(4), "second": torch.ones(2)} + result = compare_named_tensors(expected, expected, "gradient") + assert result["accepted"] and result["bitwise_equal"] + with pytest.raises(ValueError, match="names"): + compare_named_tensors({"first": torch.ones(4)}, expected, "gradient") + result = compare_named_tensors({**expected, "second": torch.zeros(2)}, expected, "gradient") + assert not result["accepted"] and not result["bitwise_equal"] + + +def test_optimizer_step_counter_must_match_exactly(): + wanted = {"weight/step": torch.tensor(40081.0)} + actual = {"weight/step": torch.tensor(40082.0)} + assert not compare_named_tensors(actual, wanted, "moment")["accepted"] + + +def test_resume_oracle_includes_dtype_signed_zero_and_metadata(): + expected = {"model": {"weight": torch.tensor([0.0])}, "rng": [torch.tensor([1, 2])], + "execution": {"identity": "version1"}} + assert exact_tree_equal(expected, expected) + assert not exact_tree_equal({**expected, "model": {"weight": torch.tensor([-0.0])}}, expected) + assert not exact_tree_equal({**expected, "execution": {"identity": "version2"}}, expected) + assert not exact_tree_equal({**expected, "rng": [torch.tensor([1.0, 2.0])]}, expected) + + +def test_cli_rejects_outputs_outside_generated_validation_directory(): + with pytest.raises(SystemExit): + arguments(["--output-dir", "/workspace/WarpState/rtx6000_40b/ops/bad-output"]) diff --git a/test_chat.py b/test_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..c99eb39bd175ef0b52cf5dc4e50ca068df1bf04d --- /dev/null +++ b/test_chat.py @@ -0,0 +1,316 @@ +"""Tiny CPU-only chat checks; all generated fixtures stay inside this workspace.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +import pytest +import torch + +import chat +from warpstate import WarpStateConfig, WarpStateLM + +ROOT = Path(__file__).resolve().parent +TOKENIZER = ROOT / 'tokenizer.model' + + +@pytest.fixture +def fixture_dir(): + root = ROOT.parent / 'tmp/chat-test-fixtures' + root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as directory: + yield Path(directory) + + +def tiny(vocab_size=32, parallel_memory=False): + torch.manual_seed(17) + cfg = WarpStateConfig(vocab_size=vocab_size, d_model=16, n_heads=2, + n_cores=1, logical_depth=3, ffn_hidden=24, chunk_size=4, + gradient_checkpointing=False, parallel_memory=parallel_memory) + return WarpStateLM(cfg).eval() + + +def source_checkpoint(directory): + model = tiny(32768) + state = dict(format='warpstate-server-v1', config=vars(model.cfg), + model=model.state_dict(), step=7, tokens_seen=112, + optimizer={'state': {'must_not_survive': torch.ones(3)}}, + signature={'data': {'signature': { + 'tokenizer_sha256': hashlib.sha256(TOKENIZER.read_bytes()).hexdigest()}}}) + source = directory / 'latest.pt' + torch.save(state, source) + return source, state + + +@pytest.mark.parametrize('parallel_memory', [False, True]) +def test_step_matches_full_forward_across_three_chunk_boundaries(parallel_memory): + model = tiny(parallel_memory=parallel_memory) + tokens = torch.randint(0, 32, (2, 12)) + caches = model.init_generation_cache(2, torch.device('cpu'), torch.float32) + with torch.inference_mode(): + expected = model(tokens) + actual = torch.cat([model.step(tokens[:, i:i+1], caches) for i in range(12)], dim=1) + torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-6) + assert all(cache.chunk_len == 0 for cache in caches) + + +def test_shared_core_has_independent_logical_depth_caches(): + model = tiny() + caches = model.init_generation_cache(1, torch.device('cpu'), torch.float32) + assert len(caches) == model.cfg.logical_depth > model.cfg.n_cores + for field in ('fast_state', 'slow_state', 'k_cache', 'v_cache', 'write_accum'): + assert len({getattr(cache, field).data_ptr() for cache in caches}) == len(caches) + for _ in range(3): + model.step(torch.tensor([[4]]), caches) + assert all(cache.chunk_len == 3 and not cache.fast_state.count_nonzero() for cache in caches) + model.step(torch.tensor([[4]]), caches) + assert all(cache.chunk_len == 0 and not cache.k_cache.count_nonzero() for cache in caches) + + +def test_sampling_greedy_masks_and_seeded_replay(): + logits = torch.tensor([0.2, 0.8, -10.0, 0.1]) + before = logits.clone() + assert chat.sample_token(logits, 0.0, 0, 1.0, torch.Generator().manual_seed(3)) == 1 + def draw(seed): + generator = torch.Generator().manual_seed(seed) + return [chat.sample_token(logits, 0.7, 3, 0.9, generator) for _ in range(12)] + assert draw(42) == draw(42) + assert 2 not in draw(42) + torch.testing.assert_close(logits, before) + + +@pytest.mark.parametrize('bad', [float('nan'), float('inf')]) +def test_sampling_rejects_nonfinite_unmasked_scores(bad): + with pytest.raises((ValueError, RuntimeError)): + chat.sample_token(torch.tensor([bad, 0.0]), 1.0, 0, 1.0, torch.Generator()) + + +def test_capture_is_model_only_and_survives_atomic_source_replacement(fixture_dir, monkeypatch): + source, state = source_checkpoint(fixture_dir) + replacement = fixture_dir / 'replacement.pt' + changed = dict(state, step=8, tokens_seen=128) + torch.save(changed, replacement) + original_load = torch.load + replaced = False + def replace_during_load(path, *args, **kwargs): + nonlocal replaced + if not replaced: + os.replace(replacement, source) + replaced = True + return original_load(path, *args, **kwargs) + monkeypatch.setattr(torch, 'load', replace_during_load) + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + monkeypatch.setattr(torch, 'load', original_load) + payload = torch.load(snapshot, map_location='cpu', weights_only=False) + assert 'optimizer' not in payload + assert torch.load(source, map_location='cpu', weights_only=False)['step'] == 8 + again = torch.load(snapshot, map_location='cpu', weights_only=False) + assert payload['step'] == again['step'] == 7 + for key in state['model']: + torch.testing.assert_close(again['model'][key], state['model'][key], rtol=0, atol=0) + model, sp, metadata = chat.load_model(snapshot, TOKENIZER) + assert not model.training and sp.vocab_size() == model.cfg.vocab_size + assert next(model.parameters()).device.type == 'cpu' + assert all(buffer.device.type == 'cpu' for buffer in model.buffers()) + tokens = torch.tensor([[3, 4, 5, 6]]) + caches = model.init_generation_cache(1, torch.device('cpu'), torch.float32) + stepped = torch.cat([model.step(tokens[:, i:i+1], caches) for i in range(4)], dim=1) + torch.testing.assert_close(stepped, model(tokens), rtol=2e-5, atol=2e-6) + + +def test_cli_snapshot_accepts_training_checkpoint_without_sidecar(fixture_dir): + source, _ = source_checkpoint(fixture_dir) + source_hash = chat.sha256(source) + snapshots = fixture_dir / 'snapshots' + run = subprocess.run([ + sys.executable, '-B', str(ROOT / 'chat.py'), '--snapshot', str(source), + '--snapshot-root', str(snapshots), '--threads', '1', '--prompt', 'Hello', + '--max-new-tokens', '2', '--max-context', '32', + '--repetition-penalty', '1.2', '--no-repeat-ngram-size', '4', + ], capture_output=True, text=True, timeout=60) + assert run.returncode == 0, run.stderr + assert 'CPU FP32 | step 7 | 112 training tokens' in run.stdout + assert 'Repetition protection: penalty=1.2, no-repeat-ngram-size=4' in run.stdout + assert 'Model: ' in run.stdout and 'Traceback' not in run.stderr + exported = list(snapshots.glob('*/model.pt')) + assert len(exported) == 1 and exported[0] != source + proof = json.loads(exported[0].with_name('snapshot.json').read_text()) + assert proof['source']['path'] == str(source.resolve()) + assert proof['snapshot_sha256'] == chat.sha256(exported[0]) + assert chat.sha256(source) == source_hash + assert not source.with_name('snapshot.json').exists() + assert 'optimizer' not in torch.load(exported[0], map_location='cpu', weights_only=False) + + +def test_resolve_reuses_sealed_snapshot_without_export(fixture_dir, monkeypatch): + source, _ = source_checkpoint(fixture_dir) + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + def unexpected_capture(*args): + pytest.fail('A sealed snapshot must be reused, not re-exported') + monkeypatch.setattr(chat, 'capture_latest', unexpected_capture) + assert chat.resolve_snapshot(snapshot, TOKENIZER, fixture_dir / 'unused') == snapshot + assert not (fixture_dir / 'unused').exists() + + +def test_snapshot_missing_metadata_is_not_resealed(fixture_dir): + source, _ = source_checkpoint(fixture_dir) + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + snapshot.with_name('snapshot.json').unlink() + with pytest.raises(ValueError, match='snapshot.json'): + chat.resolve_snapshot(snapshot, TOKENIZER, fixture_dir / 'unused') + with pytest.raises(ValueError, match='Missing snapshot metadata'): + chat.load_model(snapshot, TOKENIZER) + assert not (fixture_dir / 'unused').exists() + + +@pytest.mark.parametrize('metadata', ['{', '[]', '{}', '{"snapshot_sha256": "wrong"}']) +def test_snapshot_bad_metadata_is_not_bypassed(fixture_dir, metadata): + source, _ = source_checkpoint(fixture_dir) + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + snapshot.with_name('snapshot.json').write_text(metadata) + resolved = chat.resolve_snapshot(snapshot, TOKENIZER, fixture_dir / 'unused') + with pytest.raises(ValueError): + chat.load_model(resolved, TOKENIZER) + assert not (fixture_dir / 'unused').exists() + + +def test_missing_snapshot_path_is_a_clear_cli_error(fixture_dir): + missing = fixture_dir / 'missing.pt' + run = subprocess.run([sys.executable, '-B', str(ROOT / 'chat.py'), + '--snapshot', str(missing)], + capture_output=True, text=True, timeout=60) + assert run.returncode == 2 + assert 'Checkpoint file does not exist' in run.stderr and str(missing) in run.stderr + assert 'Traceback' not in run.stderr + + +def test_tokenizer_hash_mismatch_is_rejected(fixture_dir): + source, state = source_checkpoint(fixture_dir) + state['signature']['data']['signature']['tokenizer_sha256'] = '0' * 64 + torch.save(state, source) + with pytest.raises((ValueError, RuntimeError), match='[Tt]okenizer'): + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + chat.load_model(snapshot, TOKENIZER) + + +def test_model_load_is_strict(fixture_dir): + source, state = source_checkpoint(fixture_dir) + del state['model']['depth_bias'] + torch.save(state, source) + with pytest.raises((ValueError, RuntimeError)): + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + chat.load_model(snapshot, TOKENIZER) + + +def test_generation_stops_at_eos_without_yielding_it(fixture_dir, monkeypatch): + source, _ = source_checkpoint(fixture_dir) + snapshot = chat.capture_latest(source, TOKENIZER, fixture_dir / 'snapshots') + model, sp, _ = chat.load_model(snapshot, TOKENIZER) + logits = torch.full((1, 1, sp.vocab_size()), -100.0) + logits[0, 0, sp.eos_id()] = 100.0 + monkeypatch.setattr(model, 'step', lambda tokens, caches: logits) + output = list(chat.generate(model, sp, 'Hello', max_new_tokens=5, + temperature=0, top_k=0, top_p=1, seed=1, max_context=32)) + assert output == [] + + +@pytest.mark.parametrize('scores', [[3.0, 2.9], [-3.0, -3.4]]) +@pytest.mark.parametrize('temperature', [0.0, 0.8]) +def test_repetition_penalty_lowers_both_signs_before_top_k(scores, temperature): + logits = torch.tensor(scores) + before = logits.clone() + result = chat.sample_token(logits, temperature, 1, 1.0, torch.Generator(), + previous_tokens=[0, 0], repetition_penalty=1.2) + assert result == 1 + torch.testing.assert_close(logits, before) + + +def test_repetition_penalty_applies_once_per_distinct_token(): + assert chat.sample_token(torch.tensor([3.0, 2.4]), 0, 0, 1, torch.Generator(), + previous_tokens=[0, 0, 0], repetition_penalty=1.2) == 0 + + +@pytest.mark.parametrize('temperature', [0.0, 0.8]) +def test_ngram_mask_applies_before_greedy_or_top_k_and_handles_exhaustion(temperature): + logits = torch.tensor([100.0, 1.0]) + before = logits.clone() + assert chat.sample_token(logits, temperature, 1, 0.9, torch.Generator(), + banned_tokens={0}) == 1 + assert chat.sample_token(logits, temperature, 1, 0.9, torch.Generator(), + banned_tokens={0, 1}) is None + torch.testing.assert_close(logits, before) + + +@pytest.mark.parametrize('tokens,size,expected', [ + ([1, 2, 3, 4, 1, 2, 3], 4, {4}), + ([1, 2, 3, 4, 1, 2], 4, set()), + ([1, 1, 1], 2, {1}), + ([1, 2, 3], 1, {1, 2, 3}), + ([1, 1], 0, set()), + ([], 4, set()), + ([1, 2], 4, set()), + ([1, 2, 3, 1, 2, 4, 1, 2], 3, {3, 4}), +]) +def test_ngram_blocking_includes_overlapping_and_multiple_matches(tokens, size, expected): + assert chat.banned_ngram_tokens(tokens, size) == expected + + +class LoopTokenizer: + def encode(self, prompt, out_type): + # Repeated phrases in the prompt must still be usable in an answer. + return [3, 4, 5, 6, 3, 4, 5, 6] + + def eos_id(self): + return 2 + + +class LoopModel: + def init_generation_cache(self, *args): + return [] + + def step(self, token, caches): + next_id = {3: 4, 4: 5, 5: 6, 6: 3, 7: 2}[int(token.item())] + logits = torch.full((1, 1, 8), -100.0) + logits[0, 0, 7] = 1.0 + logits[0, 0, next_id] = 10.0 + return logits + + +@pytest.mark.parametrize('temperature', [0.0, 0.8]) +def test_generation_breaks_a_phrase_loop_preserves_prompt_and_resets_each_reply(temperature): + model, sp = LoopModel(), LoopTokenizer() + settings = dict(max_new_tokens=12, max_context=32, temperature=temperature, top_k=1) + unguarded = list(chat.generate(model, sp, 'prompt', repetition_penalty=1, + no_repeat_ngram_size=0, **settings)) + assert unguarded == [3, 4, 5, 6] * 3 + protected = list(chat.generate(model, sp, 'prompt', **settings)) + assert protected == [3, 4, 5, 6, 3, 4, 5, 7] + assert list(chat.generate(model, sp, 'prompt', **settings)) == protected + grams = [tuple(protected[i:i+4]) for i in range(len(protected) - 3)] + assert len(grams) == len(set(grams)) + + +def test_sampling_with_repetition_protection_is_seeded_without_changing_global_rng(): + logits = torch.tensor([3.0, 2.9, 2.8, 2.7]) + state = torch.random.get_rng_state().clone() + def draw(): + generator = torch.Generator().manual_seed(42) + return [chat.sample_token(logits, 0.8, 3, 0.9, generator, + previous_tokens=[0], repetition_penalty=1.2, + banned_tokens={3}) for _ in range(20)] + assert draw() == draw() + assert 3 not in draw() + assert torch.equal(torch.random.get_rng_state(), state) + + +@pytest.mark.parametrize('settings', [ + {'repetition_penalty': 0.9}, {'repetition_penalty': float('nan')}, + {'repetition_penalty': float('inf')}, {'no_repeat_ngram_size': -1}, + {'no_repeat_ngram_size': 1.5}, +]) +def test_generation_rejects_invalid_repetition_settings_before_model_use(settings): + with pytest.raises(ValueError, match='repetition-penalty|no-repeat-ngram-size'): + list(chat.generate(None, None, 'prompt', **settings)) diff --git a/test_checkpoint_writer.py b/test_checkpoint_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..78243eb1ecd9140ff47d718314f801028caaad33 --- /dev/null +++ b/test_checkpoint_writer.py @@ -0,0 +1,267 @@ +"""CPU regressions for immutable, bounded asynchronous checkpoint publication.""" +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import sys +import threading + +import numpy as np +import pytest +import torch + +from ops import checkpoint_writer +from run_40b import arguments + + +def training_state(): + model = torch.nn.Linear(2, 2) + optimizer = torch.optim.AdamW(model.parameters(), lr=0.02) + optimizer.zero_grad(set_to_none=True) + model(torch.ones(2, 2)).square().mean().backward() + optimizer.step() + state = { + "model": model.state_dict(), "optimizer": optimizer.state_dict(), + "step": 1, "tokens_seen": 8, + "rng": {"torch": torch.get_rng_state(), "numpy": np.arange(4)}, + "metadata": {"history": [{"batch": 2}], "tuple": ([1, 2], "bf16")}, + } + return model, optimizer, state + + +def assert_state_equal(actual, expected): + if isinstance(expected, torch.Tensor): + assert actual.device.type == "cpu" + assert not actual.requires_grad + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + elif isinstance(expected, np.ndarray): + np.testing.assert_array_equal(actual, expected) + elif isinstance(expected, dict): + assert actual.keys() == expected.keys() + for key in expected: + assert_state_equal(actual[key], expected[key]) + elif isinstance(expected, (tuple, list)): + assert type(actual) is type(expected) + assert len(actual) == len(expected) + for value, other in zip(actual, expected, strict=True): + assert_state_equal(value, other) + else: + assert actual == expected + + +def test_pending_save_keeps_model_optimizer_and_metadata_snapshot_immutable(tmp_path, monkeypatch): + model, optimizer, state = training_state() + expected = deepcopy(state) + entered, release = threading.Event(), threading.Event() + original = checkpoint_writer.atomic_torch_save + + def blocked_write(path, snapshot): + entered.set() + assert release.wait(10), "Test did not release checkpoint writer" + original(path, snapshot) + + monkeypatch.setattr(checkpoint_writer, "atomic_torch_save", blocked_write) + path = tmp_path / "latest.pt" + writer = checkpoint_writer.AsyncCheckpointWriter(path) + try: + writer.submit(state) + assert entered.wait(10) + # A real second AdamW update mutates parameters, moments, and step counters. + optimizer.zero_grad(set_to_none=True) + model(torch.full((2, 2), 2.0)).square().mean().backward() + optimizer.step() + state["optimizer"]["param_groups"][0]["lr"] = 0.5 + state["rng"]["numpy"][:] = -1 + state["rng"]["torch"].zero_() + state["metadata"]["history"][0]["batch"] = 99 + state["metadata"]["tuple"][0].append(3) + assert not path.exists() + release.set() + writer.flush() + actual = torch.load(path, map_location="cpu", weights_only=False) + assert_state_equal(actual, expected) + assert not torch.equal(state["model"]["weight"], expected["model"]["weight"]) + assert state["optimizer"]["state"][0]["step"].item() == 2 + finally: + release.set() + writer.close() + + +def test_atomic_save_keeps_old_checkpoint_until_complete_publication(tmp_path, monkeypatch): + path = tmp_path / "latest.pt" + torch.save({"step": 1}, path) + written, release = threading.Event(), threading.Event() + original = torch.save + + def pause_after_serialization(state, stream, *args, **kwargs): + original(state, stream, *args, **kwargs) + written.set() + assert release.wait(10), "Test did not release serialized checkpoint" + + monkeypatch.setattr(torch, "save", pause_after_serialization) + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(checkpoint_writer.atomic_torch_save, path, {"step": 2}) + try: + assert written.wait(10) + assert torch.load(path, weights_only=False) == {"step": 1} + assert not future.done() + finally: + release.set() + future.result(timeout=10) + assert torch.load(path, weights_only=False) == {"step": 2} + + +def test_failed_serialization_keeps_old_checkpoint_and_removes_partial_file(tmp_path, monkeypatch): + path = tmp_path / "latest.pt" + torch.save({"step": 1}, path) + + def partial_write_then_fail(state, stream): + stream.write(b"incomplete checkpoint") + raise OSError("serialization disk failure") + + monkeypatch.setattr(torch, "save", partial_write_then_fail) + with pytest.raises(OSError, match="serialization disk failure"): + checkpoint_writer.atomic_torch_save(path, {"step": 2}) + assert torch.load(path, weights_only=False) == {"step": 1} + assert list(tmp_path.iterdir()) == [path] + + +def test_second_submit_waits_before_allocating_another_snapshot(tmp_path, monkeypatch): + entered, release, attempting, snapshotted_second = [threading.Event() for _ in range(4)] + original_write = checkpoint_writer.atomic_torch_save + original_snapshot = checkpoint_writer.snapshot_to_cpu + active = 0 + maximum_active = 0 + lock = threading.Lock() + + def snapshot(state): + if state["step"] == 2: + snapshotted_second.set() + return original_snapshot(state) + + def blocked_write(path, state): + nonlocal active, maximum_active + with lock: + active += 1 + maximum_active = max(maximum_active, active) + try: + if state["step"] == 1: + entered.set() + assert release.wait(10), "Test did not release first writer" + original_write(path, state) + finally: + with lock: + active -= 1 + + monkeypatch.setattr(checkpoint_writer, "snapshot_to_cpu", snapshot) + monkeypatch.setattr(checkpoint_writer, "atomic_torch_save", blocked_write) + path = tmp_path / "latest.pt" + writer = checkpoint_writer.AsyncCheckpointWriter(path) + try: + writer.submit({"step": 1, "tokens_seen": 8}) + assert entered.wait(10) + + def submit_second(): + attempting.set() + writer.submit({"step": 2, "tokens_seen": 16}) + + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(submit_second) + try: + assert attempting.wait(10) + assert not snapshotted_second.wait(0.2) + assert not future.done() + finally: + release.set() + future.result(timeout=10) + writer.flush() + assert snapshotted_second.is_set() + assert maximum_active == 1 + assert torch.load(path, weights_only=False) == {"step": 2, "tokens_seen": 16} + finally: + release.set() + writer.close() + + +def test_background_failure_is_persistent_and_preserves_old_checkpoint(tmp_path, monkeypatch): + path = tmp_path / "latest.pt" + torch.save({"step": 1}, path) + + def failed_write(path, state): + raise OSError("checkpoint disk failure") + + monkeypatch.setattr(checkpoint_writer, "atomic_torch_save", failed_write) + writer = checkpoint_writer.AsyncCheckpointWriter(path) + writer.submit({"step": 2, "tokens_seen": 16}) + for action in (writer.flush, writer.check, + lambda: writer.submit({"step": 3, "tokens_seen": 24}), writer.close): + with pytest.raises(OSError, match="checkpoint disk failure"): + action() + assert torch.load(path, weights_only=False) == {"step": 1} + + +@pytest.mark.parametrize("exceptional_exit", [False, True]) +def test_context_exit_flushes_pending_checkpoint(tmp_path, monkeypatch, exceptional_exit): + entered, release, leaving_context, finished = [threading.Event() for _ in range(4)] + original = checkpoint_writer.atomic_torch_save + path = tmp_path / "latest.pt" + + def blocked_write(path, state): + entered.set() + assert release.wait(10), "Test did not release writer during exit" + original(path, state) + + monkeypatch.setattr(checkpoint_writer, "atomic_torch_save", blocked_write) + + def run_context(): + try: + with checkpoint_writer.AsyncCheckpointWriter(path) as writer: + writer.submit({"step": 7, "tokens_seen": 73}) + assert entered.wait(10) + leaving_context.set() + if exceptional_exit: + raise ValueError("training failed") + finally: + finished.set() + + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(run_context) + try: + assert leaving_context.wait(10) + assert not finished.wait(0.2) + assert not path.exists() + finally: + release.set() + if exceptional_exit: + with pytest.raises(ValueError, match="training failed"): + future.result(timeout=10) + else: + future.result(timeout=10) + assert torch.load(path, weights_only=False) == {"step": 7, "tokens_seen": 73} + + +@pytest.mark.parametrize(("flags", "enabled"), [ + ([], False), + (["--async-checkpoint"], True), + (["--async-checkpoint", "--no-async-checkpoint"], False), +]) +def test_async_checkpoint_cli_is_explicit_and_reversible(monkeypatch, flags, enabled): + monkeypatch.setenv("WORLD_SIZE", "1") + monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stage", "train", *flags]) + assert arguments().async_checkpoint is enabled + + +def test_async_checkpoint_cli_accepts_multiple_training_ranks(monkeypatch): + monkeypatch.setenv("WORLD_SIZE", "2") + monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stage", "train", "--async-checkpoint"]) + assert arguments().async_checkpoint is True + + +def test_training_exception_preserved_when_background_write_also_fails(tmp_path, monkeypatch): + def failed_write(path, state): + raise OSError("checkpoint disk failure") + + monkeypatch.setattr(checkpoint_writer, "atomic_torch_save", failed_write) + with pytest.raises(ValueError, match="training failed") as error: + with checkpoint_writer.AsyncCheckpointWriter(tmp_path / "latest.pt") as writer: + writer.submit({"step": 1}) + raise ValueError("training failed") + assert any("checkpoint disk failure" in note for note in error.value.__notes__) diff --git a/test_communication_settings_cli.py b/test_communication_settings_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..6b06bf86a594e80ad4562186f045cbd85b94062a --- /dev/null +++ b/test_communication_settings_cli.py @@ -0,0 +1,76 @@ +"""Both entry points validate and record communication experiment controls.""" + +import sys +from pathlib import Path + +import pytest + +import run_40b +from ops import benchmark_training + + +@pytest.fixture(params=[run_40b, benchmark_training], ids=["runner", "benchmark"]) +def cli(request, monkeypatch): + workspace = Path(__file__).resolve().parents[1] + for name in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR", + "TORCHINDUCTOR_CACHE_DIR"): + monkeypatch.setenv(name, str(workspace / "tmp")) + monkeypatch.delenv("WORLD_SIZE", raising=False) + + def parse(flags): + monkeypatch.setattr(sys, "argv", [request.param.__file__, "--micro-batch", "4", *flags]) + return request.param.arguments() + + return parse + + +def test_current_overlap_behavior_remains_the_default(cli): + args = cli(["--gradient-mode", "compiled-autograd-overlap"]) + assert not args.overlap_swap_collectives + assert not args.overlap_runtime_estimates + assert not args.overlap_producer_schedule + assert args.overlap_memory_budget == 0 + + +def test_overlap_experiment_is_explicit_and_in_resume_signature(cli): + args = cli(["--gradient-mode", "compiled-autograd-overlap", + "--overlap-swap-collectives", "--overlap-runtime-estimates", + "--overlap-memory-budget", "0.02"]) + assert args.overlap_swap_collectives and args.overlap_runtime_estimates + assert args.overlap_memory_budget == 0.02 + if hasattr(args, "grad_accum"): + settings = run_40b.train_signature(args, {})["settings"] + assert settings["overlap_swap_collectives"] is True + assert settings["overlap_runtime_estimates"] is True + assert settings["overlap_memory_budget"] == 0.02 + + +def test_producer_scheduling_is_explicit_and_recorded(cli): + args = cli(["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"]) + assert args.overlap_producer_schedule is True + assert not args.overlap_swap_collectives and not args.overlap_runtime_estimates + assert args.overlap_memory_budget == 0.0 + if hasattr(args, "grad_accum"): + assert run_40b.train_signature(args, {})["settings"]["overlap_producer_schedule"] is True + + +@pytest.mark.parametrize("flags", [ + ["--overlap-swap-collectives"], + ["--gradient-mode", "compiled-autograd", "--overlap-runtime-estimates"], + ["--overlap-memory-budget", "0.02"], + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "nan"], + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "-0.02"], + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "0.26"], + ["--overlap-producer-schedule"], + ["--gradient-mode", "compiled-autograd", "--overlap-producer-schedule"], + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule", + "--overlap-swap-collectives"], + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule", + "--overlap-runtime-estimates"], + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule", + "--overlap-memory-budget", "0.02"], +]) +def test_reject_silent_noops_and_unbounded_memory_before_cuda(cli, flags): + with pytest.raises(SystemExit) as error: + cli(flags) + assert error.value.code == 2 diff --git a/test_compare_communication_benchmarks.py b/test_compare_communication_benchmarks.py new file mode 100644 index 0000000000000000000000000000000000000000..a97e1b7d91bfa00463d7e98669c445913a998664 --- /dev/null +++ b/test_compare_communication_benchmarks.py @@ -0,0 +1,155 @@ +"""CPU fixtures reject misleading comparisons and preserve per-rank evidence.""" + +import copy +import hashlib +import json + +import pytest + +from ops.compare_communication_benchmarks import compare_summaries, load_run + + +def fixture(): + identity = {"path": "/workspace/WarpState/fixture", "size": 100, "inode": 1, "mtime_ns": 2} + config = {"checkpoint": identity, "checkpoint_step": 40, "checkpoint_tokens": 1280, + "data": {**identity, "path": "/workspace/WarpState/data"}, + "world_size": 2, "grad_accum": 2, "parameters": 32, + "parameter_dtype": "float32", "autocast_dtype": "bfloat16", "optimizer": "AdamW(fused=True)", + "torch": "fixture", "cuda": "fixture", "gpu": "fixture", + "update_timing_includes_gradient_zeroing": True, "checkpoint_writes": False, + "source_sha256": {"kernel.py": "old"}, "allow_bf16_reduced_precision_reduction": False, + "args": {"global_tokens_per_update": 32, "seq_len": 8, "micro_batch": 1, + "warmup_updates": 1, "measured_updates": 3, "seed": 1337, + "optimizer_state": "resume", "bf16_residuals": True, "profile": False}} + ranks = [] + for rank in range(2): + ranks.append({"configuration": {**copy.deepcopy(config), "rank": rank}, + "checkpoint_path_identity_after": copy.deepcopy(identity), "updates": [ + {"rank": rank, "update": index + 1, "phase": "warmup" if index == 0 else "measured", + "profiled": False, "global_tokens": 32, "seconds": 2.0 + rank, + "tokens_per_second": 32 / (2.0 + rank), "loss": 2.5, "grad_norm": 0.25, + "peak_allocated_gib": 60.0 + rank, "peak_reserved_gib": 70.0 + rank} + for index in range(4)]}) + return {**copy.deepcopy(ranks[0]), "ranks": ranks} + + +def alter_config(run, key, value, *, argument=False): + for summary in [run, *run["ranks"]]: + config = summary["configuration"] + (config["args"] if argument else config)[key] = value + + +def test_comparison_uses_slowest_rank_and_records_intended_source_changes(): + control, candidate = fixture(), fixture() + alter_config(candidate, "source_sha256", {"kernel.py": "new", "scheduler.py": "added"}) + for summary in [candidate, *candidate["ranks"]]: + for update in summary["updates"]: + update["seconds"] /= 2 + update["tokens_per_second"] *= 2 + update["loss"] += 0.125 + update["grad_norm"] += 0.025 + result = compare_summaries(control, candidate) + assert result["throughput_ratio"] == 2 + assert result["control_timing"]["mean_seconds"] == 3 + assert result["maximum_absolute_loss_difference"] == 0.125 + assert result["maximum_relative_grad_norm_difference"] == pytest.approx(0.1) + assert result["rank_peak_vram"][1]["candidate"]["measured_updates"]["peak_reserved_gib"] == 71 + assert result["paired_updates"][0]["token_cursor"] == 1312 + assert result["paired_updates"][0]["ranks"][1]["microbatch_cursors"] == [1320, 1336] + assert result["source_differences"]["kernel.py"]["candidate"] == "new" + assert result["candidate_timing"]["drift_windows_overlap"] + + +@pytest.mark.parametrize(("key", "value", "argument"), [ + ("checkpoint_tokens", 1281, False), ("checkpoint_step", 41, False), + ("parameter_dtype", "bfloat16", False), ("autocast_dtype", "float16", False), + ("allow_bf16_reduced_precision_reduction", True, False), + ("optimizer", "SGD", False), ("optimizer_state", "fresh", True), + ("seed", 1338, True), ("bf16_residuals", False, True), + ("warmup_updates", 0, True), ("micro_batch", 2, True), +]) +def test_incompatible_inputs_and_numerical_controls_fail(key, value, argument): + control, candidate = fixture(), fixture() + alter_config(candidate, key, value, argument=argument) + with pytest.raises(ValueError): + compare_summaries(control, candidate) + + +@pytest.mark.parametrize("kind", ["profile", "missing_rank", "missing_update", "cursor", "phase", "identity", "rank_identity"]) +def test_incomplete_or_inconsistent_recordings_fail(kind): + candidate = fixture() + if kind == "profile": + candidate["ranks"][1]["updates"][2]["profiled"] = True + elif kind == "missing_rank": + candidate["ranks"].pop() + elif kind == "missing_update": + candidate["ranks"][1]["updates"].pop() + elif kind == "cursor": + candidate["ranks"][1]["updates"][2]["token_cursor"] = 0 + elif kind == "phase": + candidate["ranks"][1]["updates"][2]["phase"] = "warmup" + elif kind == "identity": + candidate["ranks"][1]["checkpoint_path_identity_after"]["inode"] += 1 + elif kind == "rank_identity": + candidate["ranks"][1]["configuration"]["rank"] = 0 + with pytest.raises(ValueError): + compare_summaries(fixture(), candidate) + + +def test_missing_precision_controls_are_explicit_and_partial_recording_fails(): + control, candidate = fixture(), fixture() + result = compare_summaries(control, candidate) + assert "allow_tf32" in result["unrecorded_precision_optimizer_controls"] + alter_config(candidate, "allow_tf32", True) + with pytest.raises(ValueError, match="incompletely recorded"): + compare_summaries(control, candidate) + + +def test_precision_mismatch_on_only_the_second_rank_is_rejected(): + candidate = fixture() + candidate["ranks"][1]["configuration"]["allow_bf16_reduced_precision_reduction"] = True + with pytest.raises(ValueError, match="Rank 1 precision"): + compare_summaries(fixture(), candidate) + + +def test_data_identity_difference_is_rejected_even_with_equal_shapes(): + candidate = fixture() + changed = {**candidate["configuration"]["data"], "mtime_ns": 3} + alter_config(candidate, "data", changed) + with pytest.raises(ValueError, match="configuration: data"): + compare_summaries(fixture(), candidate) + + +@pytest.mark.parametrize(("argument", "value"), [ + ("overlap_swap_collectives", True), ("gradient_mode", "eager"), ("compile_mode", "default"), +]) +def test_execution_argument_changes_on_only_one_rank_are_rejected(argument, value): + candidate = fixture() + candidate["ranks"][1]["configuration"]["args"][argument] = value + with pytest.raises(ValueError, match="Rank 1 execution argument mismatch"): + compare_summaries(fixture(), candidate) + + +def test_jsonl_fallback_hashes_every_consumed_input(tmp_path): + run = fixture() + for rank_summary in run["ranks"]: + rank_summary["configuration"]["event"] = "configuration" + for update in rank_summary["updates"]: + update["event"] = "update" + summary = copy.deepcopy(run["ranks"][0]) + (tmp_path / "summary.json").write_text(json.dumps(summary)) + for rank, rank_summary in enumerate(run["ranks"]): + records = [rank_summary["configuration"], *rank_summary["updates"]] + (tmp_path / f"rank-{rank}.jsonl").write_text("\n".join(json.dumps(row) for row in records) + "\n") + loaded = load_run(tmp_path) + paths = [tmp_path / name for name in ("summary.json", "rank-0.jsonl", "rank-1.jsonl")] + assert loaded["input_files"] == [{"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + for path in paths] + assert len(loaded["summary"]["ranks"]) == 2 + before = loaded["input_files"][2]["sha256"] + # Semantically irrelevant trailing whitespace changes the exact consumed + # input hash, while preserving successful reconstruction and validation. + paths[2].write_text(paths[2].read_text().rstrip() + " \n") + after = load_run(tmp_path) + assert after["summary"] == loaded["summary"] + assert after["input_files"][2]["sha256"] != before diff --git a/test_distributed_checkpoint_writer.py b/test_distributed_checkpoint_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..3f49bce57403ac9d61ddd4530c1e4f6cbc17f646 --- /dev/null +++ b/test_distributed_checkpoint_writer.py @@ -0,0 +1,196 @@ +"""Two-process CPU/Gloo regressions for coordinated checkpoint publication.""" + +from datetime import timedelta +import json +import os +from pathlib import Path +import subprocess +import sys +import threading + +import pytest +import torch +import torch.distributed as dist + +from ops import checkpoint_writer + + +def _worker(rank, directory, scenario): + directory = Path(directory) + report = {"rank": rank, "writes": [], "syncs": 0, "factory_calls": 0} + writer = None + release = threading.Event() + original_write = checkpoint_writer.atomic_torch_save + original_snapshot = checkpoint_writer.snapshot_to_cpu + original_fsync = os.fsync + original_broadcast = dist.broadcast + + def save(path, state): + report["writes"].append({ + "rank": rank, "thread": threading.current_thread().name, + "device": str(state["weight"].device), + }) + if scenario in {"check", "submit", "flush", "close"}: + raise OSError("injected checkpoint disk failure") + assert release.wait(10), "Test did not release checkpoint serialization" + original_write(path, state) + + def snapshot(state): + if scenario == "snapshot": + raise MemoryError("injected CPU snapshot failure") + return original_snapshot(state) + + def fsync(descriptor): + original_fsync(descriptor) + report["syncs"] += 1 + + def forbidden_collective(*args, **kwargs): + raise AssertionError("Exceptional context exit attempted a collective") + + checkpoint_writer.atomic_torch_save = save + checkpoint_writer.snapshot_to_cpu = snapshot + os.fsync = fsync + try: + dist.init_process_group( + "gloo", init_method=(directory / "rendezvous").as_uri(), + rank=rank, world_size=2, timeout=timedelta(seconds=15), + ) + try: + with checkpoint_writer.DistributedCheckpointWriter( + directory / "latest.pt", rank=rank, + device=torch.device("cpu"), distributed=True, + ) as writer: + report["owns_writer"] = writer._writer is not None + state = {"weight": torch.tensor([1.0, 2.0]), "metadata": [3]} + + def build_state(): + report["factory_calls"] += 1 + assert rank == 0, "Only rank zero may construct checkpoint state" + if scenario == "factory": + raise ValueError("injected checkpoint construction failure") + return state + + report["staging_seconds"] = ( + writer.submit(state_factory=build_state) + if scenario in {"factory", "normal_factory"} + else writer.submit(state if rank == 0 else None) + ) + if scenario in {"check", "submit", "flush", "close"}: + # Ensure the disk error has happened before asking every rank + # to observe it through each distinct public operation. + if rank == 0: + try: + writer._writer._future.result(timeout=10) + except OSError: + pass + dist.barrier() + if scenario == "submit": + writer.submit(state if rank == 0 else None) + else: + getattr(writer, scenario)() + raise AssertionError("Checkpoint failure was not raised") + state["weight"].add_(10) + state["metadata"].append(4) + writer.check() # A pending write is allowed to overlap training. + report["exists_before_release"] = (directory / "latest.pt").exists() + dist.barrier() + release.set() + if scenario == "exceptional_exit": + dist.broadcast = forbidden_collective + raise ValueError("injected training failure") + report["normal_exit"] = True + except BaseException as error: + report["error"] = {"type": type(error).__name__, "message": str(error)} + if writer is not None: + report["closed"] = writer._writer is None or writer._writer._closed + finally: + release.set() + checkpoint_writer.atomic_torch_save = original_write + checkpoint_writer.snapshot_to_cpu = original_snapshot + os.fsync = original_fsync + dist.broadcast = original_broadcast + if dist.is_initialized(): + dist.destroy_process_group() + (directory / f"rank-{rank}.json").write_text(json.dumps(report)) + + +def _run_two_ranks(directory, scenario): + environment = os.environ.copy() + environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") + processes = [] + try: + for rank in range(2): + processes.append(subprocess.Popen( + [sys.executable, str(Path(__file__).resolve()), + str(rank), str(directory), scenario], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + )) + for rank, process in enumerate(processes): + output, _ = process.communicate(timeout=40) + assert process.returncode == 0, f"Rank {rank} failed:\n{output}" + finally: + for process in processes: + if process.poll() is None: + process.kill() + for process in processes: + process.wait(timeout=10) + return [json.loads((directory / f"rank-{rank}.json").read_text()) for rank in range(2)] + + +@pytest.mark.parametrize("operation", ["check", "submit", "flush", "close", "snapshot", "factory"]) +def test_rank_zero_failure_reaches_both_ranks_without_hanging(tmp_path, operation): + path = tmp_path / "latest.pt" + torch.save({"old_checkpoint": True}, path) + reports = _run_two_ranks(tmp_path, operation) + expected_type, expected_message = { + "snapshot": ("MemoryError", "CPU snapshot failure"), + "factory": ("ValueError", "checkpoint construction failure"), + }.get(operation, ("OSError", "checkpoint disk failure")) + assert reports[0]["error"]["type"] == expected_type + assert reports[1]["error"]["type"] == "RuntimeError" + for rank, report in enumerate(reports): + assert expected_message in report["error"]["message"] + assert report["owns_writer"] is (rank == 0) + assert report["closed"] + assert reports[1]["writes"] == [] + assert reports[1]["factory_calls"] == 0 + assert reports[0]["factory_calls"] == int(operation == "factory") + assert torch.load(path, weights_only=False) == {"old_checkpoint": True} + assert not path.with_suffix(".pt.tmp").exists() + + +@pytest.mark.parametrize("scenario", ["normal_exit", "normal_factory"]) +def test_normal_exit_publishes_one_immutable_cpu_snapshot_durably(tmp_path, scenario): + reports = _run_two_ranks(tmp_path, scenario) + for rank, report in enumerate(reports): + assert "error" not in report, report + assert report["owns_writer"] is (rank == 0) + assert report["normal_exit"] and report["closed"] + assert report["exists_before_release"] is False + assert reports[0]["staging_seconds"] >= 0 + assert reports[1]["staging_seconds"] is None + assert reports[1]["factory_calls"] == 0 + assert reports[0]["factory_calls"] == int(scenario == "normal_factory") + assert reports[1]["writes"] == [] + assert reports[1]["syncs"] == 0 + assert reports[0]["syncs"] == 2 # File contents, then directory publication. + assert len(reports[0]["writes"]) == 1 + write = reports[0]["writes"][0] + assert write["device"] == "cpu" and write["thread"].startswith("checkpoint-writer") + state = torch.load(tmp_path / "latest.pt", weights_only=False) + torch.testing.assert_close(state["weight"], torch.tensor([1.0, 2.0]), rtol=0, atol=0) + assert state["metadata"] == [3] + assert not (tmp_path / "latest.pt.tmp").exists() + + +def test_exceptional_exit_joins_local_writer_without_any_collectives(tmp_path): + reports = _run_two_ranks(tmp_path, "exceptional_exit") + for report in reports: + assert report["error"] == {"type": "ValueError", "message": "injected training failure"} + assert report["closed"] + assert (tmp_path / "latest.pt").is_file() + assert reports[1]["writes"] == [] + + +if __name__ == "__main__": + _worker(int(sys.argv[1]), sys.argv[2], sys.argv[3]) diff --git a/test_distributed_execution.py b/test_distributed_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..f963ad4ffea47a440d40249542fe810f10664ad8 --- /dev/null +++ b/test_distributed_execution.py @@ -0,0 +1,177 @@ +"""CPU/Gloo checks for DDP bucket execution and BF16 accumulation semantics.""" + +from contextlib import nullcontext +from copy import deepcopy +from datetime import timedelta +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import torch +import torch.distributed as dist +from torch import nn +from torch.nn.parallel import DistributedDataParallel + +from ops.distributed_execution import ddp_kwargs, ddp_runtime_settings + + +@pytest.mark.parametrize("changes", [ + {"bucket_cap_mb": 0}, {"bucket_cap_mb": -1}, {"bucket_cap_mb": True}, + {"bucket_cap_mb": float("nan")}, {"bucket_cap_mb": float("inf")}, + {"bucket_cap_mb": "128"}, {"bucket_cap_mb": 1e-12}, + {"batched_grad_copy": 1}, {"batched_grad_copy": "false"}, + {"bucket_cap_mb_list": []}, {"bucket_cap_mb_list": "64,128"}, + {"bucket_cap_mb_list": [64, 0]}, {"bucket_cap_mb_list": [64, False]}, +]) +def test_invalid_runtime_settings_are_rejected(changes): + with pytest.raises(ValueError): + ddp_runtime_settings(**changes) + + +def test_legacy_runtime_does_not_receive_unsupported_optional_arguments(monkeypatch): + def legacy_ddp(module, broadcast_buffers=False, gradient_as_bucket_view=False, + bucket_cap_mb=None): + pass + + monkeypatch.setattr(torch.nn.parallel, "DistributedDataParallel", legacy_ddp) + assert ddp_kwargs({}) == { + "broadcast_buffers": False, "gradient_as_bucket_view": True, + "bucket_cap_mb": 128, + } + for requested in [{"batched_grad_copy": True}, {"bucket_cap_mb_list": [64, 128]}]: + with pytest.raises(RuntimeError, match="Installed PyTorch DDP does not support"): + ddp_kwargs(ddp_runtime_settings(**requested)) + + +def test_capacity_list_is_copied_and_records_effective_capacity(): + capacities = [16, 64, 32] + settings = ddp_runtime_settings(bucket_cap_mb=128, batched_grad_copy=True, + bucket_cap_mb_list=capacities) + capacities[0] = 999 + assert settings["ddp_bucket_cap_mb"] == 64 + assert settings["ddp_bucket_cap_mb_list"] == [16, 64, 32] + assert json.loads(json.dumps(settings)) == settings + assert ddp_kwargs(settings)["bucket_cap_mb_list"] == [16, 64, 32] + + +@pytest.mark.parametrize("requested", [ + {"batched_grad_copy": True}, {"bucket_cap_mb_list": [64, 128]}, +]) +def test_python_reducer_rejects_inapplicable_cpp_bucket_options(requested): + with torch._dynamo.config.patch(optimize_ddp="python_reducer"): + with pytest.raises(ValueError, match="require the C\\+\\+ DDP reducer"): + ddp_kwargs(ddp_runtime_settings(**requested)) + assert ddp_kwargs(ddp_runtime_settings(bucket_cap_mb=64))["bucket_cap_mb"] == 64 + + +def _model(): + layers = [] + for _ in range(6): + layers.extend([nn.Linear(32, 32), nn.SiLU()]) + layers.append(nn.Linear(32, 8)) + return nn.Sequential(*layers) + + +def _worker(rank, directory): + torch.set_num_threads(1) + directory = Path(directory) + dist.init_process_group("gloo", init_method=(directory / "rendezvous").as_uri(), + rank=rank, world_size=2, timeout=timedelta(seconds=30)) + reports = [] + try: + cases = [(capacity, batched, None) + for capacity in [64, 128, 256, 0.002] + for batched in [False, True]] + cases.append((128, True, [0.002, 0.004])) + for capacity, batched, capacities in cases: + torch.manual_seed(321 + rank) + settings = ddp_runtime_settings(bucket_cap_mb=capacity, + batched_grad_copy=batched, + bucket_cap_mb_list=capacities) + model = DistributedDataParallel(_model(), **ddp_kwargs(settings)) + reference = deepcopy(model.module) + optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, foreach=False) + reference_optimizer = torch.optim.AdamW(reference.parameters(), lr=0.001, + foreach=False) + maximum_gradient_error = 0.0 + for update in range(3): + # Reallocation on every update exercises the production + # set_to_none=True path both before and after bucket rebuilding. + optimizer.zero_grad(set_to_none=True) + reference_optimizer.zero_grad(set_to_none=True) + assert all(parameter.grad is None for parameter in model.parameters()) + for microstep in range(8): + generator = torch.Generator().manual_seed( + 10000 + rank * 1000 + update * 8 + microstep) + inputs = torch.randn(3, 32, generator=generator) + target = torch.randn(3, 8, generator=generator) + context = model.no_sync() if microstep < 7 else nullcontext() + with context: + with torch.autocast("cpu", dtype=torch.bfloat16): + output = model(inputs) + reference_output = reference(inputs) + assert output.dtype == reference_output.dtype == torch.bfloat16 + loss = (output.float() - target).square().mean() / 8 + reference_loss = (reference_output.float() - target).square().mean() / 8 + loss.backward() + reference_loss.backward() + for actual, expected in zip(model.parameters(), reference.parameters()): + assert actual.dtype == expected.dtype == torch.float32 + assert actual.grad.dtype == expected.grad.dtype == torch.float32 + # Independent reference: accumulate each rank's distinct + # microbatches locally, then explicitly average their FP32 + # gradients. This detects missing/repeated microsteps and + # incorrect reducer scaling without assuming bucket layout. + dist.all_reduce(expected.grad) + expected.grad.div_(2) + maximum_gradient_error = max(maximum_gradient_error, + (actual.grad - expected.grad).abs().max().item()) + torch.testing.assert_close(actual.grad, expected.grad, + rtol=1e-6, atol=1e-7) + assert torch.isfinite(actual.grad).all() + optimizer.step() + reference_optimizer.step() + for actual, expected in zip(model.parameters(), reference.parameters()): + torch.testing.assert_close(actual, expected, rtol=1e-6, atol=1e-7) + for state in optimizer.state.values(): + assert state["exp_avg"].dtype == state["exp_avg_sq"].dtype == torch.float32 + reports.append({"settings": settings, + "maximum_gradient_error": maximum_gradient_error, + "updates": 3, "microsteps_per_update": 8}) + del model, reference, optimizer, reference_optimizer + (directory / f"rank-{rank}.json").write_text(json.dumps(reports)) + finally: + dist.destroy_process_group() + + +def test_two_rank_bf16_accumulation_matches_explicit_fp32_gradient_average(tmp_path): + environment = os.environ.copy() + environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") + processes = [] + try: + for rank in range(2): + processes.append(subprocess.Popen( + [sys.executable, str(Path(__file__).resolve()), str(rank), str(tmp_path)], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + )) + for rank, process in enumerate(processes): + output, _ = process.communicate(timeout=90) + assert process.returncode == 0, f"Rank {rank} failed:\n{output}" + finally: + for process in processes: + if process.poll() is None: + process.kill() + for process in processes: + process.wait(timeout=10) + for rank in range(2): + reports = json.loads((tmp_path / f"rank-{rank}.json").read_text()) + assert len(reports) == 9 + assert all(report["maximum_gradient_error"] <= 1e-7 for report in reports) + + +if __name__ == "__main__": + _worker(int(sys.argv[1]), sys.argv[2]) diff --git a/test_dpo.py b/test_dpo.py new file mode 100644 index 0000000000000000000000000000000000000000..f5ec52c2db7197a2e4ec3ab0628b57b13da8bcf5 --- /dev/null +++ b/test_dpo.py @@ -0,0 +1,196 @@ +"""CPU DPO arithmetic, frozen-reference and exact continuation checks.""" + +import copy +import json +import math +import shutil + +import numpy as np +import pytest +import torch +import torch.nn.functional as F + +import dpo +from ops.dpo_data import collate_pairs, encode_pair, load_data +from ops.dpo_preflight import make_fixture +from ops.posttrain_data import encode_messages +from test_posttrain import CharacterTokenizer, assert_tree_equal + + +def pair(): + return dict(prompt=[{'role': 'user', 'content': 'hello'}, + {'role': 'assistant', 'content': 'old answer'}, + {'role': 'user', 'content': 'two plus two?'}], + chosen=[{'role': 'assistant', 'content': 'four'}], + rejected=[{'role': 'assistant', 'content': 'five, perhaps six'}]) + + +def test_multiturn_masks_only_final_response_and_eos(): + tokenizer = CharacterTokenizer() + row = pair() + encoded = encode_pair(row, tokenizer, 512) + for branch in ('chosen', 'rejected'): + assert encoded[branch]['input_ids'] == encode_messages(row['prompt'] + row[branch], tokenizer)['input_ids'] + assert [value for value in encoded[branch]['labels'] if value != -100] == [ord(c) for c in row[branch][0]['content']] + [2] + x, y, active = collate_pairs([encoded, None], 512, 0) + assert active.tolist() == [True, False] + assert not y[2:].ne(-100).any() + assert x.shape == y.shape == (4, 512) + + +@pytest.mark.parametrize('mutation', [lambda r: r.update(prompt=[]), + lambda r: r['prompt'].append({'role': 'assistant', 'content': 'invalid end'}), + lambda r: r.update(rejected=r['chosen']), + lambda r: r['chosen'].append({'role': 'assistant', 'content': 'extra'})]) +def test_invalid_preference_pairs_rejected(mutation): + row = pair() + mutation(row) + with pytest.raises(ValueError): + encode_pair(row, CharacterTokenizer(), 512) + + +def test_length_normalized_and_summed_objectives_match_explicit_formula(): + policy = torch.tensor([[-10., -19.], [-4., -9.]], dtype=torch.float64, requires_grad=True) + reference = torch.tensor([[-12., -20.], [-4., -8.]], dtype=torch.float64, requires_grad=True) + counts = torch.tensor([[2, 10], [1, 3]]) + loss, margin, accuracy = dpo.preference_metrics(policy, reference, counts, 5., 'length-normalized-dpo') + expected = 5 * ((policy[:, 0] - reference.detach()[:, 0]) / counts[:, 0] - + (policy[:, 1] - reference.detach()[:, 1]) / counts[:, 1]) + torch.testing.assert_close(loss, -F.logsigmoid(expected)) + torch.testing.assert_close(margin, expected) + assert accuracy.tolist() == [1., 1.] + loss.sum().backward() + assert reference.grad is None + assert (policy.grad[:, 0] < 0).all() and (policy.grad[:, 1] > 0).all() + summed, _, _ = dpo.preference_metrics(policy.detach(), reference, counts, .1, 'summed-dpo') + expected = .1 * ((policy[:, 0] - reference[:, 0]) - (policy[:, 1] - reference[:, 1])) + torch.testing.assert_close(summed, -F.logsigmoid(expected)) + + +def test_global_pair_mean_handles_empty_rank_and_unequal_lengths(): + parameter = torch.tensor(.2, dtype=torch.float64, requires_grad=True) + counts = torch.tensor([[2, 10], [4, 2], [1, 6]]) + base = torch.tensor([[-10., -19.], [-5., -3.], [-2., -8.]], dtype=torch.float64) + direction = torch.tensor([[1., 2.], [3., 1.], [2., 5.]], dtype=torch.float64) + gradients = [] + for ids in ([0, 1, 2], []): + parameter.grad = None + if ids: + loss, _, _ = dpo.preference_metrics(base[ids] + parameter * direction[ids], base[ids], counts[ids], 5., 'length-normalized-dpo') + (loss.sum() * 2 / 3).backward() + else: + (parameter * 0.).backward() + gradients.append(parameter.grad.clone()) + parameter.grad = None + loss, _, _ = dpo.preference_metrics(base + parameter * direction, base, counts, 5., 'length-normalized-dpo') + loss.mean().backward() + torch.testing.assert_close(sum(gradients) / 2, parameter.grad) + + +def test_chunked_sequence_logps_and_gradients_match_full_logits(): + from warpstate import WarpStateConfig, WarpStateLM + cfg = WarpStateConfig(vocab_size=32, d_model=16, n_heads=4, n_cores=1, + logical_depth=1, ffn_hidden=32, chunk_size=4, parallel_memory=True) + torch.manual_seed(19) + original = WarpStateLM(cfg) + chunked = copy.deepcopy(original) + dpo.apply_sequence_logps(chunked, 3) + x = torch.randint(0, 32, (4, 8)) + y = torch.randint(0, 32, (4, 8)) + y[:, :2] = -100 + y[3] = -100 + logits = original(x) + expected = -F.cross_entropy(logits.reshape(-1, 32), y.reshape(-1), reduction='none').reshape(4, 8).sum(1) + _, actual = chunked(x, y, return_logits=False) + torch.testing.assert_close(actual, expected, rtol=1e-6, atol=1e-5) + expected.sum().backward() + actual.sum().backward() + for a, b in zip(original.parameters(), chunked.parameters()): + torch.testing.assert_close(a.grad, b.grad, rtol=1e-5, atol=2e-6) + assert actual[3] == 0 + + +def test_exact_bf16_resume_rebuilt_reference_and_immutable_sft(tmp_path): + args = make_fixture(tmp_path / 'fixture') + original_hash = dpo.digest(args.weights) + original = torch.load(args.weights, weights_only=False, mmap=True) + try: + dpo.train(args) + uninterrupted = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) + original_cache = dpo.digest(args.reference_cache_dir / 'manifest.json') + args.work_dir = tmp_path / 'split' + args.max_steps = 1 + dpo.train(args) + partial = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) + assert partial['step'] == 1 and not partial['completed'] + # A deterministic external-data cache can be rebuilt while resuming a + # changed policy; the source remains the frozen SFT, never latest.pt. + shutil.rmtree(args.reference_cache_dir) + args.max_steps = None + dpo.train(args) + resumed = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) + assert dpo.digest(args.reference_cache_dir / 'manifest.json') == original_cache + assert resumed['step'] == 3 and resumed['sample_cursor'] == 5 and resumed['completed'] + for key in ('model', 'optimizer', 'sample_cursor', 'tokens_seen', 'response_tokens_seen', 'validation'): + assert_tree_equal(uninterrupted[key], resumed[key]) + assert torch.equal(uninterrupted['rng_by_rank'][0]['torch'], resumed['rng_by_rank'][0]['torch']) + assert all(v.dtype == torch.float32 for v in resumed['model'].values()) + assert any(not torch.equal(original['model'][key], value) for key, value in resumed['model'].items()) + assert dpo.digest(args.weights) == original_hash + metrics = [json.loads(line) for line in (tmp_path / 'fixture/run/metrics.jsonl').read_text().splitlines()] + assert metrics[0]['val_loss'] == pytest.approx(math.log(2), abs=1e-7) + report = json.loads((args.work_dir / 'evaluation.json').read_text()) + assert report['full_holdout'] and report['evaluated_samples'] == 2 + completion = json.loads((args.work_dir / 'completion.json').read_text()) + assert completion['paused_for_evaluation'] and completion['completed'] + finally: + torch.use_deterministic_algorithms(False) + + +def test_reference_identity_tampering_and_incomplete_data_fail_closed(tmp_path): + import sentencepiece as spm + args = make_fixture(tmp_path / 'fixture') + args.reference_only = True + try: + dpo.train(args) + manifest_path = args.reference_cache_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + identity = manifest['identity'] + with pytest.raises(ValueError, match='identity'): + dpo.validate_cache(args.reference_cache_dir, identity | {'weights_sha256': 'changed'}, (5, 2)) + array = args.reference_cache_dir / 'train.npy' + original = array.read_bytes() + array.write_bytes(original + b'changed') + with pytest.raises(ValueError, match='changed'): + dpo.validate_cache(args.reference_cache_dir, identity, (5, 2)) + array.write_bytes(original) + manifest['complete'] = False + manifest_path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match='completion'): + dpo.validate_cache(args.reference_cache_dir, identity, (5, 2)) + data_manifest_path = args.data_dir / 'manifest.json' + data_manifest = json.loads(data_manifest_path.read_text()) + data_manifest['renderer_sha256'] = 'changed' + data_manifest_path.write_text(json.dumps(data_manifest)) + with pytest.raises(ValueError, match='renderer'): + load_data(args.data_dir, spm.SentencePieceProcessor(model_file=str(args.tokenizer)), dpo.digest(args.tokenizer), 128) + finally: + torch.use_deterministic_algorithms(False) + + +def test_schedule_is_linear_and_pause_independent(): + settings = dpo.DEFAULTS | dict(lr=1., min_lr=0., warmup_ratio=.1) + assert dpo.learning_rate(0, 100, settings) == .1 + assert dpo.learning_rate(9, 100, settings) == 1. + assert dpo.learning_rate(10, 100, settings) == 1. + assert dpo.learning_rate(54, 100, settings) == pytest.approx(45 / 89) + assert dpo.learning_rate(99, 100, settings) == 0. + + +@pytest.mark.parametrize('settings', [dict(beta=0), dict(objective='dpo'), dict(micro_batch=True), + dict(lr=0), dict(warmup_ratio=1), dict(unknown=True), dict(head_chunk_size=0)]) +def test_invalid_plan_rejected(tmp_path, settings): + path = tmp_path / 'plan.json' + path.write_text(json.dumps(dict(format='warpstate-dpo-plan-v1', stage='dpo', training=settings))) + with pytest.raises(ValueError): + dpo.load_plan(path) diff --git a/test_dpo_handoff.py b/test_dpo_handoff.py new file mode 100644 index 0000000000000000000000000000000000000000..03519c9de906ae24c8e01460403bcee2b2508713 --- /dev/null +++ b/test_dpo_handoff.py @@ -0,0 +1,249 @@ +"""CPU-only proof that DPO starts only after immutable full-state SFT archival.""" +import copy +import json +from pathlib import Path + +import pytest +import torch + +from ops import dpo_handoff as handoff + + +@pytest.fixture +def fixture(tmp_path): + base = tmp_path / 'base' + sft = base / 'posttrain/sft' + run = sft / 'run' + run.mkdir(parents=True) + (sft / 'control').mkdir() + (base / 'control').mkdir() + tokenizer = tmp_path / 'tokenizer.model' + tokenizer.write_bytes(b'original tokenizer fixture') + tokenizer_hash = handoff.sha256(tokenizer) + sft_plan = dict(format='warpstate-posttrain-plan-v1', stage='sft', + after_completion='pause_for_evaluation', base=dict(step=76294, + tokens_seen=10000007168, tokenizer_sha256=tokenizer_hash)) + sft_plan_path = tmp_path / 'sft-plan.json' + sft_plan_path.write_text(json.dumps(sft_plan)) + data = base / 'posttrain/data' + data.mkdir() + outputs = {} + for name in ('dpo_train.jsonl', 'dpo_val.jsonl'): + (data / name).write_text('{}\n') + outputs[name] = dict(rows=1, tokens=10, supervised_tokens=4, sha256=handoff.sha256(data / name)) + manifest = dict(format='warpstate-posttrain-data-v1', complete=True, + tokenizer_sha256=tokenizer_hash, max_length=2048, outputs=outputs) + (data / 'manifest.json').write_text(json.dumps(manifest)) + data_hash = handoff.sha256(data / 'manifest.json') + expected = dict(step=10312, train_rows=329973, validation_rows=1603, + tokens_seen=212321372, assistant_tokens_seen=143695039, + base_step=76294, base_tokens_seen=10000007168, + raw_archive_sha256='raw-base-hash', tokenizer_sha256=tokenizer_hash) + work = base / 'posttrain/dpo' + archive = base / 'archives/sft-final' + plan = dict(format='warpstate-dpo-plan-v1', stage='dpo', after_completion='pause_for_evaluation', + sft=expected, handoff=dict(base_work_dir=str(base), sft_work_dir=str(sft), + sft_plan=str(sft_plan_path), sft_program='warpstate-posttrain', + supervisor_socket=str(base / 'control/supervisor.sock'), archive_dir=str(archive), + work_dir=str(work), data_dir=str(data), data_manifest_sha256=data_hash, + tokenizer=str(tokenizer), processes=2, poll_seconds=1), + training=dict(sequence_length=2048)) + config = tmp_path / 'dpo-plan.json' + config.write_text(json.dumps(plan)) + checkpoint = dict(format='warpstate-posttrain-v1', completed=True, step=10312, + total_steps=10312, sample_cursor=329973, tokens_seen=212321372, + assistant_tokens_seen=143695039, val_loss=1.2, val_step=10312, config={'d_model': 2}, + model={'weight': torch.arange(6, dtype=torch.float32).reshape(2, 3)}, + optimizer={'state': {0: {'step': torch.tensor(10312.), 'exp_avg': torch.arange(6.)}}}, + rng_by_rank=[{'torch': torch.tensor([1, 2], dtype=torch.uint8)}, + {'torch': torch.tensor([3, 4], dtype=torch.uint8)}], + signature=dict(plan=sft_plan, base_sha256='raw-base-hash', tokenizer_sha256=tokenizer_hash, + data_sha256=data_hash), + origin=dict(step=76294, tokens_seen=10000007168, weights_sha256='raw-base-hash')) + completion = dict(format='warpstate-posttrain-completion-v1', stage='sft', completed=True, + paused_for_evaluation=True, step=10312, total_steps=10312, base_step=76294) + evaluation = dict(stage='sft', step=10312, full_holdout=True, evaluated_samples=1603, + available_validation_samples=1603, val_loss=1.2, base_sha256='raw-base-hash') + torch.save(checkpoint, run / 'latest.pt') + for name, obj in [('completion.json', completion), ('evaluation.json', evaluation), + ('settings.json', {'signature': checkpoint['signature']})]: + (run / name).write_text(json.dumps(obj)) + (run / 'metrics.jsonl').write_text(json.dumps(evaluation) + '\n') + (sft / 'control/handoff.json').write_text(json.dumps({'phase': 'complete'})) + (sft / 'control/execution.json').write_text(json.dumps({'plan': sft_plan})) + return dict(base=base, sft=sft, run=run, work=work, archive=archive, tokenizer=tokenizer, + plan=plan, sft_plan=sft_plan, checkpoint=checkpoint, completion=completion, + evaluation=evaluation, config=config, data=data) + + +@pytest.mark.parametrize('field,value', [('completed', False), ('step', 10311), + ('step', 10313), ('sample_cursor', 329972), ('tokens_seen', 212321371), + ('assistant_tokens_seen', 143695038), ('val_step', 10300), ('val_loss', float('nan'))]) +def test_only_complete_exact_sft_state_is_accepted(fixture, field, value): + checkpoint = copy.deepcopy(fixture['checkpoint']) + checkpoint[field] = value + with pytest.raises(ValueError): + handoff.validate_sft(checkpoint, fixture['evaluation'], fixture['completion'], + fixture['plan'], fixture['sft_plan']) + + +def test_full_sft_archive_retains_optimizer_rng_exactly_and_model_copy_is_independent(fixture): + source = fixture['run'] / 'latest.pt' + original = source.read_bytes() + metadata, entries = handoff.archive_sft(fixture['sft'], fixture['tokenizer'], fixture['archive'], + fixture['plan'], fixture['sft_plan'], 'test-revision') + assert (fixture['archive'] / 'checkpoint.pt').read_bytes() == original == source.read_bytes() + full = torch.load(fixture['archive'] / 'checkpoint.pt', weights_only=False, mmap=True) + model = torch.load(fixture['archive'] / 'model.pt', weights_only=False, mmap=True) + assert full['optimizer']['state'][0]['step'].item() == 10312 + assert len(full['rng_by_rank']) == 2 and 'optimizer' not in model + assert model['stage'] == 'sft' and model['step'] == 10312 and model['origin']['step'] == 76294 + assert torch.equal(model['model']['weight'], full['model']['weight']) + copied = handoff.initialize_copy(fixture['archive'], fixture['work'] / 'initialization', metadata) + assert handoff.sha256(copied) == entries['model.pt'] + assert copied.stat().st_ino != (fixture['archive'] / 'model.pt').stat().st_ino + assert all(path.stat().st_mode & 0o222 == 0 for path in fixture['archive'].iterdir()) + inodes = {path.name: path.stat().st_ino for path in fixture['archive'].iterdir()} + assert handoff.archive_sft(fixture['sft'], fixture['tokenizer'], fixture['archive'], + fixture['plan'], fixture['sft_plan'], 'different-revision') == (metadata, entries) + assert {path.name: path.stat().st_ino for path in fixture['archive'].iterdir()} == inodes + assert handoff.initialize_copy(fixture['archive'], copied.parent, metadata) == copied + + +def test_existing_incomplete_archive_is_preserved(fixture): + fixture['archive'].mkdir(parents=True) + note = fixture['archive'] / 'keep.txt' + note.write_text('user file') + with pytest.raises(ValueError, match='checksum'): + handoff.archive_sft(fixture['sft'], fixture['tokenizer'], fixture['archive'], + fixture['plan'], fixture['sft_plan'], 'test') + assert list(fixture['archive'].iterdir()) == [note] + assert note.read_text() == 'user file' + + +def test_subset_evaluation_does_not_qualify_for_archival(fixture): + fixture['evaluation']['full_holdout'] = False + with pytest.raises(ValueError, match='full-holdout'): + handoff.validate_sft(fixture['checkpoint'], fixture['evaluation'], fixture['completion'], + fixture['plan'], fixture['sft_plan']) + + +def test_preference_data_tamper_blocks_dpo(fixture): + assert handoff.data_readiness(fixture['data'], fixture['plan'], verify=True) + (fixture['data'] / 'dpo_train.jsonl').write_text('changed') + with pytest.raises(ValueError, match='checksum'): + handoff.data_readiness(fixture['data'], fixture['plan'], verify=True) + + +def test_cannot_target_sft_or_historical_archives(fixture): + for path in (fixture['sft'], fixture['run'], fixture['archive']): + plan = copy.deepcopy(fixture['plan']) + plan['handoff']['work_dir'] = str(path) + fixture['config'].write_text(json.dumps(plan)) + with pytest.raises(ValueError): + handoff.load_plan(fixture['config']) + + +def test_archive_completed_sft_before_starting_any_gpu_job(fixture, monkeypatch): + monkeypatch.setattr(handoff, 'STOPPED', False) + monkeypatch.setattr(handoff, 'source_identity', lambda *a: {'code': 'pinned'}) + monkeypatch.setattr(handoff, 'verify_preflight', lambda *a: {'accepted': True}) + monkeypatch.setattr(handoff, 'verify_dpo_completion', lambda *a: {'completed': True}) + statuses = iter([{'statename': 'RUNNING', 'exitstatus': 0}]) + monkeypatch.setattr(handoff, 'process_status', lambda *a: next(statuses, {'statename': 'EXITED', 'exitstatus': 0})) + monkeypatch.setattr(handoff.time, 'sleep', lambda *a: None) + commands = [] + def fake_supervise(command, stop_file, environment, *, pass_fds): + commands.append(command) + assert len(pass_fds) == 3 + # Complete full-state archive must exist before even the tiny GPU preflight. + assert (fixture['archive'] / 'checkpoint.pt').read_bytes() == (fixture['run'] / 'latest.pt').read_bytes() + for path in (fixture['sft'] / 'control/handoff.lock', fixture['base'] / 'control/training-launcher.lock'): + with pytest.raises(BlockingIOError): + with handoff.exclusive_lock(path): + pass + with handoff.exclusive_lock(fixture['base'] / 'pipeline.lock'): + pass # Released for all-rank shared locks immediately before launch. + if 'dpo_preflight.py' in ' '.join(command): + return 0 + work = Path(command[command.index('--work-dir') + 1]) + weights = Path(command[command.index('--weights') + 1]) + cache = Path(command[command.index('--reference-cache-dir') + 1]) + assert not weights.is_relative_to(work) and not cache.is_relative_to(work) + assert Path(command[command.index('--stop-file') + 1]) == stop_file + work.mkdir() + model_hash = handoff.sha256(weights) + checkpoint = dict(format='warpstate-dpo-v1', completed=True, step=2, total_steps=2, + val_step=2, signature=dict(plan=fixture['plan'], base_sha256=model_hash)) + torch.save(checkpoint, work / 'latest.pt') + (work / 'evaluation.json').write_text(json.dumps(dict(stage='dpo', step=2, full_holdout=True))) + (work / 'completion.json').write_text(json.dumps(dict(stage='dpo', step=2, + completed=True, paused_for_evaluation=True))) + return 0 + monkeypatch.setattr(handoff, 'supervise', fake_supervise) + assert handoff.run_handoff(fixture['config']) == 0 + assert len(commands) == 2 + assert json.loads((fixture['work'] / 'control/handoff.json').read_text())['phase'] == 'complete' + assert handoff.run_handoff(fixture['config']) == 0 + assert len(commands) == 2 # Completed jobs never retrain automatically. + + +def test_preflight_stop_never_starts_full_dpo(fixture, monkeypatch): + monkeypatch.setattr(handoff, 'STOPPED', False) + monkeypatch.setattr(handoff, 'source_identity', lambda *a: {'code': 'pinned'}) + monkeypatch.setattr(handoff, 'process_status', lambda *a: {'statename': 'EXITED', 'exitstatus': 0}) + calls = [] + def stopped(command, stop, env, **kwargs): + calls.append(command) + stop.touch() + return 0 + monkeypatch.setattr(handoff, 'supervise', stopped) + assert handoff.run_handoff(fixture['config']) == 0 + assert len(calls) == 1 and 'dpo_preflight.py' in ' '.join(calls[0]) + assert json.loads((fixture['work'] / 'control/handoff.json').read_text())['phase'] == 'paused' + + +@pytest.mark.parametrize('corruption', [None, 'incomplete_epoch', 'subset_validation', 'cache_tamper']) +def test_dpo_completion_checks_epoch_full_evaluation_and_fixed_reference(fixture, corruption): + import dpo + import numpy as np + run = fixture['work'] / 'run' + cache = fixture['work'] / 'reference-cache' + run.mkdir(parents=True) + cache.mkdir() + settings = dpo.DEFAULTS | fixture['plan']['training'] + model_hash = 'immutable-sft-model' + data_hash = fixture['plan']['handoff']['data_manifest_sha256'] + identity = dpo.reference_identity(model_hash, data_hash, fixture['plan']['sft']['tokenizer_sha256'], + settings, torch.device('cuda'), 2) + files = {} + for name in ('train.npy', 'val.npy'): + dpo.atomic_array(cache / name, np.array([[-3., -4.]], dtype=np.float64)) + files[name] = dict(rows=1, sha256=handoff.sha256(cache / name)) + handoff.atomic_json(cache / 'manifest.json', dict(format=dpo.CACHE_FORMAT, complete=True, + identity=identity, identity_sha256=dpo.identity_digest(identity), files=files)) + reference_hash = handoff.sha256(cache / 'manifest.json') + checkpoint = dict(format='warpstate-dpo-v1', completed=True, step=1, total_steps=1, + sample_cursor=1, tokens_seen=8, response_tokens_seen=4, val_step=1, val_loss=.69, + signature=dict(plan=fixture['plan'], settings=settings, base_sha256=model_hash, + data_sha256=data_hash, reference_sha256=reference_hash)) + completion = dict(format='warpstate-dpo-completion-v1', stage='dpo', completed=True, + paused_for_evaluation=True, step=1, total_steps=1, sample_cursor=1, + tokens_seen=8, response_tokens_seen=4, val_loss=.69, reference_sha256=reference_hash) + evaluation = dict(stage='dpo', step=1, full_holdout=True, evaluated_samples=1, + available_validation_samples=1, val_loss=.69, reward_margin=.01, reward_accuracy=1., + chosen_nll=1., base_sha256=model_hash, data_sha256=data_hash, reference_sha256=reference_hash) + if corruption == 'incomplete_epoch': + checkpoint['sample_cursor'] = 0 + elif corruption == 'subset_validation': + evaluation['evaluated_samples'] = 0 + elif corruption == 'cache_tamper': + dpo.atomic_array(cache / 'train.npy', np.array([[-6., -7.]], dtype=np.float64)) + torch.save(checkpoint, run / 'latest.pt') + handoff.atomic_json(run / 'completion.json', completion) + handoff.atomic_json(run / 'evaluation.json', evaluation) + if corruption: + with pytest.raises(ValueError): + handoff.verify_dpo_completion(run, fixture['plan'], model_hash) + else: + assert handoff.verify_dpo_completion(run, fixture['plan'], model_hash) == completion diff --git a/test_dpo_preflight.py b/test_dpo_preflight.py new file mode 100644 index 0000000000000000000000000000000000000000..ebf0b94ec0a87aef3171c4e5411a599aeade07e7 --- /dev/null +++ b/test_dpo_preflight.py @@ -0,0 +1,181 @@ +"""CPU-only checks of deferred DPO fixtures and acceptance decisions.""" + +import copy +import json +import math +from pathlib import Path + +import numpy as np +import pytest +import torch + +from ops import dpo_preflight as preflight +from posttrain import atomic_json, digest + + +def test_fixture_is_source_pinned_cpu_only_and_contains_multiturn_pairs(tmp_path, monkeypatch): + import sentencepiece as spm + from ops.posttrain_data import encode_messages + + def no_cuda(*args, **kwargs): + raise AssertionError('Fixture preparation must not initialize CUDA') + + monkeypatch.setattr(torch.cuda, '_lazy_init', no_cuda) + fixture = preflight.make_fixture(tmp_path, device='cuda', compile_core=True) + assert preflight.make_fixture(tmp_path, device='cuda', compile_core=True).weights == fixture.weights + export = torch.load(fixture.weights, weights_only=False, mmap=True) + assert export['format'] == 'warpstate-chat-v1' and export['stage'] == 'sft' + assert export['config']['n_heads'] == 4 and export['config']['d_model'] == 32 + assert export['config']['dropout'] == 0.0 + rows = [json.loads(line) for line in (fixture.data_dir / 'dpo_train.jsonl').read_text().splitlines()] + assert len(rows) == 5 + assert any(message['role'] == 'assistant' for message in rows[0]['prompt']) + sp = spm.SentencePieceProcessor(model_file=str(fixture.tokenizer)) + for row in rows: + for branch in ('chosen', 'rejected'): + full = encode_messages(row['prompt'] + row[branch], sp, max_length=128) + response = encode_messages(row[branch], sp, max_length=128) + assert row['tokens'][branch] == len(full['input_ids']) + assert row['supervised_tokens'][branch] == sum(label != -100 for label in response['labels'][1:]) + full = encode_messages(rows[0]['prompt'] + rows[0]['chosen'], sp) + assert sum(label != -100 for label in full['labels'][1:]) > rows[0]['supervised_tokens']['chosen'] + assert any(row['supervised_tokens']['chosen'] != row['supervised_tokens']['rejected'] for row in rows) + marker = json.loads((tmp_path / 'fixture.json').read_text()) + for name, expected in marker['files'].items(): + assert digest(tmp_path / name) == expected + with pytest.raises(ValueError, match='settings/source differ'): + preflight.make_fixture(tmp_path, device='cpu', compile_core=True) + (fixture.data_dir / 'dpo_val.jsonl').write_text('changed\n') + with pytest.raises(ValueError, match='input changed'): + preflight.make_fixture(tmp_path, device='cuda', compile_core=True) + + +def test_fixture_refuses_unrecognized_contents_and_base_run(tmp_path): + (tmp_path / 'unrecognized').write_text('preserve') + with pytest.raises(ValueError, match='nonempty'): + preflight.make_fixture(tmp_path) + with pytest.raises(ValueError, match='overwrite the base run'): + preflight.make_fixture(tmp_path / 'run/preflight', base_work_dir=tmp_path) + with pytest.raises(ValueError, match='execution mode'): + preflight.make_fixture(tmp_path / 'fixture', device='cuda', compile_core=1) + + +def test_distributed_commands_use_fixed_reference_and_independent_output(tmp_path): + fixture = preflight.make_fixture(tmp_path) + command = preflight.training_command(fixture, 2, tmp_path / 'split', tmp_path / 'stop', 1) + assert command[-2:] == ['--max-steps', '1'] + assert command[command.index('--reference-cache-dir') + 1] == str(fixture.reference_cache_dir) + assert command[command.index('--work-dir') + 1] == str(tmp_path / 'split') + assert command[command.index('--nproc-per-node') + 1] == '2' + reference = preflight.training_command(fixture, 2, tmp_path / 'scoring', tmp_path / 'stop', reference_only=True) + assert reference[-1] == '--reference-only' and '--max-steps' not in reference + with pytest.raises(ValueError, match='two ranks'): + preflight.training_command(fixture, 1, tmp_path / 'run', tmp_path / 'stop') + with pytest.raises(ValueError, match='cannot specify training steps'): + preflight.training_command(fixture, 2, tmp_path / 'run', tmp_path / 'stop', 1, reference_only=True) + + +def make_reference(directory, *, dtype=np.float64, complete=True, shape=None, value=-2.0): + directory.mkdir(parents=True, exist_ok=True) + files = {} + for name, count in [('train.npy', 5), ('val.npy', 2)]: + np.save(directory / name, np.full(shape or (count, 2), value, dtype=dtype)) + files[name] = {'sha256': digest(directory / name), 'rows': count} + atomic_json(directory / 'manifest.json', dict(format='warpstate-dpo-reference-v1', + complete=complete, files=files)) + + +def test_reference_artifacts_are_complete_finite_frozen_and_hash_checked(tmp_path): + make_reference(tmp_path) + hashes = preflight.reference_artifacts(tmp_path) + assert set(hashes) == {'manifest.json', 'train.npy', 'val.npy'} + assert preflight.reference_artifacts(tmp_path) == hashes + with (tmp_path / 'train.npy').open('ab') as stream: + stream.write(b'changed') + with pytest.raises(ValueError, match='hash differs'): + preflight.reference_artifacts(tmp_path) + + +@pytest.mark.parametrize(('kwargs', 'message'), [ + ({'complete': False}, 'incomplete'), + ({'dtype': np.float32}, 'invalid scores'), + ({'shape': (5, 3)}, 'shape differs'), + ({'value': math.nan}, 'invalid scores'), + ({'value': 0.0}, 'summed log probabilities'), +]) +def test_reference_artifacts_reject_invalid_arrays(tmp_path, kwargs, message): + make_reference(tmp_path, **kwargs) + with pytest.raises(ValueError, match=message): + preflight.reference_artifacts(tmp_path) + + +def sample_checkpoint(): + rng = dict(torch=torch.tensor([1, 2], dtype=torch.uint8), cuda=None, + python=(3, (1, 2), None), numpy=('MT19937', np.array([1, 2], dtype=np.uint32), 2, 0, 0.0)) + return dict(format='warpstate-dpo-v1', step=2, completed=True, sample_cursor=5, + tokens_seen=123, response_tokens_seen=45, val_loss=0.65, + signature={'world_size': 2}, model={'weight': torch.tensor([1.0], dtype=torch.float32)}, + optimizer={'state': {0: {'step': torch.tensor(2.), 'exp_avg': torch.tensor([0.1]), + 'exp_avg_sq': torch.tensor([0.01])}}, 'param_groups': [{'lr': 0.001}]}, + rng_by_rank=[copy.deepcopy(rng), copy.deepcopy(rng)]) + + +def make_finished_run(directory): + directory.mkdir(exist_ok=True) + state = sample_checkpoint() + torch.save(state, directory / 'latest.pt') + rows = [dict(event='validation', step=0, val_loss=math.log(2)), + dict(event='train', step=1, update_samples=4, loss=math.log(2), grad_norm=0.1, lr=0.001), + dict(event='train', step=2, update_samples=1, loss=0.65, grad_norm=0.1, lr=0.0001)] + (directory / 'metrics.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows)) + atomic_json(directory / 'completion.json', dict(completed=True, paused_for_evaluation=True)) + atomic_json(directory / 'evaluation.json', dict(step=2, full_holdout=True, evaluated_samples=2, + val_loss=0.65, reward_accuracy=0.5, reward_margin=0.01, chosen_nll=10.0)) + return rows + + +def test_finished_run_requires_initial_reference_equality_and_masked_tail(tmp_path): + rows = make_finished_run(tmp_path) + assert preflight.verify_fixture_run(tmp_path, expected_step=2, completed=True)['sample_cursor'] == 5 + rows[-1]['update_samples'] = 2 + (tmp_path / 'metrics.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows)) + with pytest.raises(ValueError, match='masked pair tail'): + preflight.verify_fixture_run(tmp_path, expected_step=2, completed=True) + rows[-1]['update_samples'] = 1 + rows[0]['val_loss'] = 0.68 + (tmp_path / 'metrics.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows)) + with pytest.raises(ValueError, match=r'equal log\(2\)'): + preflight.verify_fixture_run(tmp_path, expected_step=2, completed=True) + + +def test_resume_compares_counters_rng_exactly_and_model_optimizer_with_tolerance(): + resumed = sample_checkpoint() + continuous = copy.deepcopy(resumed) + continuous['model']['weight'] += 1e-6 + comparison = preflight.verify_resume(resumed, continuous) + assert not comparison['model']['bitwise_equal'] + assert comparison['optimizer']['bitwise_equal'] + continuous['tokens_seen'] += 1 + with pytest.raises(ValueError, match='tokens_seen'): + preflight.verify_resume(resumed, continuous) + continuous = copy.deepcopy(resumed) + continuous['rng_by_rank'][1]['torch'][0] += 1 + with pytest.raises(ValueError, match='rng.1.torch'): + preflight.verify_resume(resumed, continuous) + continuous = copy.deepcopy(resumed) + continuous['optimizer']['state'][0]['exp_avg'] += 0.01 + with pytest.raises(ValueError, match='optimizer'): + preflight.verify_resume(resumed, continuous) + + +def test_prepare_only_never_supervises_a_training_process(tmp_path, monkeypatch, capsys): + def forbidden(*args, **kwargs): + raise AssertionError('Preparation-only invocation cannot launch subprocesses') + + monkeypatch.setattr(preflight, 'supervise', forbidden) + monkeypatch.setattr(torch.cuda, '_lazy_init', forbidden) + monkeypatch.setattr('sys.argv', ['dpo_preflight.py', '--work-dir', str(tmp_path / 'fixture'), + '--base-work-dir', str(tmp_path), '--prepare-only']) + assert preflight.main() == 0 + assert json.loads(capsys.readouterr().out)['prepared'] is True + assert not (tmp_path / 'fixture/reference').exists() diff --git a/test_gradient_execution.py b/test_gradient_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..09329379b8ea07004b3df35b1190da2ee0d894d8 --- /dev/null +++ b/test_gradient_execution.py @@ -0,0 +1,441 @@ +"""Numerical, accumulation, tied-weight, and DDP checks for compiled autograd.""" + +from __future__ import annotations + +from contextlib import nullcontext +from copy import deepcopy +from dataclasses import FrozenInstanceError +from datetime import timedelta +import json +from pathlib import Path + +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F + +from ops.gradient_execution import OverlapSettings, make_backward, scale_loss + + +@pytest.fixture(autouse=True) +def compiler_settings(): + # make_backward intentionally configures the lifetime of a training process. + # Tests share a process, so restore configuration and graph caches between them. + from torch._inductor import config as inductor_config, config_comms + + with torch._dynamo.config.patch( + compiled_autograd=False, compiled_autograd_kwargs_override={}, optimize_ddp=True + ), inductor_config.patch( + reorder_for_compute_comm_overlap=False, reorder_for_compute_comm_overlap_passes=[], + deterministic=False, runtime_estimations_mms_benchmark=False, + ), config_comms.patch( + sink_iterative_peak_memory_budget=0.2, + sink_waits_iterative_unsafe_collectives_reorder=True, + sink_waits_iterative_swap_with_collectives=False, + sink_iterative_use_runtime_estimations=False, + runtime_estimations_align_across_all_distributed_ranks=False, + reorder_iterative_unsafe_collectives_reorder=True, + reorder_iterative_peak_memory_budget=0.2, + reorder_iterative_group_with_collectives=False, + reorder_iterative_use_runtime_estimations=False, + ): + yield + torch._dynamo.reset() + + +def test_eager_selection_does_not_change_compiler_configuration(): + backward = make_backward() + parameter = torch.nn.Parameter(torch.tensor([2.0])) + backward(parameter.square().sum()) + assert parameter.grad.item() == 4.0 + assert torch._dynamo.config.compiled_autograd is False + assert torch._dynamo.config.optimize_ddp is True + with pytest.raises(ValueError, match="Unknown gradient"): + make_backward("not-a-mode") + with pytest.raises(ValueError, match="Unknown gradient"): + scale_loss(parameter, 0.125, "not-a-mode") + + +@pytest.mark.parametrize("mode", ["eager", "compiled-autograd", "compiled-autograd-overlap"]) +def test_cpu_loss_scaling_preserves_masked_and_partial_batch_gradients(mode): + parameter = torch.nn.Parameter(torch.tensor([1.0, 2.0, 3.0])) + for factor in (0.0, 0.125, 7.0 / 19.0): + parameter.grad = None + scale_loss(parameter.sum(), factor, mode).backward() + torch.testing.assert_close(parameter.grad, torch.full_like(parameter, factor), rtol=0, atol=0) + + +def test_overlap_mode_preserves_collective_order_and_peak_memory_budget(): + from torch._inductor import config, config_comms + + operation = make_backward("compiled-autograd-overlap", distributed=True, backend="eager") + assert callable(operation) + assert torch._dynamo.config.optimize_ddp == "python_reducer" + assert torch._dynamo.config.compiled_autograd is True + assert config.reorder_for_compute_comm_overlap is True + assert config.reorder_for_compute_comm_overlap_passes == ["sink_waits_iterative"] + assert config_comms.sink_iterative_peak_memory_budget == 0.0 + assert config_comms.sink_waits_iterative_unsafe_collectives_reorder is False + assert config_comms.sink_waits_iterative_swap_with_collectives is False + assert config_comms.sink_iterative_use_runtime_estimations is False + assert config.runtime_estimations_mms_benchmark is False + assert config_comms.runtime_estimations_align_across_all_distributed_ranks is False + assert OverlapSettings().producer_schedule is False + + +def test_overlap_options_are_immutable_and_have_stable_json_metadata(): + options = OverlapSettings(True, True, 0.02) + assert json.loads(json.dumps(options.metadata)) == { + "swap_with_collectives": True, "use_runtime_estimations": True, + "peak_memory_budget": 0.02, "producer_schedule": False} + with pytest.raises(FrozenInstanceError): + options.peak_memory_budget = 0.1 + modified_copy = options.metadata + modified_copy["peak_memory_budget"] = 1 + assert options.peak_memory_budget == 0.02 + assert OverlapSettings(peak_memory_budget=0).metadata["peak_memory_budget"] == 0.0 + assert OverlapSettings(peak_memory_budget=0.25).peak_memory_budget == 0.25 + + +@pytest.mark.parametrize("settings", [ + {"swap_with_collectives": 1}, {"swap_with_collectives": "false"}, + {"use_runtime_estimations": 0}, {"use_runtime_estimations": None}, + {"producer_schedule": 1}, {"producer_schedule": "true"}, {"producer_schedule": None}, + {"peak_memory_budget": True}, {"peak_memory_budget": "0.02"}, + {"peak_memory_budget": -0.01}, {"peak_memory_budget": 0.251}, + {"peak_memory_budget": float("nan")}, {"peak_memory_budget": float("inf")}, + {"peak_memory_budget": 10**999}, +]) +def test_invalid_overlap_options_fail_before_global_configuration(settings): + with pytest.raises(ValueError): + OverlapSettings(**settings) + assert torch._dynamo.config.compiled_autograd is False + + +@pytest.mark.parametrize("other", [ + {"swap_with_collectives": True}, + {"use_runtime_estimations": True}, + {"peak_memory_budget": 0.02}, +]) +def test_producer_scheduling_is_a_separate_bounded_experiment(other): + with pytest.raises(ValueError, match="producer_schedule initially requires"): + OverlapSettings(producer_schedule=True, **other) + assert torch._dynamo.config.compiled_autograd is False + + +@pytest.mark.parametrize("distributed", [False, True]) +def test_producer_scheduling_enforces_order_and_memory_guards(distributed): + from torch._inductor import config, config_comms + + # These are unsafe installed defaults; selecting the pass alone is not safe. + assert config_comms.reorder_iterative_unsafe_collectives_reorder is True + assert config_comms.reorder_iterative_peak_memory_budget == 0.2 + options = OverlapSettings(producer_schedule=True) + assert options.metadata["producer_schedule"] is True + make_backward("compiled-autograd-overlap", distributed=distributed, + backend="eager", overlap_settings=options) + assert config.reorder_for_compute_comm_overlap_passes == [ + "reorder_communication_preserving_peak_memory", "sink_waits_iterative"] + assert config_comms.reorder_iterative_unsafe_collectives_reorder is False + assert config_comms.reorder_iterative_peak_memory_budget == 0.0 + assert config_comms.reorder_iterative_group_with_collectives is False + assert config_comms.reorder_iterative_use_runtime_estimations is False + assert config_comms.sink_iterative_peak_memory_budget == 0.0 + assert config_comms.sink_waits_iterative_unsafe_collectives_reorder is False + assert config_comms.sink_waits_iterative_swap_with_collectives is False + assert config_comms.sink_iterative_use_runtime_estimations is False + assert config.runtime_estimations_mms_benchmark is False + assert config_comms.runtime_estimations_align_across_all_distributed_ranks is False + + +@pytest.mark.parametrize("missing", [ + "reorder_communication_preserving_peak_memory", "sink_waits_iterative"]) +def test_producer_scheduling_requires_both_passes_before_global_changes(monkeypatch, missing): + from torch._inductor import comms, config, config_comms + + monkeypatch.setattr(comms, missing, None) + with pytest.raises(RuntimeError, match="lacks the " + missing): + make_backward("compiled-autograd-overlap", overlap_settings=OverlapSettings(producer_schedule=True)) + assert config.reorder_for_compute_comm_overlap is False + assert config_comms.reorder_iterative_unsafe_collectives_reorder is True + assert config_comms.reorder_iterative_peak_memory_budget == 0.2 + assert torch._dynamo.config.compiled_autograd is False + + +@pytest.mark.parametrize("mode", ["eager", "compiled-autograd"]) +def test_overlap_options_rejected_outside_overlap_mode(mode): + with pytest.raises(ValueError, match="requires compiled-autograd-overlap"): + make_backward(mode, overlap_settings=OverlapSettings()) + assert torch._dynamo.config.compiled_autograd is False + + +def test_overlap_options_require_validated_type(): + with pytest.raises(TypeError, match="OverlapSettings instance"): + make_backward("compiled-autograd-overlap", overlap_settings={"peak_memory_budget": 0.02}) + + +@pytest.mark.parametrize("distributed", [False, True]) +@pytest.mark.parametrize("runtime_estimates", [False, True]) +def test_explicit_overlap_options_keep_launch_order_and_map_runtime_controls(distributed, runtime_estimates): + from torch._inductor import config, config_comms + + options = OverlapSettings(swap_with_collectives=True, + use_runtime_estimations=runtime_estimates, peak_memory_budget=0.02) + make_backward("compiled-autograd-overlap", distributed=distributed, + backend="eager", overlap_settings=options) + assert config_comms.sink_waits_iterative_swap_with_collectives is True + assert config_comms.sink_waits_iterative_unsafe_collectives_reorder is False + assert config_comms.sink_iterative_peak_memory_budget == 0.02 + assert config_comms.sink_iterative_use_runtime_estimations is runtime_estimates + assert config.runtime_estimations_mms_benchmark is runtime_estimates + assert config_comms.runtime_estimations_align_across_all_distributed_ranks is ( + distributed and runtime_estimates) + + +def test_runtime_estimate_alignment_must_exist_before_settings_change(monkeypatch): + from torch._inductor import comms, config + + monkeypatch.setattr(comms, "align_runtime_estimations_across_all_distributed_ranks", None) + with pytest.raises(RuntimeError, match="lacks distributed runtime-estimate alignment"): + make_backward("compiled-autograd-overlap", distributed=True, + overlap_settings=OverlapSettings(use_runtime_estimations=True)) + assert torch._dynamo.config.compiled_autograd is False + assert config.reorder_for_compute_comm_overlap is False + assert config.runtime_estimations_mms_benchmark is False + + +def test_overlap_mode_rejects_a_disabled_or_missing_scheduler(monkeypatch): + from torch._inductor import comms, config + + with config.patch(deterministic=True): + with pytest.raises(ValueError, match="deterministic mode disables"): + make_backward("compiled-autograd-overlap") + monkeypatch.setattr(comms, "sink_waits_iterative", None) + with pytest.raises(RuntimeError, match="lacks the sink_waits_iterative"): + make_backward("compiled-autograd-overlap") + assert torch._dynamo.config.compiled_autograd is False + + +@pytest.mark.parametrize("reset_to_none", [True, False]) +@pytest.mark.parametrize("mode", ["compiled-autograd", "compiled-autograd-overlap"]) +def test_bf16_gemm_rounding_and_fp32_accumulation_are_preserved(reset_to_none, mode): + from torch._dynamo.utils import counters + + captures_before = counters["compiled_autograd"]["captures"] + torch.manual_seed(901) + expected = torch.nn.Parameter(torch.randn(12, 16)) + actual = torch.nn.Parameter(expected.detach().clone()) + backward = make_backward(mode) + expected_optimizer = torch.optim.AdamW([expected], lr=0.01) + actual_optimizer = torch.optim.AdamW([actual], lr=0.01) + for update in range(2): + expected_optimizer.zero_grad(set_to_none=reset_to_none) + actual_optimizer.zero_grad(set_to_none=reset_to_none) + reference = torch.zeros_like(expected) + for microbatch in range(4): + inputs = torch.randn(8, 16, dtype=torch.bfloat16) + derivative = torch.randn(8, 12, dtype=torch.bfloat16) + with torch.autocast("cpu", dtype=torch.bfloat16): + expected_loss = (F.linear(inputs, expected) * derivative).sum() + actual_loss = (F.linear(inputs, actual) * derivative).sum() + expected_loss.backward() + backward(actual_loss) + # This reference deliberately rounds dW to BF16 before FP32 addition. + reference.add_((derivative.T @ inputs).float()) + torch.testing.assert_close(expected.grad, reference, rtol=0, atol=0) + torch.testing.assert_close(actual.grad, reference, rtol=0, atol=0) + assert actual.grad.dtype == torch.float32 + for parameter in (expected, actual): + torch.nn.utils.clip_grad_norm_([parameter], 1.0, error_if_nonfinite=True) + expected_optimizer.step() + actual_optimizer.step() + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + for key in ("exp_avg", "exp_avg_sq"): + torch.testing.assert_close(actual_optimizer.state[actual][key], + expected_optimizer.state[expected][key], rtol=0, atol=0) + assert counters["compiled_autograd"]["captures"] >= captures_before + 2 + + +class TiedProjection(torch.nn.Module): + """Tied embedding/head and repeated linear calls, with FP32 master leaves.""" + + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(16, 8) + self.projection = torch.nn.Linear(8, 8, bias=False) + + def forward(self, tokens): + weight = self.embed.weight.to(torch.bfloat16) + value = F.embedding(tokens, weight) + value = self.projection(value) + self.projection(value * 0.5) + return F.linear(value, weight).float() + + +def test_compiled_forward_tied_weights_and_checkpoint_continuation(tmp_path): + torch.manual_seed(112) + expected = TiedProjection() + actual = deepcopy(expected) + backward = make_backward("compiled-autograd") + expected.forward = torch.compile(expected.forward, fullgraph=True, dynamic=False) + actual.forward = torch.compile(actual.forward, fullgraph=True, dynamic=False) + expected_optimizer = torch.optim.AdamW(expected.parameters(), lr=0.002) + actual_optimizer = torch.optim.AdamW(actual.parameters(), lr=0.002) + tokens = torch.tensor([[0, 2, 0, 1], [5, 3, 9, 3]]) + for _ in range(3): + for model, operation in ((expected, lambda loss: loss.backward()), (actual, backward)): + with torch.autocast("cpu", dtype=torch.bfloat16): + loss = model(tokens).square().mean() + operation(loss) + for wanted, observed in zip(expected.parameters(), actual.parameters(), strict=True): + torch.testing.assert_close(observed.grad, wanted.grad, rtol=0, atol=0) + for model, optimizer in ((expected, expected_optimizer), (actual, actual_optimizer)): + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) + optimizer.step() + path = tmp_path / "resume.pt" + torch.save({"model": actual.state_dict(), "optimizer": actual_optimizer.state_dict()}, path) + state = torch.load(path, weights_only=True) + resumed = TiedProjection() + resumed.load_state_dict(state["model"], strict=True) + resumed_optimizer = torch.optim.AdamW(resumed.parameters(), lr=0.002) + resumed_optimizer.load_state_dict(state["optimizer"]) + resumed.forward = torch.compile(resumed.forward, fullgraph=True, dynamic=False) + assert tuple(state["model"]) == ("embed.weight", "projection.weight") + for wanted, observed in zip(expected.parameters(), resumed.parameters(), strict=True): + torch.testing.assert_close(observed, wanted, rtol=0, atol=0) + # Resume with ordinary autograd: state contains no compiler-specific objects. + for model, optimizer in ((expected, expected_optimizer), (resumed, resumed_optimizer)): + optimizer.zero_grad(set_to_none=True) + with torch.autocast("cpu", dtype=torch.bfloat16): + loss = model(tokens).square().mean() + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) + optimizer.step() + for wanted, observed in zip(expected.parameters(), resumed.parameters(), strict=True): + torch.testing.assert_close(observed, wanted, rtol=0, atol=0) + + +def _distributed_worker(rank, rendezvous, output_directory, explicit_overlap=False): + torch.set_num_threads(1) + dist.init_process_group("gloo", init_method="file://" + rendezvous, rank=rank, world_size=2, + timeout=timedelta(seconds=30)) + try: + torch.manual_seed(31) + expected = TiedProjection() + actual = deepcopy(expected) + eager = torch.nn.parallel.DistributedDataParallel(expected, bucket_cap_mb=1) + # Eager backend keeps this a lightweight CPU distributed-semantics test; + # the separate tests above exercise Inductor's generated accumulation. + if explicit_overlap: + from torch._inductor.comms import align_runtime_estimations_across_all_distributed_ranks + + backward = make_backward("compiled-autograd-overlap", distributed=True, backend="eager", + overlap_settings=OverlapSettings(True, True, 0.02)) + + class Estimate: + def __init__(self, value): + self.value = value + + def get_estimated_runtime(self): + return self.value + + estimates = [Estimate(1.0 + rank), Estimate(2.0 + 2 * rank)] + # Exercise the actual installed scheduler's cross-rank alignment + # collective on CPU, separately from the lightweight eager backend. + align_runtime_estimations_across_all_distributed_ranks(estimates) + assert [node.override_estimated_runtime for node in estimates] == [1.0, 2.0] + else: + backward = make_backward("compiled-autograd", distributed=True, backend="eager") + compiled = torch.nn.parallel.DistributedDataParallel(actual, bucket_cap_mb=1) + for microbatch in range(3): + tokens = (torch.arange(8).reshape(2, 4) + rank * 2 + microbatch) % 16 + for model, operation in ((eager, lambda loss: loss.backward()), (compiled, backward)): + synchronization = model.no_sync() if microbatch < 2 else nullcontext() + with synchronization: + with torch.autocast("cpu", dtype=torch.bfloat16): + loss = model(tokens).square().mean() / 3 + operation(loss) + for wanted, observed in zip(expected.parameters(), actual.parameters(), strict=True): + torch.testing.assert_close(observed.grad, wanted.grad, rtol=0, atol=0) + torch.save({name: parameter.grad for name, parameter in actual.named_parameters()}, + Path(output_directory) / f"rank-{rank}.pt") + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("explicit_overlap", [False, True]) +def test_distributed_no_sync_and_fp32_reduction_match_eager(tmp_path, explicit_overlap): + torch.multiprocessing.spawn(_distributed_worker, + args=(str(tmp_path / "rendezvous"), str(tmp_path), explicit_overlap), nprocs=2, join=True) + ranks = [torch.load(tmp_path / f"rank-{rank}.pt", weights_only=True) for rank in range(2)] + for name in ranks[0]: + assert ranks[0][name].dtype == torch.float32 + torch.testing.assert_close(ranks[0][name], ranks[1][name], rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA correctness fixture") +def test_cuda_warpstate_compiled_cores_loss_residuals_and_accumulation(): + from ops.loss_experiment import apply_loss_execution + from ops.residual_experiment import apply_bf16_residuals + from warpstate import WarpStateConfig, WarpStateLM + from run_40b import compile_training_cores + + torch.manual_seed(119) + cfg = WarpStateConfig(vocab_size=128, d_model=128, n_heads=2, n_cores=1, + logical_depth=2, ffn_hidden=256, chunk_size=16, + gradient_checkpointing=False, parallel_memory=True) + expected = WarpStateLM(cfg).cuda() + actual = deepcopy(expected) + backward = make_backward("compiled-autograd") + for model in (expected, actual): + compile_training_cores(model) + apply_loss_execution(model, "compiled-full") + apply_bf16_residuals(model) + expected_optimizer = torch.optim.AdamW(expected.parameters(), lr=0.0001, fused=True) + actual_optimizer = torch.optim.AdamW(actual.parameters(), lr=0.0001, fused=True) + # Exercise continuation with existing FP32 Adam moments. Fresh Adam can + # turn a tiny BF16 cancellation around zero into an entire signed learning + # rate on its first update; it is not representative of this resumed run. + for model, optimizer in ((expected, expected_optimizer), (actual, actual_optimizer)): + for parameter in model.parameters(): + optimizer.state[parameter] = { + "step": torch.tensor(40081.0, device="cuda"), + "exp_avg": torch.full_like(parameter, 0.001), + "exp_avg_sq": torch.full_like(parameter, 0.0001), + } + for update in range(2): + expected_optimizer.zero_grad(set_to_none=True) + actual_optimizer.zero_grad(set_to_none=True) + for microbatch in range(3): + tokens = torch.randint(128, (2, 32), device="cuda") + labels = torch.randint(128, (2, 32), device="cuda") + for model, operation in ((expected, lambda loss: loss.backward()), (actual, backward)): + with torch.autocast("cuda", dtype=torch.bfloat16, cache_enabled=False): + _, loss = model(tokens, labels, return_logits=False) + operation(scale_loss(loss, 1.0 / 3.0, "compiled-autograd")) + for (name, wanted), (_, observed) in zip(expected.named_parameters(), + actual.named_parameters(), strict=True): + assert observed.dtype == observed.grad.dtype == torch.float32, name + assert torch.isfinite(observed.grad).all(), name + # CA also fuses BF16 work across the old core/loss boundaries. + # Individual BF16 rounding can differ: bound both total error + # and worst-element error instead of claiming bitwise identity. + difference = observed.grad - wanted.grad + relative = difference.norm() / wanted.grad.norm().clamp_min(1e-12) + assert difference.norm().item() <= wanted.grad.norm().item() * 0.015 + 1e-8, (name, relative.item()) + assert difference.abs().max().item() <= wanted.grad.abs().max().item() * 0.02 + 1e-8, name + for model, optimizer in ((expected, expected_optimizer), (actual, actual_optimizer)): + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) + optimizer.step() + for wanted, observed in zip(expected.parameters(), actual.parameters(), strict=True): + assert torch.isfinite(observed).all() + torch.testing.assert_close(observed, wanted, rtol=0, atol=2e-5) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA scalar capture workaround") +def test_cuda_loss_scalar_materialization_preserves_fp32_arithmetic(): + torch.manual_seed(711) + values = torch.randn(97, device="cuda", dtype=torch.float32) + for factor in (0.0, 0.125, 1.0 / 3.0, 7.0 / 19.0, 1.0): + expected = scale_loss(values, factor, "eager") + actual = scale_loss(values, factor, "compiled-autograd") + torch.testing.assert_close(actual, expected, rtol=0, atol=0) diff --git a/test_loss_execution.py b/test_loss_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..b1ccd9356b9eeba484c381735423d0d9e26dde89 --- /dev/null +++ b/test_loss_execution.py @@ -0,0 +1,72 @@ +"""The experimental loss retains the full vocabulary objective and model state.""" + +import copy + +import pytest +import torch + +from ops.loss_experiment import apply_loss_execution +from warpstate.config import WarpStateConfig +from warpstate.model import WarpStateLM + + +@pytest.mark.parametrize("mode", ["full", "compiled-full"]) +@pytest.mark.parametrize("mask", ["none", "partial", "all"]) +def test_full_loss_preserves_objective_gradients_and_parameters(mode, mask, monkeypatch): + # Exercise full graph capture on CPU without hardware-specific Inductor code. + if mode == "compiled-full": + actual_compile = torch.compile + monkeypatch.setattr(torch, "compile", lambda fn, **kw: actual_compile(fn, backend="eager", **kw)) + torch.manual_seed(572) + cfg = WarpStateConfig(vocab_size=32, d_model=16, n_heads=2, n_cores=1, + logical_depth=2, ffn_hidden=24, chunk_size=4, + gradient_checkpointing=False, parallel_memory=True, + loss_chunk_size=3) + reference = WarpStateLM(cfg).train() + experiment = copy.deepcopy(reference) + keys = list(experiment.state_dict()) + parameters = list(experiment.parameters()) + embedding = experiment.embed.weight + apply_loss_execution(experiment, mode) + assert list(experiment.state_dict()) == keys + assert all(a is b for a, b in zip(parameters, experiment.parameters(), strict=True)) + assert experiment.embed.weight is embedding + inputs = torch.randint(0, cfg.vocab_size, (2, 8)) + labels = torch.randint(0, cfg.vocab_size, (2, 8)) + if mask == "partial": + labels.flatten()[3:11] = -100 + elif mask == "all": + labels.fill_(-100) + ref_logits, ref_loss = reference(inputs, labels, return_logits=False) + logits, loss = experiment(inputs, labels, return_logits=False) + assert ref_logits is None and logits is None + assert torch.isfinite(loss) + torch.testing.assert_close(loss, ref_loss, rtol=1e-6, atol=1e-6) + ref_loss.backward() + loss.backward() + for (name, expected), (other_name, actual) in zip(reference.named_parameters(), experiment.named_parameters(), strict=True): + assert name == other_name + assert actual.grad is not None and torch.isfinite(actual.grad).all() + torch.testing.assert_close(actual.grad, expected.grad, rtol=1e-6, atol=1e-6) + if mask == "all": + assert loss.item() == 0.0 + assert all(torch.count_nonzero(p.grad) == 0 for p in experiment.parameters()) + with torch.no_grad(): + reference.eval() + experiment.eval() + torch.testing.assert_close(experiment(inputs), reference(inputs), rtol=0, atol=0) + torch.testing.assert_close(experiment(inputs, labels, return_logits=False)[1], ref_loss.detach(), rtol=1e-6, atol=1e-6) + + +def test_loss_mode_validation_and_checkpointed_noop(): + model = WarpStateLM(WarpStateConfig(vocab_size=16, d_model=8, n_heads=1, + n_cores=1, logical_depth=1, ffn_hidden=8, + chunk_size=4)) + forward = model.forward + apply_loss_execution(model, "checkpointed") + assert model.forward == forward + with pytest.raises(ValueError, match="Unknown loss"): + apply_loss_execution(model, "invalid") + apply_loss_execution(model, "full") + with pytest.raises(ValueError, match="already been patched"): + apply_loss_execution(model, "full") diff --git a/test_matmul_epilogue.py b/test_matmul_epilogue.py new file mode 100644 index 0000000000000000000000000000000000000000..1be31a6caea98b8530749a86e7b3d593e8a6183b --- /dev/null +++ b/test_matmul_epilogue.py @@ -0,0 +1,84 @@ +"""Projection rounding, masking and output ownership for the fused FFN GEMM.""" + +import os + +import pytest +import torch +import torch.nn.functional as F + +from ops.matmul_epilogue import SUPPORTED_TILES, launch_swiglu_mm + + +@pytest.mark.parametrize("eager_rounding", [False, True]) +def test_reference_preserves_projection_and_activation_rounding(eager_rounding): + torch.manual_seed(554) + a = torch.randn(7, 31, dtype=torch.bfloat16) + b = torch.randn(46, 31, dtype=torch.bfloat16).T + packed, activation = launch_swiglu_mm(a, b, "aten", SUPPORTED_TILES[0], eager_rounding) + expected = a @ b + up, gate = expected.chunk(2, dim=-1) + wanted = F.silu(gate) * up if eager_rounding else (F.silu(gate.float()) * up.float()).bfloat16() + torch.testing.assert_close(packed, expected, rtol=0, atol=0) + torch.testing.assert_close(activation, wanted, rtol=0, atol=0) + assert packed.is_contiguous() and activation.is_contiguous() + assert packed.untyped_storage().data_ptr() != activation.untyped_storage().data_ptr() + + +@pytest.mark.parametrize("case", ["dtype", "shape", "odd_width", "empty", "cpu", "tile", "algorithm"]) +def test_unsupported_execution_fails_before_a_kernel_launch(case): + a = torch.ones(4, 8, dtype=torch.bfloat16) + b = torch.ones(8, 16, dtype=torch.bfloat16) + algorithm, tile = "aten", SUPPORTED_TILES[0] + if case == "dtype": + a = a.float() + elif case == "shape": + b = b[:7] + elif case == "odd_width": + b = b[:, :15] + elif case == "empty": + a = a[:0] + elif case == "cpu": + algorithm = "tiled" + elif case == "tile": + tile = (256, 256, 64, 8, 4) + elif case == "algorithm": + algorithm = "unsupported" + with pytest.raises(ValueError): + launch_swiglu_mm(a, b, algorithm, tile) + + +cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1", + reason="requires an explicitly reserved CUDA device") + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["tiled", "persistent"]) +@pytest.mark.parametrize("eager_rounding", [False, True]) +@pytest.mark.parametrize("shape", [(13, 46, 31), (128, 256, 128), (257, 192, 193)]) +@pytest.mark.parametrize("tile", [SUPPORTED_TILES[0], SUPPORTED_TILES[-1]]) +def test_cuda_projection_and_saved_activation_match_reference(algorithm, eager_rounding, shape, tile): + m, packed_width, k = shape + torch.manual_seed(561) + a = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1 + b = (torch.randn(packed_width, k, device="cuda", dtype=torch.bfloat16) * 0.1).T + packed, activation = launch_swiglu_mm(a, b, algorithm, tile, eager_rounding) + expected, wanted = launch_swiglu_mm(a, b, "aten", SUPPORTED_TILES[0], eager_rounding) + for result, reference in ((packed, expected), (activation, wanted)): + assert result.dtype == torch.bfloat16 and result.is_contiguous() + assert torch.isfinite(result).all() + delta = result.float() - reference.float() + relative_l2 = delta.norm() / reference.float().norm().clamp_min(1e-12) + # cuBLAS may use reduced-precision reduction for ragged BF16 GEMMs. + # The packed error can grow through the activation/product; bound both + # and independently check our FP32 accumulation and saved projection. + assert relative_l2.item() < 0.006 + torch.testing.assert_close(result, reference, rtol=0.02, atol=0.001) + accumulated = (a.float() @ b.float()).bfloat16() + projection_relative_l2 = ((packed.float() - accumulated.float()).norm() + / accumulated.float().norm().clamp_min(1e-12)) + assert projection_relative_l2.item() < 0.001 + # The activation must consume the RETURNED rounded projection. This detects + # an incorrect shortcut that activates unrounded FP32 tensor-core accumulators. + up, gate = packed.chunk(2, dim=-1) + packed_reference = F.silu(gate) * up if eager_rounding else (F.silu(gate.float()) * up.float()).bfloat16() + torch.testing.assert_close(activation, packed_reference, rtol=0.01, atol=0.0002) diff --git a/test_matmul_execution.py b/test_matmul_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..314ab36be2da945ac669c85d2b2d51c77eef3b99 --- /dev/null +++ b/test_matmul_execution.py @@ -0,0 +1,300 @@ +"""Exact plan selection and autograd ownership for optional BF16 matrix kernels.""" + +import copy +import json +import os + +import pytest +import torch + +from ops.matmul_execution import ( + PLAN_FORMAT, apply_matmul_execution, load_matmul_plan, + matmul_plan_from_dict, planned_matmul, +) +from warpstate import WarpStateConfig, WarpStateLM + + +def _entry(*, kind="mm", m=8, n=32, k=16, algorithm="tiled"): + return {"kind": kind, "m": m, "n": n, "k": k, + "a_stride": [k, 1], "b_stride": [1, k], "dtype": "bfloat16", + "algorithm": algorithm, + "tile": {"block_m": 32, "block_n": 32, "block_k": 32, + "num_warps": 4, "num_stages": 3}} + + +def _document(entries=None): + return {"format": PLAN_FORMAT, "entries": entries if entries is not None else [_entry()]} + + +def _model(): + torch.manual_seed(492) + return WarpStateLM(WarpStateConfig(vocab_size=48, d_model=16, n_heads=2, + n_cores=1, logical_depth=1, ffn_hidden=32, chunk_size=4, + parallel_memory=True, gradient_checkpointing=False)).train() + + +def test_plan_exact_shape_stride_key_and_source_provenance(tmp_path): + path = tmp_path / "plan.json" + path.write_text(json.dumps(_document())) + plan = load_matmul_plan(path) + assert plan.select("mm", 8, 32, 16, (16, 1), (1, 16)).algorithm == "tiled" + assert plan.select("mm", 8, 32, 16, (16, 1), (32, 1)) is None + assert plan.select("mm", 7, 32, 16, (16, 1), (1, 16)) is None + assert plan.select("swiglu", 8, 32, 16, (16, 1), (1, 16)) is None + assert len(plan.metadata["sha256"]) == 64 + assert "ops/matmul_execution.py" in plan.metadata["source_sha256"] + assert "ops/matmul_kernels.py" in plan.metadata["source_sha256"] + path.write_text(json.dumps(_document(), indent=4)) + assert load_matmul_plan(path).metadata == plan.metadata + altered = _document() + altered["entries"][0]["tile"]["block_m"] = 64 + assert matmul_plan_from_dict(altered).sha256 != plan.sha256 + fused = matmul_plan_from_dict(_document([_entry(kind="swiglu")])) + assert "ops/matmul_epilogue.py" in fused.metadata["source_sha256"] + assert "ops/activation_execution.py" in fused.metadata["source_sha256"] + + +@pytest.mark.parametrize("mutation", [ + lambda d: d.update(format="other"), + lambda d: d.update(unknown=True), + lambda d: d["entries"][0].update(dtype="float16"), + lambda d: d["entries"][0].update(algorithm="unknown"), + lambda d: d["entries"][0].update(m=True), + lambda d: d["entries"][0].update(n=0), + lambda d: d["entries"][0].update(a_stride=[0, 1]), + lambda d: d["entries"][0].update(b_stride=[1]), + lambda d: d["entries"][0].update(kind="swiglu", n=31), + lambda d: d["entries"][0].update(kind="swiglu", algorithm="persistent_tma"), + lambda d: d["entries"][0]["tile"].update(block_k=0), + lambda d: d["entries"].append(copy.deepcopy(d["entries"][0])), +]) +def test_invalid_plan_fails_before_model_or_gpu_work(mutation): + document = _document() + mutation(document) + with pytest.raises(ValueError): + matmul_plan_from_dict(document) + + +def test_duplicate_json_keys_rejected(tmp_path): + path = tmp_path / "duplicate.json" + path.write_text('{"format":"warpstate-matmul-plan-v1","entries":[],"entries":[]}') + with pytest.raises(ValueError, match="Duplicate JSON"): + load_matmul_plan(path) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_unsupported_cpu_matmul_retains_cast_and_fp32_gradient_accumulation(compiled): + torch.manual_seed(495) + plan = matmul_plan_from_dict(_document()) + expected = torch.nn.Parameter(torch.randn(32, 16)) + actual = torch.nn.Parameter(expected.detach().clone()) + original = lambda x, weight: torch.mm(x, weight.to(torch.bfloat16).T) + selected = lambda x, weight: planned_matmul(x, weight.to(torch.bfloat16).T, plan) + if compiled: + original = torch.compile(original, backend="aot_eager", fullgraph=True) + selected = torch.compile(selected, backend="aot_eager", fullgraph=True) + for _ in range(8): + x = torch.randn(8, 16, dtype=torch.bfloat16) + grad = torch.randn(8, 32, dtype=torch.bfloat16) + y = original(x, expected) + z = selected(x, actual) + torch.testing.assert_close(z, y, rtol=0, atol=0) + y.backward(grad) + z.backward(grad) + assert actual.grad.dtype == torch.float32 + torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0) + + +def test_model_patch_preserves_parameters_state_and_cpu_eval(): + reference = _model() + actual = copy.deepcopy(reference) + masters = tuple(actual.parameters()) + keys = tuple(actual.state_dict()) + apply_matmul_execution(actual, matmul_plan_from_dict(_document([_entry(kind="swiglu")]))) + inputs = torch.randint(48, (2, 8)) + for training in [True, False]: + reference.train(training) + actual.train(training) + with torch.set_grad_enabled(training): + torch.testing.assert_close(actual(inputs), reference(inputs), rtol=0, atol=0) + assert tuple(actual.state_dict()) == keys + assert all(a is b for a, b in zip(masters, actual.parameters(), strict=True)) + with pytest.raises(ValueError, match="already been applied"): + apply_matmul_execution(actual, matmul_plan_from_dict(_document())) + with pytest.raises(ValueError, match="FP32 master"): + apply_matmul_execution(_model().bfloat16(), matmul_plan_from_dict(_document())) + + +def test_shape_policy_only_patches_relevant_projection_modules(): + model = _model() + core = model.cores[0] + originals = {name: module.forward for name, module in core.named_modules() + if isinstance(module, torch.nn.Linear)} + apply_matmul_execution(model, matmul_plan_from_dict( + _document([_entry(m=16, n=64, k=16)]))) + assert core.ffn.up_gate.forward != originals["ffn.up_gate"] + assert core.ffn.down.forward == originals["ffn.down"] + assert core.in_proj.forward == originals["in_proj"] + assert core.out_proj.forward == originals["out_proj"] + + +cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1", + reason="requires an explicit exclusive GPU test window") + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["tiled", "persistent", "persistent_tma"]) +@pytest.mark.parametrize("compiled", [False, True]) +def test_cuda_selected_forward_dx_dw_keep_fp32_leaf_gradients(algorithm, compiled): + torch.manual_seed(497) + forward = _entry(m=64, n=128, k=64, algorithm=algorithm) + backward = _entry(m=128, n=64, k=64, algorithm=algorithm) + # Match the original Linear dW orientation: dY.T @ X. + backward["a_stride"] = [1, 128] + backward["b_stride"] = [64, 1] + activation_backward = _entry(m=64, n=64, k=128, algorithm=algorithm) + activation_backward["b_stride"] = [64, 1] + plan = matmul_plan_from_dict(_document([forward, backward, activation_backward])) + reference = torch.nn.Parameter(torch.randn(128, 64, device="cuda") * 0.02) + actual = torch.nn.Parameter(reference.detach().clone()) + expected_fn = lambda x, w: x.to(torch.bfloat16) @ w.to(torch.bfloat16).T + selected_fn = lambda x, w: planned_matmul(x.to(torch.bfloat16), w.to(torch.bfloat16).T, plan) + if compiled: + expected_fn = torch.compile(expected_fn, fullgraph=True, dynamic=False) + selected_fn = torch.compile(selected_fn, fullgraph=True, dynamic=False) + for _ in range(8): + x = torch.randn(64, 64, device="cuda", requires_grad=True) + selected_x = x.detach().clone().requires_grad_(True) + dy = torch.randn(64, 128, device="cuda", dtype=torch.bfloat16) + expected = expected_fn(x, reference) + result = selected_fn(selected_x, actual) + torch.testing.assert_close(result, expected, rtol=0.015, atol=0.015) + expected.backward(dy) + result.backward(dy) + assert selected_x.grad.dtype == torch.float32 + assert ((selected_x.grad - x.grad).norm() / x.grad.norm()).item() < 0.005 + assert actual.dtype == actual.grad.dtype == torch.float32 + relative = (actual.grad - reference.grad).norm() / reference.grad.norm() + assert relative.item() < 0.005 + + +@cuda_window +def test_cuda_unsupported_tma_layout_falls_back_to_aten(): + a = torch.randn(32, 32, device="cuda", dtype=torch.bfloat16) + b = torch.randn(32, 64, device="cuda", dtype=torch.bfloat16)[:, ::2] + entry = _entry(m=32, n=32, k=32, algorithm="persistent_tma") + entry["b_stride"] = list(b.stride()) + plan = matmul_plan_from_dict(_document([entry])) + torch.testing.assert_close(planned_matmul(a, b, plan), a @ b, rtol=0, atol=0) + + +@cuda_window +@pytest.mark.parametrize("misaligned_operand", ["a", "b"]) +def test_cuda_compiled_tma_rechecks_storage_offset_before_launch(misaligned_operand): + a = torch.randn(64, 64, device="cuda", dtype=torch.bfloat16) + b = torch.randn(128, 64, device="cuda", dtype=torch.bfloat16).T + plan = matmul_plan_from_dict(_document([ + _entry(m=64, n=128, k=64, algorithm="persistent_tma")])) + compiled = torch.compile(lambda x, y: planned_matmul(x, y, plan), + fullgraph=True, dynamic=False) + control = torch.compile(lambda x, y: x @ y, fullgraph=True, dynamic=False) + torch.testing.assert_close(compiled(a, b), a @ b, rtol=0.015, atol=0.015) + original = a if misaligned_operand == "a" else b + storage = torch.randn(original.numel() + 1, device="cuda", dtype=torch.bfloat16) + shifted = storage.as_strided(original.shape, original.stride(), storage_offset=1) + assert shifted.shape == original.shape and shifted.stride() == original.stride() + assert shifted.data_ptr() % 16 == 2 + a, b = (shifted, b) if misaligned_operand == "a" else (a, shifted) + torch.testing.assert_close(planned_matmul(a, b, plan), a @ b, rtol=0, atol=0) + # Use the compiled ATen control: Inductor may realign an input before its + # fallback call, and aligned cuBLAS can differ from eager unaligned cuBLAS. + # Realignment may also make TMA valid. Check the compiled result without + # assuming the caller's original offset survives to the opaque boundary. + torch.testing.assert_close(compiled(a, b), control(a, b), rtol=0, atol=0) + + +@cuda_window +@pytest.mark.parametrize("fused", [False, True]) +def test_cuda_compiled_model_selected_matmul_preserves_fp32_adam_and_reload(fused): + from ops.residual_experiment import apply_bf16_residuals + from run_40b import compile_training_cores + + reference = _model().cuda() + actual = copy.deepcopy(reference) + entries = [_entry(kind="swiglu" if fused else "mm", m=16, n=64, k=16)] + plan = matmul_plan_from_dict(_document(entries)) + keys = tuple(actual.state_dict()) + masters = tuple(actual.parameters()) + apply_matmul_execution(actual, plan) + for model in [reference, actual]: + compile_training_cores(model) + apply_bf16_residuals(model) + optimizer = torch.optim.AdamW(actual.parameters(), lr=1e-4) + inputs = torch.randint(48, (2, 8), device="cuda") + with torch.autocast("cuda", dtype=torch.bfloat16): + expected = reference(inputs, inputs, return_logits=False)[1] + result = actual(inputs, inputs, return_logits=False)[1] + expected.backward() + result.backward() + torch.testing.assert_close(result, expected, rtol=0.001, atol=0.001) + grads = torch.cat([p.grad.flatten() for p in actual.parameters()]) + ref_grads = torch.cat([p.grad.flatten() for p in reference.parameters()]) + assert torch.nn.functional.cosine_similarity(grads, ref_grads, dim=0).item() > 0.999 + assert (grads - ref_grads).norm().item() / ref_grads.norm().item() < 0.02 + optimizer.step() + assert tuple(actual.state_dict()) == keys + assert all(a is b for a, b in zip(masters, actual.parameters(), strict=True)) + assert all(p.dtype == p.grad.dtype == optimizer.state[p]["exp_avg"].dtype + == optimizer.state[p]["exp_avg_sq"].dtype == torch.float32 for p in masters) + restored = _model().cuda().eval() + restored.load_state_dict(actual.state_dict(), strict=True) + actual.eval() + with torch.no_grad(): + torch.testing.assert_close(actual(inputs), restored(inputs), rtol=0, atol=0) + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["tiled", "persistent", "persistent_tma"]) +@pytest.mark.parametrize("recompute", [False, True]) +def test_cuda_compiled_autograd_fused_projection_dx_dw_and_recomputation(algorithm, recompute): + from ops.gradient_execution import make_backward + from torch.utils.checkpoint import checkpoint + + # Ordinary AOT forward/backward testing misses Dynamo re-tracing AOT's + # functionalized Triton HOP during Compiled Autograd. Exercise that second + # trace and activation-checkpoint recomputation with every matrix algorithm. + with torch._dynamo.config.patch(compiled_autograd=False, + compiled_autograd_kwargs_override={}, optimize_ddp=True): + torch._dynamo.reset() + backward = make_backward("compiled-autograd") + reference_model = _model().cuda() + selected_model = copy.deepcopy(reference_model) + fused = _entry(kind="swiglu", m=16, n=64, k=16, algorithm="persistent") + dx = _entry(m=16, n=16, k=64, algorithm=algorithm) + dx["b_stride"] = [16, 1] + dw = _entry(m=64, n=16, k=16, algorithm=algorithm) + dw["a_stride"], dw["b_stride"] = [1, 64], [16, 1] + apply_matmul_execution(selected_model, matmul_plan_from_dict(_document([fused, dx, dw]))) + reference = reference_model.cores[0].ffn + selected = selected_model.cores[0].ffn + # Capture recomputation inside AOT. An eager checkpoint outside the + # compiled forward has a separate installed-PyTorch CA limitation: + # Dynamo cannot trace its saved-tensor unpack hook. + wanted_eager = (lambda x: checkpoint(reference, x, use_reentrant=False)) if recompute else reference + actual_eager = (lambda x: checkpoint(selected, x, use_reentrant=False)) if recompute else selected + wanted_fn = torch.compile(wanted_eager, fullgraph=True, dynamic=False) + actual_fn = torch.compile(actual_eager, fullgraph=True, dynamic=False) + for _ in range(2): + x = torch.randn(16, 16, device="cuda", requires_grad=True) + actual_x = x.detach().clone().requires_grad_(True) + with torch.autocast("cuda", dtype=torch.bfloat16): + wanted = wanted_fn(x) + actual = actual_fn(actual_x) + torch.testing.assert_close(actual, wanted, rtol=0.015, atol=0.0001) + wanted.float().square().mean().backward() + backward(actual.float().square().mean()) + for p, q in [(x, actual_x), *zip(reference.parameters(), selected.parameters(), strict=True)]: + assert q.grad.dtype == torch.float32 + relative = (q.grad - p.grad).norm() / p.grad.norm().clamp_min(1e-12) + assert relative.item() < 0.02 + torch._dynamo.reset() diff --git a/test_matmul_kernels.py b/test_matmul_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..b341dbc01cfef6a8317975ea23f55ea260ec2658 --- /dev/null +++ b/test_matmul_kernels.py @@ -0,0 +1,188 @@ +"""Matrix tile validation, dispatch signatures, and explicit CUDA correctness.""" + +import json +import os + +import pytest +import torch +from torch.library import triton_op + +from ops.matmul_kernels import ( + DEFAULT_TILE, TUNING_TILES, launch_mm, matrix_signature, normalize_tile, + policy_key, tile_dict, +) + + +@pytest.mark.parametrize("tile", [ + [], (128, 128, 32, 4), (128, 128, 32, True, 3), (120, 128, 32, 4, 3), + (128, 128, 32, 2, 3), (128, 128, 32, 4, 8), (256, 256, 128, 8, 4), + {"block_m": 128}, +]) +def test_invalid_or_oversized_tiles_fail(tile): + with pytest.raises(ValueError): + normalize_tile(tile) + + +def test_tuning_tiles_and_json_round_trip(): + for tile in TUNING_TILES: + assert normalize_tile(json.loads(json.dumps(tile_dict(tile)))) == tile + assert normalize_tile() == DEFAULT_TILE + + +def test_signature_distinguishes_strides_even_when_shapes_match(): + a = torch.empty((16, 32), dtype=torch.bfloat16) + b = torch.empty((64, 32), dtype=torch.bfloat16).T + signature = matrix_signature(a, b) + assert signature == {"m": 16, "n": 64, "k": 32, "a_stride": [32, 1], + "b_stride": [1, 32], "dtype": "bfloat16"} + assert json.loads(policy_key(a, b)) == signature + assert policy_key(a, b) != policy_key(a, b.contiguous()) + + +def test_no_cpu_or_unknown_algorithm_fallback(): + a = torch.empty((16, 32), dtype=torch.bfloat16) + b = torch.empty((32, 16), dtype=torch.bfloat16) + with pytest.raises(ValueError, match="CUDA"): + launch_mm(a, b) + with pytest.raises(ValueError, match="Unknown"): + launch_mm(a, b, "fp8") + + +def test_tuner_kind_defaults_and_selected_tiles(): + from ops.tune_matmul import arguments, cases + + generic = arguments(["--list-cases"]) + assert generic.operations == ["fwd", "dw"] + assert generic.algorithms == ["tiled", "persistent", "persistent_tma"] + fused = arguments(["--list-cases", "--kind", "swiglu", "--tile", "128,128,32,8,3"]) + assert fused.operations == ["fwd"] + assert fused.algorithms == ["tiled", "persistent"] + assert fused.tile == [(128, 128, 32, 8, 3)] + forward, dw = cases(8192, generic.operations) + assert forward["a_stride"] == [2048, 1] and forward["b_stride"] == [1, 2048] + assert dw["a_stride"] == [1, 14336] and dw["b_stride"] == [2048, 1] + + +@pytest.mark.parametrize("flags", [ + ["--kind", "swiglu", "--operations", "dw"], + ["--kind", "swiglu", "--algorithms", "persistent_tma"], + ["--kind", "swiglu", "--tile", "256,256,32,8,3"], + ["--tile", "128,128,32,4"], + ["--input-scale", "nan"], ["--input-scale", "0"], + ["--operations", "fwd", "fwd"], + ["--algorithms", "tiled", "tiled"], + ["--tile", "128,128,32,4,3", "--tile", "128,128,32,4,3"], + ["--output-dir", "/tmp/outside-workspace"], +]) +def test_tuner_rejects_unsupported_kind_or_unsafe_output(flags): + from ops.tune_matmul import arguments + + with pytest.raises(SystemExit): + arguments(["--list-cases", *flags]) + + +def test_tuner_checks_all_outputs_and_preserves_bf16(): + from ops.tune_matmul import output_metrics + + expected = torch.ones((4, 8), dtype=torch.bfloat16) + metrics = output_metrics((expected.clone(), expected * 2), (expected, expected)) + assert metrics["output_metrics"][0]["relative_rms_error"] == 0 + assert metrics["relative_rms_error"] == 1 + with pytest.raises(ValueError, match="BF16"): + output_metrics(expected.float(), expected) + with pytest.raises(ValueError, match="Nonfinite"): + output_metrics((expected, expected * float("nan")), (expected, expected)) + + +def test_tuner_cannot_select_a_median_winner_with_a_losing_trial(): + from ops.tune_matmul import paired_timing_metrics + + trials = [{"aten_before_ms": 2.0, "candidate_ms": candidate, "aten_after_ms": 2.0} + for candidate in (1.5, 1.5, 2.1)] + metrics = paired_timing_metrics(trials, minimum_gain=0.01) + assert metrics["paired_gain"] > 0.3 + assert metrics["minimum_trial_gain"] < 0 + assert not metrics["timing_eligible"] + drifted = [{"aten_before_ms": 1.0, "candidate_ms": 1.4, "aten_after_ms": 2.0}] + metrics = paired_timing_metrics(drifted, minimum_gain=0.01) + assert metrics["timing_eligible"] + assert metrics["maximum_control_drift_fraction"] == pytest.approx(2 / 3) + + +@pytest.mark.parametrize("timing", [0, -1, float("nan"), float("inf")]) +def test_tuner_rejects_invalid_timing_metrics(timing): + from ops.tune_matmul import paired_timing_metrics + + with pytest.raises(ValueError, match="finite and positive"): + paired_timing_metrics([{"aten_before_ms": 1, "candidate_ms": timing, + "aten_after_ms": 1}], minimum_gain=0.01) + + +@pytest.mark.parametrize("offset_operand", ["a", "b"]) +def test_tma_rejects_misaligned_offset_without_launching(offset_operand): + from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode + + mode = FakeTensorMode() + # Construct fake CUDA operands from meta storage; this test never starts + # CUDA or reaches the custom-op launch, including on CPU-only machines. + tensors = [] + for name, shape in (("a", (32, 64)), ("b", (64, 32))): + offset = 1 if name == offset_operand else 0 + storage = torch.empty(shape[0] * shape[1] + offset, device="meta", dtype=torch.bfloat16) + view = storage.as_strided(shape, (shape[1], 1), offset) + tensors.append(FakeTensor(mode, view, torch.device("cuda:0"))) + with pytest.raises(ValueError, match="16-byte-aligned storage offsets"): + launch_mm(*tensors, "persistent_tma") + + +cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1", + reason="requires an explicitly reserved CUDA device") + + +@triton_op("warpstate_test::candidate_mm", mutates_args={}) +def _test_mm(a: torch.Tensor, b: torch.Tensor, algorithm: str) -> torch.Tensor: + return launch_mm(a, b, algorithm, DEFAULT_TILE) + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["tiled", "persistent"]) +@pytest.mark.parametrize("layouts", [(False, False), (False, True), (True, False), (True, True)]) +def test_cuda_odd_shapes_and_transposed_operands(algorithm, layouts): + torch.manual_seed(77) + m, n, k = 97, 113, 129 + a = torch.randn((k, m) if layouts[0] else (m, k), device="cuda", dtype=torch.bfloat16) + b = torch.randn((n, k) if layouts[1] else (k, n), device="cuda", dtype=torch.bfloat16) + a = a.T if layouts[0] else a + b = b.T if layouts[1] else b + acopy, bcopy = a.clone(), b.clone() + with torch.inference_mode(): + expected = torch.mm(a, b) + output = launch_mm(a, b, algorithm) + error = (output.float() - expected.float()).square().mean().sqrt() + assert output.dtype == torch.bfloat16 and torch.isfinite(output).all() + assert error / expected.float().square().mean().sqrt() < 0.003 + torch.testing.assert_close(a, acopy, rtol=0, atol=0) + torch.testing.assert_close(b, bcopy, rtol=0, atol=0) + + +@cuda_window +@pytest.mark.parametrize("algorithm", ["tiled", "persistent", "persistent_tma"]) +def test_cuda_launch_is_usable_under_compiled_triton_op(algorithm): + a = torch.randn((128, 64), device="cuda", dtype=torch.bfloat16) + b = torch.randn((96, 64), device="cuda", dtype=torch.bfloat16).T + compiled = torch.compile(lambda x, y: _test_mm(x, y, algorithm), fullgraph=True) + torch.testing.assert_close(compiled(a, b), _test_mm(a, b, algorithm), rtol=0, atol=0) + + +@cuda_window +@pytest.mark.parametrize("layouts", [(False, False), (False, True), (True, False), (True, True)]) +def test_cuda_tma_layouts_with_masked_output_tiles(layouts): + m, n, k = 96, 112, 128 + a = torch.randn((k, m) if layouts[0] else (m, k), device="cuda", dtype=torch.bfloat16) + b = torch.randn((n, k) if layouts[1] else (k, n), device="cuda", dtype=torch.bfloat16) + a, b = a.T if layouts[0] else a, b.T if layouts[1] else b + expected = torch.mm(a, b) + actual = launch_mm(a, b, "persistent_tma") + difference = actual.float() - expected.float() + assert torch.isfinite(actual).all() + assert difference.square().mean().sqrt() / expected.float().square().mean().sqrt() < 0.003 diff --git a/test_posttrain.py b/test_posttrain.py new file mode 100644 index 0000000000000000000000000000000000000000..90283bdaccc113503a0ac2aeab1d7853a0e7ecf8 --- /dev/null +++ b/test_posttrain.py @@ -0,0 +1,238 @@ +"""CPU correctness tests; no GPU allocation or production state is involved.""" + +import copy +import json +from pathlib import Path + +import pytest +import torch + +import posttrain +from ops.posttrain_data import encode_messages, renderer_sha256 +from ops.posttrain_preflight import make_fixture +from warpstate import WarpStateConfig, WarpStateLM + + +class CharacterTokenizer: + def eos_id(self): + return 2 + + def bos_id(self): + return 1 + + def vocab_size(self): + return 256 + + def encode(self, text, out_type=int): + return [ord(char) for char in text] + + +def messages(answer='four'): + return [{'role': 'system', 'content': 'Be concise.'}, + {'role': 'user', 'content': 'Two plus two?'}, + {'role': 'assistant', 'content': answer}] + + +def test_assistant_content_and_real_eos_only(): + rows = messages() + [{'role': 'tool', 'content': 'untrusted result'}, + {'role': 'assistant', 'content': 'done'}] + encoded = encode_messages(rows, CharacterTokenizer()) + targets = [value for value in encoded['labels'] if value != -100] + assert targets == [ord(c) for c in 'four'] + [2] + [ord(c) for c in 'done'] + [2] + assert encoded['input_ids'][0] == 1 + assert len(encoded['input_ids']) == len(encoded['labels']) + with pytest.raises(ValueError, match='exceeds'): + encode_messages(rows, CharacterTokenizer(), max_length=len(encoded['input_ids']) - 1) + assert encode_messages(rows, CharacterTokenizer(), max_length=len(encoded['input_ids'])) == encoded + + +@pytest.mark.parametrize('bad', [[], [{'role': 'user', 'content': 'no answer'}], + [{'role': 'unknown', 'content': 'text'}], + [{'role': 'assistant', 'content': ''}], + [{'role': 'assistant', 'content': 'call', 'tool_calls': [{'name': 'x'}]}]]) +def test_invalid_conversations_rejected(bad): + with pytest.raises(ValueError): + encode_messages(bad, CharacterTokenizer()) + + +def test_collation_shifts_and_masks_independent_padding(): + a = {'input_ids': [1, 4, 5, 2], 'labels': [-100, -100, 5, 2]} + x, y = posttrain.collate([a, None], 8, 0) + assert x.tolist() == [[1, 4, 5, 0, 0, 0, 0, 0], [0] * 8] + assert y.tolist() == [[-100, 5, 2, -100, -100, -100, -100, -100], [-100] * 8] + + +def test_sample_order_resume_and_tail_do_not_duplicate(): + cursor, groups = 0, [] + for _ in range(6): + values, cursor = posttrain.update_indices(5, 17, cursor, 2, 2) + groups.append(values) + assert sorted(sum(groups[:3], [])) == list(range(5)) + assert sorted(sum(groups[3:], [])) == list(range(5)) + assert [len(v) for v in groups] == [2, 2, 1, 2, 2, 1] + assert posttrain.update_indices(5, 17, 4, 2, 2)[0] == groups[2] + assert posttrain.update_indices(5, 17, cursor, 2, 2) == ([], 10) + + +def test_ddp_token_weighting_matches_global_objective_with_empty_rank(): + # Simulate DDP's gradient average with unequal rank/microbatch target counts. + parameter = torch.tensor(0.7, dtype=torch.float64, requires_grad=True) + partitions = [[torch.tensor([1., 2., 4.]), torch.tensor([7.])], + [torch.tensor([]), torch.tensor([3., 8.])]] + gradients = [] + for rank in partitions: + parameter.grad = None + for target in rank: + target = target.double() + mean = ((parameter - target) ** 2).sum() / max(1, len(target)) + posttrain.token_weighted_loss(mean, len(target), 6, 2).backward() + gradients.append(parameter.grad.clone()) + parameter.grad = None + target = torch.tensor([1., 2., 4., 7., 3., 8.], dtype=torch.float64) + ((parameter - target) ** 2).mean().backward() + torch.testing.assert_close(sum(gradients) / 2, parameter.grad, rtol=1e-14, atol=1e-14) + + +def assert_tree_equal(a, b): + if isinstance(a, torch.Tensor): + assert torch.equal(a, b) + elif isinstance(a, dict): + assert a.keys() == b.keys() + for key in a: + assert_tree_equal(a[key], b[key]) + elif isinstance(a, (list, tuple)): + assert len(a) == len(b) + for x, y in zip(a, b): + assert_tree_equal(x, y) + else: + assert a == b + + +def test_real_bf16_training_exact_resume_and_frozen_base(tmp_path): + args = make_fixture(tmp_path) + original_hash = posttrain.digest(args.weights) + original = torch.load(args.weights, weights_only=False, mmap=True) + try: + posttrain.train(args) + uninterrupted = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) + args.work_dir = tmp_path / 'resumed' + args.max_steps = 1 + posttrain.train(args) + partial = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) + assert partial['step'] == 1 and partial['completed'] is False + assert not (args.work_dir / 'completion.json').exists() + args.max_steps = None + posttrain.train(args) + resumed = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) + assert resumed['step'] == 3 and resumed['sample_cursor'] == 5 + assert resumed['completed'] is True + for key in ('model', 'optimizer', 'sample_cursor', 'tokens_seen', 'assistant_tokens_seen', 'val_loss'): + assert_tree_equal(uninterrupted[key], resumed[key]) + assert any(not torch.equal(original['model'][k], v) for k, v in resumed['model'].items()) + assert all(v.dtype == torch.float32 for v in resumed['model'].values()) + assert resumed['origin']['step'] == 76294 + report = json.loads((args.work_dir / 'evaluation.json').read_text()) + assert report['step'] == 3 and report['evaluated_samples'] == 2 + assert report['objective'] == 'assistant-token-mean' + completion = json.loads((args.work_dir / 'completion.json').read_text()) + assert completion['completed'] is True and completion['paused_for_evaluation'] is True + assert completion['step'] == resumed['step'] + assert completion['val_loss'] == resumed['val_loss'] + assert torch.equal(uninterrupted['rng_by_rank'][0]['torch'], resumed['rng_by_rank'][0]['torch']) + assert posttrain.digest(args.weights) == original_hash + bad = copy.deepcopy(resumed['signature']) + bad['world_size'] = 2 + with pytest.raises(ValueError, match='signature'): + posttrain.validate_resume(resumed, bad, 2) + finally: + torch.use_deterministic_algorithms(False) + + +def test_ready_manifest_and_per_record_counts_are_enforced(tmp_path): + import sentencepiece as spm + args = make_fixture(tmp_path) + sp = spm.SentencePieceProcessor(model_file=str(args.tokenizer)) + path = args.data_dir / 'manifest.json' + manifest = json.loads(path.read_text()) + manifest['complete'] = False + path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match='not complete'): + posttrain.load_data(args.data_dir, sp, posttrain.digest(args.tokenizer), 128) + manifest['complete'] = True + manifest['renderer_sha256'] = 'wrong' + path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match='renderer'): + posttrain.load_data(args.data_dir, sp, posttrain.digest(args.tokenizer), 128) + + +def test_inputs_cannot_be_training_outputs(tmp_path): + args = make_fixture(tmp_path) + for output in (args.data_dir, args.weights.parent / 'run', tmp_path): + with pytest.raises(ValueError, match='separate|inside'): + posttrain.validate_paths(output, args.data_dir, args.weights, args.tokenizer, args.config) + + +def test_wrong_base_token_boundary_is_rejected(tmp_path): + args = make_fixture(tmp_path) + plan = json.loads(args.config.read_text()) + plan['base']['tokens_seen'] = 999 + args.config.write_text(json.dumps(plan)) + try: + with pytest.raises(ValueError, match='tokens_seen differs'): + posttrain.train(args) + finally: + torch.use_deterministic_algorithms(False) + + +@pytest.mark.parametrize('bad', [{'stage': 'dpo'}, {'training': {'min_lr': -1}}, + {'training': {'warmup_ratio': 1}}, {'training': {'lr': 0}}, + {'training': {'bogus': True}}, {'training': {'micro_batch': True}}]) +def test_plan_rejects_unsupported_settings(tmp_path, bad): + plan = dict(format='warpstate-posttrain-plan-v1', stage='sft') | bad + path = tmp_path / 'plan.json' + path.write_text(json.dumps(plan)) + with pytest.raises(ValueError): + posttrain.load_plan(path) + + +def test_schedule_is_fixed_by_plan_not_pause_boundary(): + cfg = posttrain.DEFAULTS | dict(warmup_ratio=.1) + assert posttrain.learning_rate(0, 100, cfg) == pytest.approx(cfg['lr'] / 10) + assert posttrain.learning_rate(9, 100, cfg) == pytest.approx(cfg['lr']) + assert posttrain.learning_rate(99, 100, cfg) == 0 + + +def test_preflight_fixture_identity_and_command_are_cpu_only(tmp_path): + from ops.posttrain_preflight import training_command, compare_tree + fixture = make_fixture(tmp_path, device='cuda', compile_core=True) + repeated = make_fixture(tmp_path, device='cuda', compile_core=True) + assert repeated.weights == fixture.weights + cfg = torch.load(fixture.weights, weights_only=False, mmap=True)['config'] + assert cfg['d_model'] == 32 and cfg['n_heads'] == 4 + command = training_command(fixture, 2, tmp_path / 'split', tmp_path / 'stop', 1) + assert command[-2:] == ['--max-steps', '1'] + assert '--device' in command and command[command.index('--device') + 1] == 'cuda' + with pytest.raises(ValueError, match='two ranks'): + training_command(fixture, 1, tmp_path / 'split', tmp_path / 'stop') + with pytest.raises(ValueError, match='settings/source differ'): + make_fixture(tmp_path, device='cpu', compile_core=True) + a = torch.tensor([1.], dtype=torch.float32) + b = a + 1e-6 + report = compare_tree(a, b, tolerant=True) + assert report['bitwise_equal'] is False and report['max_absolute_difference'] > 0 + with pytest.raises(ValueError, match='tolerance'): + compare_tree(a, a + 1e-3, tolerant=True) + with pytest.raises(ValueError, match='tolerance'): + compare_tree(a, b) + + +def test_evaluation_full_holdout_exceeds_intermediate_subset(): + class ConstantLoss(torch.nn.Module): + def forward(self, x, y, return_logits=False): + return None, torch.tensor(2.) + row = dict(input_ids=[1, 3, 4, 2], labels=[-100, -100, 4, 2]) + settings = posttrain.DEFAULTS | dict(eval_batches=1, micro_batch=1, sequence_length=8) + model = ConstantLoss() + assert posttrain.evaluate(model, [row] * 3, settings, torch.device('cpu'), 0) == (2., 2) + assert posttrain.evaluate(model, [row] * 3, settings, torch.device('cpu'), 0, full=True) == (2., 6) + assert model.training diff --git a/test_posttrain_chat.py b/test_posttrain_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..508e427b4954688b4a6b207479ecaf2fdcffe538 --- /dev/null +++ b/test_posttrain_chat.py @@ -0,0 +1,205 @@ +"""CPU tests for posttraining prompt serialization and cached inference math.""" + +import copy +import json + +import pytest +import torch + +from ops import posttrain_chat as chat +from ops.posttrain_data import encode_messages +from ops.residual_experiment import apply_bf16_residuals +from warpstate import WarpStateConfig, WarpStateLM + + +class SegmentTokenizer: + def vocab_size(self): + return 256 + + def eos_id(self): + return 2 + + def bos_id(self): + return 1 + + def encode(self, text, out_type=int): + # Segment marker makes concat-then-tokenize observably different. + return [3] + [ord(character) + 10 for character in text] + + def decode(self, tokens): + return ','.join(str(value) for value in tokens) + + +def conversation(): + return [{'role': 'system', 'content': 'Be concise.'}, + {'role': 'user', 'content': 'Say hello.'}, + {'role': 'assistant', 'content': 'Hello.'}, + {'role': 'user', 'content': 'Two plus two?'}] + + +@pytest.mark.parametrize('original_tokenizer', [False, True]) +def test_prompt_is_exact_training_prefix_with_real_eos_only_for_history(original_tokenizer): + if original_tokenizer: + import sentencepiece as spm + tokenizer = spm.SentencePieceProcessor(model_file=str(chat.PACKAGE / 'tokenizer.model')) + else: + tokenizer = SegmentTokenizer() + messages = conversation() + prompt = chat.encode_prompt(messages, tokenizer) + encoded = encode_messages(messages + [{'role': 'assistant', 'content': 'Four.'}], tokenizer) + answer_tokens = tokenizer.encode('Four.', out_type=int) + assert encoded['input_ids'][:len(prompt)] == prompt + assert encoded['input_ids'][len(prompt):len(prompt) + len(answer_tokens)] == answer_tokens + assert encoded['labels'][len(prompt)] != -100 + assert prompt[0] == tokenizer.bos_id() + assert prompt.count(tokenizer.eos_id()) == 1 + assert prompt[-len(tokenizer.encode('<|assistant|>\n', out_type=int)):] == tokenizer.encode('<|assistant|>\n', out_type=int) + + +@pytest.mark.parametrize('messages', [[], ['bad'], [{'role': 'assistant', 'content': 'bad'}], + [{'role': 'user', 'content': ''}], [{'role': 'user', 'content': 'call', 'tool_calls': [1]}]]) +def test_prompt_rejects_invalid_conversations(messages): + with pytest.raises(ValueError): + chat.encode_prompt(messages, SegmentTokenizer()) + + +def tiny_model(parallel=True): + with torch.random.fork_rng(devices=[]): + torch.manual_seed(23) + cfg = WarpStateConfig(vocab_size=64, d_model=32, n_heads=4, n_cores=2, + logical_depth=3, ffn_hidden=64, chunk_size=4, dropout=0.0, + parallel_memory=parallel, gradient_checkpointing=False) + model = WarpStateLM(cfg).eval().requires_grad_(False) + model.embed.weight.normal_(std=0.2) + model.depth_scale.normal_(std=0.05) + model.depth_bias.normal_(std=0.01) + for core in model.cores: + core.in_proj.weight.normal_(std=0.15) + core.out_proj.weight.normal_(std=0.12) + return model + + +@pytest.mark.parametrize('parallel', [False, True]) +@pytest.mark.parametrize('dtype', [torch.float32, torch.bfloat16]) +def test_cached_logits_match_full_forward_across_seven_chunks(parallel, dtype): + torch.set_num_threads(1) + model = tiny_model(parallel) + original = {name: parameter.clone() for name, parameter in model.named_parameters()} + full = copy.deepcopy(model) + apply_bf16_residuals(full) + decoder = chat.CachedDecoder(model, dtype=dtype) + tokens = torch.arange(28).reshape(1, -1) % model.cfg.vocab_size + with torch.inference_mode(), torch.autocast('cpu', dtype=torch.bfloat16, + enabled=dtype == torch.bfloat16, cache_enabled=False): + expected = full(tokens) + caches = decoder.new_caches() + actual = torch.cat([decoder.step(tokens[:, index:index + 1], caches) + for index in range(tokens.shape[1])], dim=1) + # Shape-dependent GEMM/SDPA rounding is expected in BF16. FP32 verifies the + # recurrence itself tightly, including shared cores with separate caches. + tolerance = dict(rtol=0.025, atol=0.025) if dtype == torch.bfloat16 else dict(rtol=1e-5, atol=2e-6) + torch.testing.assert_close(actual, expected, **tolerance) + assert all(cache.position == 0 and cache.fast.dtype == dtype for cache in caches) + assert all(len(cache.writes) == (7 if parallel else 0) for cache in caches) + for name, parameter in model.named_parameters(): + assert parameter.dtype == torch.float32 and torch.equal(parameter, original[name]) + + +class FakeDecoder: + device = torch.device('cpu') + + def __init__(self, prefix_length, sequence): + self.prefix_length, self.sequence, self.calls = prefix_length, sequence, 0 + + def new_caches(self): + return [] + + def step(self, token, caches): + self.calls += 1 + index = max(0, self.calls - self.prefix_length) + value = self.sequence[min(index, len(self.sequence) - 1)] + logits = torch.full((1, 1, 256), -100.) + logits[0, 0, value] = 100. + return logits + + +def test_generation_stops_on_eos_and_excludes_eos_from_response(): + tokenizer = SegmentTokenizer() + messages = [{'role': 'user', 'content': 'Question'}] + prefix = chat.encode_prompt(messages, tokenizer) + decoder = FakeDecoder(len(prefix), [7, 8, tokenizer.eos_id()]) + emitted = [] + result = chat.generate(decoder, tokenizer, messages, max_new_tokens=10, + on_token=lambda tokens: emitted.append((tokens, decoder.calls))) + assert result['token_ids'] == [7, 8] and result['response'] == '7,8' + assert result['generated_tokens'] == 2 and result['sampled_tokens'] == 3 + assert result['stop_reason'] == 'eos' + assert decoder.calls == len(prefix) + 2 + assert emitted == [((7,), len(prefix)), ((7, 8), len(prefix) + 1)] + assert result['elapsed_seconds'] >= result['first_token_seconds'] >= 0 + + +def test_generation_preserves_prompt_and_respects_context_limit(): + tokenizer = SegmentTokenizer() + messages = [{'role': 'user', 'content': 'Question'}] + prefix = chat.encode_prompt(messages, tokenizer) + decoder = FakeDecoder(len(prefix), [7]) + result = chat.generate(decoder, tokenizer, messages, max_new_tokens=10, max_context=len(prefix) + 2) + assert result['prompt_tokens'] == len(prefix) and result['generated_tokens'] == 2 + assert result['stop_reason'] == 'context_limit' + with pytest.raises(ValueError, match='context limit'): + chat.generate(decoder, tokenizer, messages, max_context=len(prefix)) + + +def test_suite_preserves_ids_and_ignores_rubric_fields(tmp_path): + path = tmp_path / 'prompts.json' + rows = [{'id': 'math', 'prompt': 'Two plus two?', 'expected_exact': '4', 'rubric': 'Correct arithmetic'}] + path.write_text(json.dumps(rows)) + assert chat.read_prompts(path) == rows + path.write_text(json.dumps({'prompts': rows})) + assert chat.read_prompts(path) == rows + path.write_text(json.dumps(rows * 2)) + with pytest.raises(ValueError, match='distinct'): + chat.read_prompts(path) + + +def test_ctrl_c_cancels_typing_and_generation_preserving_completed_history(monkeypatch, capsys): + inputs = iter(['first', KeyboardInterrupt(), 'cancel reply', 'next', '/quit']) + calls = [] + + def read(prompt): + value = next(inputs) + if isinstance(value, BaseException): + raise value + return value + + def reply(messages, *, seed): + calls.append(copy.deepcopy(messages)) + if messages[-1]['content'] == 'cancel reply': + raise KeyboardInterrupt() + return {'response': 'completed'} + + monkeypatch.setattr('builtins.input', read) + assert chat.interactive_chat(reply, lambda prompt: [{'role': 'user', 'content': prompt}], seed=42) == 0 + assert calls[-1] == [ + {'role': 'user', 'content': 'first'}, + {'role': 'assistant', 'content': 'completed'}, + {'role': 'user', 'content': 'next'}] + assert capsys.readouterr().out.count('Cancelled.') == 2 + + +def test_ctrl_c_unwinds_cached_parameter_views_and_allows_next_generation(monkeypatch): + model = tiny_model() + original = dict(model.named_parameters()) + decoder = chat.CachedDecoder(model) + step = chat._core_step + + def interrupt(*args): + raise KeyboardInterrupt() + + monkeypatch.setattr(chat, '_core_step', interrupt) + with pytest.raises(KeyboardInterrupt): + decoder.step(torch.tensor([[4]]), decoder.new_caches()) + assert all(parameter is original[name] for name, parameter in model.named_parameters()) + monkeypatch.setattr(chat, '_core_step', step) + assert torch.isfinite(decoder.step(torch.tensor([[4]]), decoder.new_caches())).all() diff --git a/test_posttrain_handoff.py b/test_posttrain_handoff.py new file mode 100644 index 0000000000000000000000000000000000000000..3f16f1d238e89f0cc4a331dbf11c42e0d3bd31d5 --- /dev/null +++ b/test_posttrain_handoff.py @@ -0,0 +1,202 @@ +"""CPU tests for the unattended base -> immutable archive -> separate SFT gate.""" +import copy +import json +from pathlib import Path +import stat + +import pytest +import torch + +from ops import posttrain_handoff as handoff + + +@pytest.fixture +def fixture(tmp_path): + base_work = tmp_path / 'base' + (base_work / 'run').mkdir(parents=True) + (base_work / 'control').mkdir() + tokenizer = tmp_path / 'tokenizer.model' + tokenizer.write_bytes(b'original required tokenizer fixture') + base = dict(step=76294, tokens_seen=10000007168, tokenizer_sha256=handoff.sha256(tokenizer)) + checkpoint = dict(format='warpstate-server-v1', config=dict(d_model=2), + step=base['step'], tokens_seen=base['tokens_seen'], + val_step=base['step'], val_tokens=base['tokens_seen'], val_loss=2.3, + signature=dict(data=dict(signature=dict(tokenizer_sha256=base['tokenizer_sha256']))), + model={'weights': torch.arange(6, dtype=torch.float32).reshape(2, 3)}, + optimizer={'unmodified': torch.tensor([12.0])}) + source = base_work / 'run/latest.pt' + torch.save(checkpoint, source) + plan = dict(format='warpstate-posttrain-plan-v1', stage='sft', after_completion='pause_for_evaluation', + base=base, handoff=dict(base_work_dir=str(base_work), base_program='warpstate-train', + supervisor_socket=str(base_work / 'control/supervisor.sock'), + archive_dir=str(tmp_path / 'archives/step-00076294'), + work_dir=str(tmp_path / 'posttrain'), data_dir=str(tmp_path / 'data'), + tokenizer=str(tokenizer), processes=2, poll_seconds=1), + training=dict(sequence_length=2048)) + config = tmp_path / 'plan.json' + config.write_text(json.dumps(plan)) + return dict(root=tmp_path, source=source, tokenizer=tokenizer, base=base, checkpoint=checkpoint, + archive=tmp_path / 'archives/step-00076294', plan=plan, config=config) + + +@pytest.mark.parametrize('state,exitcode,expected', [ + ('RUNNING', 0, False), ('STOPPED', 0, False), ('STOPPING', 0, False), + ('EXITED', 1, False), ('FATAL', 0, False), ('EXITED', 0, True)]) +def test_only_clean_supervisor_exit_opens_gate(state, exitcode, expected): + assert handoff.clean_exit(dict(statename=state, exitstatus=exitcode)) is expected + + +@pytest.mark.parametrize('field,value', [('step', 76293), ('step', 76295), + ('tokens_seen', 10000000000), ('val_step', 76200), ('val_tokens', 0), + ('val_loss', float('nan')), ('format', 'warpstate-chat-v1')]) +def test_wrong_or_stale_boundary_rejected(fixture, field, value): + checkpoint = copy.deepcopy(fixture['checkpoint']) + checkpoint[field] = value + with pytest.raises(ValueError): + handoff.validate_boundary(checkpoint, fixture['base'], fixture['base']['tokenizer_sha256']) + + +def test_archive_and_copy_preserve_exact_raw_tensors_and_bytes(fixture): + source_hash = handoff.sha256(fixture['source']) + metadata = handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], + fixture['base'], 'test-revision') + original = torch.load(fixture['source'], weights_only=False, mmap=True) + exported = torch.load(fixture['archive'] / 'model.pt', weights_only=False, mmap=True) + assert torch.equal(original['model']['weights'], exported['model']['weights']) + assert 'optimizer' not in exported and 'rng_by_rank' not in exported + assert handoff.sha256(fixture['source']) == source_hash + hashes = {p.name: handoff.sha256(p) for p in fixture['archive'].iterdir()} + assert all(not stat.S_IMODE(p.stat().st_mode) & 0o222 for p in fixture['archive'].iterdir()) + model = handoff.independent_initialization(fixture['archive'], fixture['root'] / 'init', + fixture['base'], metadata['snapshot_sha256']) + assert handoff.sha256(model) == metadata['snapshot_sha256'] + assert model.stat().st_ino != (fixture['archive'] / 'model.pt').stat().st_ino + # Repeated handoff reuses the original archive/copy without changing bytes. + inode = model.stat().st_ino + assert handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], + fixture['base'], 'different-revision') == metadata + handoff.independent_initialization(fixture['archive'], model.parent, fixture['base'], + metadata['snapshot_sha256']) + assert model.stat().st_ino == inode + assert {p.name: handoff.sha256(p) for p in fixture['archive'].iterdir()} == hashes + + +def test_incomplete_existing_archive_is_not_replaced(fixture): + fixture['archive'].mkdir(parents=True) + marker = fixture['archive'] / 'user-file' + marker.write_text('keep') + with pytest.raises(ValueError, match='checksum'): + handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], + fixture['base'], 'test') + assert marker.read_text() == 'keep' + assert list(fixture['archive'].iterdir()) == [marker] + + +def test_publishing_does_not_replace_even_an_empty_directory(tmp_path): + staging, target = tmp_path / 'staging', tmp_path / 'target' + staging.mkdir() + target.mkdir() + (staging / 'payload').write_text('complete') + inode = target.stat().st_ino + with pytest.raises(FileExistsError): + handoff.publish_directory(staging, target) + assert target.stat().st_ino == inode and (staging / 'payload').exists() + + +def prepare_data(fixture): + directory = Path(fixture['plan']['handoff']['data_dir']) + directory.mkdir() + outputs = {} + for name in ('sft_train.jsonl', 'sft_val.jsonl'): + (directory / name).write_text('{}\n') + outputs[name] = dict(rows=1, sha256=handoff.sha256(directory / name)) + manifest = dict(format='warpstate-posttrain-data-v1', complete=True, + tokenizer_sha256=fixture['base']['tokenizer_sha256'], max_length=2048, outputs=outputs) + (directory / 'manifest.json').write_text(json.dumps(manifest)) + return directory + + +def test_data_hashes_must_match_before_launch(fixture): + directory = prepare_data(fixture) + assert handoff.data_readiness(directory, fixture['plan'], verify=True)[0] + (directory / 'sft_train.jsonl').write_text('changed\n') + with pytest.raises(ValueError, match='checksum'): + handoff.data_readiness(directory, fixture['plan'], verify=True) + + +def test_corrupted_raw_tensor_cannot_reuse_existing_archive(fixture): + handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], fixture['base'], 'test') + fixture['checkpoint']['model']['weights'][0, 0] = 99 + torch.save(fixture['checkpoint'], fixture['source']) + with pytest.raises(ValueError, match='tensor'): + handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], fixture['base'], 'test') + + +def test_complete_handoff_waits_archives_and_passes_separate_output_paths(fixture, monkeypatch): + prepare_data(fixture) + monkeypatch.setattr(handoff, 'STOPPED', False) + monkeypatch.setattr(handoff, 'source_identity', lambda config: {'fixture': 'immutable'}) + monkeypatch.setattr(handoff, 'verify_preflight', lambda *args: {'accepted': True}) + statuses = iter([dict(statename='RUNNING', exitstatus=0), dict(statename='EXITED', exitstatus=0)]) + monkeypatch.setattr(handoff, 'process_status', lambda *args: next(statuses, dict(statename='EXITED', exitstatus=0))) + monkeypatch.setattr(handoff.time, 'sleep', lambda seconds: None) + commands = [] + base_work = Path(fixture['plan']['handoff']['base_work_dir']) + def fake_train(command, stop, env, *, pass_fds): + commands.append(command) + assert len(pass_fds) == 2 + with pytest.raises(BlockingIOError): + with handoff.exclusive_lock(base_work / 'control/training-launcher.lock'): + pass + # Released precisely before worker SH acquisition. + with handoff.exclusive_lock(base_work / 'pipeline.lock'): + pass + if 'posttrain_preflight.py' in ' '.join(command): + return 0 + work = Path(command[command.index('--work-dir') + 1]) + weights = Path(command[command.index('--weights') + 1]) + assert not weights.is_relative_to(work) + assert Path(command[command.index('--stop-file') + 1]) == stop + assert work.name == 'run' + work.mkdir(parents=True) + model_hash = handoff.sha256(weights) + completion = dict(format='warpstate-posttrain-completion-v1', stage='sft', + completed=True, paused_for_evaluation=True, step=1, + total_steps=1, base_step=fixture['base']['step']) + checkpoint = dict(format='warpstate-posttrain-v1', completed=True, step=1, total_steps=1, + val_step=1, val_loss=2., signature=dict(plan=fixture['plan'], base_sha256=model_hash), + origin=dict(step=fixture['base']['step'])) + torch.save(checkpoint, work / 'latest.pt') + (work / 'completion.json').write_text(json.dumps(completion)) + (work / 'evaluation.json').write_text(json.dumps(dict(stage='sft', step=1, + base_sha256=model_hash, val_loss=2.))) + return 0 + monkeypatch.setattr(handoff, 'supervise', fake_train) + assert handoff.run_handoff(fixture['config']) == 0 + state = json.loads((fixture['root'] / 'posttrain/control/handoff.json').read_text()) + assert state['phase'] == 'complete' + count = len(commands) + assert handoff.run_handoff(fixture['config']) == 0 + assert len(commands) == count # Never retrain after terminal completion. + + +def test_overshoot_never_archives_or_trains(fixture, monkeypatch): + prepare_data(fixture) + fixture['checkpoint']['step'] = 76295 + torch.save(fixture['checkpoint'], fixture['source']) + monkeypatch.setattr(handoff, 'STOPPED', False) + monkeypatch.setattr(handoff, 'source_identity', lambda config: {'fixture': 'immutable'}) + monkeypatch.setattr(handoff, 'process_status', lambda *args: dict(statename='EXITED', exitstatus=0)) + monkeypatch.setattr(handoff, 'supervise', lambda *a, **kw: pytest.fail('Must not start SFT')) + with pytest.raises(ValueError, match='step must equal'): + handoff.run_handoff(fixture['config']) + assert not fixture['archive'].exists() + + +def test_sft_completion_requires_matching_durable_checkpoint(fixture, tmp_path): + (tmp_path / 'completion.json').write_text(json.dumps(dict(completed=True))) + torch.save(dict(format='warpstate-posttrain-v1', completed=False, step=1, total_steps=2, + val_step=1, val_loss=2.), tmp_path / 'latest.pt') + (tmp_path / 'evaluation.json').write_text('{}') + with pytest.raises(ValueError, match='fully evaluated'): + handoff.verify_completion(tmp_path, fixture['plan'], 'hash') diff --git a/test_posttrain_worker_lock.py b/test_posttrain_worker_lock.py new file mode 100644 index 0000000000000000000000000000000000000000..249f630d321d51947bd776e5c96f9faed8ae2e4e --- /dev/null +++ b/test_posttrain_worker_lock.py @@ -0,0 +1,156 @@ +"""CPU process test: orphan SFT ranks continue excluding base training.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + +import pytest + +from run_40b import process_lock + + +ROOT = Path(__file__).resolve().parent +WORKSPACE = ROOT.parent + + +def _identity(pid): + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return fields[0], int(fields[19]) + except (OSError, ValueError, IndexError): + return None + + +def _wait(predicate, seconds=20): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.02) + raise AssertionError("Timed out waiting for the CPU lock-test process") + + +def _kill(identity): + pid, created = identity + current = _identity(pid) + if current is not None and current[0] != "Z" and current[1] == created: + os.kill(pid, signal.SIGKILL) + + +def _dead(identity): + current = _identity(identity[0]) + return current is None or current[0] == "Z" or current[1] != identity[1] + + +def _exclusive_available(path): + try: + with process_lock(path): + return True + except BlockingIOError: + return False + + +def test_detached_shared_ranks_exclude_base_after_both_ancestors_die(): + """Exercise the real worker helper and base lock across a fatal orphan tree. + + Fake torchrun deliberately closes inherited descriptors and starts workers + in separate sessions, matching the installed elastic subprocess launcher. + Either surviving rank must independently prevent an exclusive base/handoff + lock, even after the guardian and torchrun process have both been SIGKILLed. + """ + temporary_root = WORKSPACE / "tmp" + temporary_root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="posttrain-worker-lock-", dir=temporary_root) as folder: + directory = Path(folder) + pipeline = directory / "pipeline.lock" + pipeline.write_text("Existing lock inode must be preserved.\n") + initial_inode = pipeline.stat().st_ino + initial_bytes = pipeline.read_bytes() + helper = directory / "tree.py" + helper.write_text( + "import json, os, pathlib, subprocess, sys, time\n" + "mode, root = sys.argv[1], pathlib.Path(sys.argv[2])\n" + "def ready(name):\n" + " fields = pathlib.Path('/proc/self/stat').read_text().rsplit(')', 1)[1].split()\n" + " target = root / (name + '.json')\n" + " temporary = target.with_suffix('.tmp')\n" + " temporary.write_text(json.dumps([os.getpid(), int(fields[19])]))\n" + " temporary.replace(target)\n" + "if mode == 'guardian':\n" + " child = subprocess.Popen([sys.executable, __file__, 'agent', str(root)],\n" + " start_new_session=True, close_fds=True)\n" + " ready('guardian')\n" + " sys.exit(child.wait())\n" + "elif mode == 'agent':\n" + " ready('agent')\n" + " children = [subprocess.Popen([sys.executable, __file__, 'rank' + str(i), str(root)],\n" + " start_new_session=True, close_fds=True) for i in range(2)]\n" + " sys.exit(max(child.wait() for child in children))\n" + "else:\n" + " from posttrain import shared_pipeline_lock\n" + " with shared_pipeline_lock(root / 'pipeline.lock'):\n" + " ready(mode)\n" + " while True: time.sleep(0.05)\n" + ) + environment = os.environ.copy() + environment.update(PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE="1", + CUDA_VISIBLE_DEVICES="", TMPDIR=str(temporary_root)) + identities = {} + with (directory / "process.log").open("w+") as log: + guardian = subprocess.Popen([sys.executable, str(helper), "guardian", str(directory)], + env=environment, stdout=log, stderr=log) + try: + _wait(lambda: all((directory / f"{name}.json").exists() + for name in ("guardian", "agent", "rank0", "rank1"))) + identities = {name: json.loads((directory / f"{name}.json").read_text()) + for name in ("guardian", "agent", "rank0", "rank1")} + assert len({os.getsid(value[0]) for value in identities.values()}) == 4 + with pytest.raises(BlockingIOError), process_lock(pipeline): + pass + guardian.kill() + guardian.wait(timeout=5) + assert not _dead(identities["agent"]) + with pytest.raises(BlockingIOError), process_lock(pipeline): + pass + _kill(identities["agent"]) + _wait(lambda: _dead(identities["agent"])) + assert all(not _dead(identities[name]) for name in ("rank0", "rank1")) + with pytest.raises(BlockingIOError), process_lock(pipeline): + pass + _kill(identities["rank0"]) + _wait(lambda: _dead(identities["rank0"])) + assert not _dead(identities["rank1"]) + with pytest.raises(BlockingIOError), process_lock(pipeline): + pass + _kill(identities["rank1"]) + _wait(lambda: _dead(identities["rank1"])) + # A killed process's main task may be a zombie while other + # threads are still closing the shared descriptor table. + _wait(lambda: _exclusive_available(pipeline), seconds=5) + with process_lock(pipeline): + assert pipeline.stat().st_ino == initial_inode + assert pipeline.read_bytes() == initial_bytes + except Exception as error: + log.flush() + log.seek(0) + error.add_note(log.read()) + raise + finally: + # Recover ready identities even if setup failed midway. Creation + # times ensure cleanup never signals an unrelated reused PID. + for name in ("guardian", "agent", "rank0", "rank1"): + ready = directory / f"{name}.json" + if ready.exists(): + identities[name] = json.loads(ready.read_text()) + for identity in identities.values(): + _kill(identity) + if guardian.poll() is None: + guardian.kill() + guardian.wait(timeout=5) + for identity in identities.values(): + _wait(lambda identity=identity: _dead(identity), seconds=5) diff --git a/test_residual_execution.py b/test_residual_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..abafb2a4ec1db13c9aeff9dc140465d0da208ecd --- /dev/null +++ b/test_residual_execution.py @@ -0,0 +1,124 @@ +"""BF16 residual execution keeps FP32 masters and changes only compute rounding.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest +import torch +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel + +from ops.residual_experiment import apply_bf16_residuals +from run_40b import compile_training_cores +from warpstate import WarpStateConfig, WarpStateLM + + +def _model(): + torch.manual_seed(685) + config = WarpStateConfig(vocab_size=128, d_model=64, n_heads=4, + n_cores=1, logical_depth=2, ffn_hidden=112, + chunk_size=4, parallel_memory=True, + gradient_checkpointing=False, loss_chunk_size=8) + return WarpStateLM(config).train() + + +@pytest.mark.parametrize("compile_cores", [False, True]) +def test_bf16_residual_numerics_and_fp32_state(compile_cores, tmp_path, monkeypatch): + # CPU eager backend checks Dynamo capture and functional parameter swapping; + # production CUDA Inductor speed and numerics require separate GPU screening. + if compile_cores: + actual_compile = torch.compile + monkeypatch.setattr(torch, "compile", lambda fn, **kw: actual_compile(fn, backend="eager", **kw)) + reference = _model() + experiment = copy.deepcopy(reference) + parameters = tuple(experiment.parameters()) + keys = tuple(experiment.state_dict()) + optimizer = torch.optim.AdamW(experiment.parameters(), lr=1e-4) + apply_bf16_residuals(experiment) + if compile_cores: + compile_training_cores(reference) + compile_training_cores(experiment) + inputs = torch.randint(128, (2, 16)) + targets = torch.randint(128, inputs.shape) + targets[:, -3:] = -100 + residual_dtypes = [] + hook = experiment.cores[0].register_forward_pre_hook(lambda _module, args: residual_dtypes.append(args[0].dtype)) + with torch.autocast("cpu", dtype=torch.bfloat16): + _, ref_loss = reference(inputs, targets, return_logits=False) + _, loss = experiment(inputs, targets, return_logits=False) + hook.remove() + assert residual_dtypes == [torch.bfloat16] * experiment.cfg.logical_depth + ref_loss.backward() + loss.backward() + assert tuple(experiment.state_dict()) == keys + assert all(a is b for a, b in zip(parameters, experiment.parameters(), strict=True)) + assert all(parameter.dtype == parameter.grad.dtype == torch.float32 for parameter in parameters) + assert all(torch.isfinite(parameter.grad).all() for parameter in parameters) + actual_gradients = torch.cat([parameter.grad.flatten() for parameter in parameters]) + reference_gradients = torch.cat([parameter.grad.flatten() for parameter in reference.parameters()]) + cosine = float(torch.nn.functional.cosine_similarity(actual_gradients, reference_gradients, dim=0)) + norm_ratio = float(actual_gradients.norm() / reference_gradients.norm()) + relative_loss_difference = float((loss.detach() - ref_loss.detach()).abs() / ref_loss.detach().abs()) + report = { + "compiled_cores": compile_cores, "reference_loss": float(ref_loss.detach()), + "loss": float(loss.detach()), "relative_loss_difference": relative_loss_difference, + "gradient_cosine": cosine, "gradient_norm_ratio": norm_ratio, + } + print("BF16_RESIDUAL_NUMERICS " + json.dumps(report, sort_keys=True), flush=True) + (tmp_path / "bf16-residual-numerics.json").write_text(json.dumps(report, indent=2) + "\n") + assert relative_loss_difference <= 0.01, report + assert cosine >= 0.99, report + assert 0.95 <= norm_ratio <= 1.05, report + optimizer.step() + for parameter in parameters: + assert parameter.dtype == torch.float32 + assert optimizer.state[parameter]["exp_avg"].dtype == torch.float32 + assert optimizer.state[parameter]["exp_avg_sq"].dtype == torch.float32 + restored = _model() + restored.load_state_dict(experiment.state_dict(), strict=True) + assert tuple(restored.state_dict()) == keys + for name, expected in experiment.state_dict().items(): + torch.testing.assert_close(restored.state_dict()[name], expected, rtol=0, atol=0) + + +def test_bf16_residual_restores_parameters_on_error_and_preserves_non_autocast(): + model = _model() + reference = copy.deepcopy(model) + original_parameters = tuple(model.parameters()) + apply_bf16_residuals(model) + inputs = torch.randint(128, (1, 16)) + torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0) + with pytest.raises(ValueError, match="must be divisible"): + with torch.autocast("cpu", dtype=torch.bfloat16): + model(inputs[:, :-1]) + assert all(a is b for a, b in zip(original_parameters, model.parameters(), strict=True)) + assert all(parameter.dtype == torch.float32 for parameter in model.parameters()) + with pytest.raises(ValueError, match="already been applied"): + apply_bf16_residuals(model) + + +def test_bf16_residual_ddp_reducer_owns_fp32_master_gradients(tmp_path): + if not dist.is_available() or not dist.is_gloo_available(): + pytest.skip("requires the CPU Gloo distributed backend") + if dist.is_initialized(): + pytest.skip("requires ownership of an isolated default process group") + assert tmp_path.resolve().is_relative_to(Path(__file__).resolve().parents[1]) + dist.init_process_group("gloo", init_method=(tmp_path / "ddp-init").as_uri(), rank=0, world_size=1) + try: + model = _model() + masters = tuple(model.parameters()) + apply_bf16_residuals(model) + distributed = DistributedDataParallel(model) + inputs = torch.randint(128, (2, 16)) + for _ in range(2): # Second forward detects unfinished reducer work. + distributed.zero_grad(set_to_none=True) + with torch.autocast("cpu", dtype=torch.bfloat16): + _, loss = distributed(inputs, inputs, return_logits=False) + loss.backward() + assert all(parameter.grad is not None and parameter.grad.dtype == torch.float32 for parameter in masters) + assert all(a is b for a, b in zip(masters, model.parameters(), strict=True)) + finally: + dist.destroy_process_group() diff --git a/test_server.py b/test_server.py new file mode 100644 index 0000000000000000000000000000000000000000..ec89c8ea965c58cf7e0c27d033a95dd1874e4475 --- /dev/null +++ b/test_server.py @@ -0,0 +1,125 @@ +"""Run with pytest. CPU preparation tests plus a small CUDA full-state resume test.""" +import importlib.util +import json +from pathlib import Path +import pickle +import subprocess +import sys +import types +import numpy as np +import pytest +import torch + +ROOT = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('server_pipeline', ROOT/'run_40b.py') +pipeline = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pipeline) + + +def args(tmp_path): + return types.SimpleNamespace(work_dir=tmp_path, tokenizer=ROOT/'tokenizer.model', + dataset='fixture',dataset_config='fixture',revision='main',train_tokens=300,val_tokens=100, + workers=2,documents_per_batch=7,test_small_gpu=True,seq_len=8) + + +def test_masked_final_batch_and_wrap(): + data = np.arange(19,dtype=np.uint16) + x,y = pipeline.batches(data,17,2,4,5,torch.device('cpu')) + assert x.flatten().tolist() == [17,18,0,1,2,3,4,5] + assert y.flatten().tolist() == [18,0,1,2,3,-100,-100,-100] + + +def test_preparation_resume_truncates_uncommitted_tail(tmp_path,monkeypatch): + a = args(tmp_path) + holdout = next('document '+str(i) for i in range(100000) if pipeline.is_holdout('document '+str(i))) + normal = next('other '+str(i) for i in range(100) if not pipeline.is_holdout('other '+str(i))) + documents = [{'text':holdout if i%3==0 else normal} for i in range(500)] + class Source: + def __init__(self): self.offset=0 + def __iter__(self): + while self.offset < len(documents): + row=documents[self.offset] + self.offset+=1 + yield row + def state_dict(self): return {'offset':self.offset} + def load_state_dict(self,state): self.offset=state['offset'] + monkeypatch.setitem(sys.modules,'datasets',types.SimpleNamespace(__version__='fixture',load_dataset=lambda *a,**kw:Source())) + monkeypatch.setitem(sys.modules,'huggingface_hub',types.SimpleNamespace(HfApi=lambda:types.SimpleNamespace(dataset_info=lambda *a,**kw:types.SimpleNamespace(sha='fixed-commit')))) + pipeline.STOP=True + assert not pipeline.prepare(a) + with (tmp_path/'data/prepare-state.pkl').open('rb') as f: + state=pickle.load(f) + with (tmp_path/'data/train.bin').open('ab') as f: f.write(b'uncommitted-tail') + pipeline.STOP=False + assert pipeline.prepare(a) + pipeline.check_manifest(a) + other=tmp_path/'continuous' + b=args(other) + assert pipeline.prepare(b) + for name in ('train.bin','val.bin'): + assert (tmp_path/'data'/name).read_bytes() == (other/'data'/name).read_bytes() + assert state['iterator']['offset'] == 7 + a.train_tokens+=1 + with pytest.raises(ValueError): pipeline.check_manifest(a) + + +def test_memory_cap_refuses_insufficient_headroom(monkeypatch): + a = types.SimpleNamespace(vram_limit_gib=33.5, vram_headroom_gib=3, autocast_cache=False) + calls = [] + monkeypatch.setattr(torch.cuda, 'mem_get_info', lambda device: (36*1024**3, 96*1024**3)) + monkeypatch.setattr(torch.cuda, 'set_per_process_memory_fraction', lambda *args: calls.append(args)) + with pytest.raises(RuntimeError, match='requested allocator cap and headroom'): + pipeline.configure_memory(a) + assert not calls + monkeypatch.setattr(torch.cuda, 'mem_get_info', lambda device: (37*1024**3, 96*1024**3)) + pipeline.configure_memory(a) + assert calls == [(33.5/96, torch.device('cuda:0'))] + + +@pytest.mark.skipif(not torch.cuda.is_available(),reason='CUDA required') +@pytest.mark.parametrize('initialization', ['weights', 'fresh']) +def test_gpu_full_state_resume_and_exact_partial_update(tmp_path, initialization): + sys.path.insert(0,str(ROOT)) + from warpstate import WarpStateConfig,WarpStateLM + cfg=WarpStateConfig(vocab_size=32768,d_model=32,n_heads=2,n_cores=2,logical_depth=4, + ffn_hidden=64,chunk_size=8,parallel_memory=True) + config_path=tmp_path/'config.json' + cfg.to_json(config_path) + torch.manual_seed(133) + original=WarpStateLM(cfg) + weights=tmp_path/'initial.pt' + torch.save({'config':vars(cfg),'model':original.state_dict(),'tokens_seen':100},weights) + dirs=[tmp_path/'continuous',tmp_path/'resumed'] + common=[sys.executable,str(ROOT/'run_40b.py'),'--no-bootstrap','--test-small-gpu', + '--stage','train','--config',str(config_path), + '--train-tokens','73','--val-tokens','256','--seq-len','8','--micro-batch','2', + '--grad-accum','2','--loss-chunk-size','5','--workers','2','--eval-batches','2','--eval-every','1', + '--no-autocast-cache','--vram-limit-gib','0.5','--vram-headroom-gib','1'] + common += ['--weights',str(weights)] if initialization == 'weights' else ['--from-scratch'] + for directory in dirs: + (directory/'data').mkdir(parents=True) + np.arange(73,dtype=' 1: + command += ["-m", "torch.distributed.run", "--standalone", "--nnodes", "1", + "--nproc-per-node", str(world_size)] + command += common + ["--work-dir", str(directory), "--grad-accum", + str(2 // world_size), *extra] + environment = os.environ.copy() + for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", + "GROUP_RANK", "ROLE_RANK", "ROLE_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT"): + environment.pop(key, None) + result = subprocess.run(command, capture_output=True, text=True, env=environment, timeout=timeout) + assert result.returncode == 0, result.stdout + "\n" + result.stderr + if expect_validation_display: + records = {record["step"]: record for record in map( + json.loads, (directory / "run/metrics.jsonl").read_text().splitlines())} + step_lines = [line for line in result.stdout.splitlines() if line.startswith("step ")] + assert step_lines, result.stdout + for line in step_lines: + step = int(line.split()[1].split("/")[0]) + record = records[step] + if record["val_step"] == step: + assert f"val {record['val_loss']:.5f} (step {record['val_step']})" in line, line + else: + assert " | val " not in line, line + if expect_async: + assert "Staging full CPU checkpoint" in result.stdout, result.stdout + assert "for asynchronous checkpoint" in result.stdout, result.stdout + assert "background write" in result.stdout, result.stdout + runtime = json.loads((directory / "run/settings.json").read_text())["runtime"] + assert runtime["async_checkpoint"] is True + checkpoint = torch.load(directory / "run/latest.pt", map_location="cpu", weights_only=False) + if expect_async: + # The final full-state checkpoint must already be published when the + # trainer exits; a queued write or leftover temporary file is insufficient. + final_progress = f'step {checkpoint["step"]}, tokens={checkpoint["tokens_seen"]:,}; ' + assert final_progress + "background write" in result.stdout, result.stdout + assert not (directory / "run/latest.pt.tmp").exists() + return checkpoint + + +def assert_tensors_equal(first, second): + if isinstance(first, torch.Tensor): + torch.testing.assert_close(first, second, rtol=0, atol=0) + elif isinstance(first, np.ndarray): + np.testing.assert_array_equal(first, second) + elif isinstance(first, dict): + assert first.keys() == second.keys() + for key in first: + assert_tensors_equal(first[key], second[key]) + elif isinstance(first, (tuple, list)): + assert len(first) == len(second) + for value, other in zip(first, second, strict=True): + assert_tensors_equal(value, other) + else: + assert first == second + + +TWO_GPUS = pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Two visible CUDA GPUs required") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize(("compile_mode", "async_checkpoint"), [ + pytest.param("default", False, id="default"), + pytest.param("reduce-overhead", False, id="reduce-overhead"), + pytest.param("max-autotune-no-cudagraphs", False, id="max-autotune-no-cudagraphs"), + pytest.param("max-autotune-no-cudagraphs", True, id="max-autotune-no-cudagraphs-async"), +]) +def test_single_gpu_compiled_full_state_resume(tmp_path, compile_mode, async_checkpoint): + (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) + common += ["--loss-mode", "compiled-full", "--compile-core", + "--compile-mode", compile_mode, "--no-activation-checkpointing", "--autocast-cache"] + if async_checkpoint: + # Exercise overlapping periodic saves, then flush at max-steps and final exit. + common += ["--save-every", "1"] + initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False) + # Allow cold Inductor/autotuning compilation while keeping each subprocess bounded. + # The continuous reference always saves synchronously, including the async case. + expected = run_training(continuous, common, 1, timeout=300) + resumed_common = common + (["--async-checkpoint"] if async_checkpoint else []) + partial = run_training(resumed, resumed_common, 1, ["--max-steps", "1"], + timeout=300, expect_async=async_checkpoint) + assert partial["tokens_seen"] == 32 and partial["step"] == 1 + actual = run_training(resumed, resumed_common, 1, timeout=300, expect_async=async_checkpoint) + + assert actual["format"] == expected["format"] == "warpstate-server-v1" + assert actual["tokens_seen"] == expected["tokens_seen"] == 73 + assert actual["step"] == expected["step"] == 3 + assert actual["origin"] == partial["origin"] == expected["origin"] + assert actual["signature"] == partial["signature"] == expected["signature"] + settings = actual["signature"]["settings"] + assert settings["world_size"] == 1 + assert settings["loss_mode"] == "compiled-full" + assert settings["compile_core"] is True + assert settings["compile_mode"] == compile_mode + assert settings["activation_checkpointing"] is False + assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] == 32 + assert len(actual["rng_by_rank"]) == 1 + for key in ("model", "optimizer", "rng_by_rank"): + assert_tensors_equal(actual[key], expected[key]) + assert actual["model"].keys() == initial["model"].keys() + assert all(not key.startswith(("module.", "_orig_mod.")) for key in actual["model"]) + assert all(value.dtype == torch.float32 and torch.isfinite(value).all() + for value in actual["model"].values()) + assert actual["optimizer"]["state"] + for state in actual["optimizer"]["state"].values(): + assert state["step"].item() == 3 + for key in ("exp_avg", "exp_avg_sq"): + assert state[key].dtype == torch.float32 and torch.isfinite(state[key]).all() + assert math.isfinite(actual["val_loss"]) + assert actual["val_loss"] == expected["val_loss"] + + records = [json.loads(line) for line in (resumed / "run/metrics.jsonl").read_text().splitlines()] + assert [record["step"] for record in records] == [1, 2, 3] + assert [record["tokens"] for record in records] == [32, 64, 73] + assert [record["lr"] for record in records] == [0.001, 0.002, 0.002] + assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"]) + for record in records) + validations = [json.loads(line) for line in (resumed / "run/validation.jsonl").read_text().splitlines()] + assert [record["tokens"] for record in validations] == [32, 64, 73] + assert all(math.isfinite(record["loss"]) for record in validations) + + +@TWO_GPUS +@pytest.mark.parametrize("loss_mode", ["checkpointed", "compiled-full"]) +def test_two_gpu_full_state_resume_and_empty_final_rank(tmp_path, loss_mode): + (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) + common += ["--loss-mode", loss_mode] + if loss_mode == "compiled-full": + common += ["--compile-core", "--compile-mode", "default", "--no-activation-checkpointing"] + expected = run_training(continuous, common, 2) + partial = run_training(resumed, common, 2, ["--max-steps", "1"]) + assert partial["tokens_seen"] == 32 and partial["step"] == 1 + actual = run_training(resumed, common, 2) + assert actual["tokens_seen"] == expected["tokens_seen"] == 73 + assert actual["step"] == expected["step"] == 3 + assert actual["origin"] == expected["origin"] + assert actual["signature"]["settings"]["world_size"] == 2 + assert actual["signature"]["settings"]["loss_mode"] == loss_mode + if loss_mode == "compiled-full": + assert actual["signature"]["settings"]["compile_core"] is True + assert actual["signature"]["settings"]["activation_checkpointing"] is False + assert len(actual["rng_by_rank"]) == 2 + for key in ("model", "optimizer", "rng_by_rank"): + assert_tensors_equal(actual[key], expected[key]) + assert all(not key.startswith(("module.", "_orig_mod.")) for key in actual["model"]) + records = [json.loads(line) for line in (resumed / "run/metrics.jsonl").read_text().splitlines()] + assert [record["tokens"] for record in records] == [32, 64, 73] + assert all(math.isfinite(record["loss"]) for record in records) + + +@TWO_GPUS +def test_two_gpu_bf16_residual_async_full_state_resume_and_empty_final_rank(tmp_path): + (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) + common += ["--loss-mode", "compiled-full", "--compile-core", "--compile-mode", "default", + "--no-activation-checkpointing", "--autocast-cache", "--bf16-residuals", + "--eval-every", "2", "--save-every", "2"] + initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False) + # A synchronous continuous reference catches snapshot aliasing while updates + # run during periodic writes. The split run also tests the final flush on + # max-steps and the masked final rank when nine tokens remain across two GPUs. + expected = run_training(continuous, common, 2, timeout=300, expect_validation_display=True) + async_common = common + ["--async-checkpoint"] + partial = run_training(resumed, async_common, 2, ["--max-steps", "1"], + timeout=300, expect_async=True, expect_validation_display=True) + assert partial["tokens_seen"] == 32 and partial["step"] == 1 + assert partial["val_step"] == 1 and partial["val_tokens"] == 32 + actual = run_training(resumed, async_common, 2, timeout=300, expect_async=True, + expect_validation_display=True) + + assert actual["format"] == expected["format"] == "warpstate-server-v1" + assert actual["tokens_seen"] == expected["tokens_seen"] == 73 + assert actual["step"] == expected["step"] == 3 + assert actual["origin"] == partial["origin"] == expected["origin"] + assert actual["signature"] == partial["signature"] == expected["signature"] + settings = actual["signature"]["settings"] + assert settings["world_size"] == 2 + assert settings["loss_mode"] == "compiled-full" + assert settings["compile_core"] is True + assert settings["compile_mode"] == "default" + assert settings["activation_checkpointing"] is False + assert settings["bf16_residuals"] is True + assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32 + assert len(actual["rng_by_rank"]) == 2 + for key in ("model", "optimizer", "rng_by_rank"): + assert_tensors_equal(actual[key], expected[key]) + assert actual["model"].keys() == initial["model"].keys() + assert all(not key.startswith(("module.", "_orig_mod.")) for key in actual["model"]) + assert all(value.dtype == torch.float32 and torch.isfinite(value).all() + for value in actual["model"].values()) + assert actual["optimizer"]["state"] + for state in actual["optimizer"]["state"].values(): + assert state["step"].item() == 3 + for key in ("exp_avg", "exp_avg_sq"): + assert state[key].dtype == torch.float32 and torch.isfinite(state[key]).all() + assert math.isfinite(actual["val_loss"]) + assert actual["val_loss"] == expected["val_loss"] + assert actual["val_step"] == expected["val_step"] == 3 + assert actual["val_tokens"] == expected["val_tokens"] == 73 + + continuous_records = [json.loads(line) for line in + (continuous / "run/metrics.jsonl").read_text().splitlines()] + assert [record["val_step"] for record in continuous_records] == [0, 2, 3] + assert [record["val_tokens"] for record in continuous_records] == [0, 64, 73] + continuous_validations = [json.loads(line) for line in + (continuous / "run/validation.jsonl").read_text().splitlines()] + assert [record["step"] for record in continuous_validations] == [2, 3] + assert all(record["val_loss"] == validation["loss"] for record, validation in + zip(continuous_records[1:], continuous_validations, strict=True)) + assert math.isfinite(continuous_records[0]["val_loss"]) + for directory in (continuous, resumed): + runtime = json.loads((directory / "run/settings.json").read_text())["runtime"] + assert runtime["eval_every"] == runtime["save_every"] == 2 + + records = [json.loads(line) for line in (resumed / "run/metrics.jsonl").read_text().splitlines()] + assert [record["step"] for record in records] == [1, 2, 3] + assert [record["tokens"] for record in records] == [32, 64, 73] + assert [record["lr"] for record in records] == [0.001, 0.002, 0.002] + assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"]) + for record in records) + validations = [json.loads(line) for line in (resumed / "run/validation.jsonl").read_text().splitlines()] + assert [record["tokens"] for record in validations] == [32, 64, 73] + assert all(math.isfinite(record["loss"]) for record in validations) + assert [record["val_step"] for record in records] == [1, 2, 3] + assert [record["val_tokens"] for record in records] == [32, 64, 73] + assert all(record["val_loss"] == validation["loss"] for record, validation in + zip(records, validations, strict=True)) + + +@TWO_GPUS +def test_one_to_two_gpu_reconfiguration_preserves_training_state(tmp_path): + (serial, distributed), common = fixture_runs(tmp_path, ["serial", "distributed"]) + expected = run_training(serial, common, 1) + partial = run_training(distributed, common, 1, ["--max-steps", "1"]) + assert partial["tokens_seen"] == 32 and partial["step"] == 1 + actual = run_training(distributed, common, 2, ["--reconfigure"]) + assert actual["tokens_seen"] == expected["tokens_seen"] == 73 + assert actual["step"] == expected["step"] == 3 + assert actual["origin"] == partial["origin"] == expected["origin"] + settings = actual["signature"]["settings"] + assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32 + assert settings["world_size"] == 2 + assert len(actual["rng_by_rank"]) == 2 + assert actual["optimizer"]["param_groups"] == expected["optimizer"]["param_groups"] + for key, value in actual["model"].items(): + torch.testing.assert_close(value, expected["model"][key], rtol=0.01, atol=2e-5) + for parameter, state in actual["optimizer"]["state"].items(): + reference = expected["optimizer"]["state"][parameter] + assert state["step"].item() == reference["step"].item() == 3 + for key in ("exp_avg", "exp_avg_sq"): + torch.testing.assert_close(state[key], reference[key], rtol=0.03, atol=2e-7) + records = [json.loads(line) for line in (distributed / "run/metrics.jsonl").read_text().splitlines()] + assert [record["tokens"] for record in records] == [32, 64, 73] + assert [record["lr"] for record in records] == [0.001, 0.002, 0.002] diff --git a/test_training_launcher.py b/test_training_launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..d3e2bd461337bd07ec0f5de85ddec00384c07877 --- /dev/null +++ b/test_training_launcher.py @@ -0,0 +1,95 @@ +"""CPU-only regression tests for graceful and forced supervised shutdown.""" +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + +import pytest + + +ROOT = Path(__file__).resolve().parent +WORKSPACE = ROOT.parent +LAUNCHER = ROOT / "ops/launch_training.py" + + +@pytest.fixture +def workspace_tmp(): + directory = WORKSPACE / "tmp" + directory.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="launcher-test-", dir=directory) as temporary: + yield Path(temporary) + + +@pytest.mark.parametrize("cooperative", [True, False]) +def test_stop_reaches_or_cleans_up_worker_in_another_session(workspace_tmp, cooperative): + stop_file = workspace_tmp / "stop-request" + ready_file = workspace_tmp / "ready" + observed_file = workspace_tmp / "observed" + worker = workspace_tmp / "worker.py" + worker.write_text( + "import os, pathlib, signal, sys, time\n" + f"cooperative = {cooperative!r}\n" + "if not cooperative: signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "stop, ready, observed = map(pathlib.Path, sys.argv[1:])\n" + "ready.write_text(str(os.getpid()))\n" + "while not cooperative or not stop.exists(): time.sleep(0.02)\n" + "observed.write_text('graceful stop')\n" + ) + agent = workspace_tmp / "agent.py" + agent.write_text( + "import signal, subprocess, sys\n" + f"if not {cooperative!r}: signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "child = subprocess.Popen([sys.executable, *sys.argv[1:]], start_new_session=True)\n" + "sys.exit(child.wait())\n" + ) + harness = ( + "import importlib.util, os, pathlib, sys; " + "spec = importlib.util.spec_from_file_location('launcher', sys.argv[1]); " + "module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); " + "sys.exit(module.supervise([sys.executable, *sys.argv[3:]], pathlib.Path(sys.argv[2]), " + "os.environ.copy(), grace_seconds=0.3, terminate_seconds=0.3))" + ) + environment = os.environ.copy() + environment.update(TMPDIR=str(WORKSPACE / "tmp"), PYTHONDONTWRITEBYTECODE="1") + command = [sys.executable, "-c", harness, str(LAUNCHER), str(stop_file), + str(agent), str(worker), str(stop_file), str(ready_file), str(observed_file)] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, env=environment) + try: + deadline = time.monotonic() + 10 + while not ready_file.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.02) + assert ready_file.exists(), "Fake distributed worker did not start" + worker_pid = int(ready_file.read_text()) + assert os.getpgid(worker_pid) != os.getpgid(process.pid) + process.send_signal(signal.SIGUSR1) + stdout, stderr = process.communicate(timeout=10) + assert stop_file.exists() + if cooperative: + assert process.returncode == 0, stdout + "\n" + stderr + assert observed_file.read_text() == "graceful stop" + else: + assert process.returncode != 0, stdout + "\n" + stderr + assert "Termination deadline exceeded" in stdout + stat = Path(f"/proc/{worker_pid}/stat") + assert not stat.exists() or stat.read_text().rsplit(")", 1)[1].split()[0] == "Z" + finally: + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def test_launcher_refuses_work_outside_workspace(): + result = subprocess.run( + [sys.executable, str(LAUNCHER), "--work-dir", "/tmp"], + capture_output=True, text=True, timeout=10, + ) + assert result.returncode != 0 + assert "Created paths must stay inside" in result.stderr diff --git a/test_training_optimization.py b/test_training_optimization.py new file mode 100644 index 0000000000000000000000000000000000000000..96c7afb5bb3f10ef598d790e971ee30cec8ac84f --- /dev/null +++ b/test_training_optimization.py @@ -0,0 +1,159 @@ +"""Two-GPU full-state continuation checks for optional training optimizations.""" + +import json +import math + +import pytest +import torch + +from test_training_execution import ( + TWO_GPUS, + assert_tensors_equal, + fixture_runs, + run_training, +) + + +CASES = [ + pytest.param( + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"], + {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", + "overlap_producer_schedule": True, "overlap_swap_collectives": False, + "overlap_runtime_estimates": False, "overlap_memory_budget": 0.0, + "ddp_bucket_cap_mb": 128}, + id="producer-scheduled-overlap", + ), + pytest.param( + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives", + "--overlap-runtime-estimates"], + {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", + "overlap_swap_collectives": True, "overlap_runtime_estimates": True, + "overlap_memory_budget": 0.0, "ddp_bucket_cap_mb": 128}, + id="runtime-estimated-overlap", + ), + pytest.param( + ["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives", + "--overlap-memory-budget", "0.02", "--ddp-bucket-cap-mb", "64"], + {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", + "overlap_swap_collectives": True, "overlap_runtime_estimates": False, + "overlap_memory_budget": 0.02, "ddp_bucket_cap_mb": 64}, + id="waits-past-collectives", + ), + pytest.param( + ["--gradient-mode", "compiled-autograd-overlap", "--matmul-plan", "__fixture_plan__"], + {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", + "cache_core_weight_casts": False, "ddp_batched_grad_copy": False, "ddp_bucket_cap_mb": 128}, + id="matmul-epilogue-with-communication-overlap", + ), + pytest.param( + ["--gradient-mode", "eager", "--activation-mode", "all", + "--cache-core-weight-casts", "--ddp-batched-grad-copy", + "--ddp-bucket-cap-mb", "64"], + {"gradient_mode": "eager", "activation_mode": "all", + "cache_core_weight_casts": True, "ddp_batched_grad_copy": True, + "ddp_bucket_cap_mb": 64}, + id="activations-cached-casts-batched-copy", + ), + pytest.param( + ["--gradient-mode", "compiled-autograd-overlap", "--activation-mode", "baseline", + "--no-cache-core-weight-casts", "--no-ddp-batched-grad-copy"], + {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", + "cache_core_weight_casts": False, "ddp_batched_grad_copy": False, + "ddp_bucket_cap_mb": 128}, + id="compiled-autograd-overlap", + ), +] + + +@TWO_GPUS +@pytest.mark.parametrize(("flags", "selected"), CASES) +def test_optimized_two_gpu_async_resume_preserves_full_state(tmp_path, flags, selected): + (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) + common += [ + "--loss-mode", "compiled-full", "--compile-core", "--compile-mode", "default", + "--bf16-residuals", "--no-activation-checkpointing", "--autocast-cache", + "--eval-every", "2", "--save-every", "2", *flags, + ] + selected = dict(selected) + if "__fixture_plan__" in common: + from ops.matmul_execution import load_matmul_plan + tile = {"block_m": 16, "block_n": 32, "block_k": 16, "num_warps": 4, "num_stages": 2} + document = {"format": "warpstate-matmul-plan-v1", "entries": [ + {"kind": "swiglu", "m": 16, "n": 64, "k": 16, + "a_stride": [16, 1], "b_stride": [1, 16], "dtype": "bfloat16", + "algorithm": "persistent", "tile": tile}, + {"kind": "mm", "m": 64, "n": 16, "k": 16, + "a_stride": [1, 64], "b_stride": [16, 1], "dtype": "bfloat16", + "algorithm": "persistent_tma", "tile": tile}, + ]} + recipe = tmp_path / "matmul-plan.json" + recipe.write_text(json.dumps(document)) + common[common.index("__fixture_plan__")] = str(recipe) + selected["matmul_execution"] = load_matmul_plan(recipe).metadata + initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False) + # The 73-token fixture consumes 32, 32, then 9 tokens. On its final update, + # rank zero consumes the remaining tokens and rank one has entirely masked + # targets. Resume must preserve that weighting and every rank's RNG state. + expected = run_training(continuous, common, 2, timeout=300, + expect_validation_display=True) + async_common = common + ["--async-checkpoint"] + partial = run_training(resumed, async_common, 2, ["--max-steps", "1"], + timeout=300, expect_async=True, expect_validation_display=True) + assert partial["tokens_seen"] == 32 and partial["step"] == 1 + assert partial["val_step"] == 1 and partial["val_tokens"] == 32 + actual = run_training(resumed, async_common, 2, timeout=300, expect_async=True, + expect_validation_display=True) + + assert actual["format"] == expected["format"] == "warpstate-server-v1" + assert actual["step"] == expected["step"] == 3 + assert actual["tokens_seen"] == expected["tokens_seen"] == 73 + assert actual["origin"] == partial["origin"] == expected["origin"] + assert actual["execution_history"] == partial["execution_history"] == expected["execution_history"] + assert actual["signature"] == partial["signature"] == expected["signature"] + settings = actual["signature"]["settings"] + for name, value in selected.items(): + assert settings[name] == value + assert settings["world_size"] == 2 + assert settings["compile_core"] is True + assert settings["compile_mode"] == "default" + assert settings["loss_mode"] == "compiled-full" + assert settings["bf16_residuals"] is True + assert settings["activation_checkpointing"] is False + assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32 + assert len(actual["rng_by_rank"]) == len(partial["rng_by_rank"]) == 2 + for name in ("model", "optimizer", "rng_by_rank"): + assert_tensors_equal(actual[name], expected[name]) + + assert actual["model"].keys() == initial["model"].keys() + assert all(not name.startswith(("module.", "_orig_mod.")) for name in actual["model"]) + assert all(value.dtype == torch.float32 and torch.isfinite(value).all() + for value in actual["model"].values()) + assert actual["optimizer"]["state"] + for state in actual["optimizer"]["state"].values(): + assert state["step"].item() == 3 + for name in ("exp_avg", "exp_avg_sq"): + assert state[name].dtype == torch.float32 and torch.isfinite(state[name]).all() + assert actual["val_loss"] == expected["val_loss"] + assert math.isfinite(actual["val_loss"]) + assert actual["val_step"] == expected["val_step"] == 3 + assert actual["val_tokens"] == expected["val_tokens"] == 73 + + for directory, async_enabled in [(continuous, False), (resumed, True)]: + runtime = json.loads((directory / "run/settings.json").read_text())["runtime"] + for name, value in selected.items(): + assert runtime[name] == value + assert runtime["async_checkpoint"] is async_enabled + assert runtime["bf16_residuals"] is True + assert runtime["eval_every"] == runtime["save_every"] == 2 + assert not (directory / "run/latest.pt.tmp").exists() + records = [json.loads(line) for line in + (directory / "run/metrics.jsonl").read_text().splitlines()] + assert [record["step"] for record in records] == [1, 2, 3] + assert [record["tokens"] for record in records] == [32, 64, 73] + assert [record["lr"] for record in records] == [0.001, 0.002, 0.002] + assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"]) + for record in records) + validations = [json.loads(line) for line in + (resumed / "run/validation.jsonl").read_text().splitlines()] + assert [record["tokens"] for record in validations] == [32, 64, 73] + assert all(math.isfinite(record["loss"]) for record in validations) diff --git a/test_training_stop.py b/test_training_stop.py new file mode 100644 index 0000000000000000000000000000000000000000..20d39eb8fb64ecfbe8bfd0c46a7cf257eb5afd5f --- /dev/null +++ b/test_training_stop.py @@ -0,0 +1,169 @@ +"""Absolute training stops preserve the objective, checkpoint, and continuation.""" + +import json +import math +import sys + +import numpy as np +import pytest +import torch + +from test_training_execution import ( + TWO_GPUS, + assert_tensors_equal, + fixture_runs, + pipeline, + run_training, +) +from warpstate import WarpStateConfig, WarpStateLM + + +def test_stop_at_step_cli_defaults_to_no_absolute_limit(monkeypatch): + monkeypatch.setattr(sys, "argv", ["run_40b.py"]) + args = pipeline.arguments() + assert args.stop_at_step is None + assert args.max_steps is None + assert args.train_tokens == 40_000_000_000 + + +@pytest.mark.parametrize("target", [1, 64_000, 2_048_000]) +def test_stop_at_step_cli_accepts_positive_absolute_target(monkeypatch, target): + monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stop-at-step", str(target)]) + args = pipeline.arguments() + assert args.stop_at_step == target + assert args.max_steps is None + + +@pytest.mark.parametrize("target", ["0", "-1", "2.5", "nan"]) +def test_stop_at_step_cli_rejects_nonpositive_or_noninteger_target(monkeypatch, target): + monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stop-at-step", target]) + with pytest.raises(SystemExit) as error: + pipeline.arguments() + assert error.value.code == 2 + + +def test_stop_at_step_and_additional_limit_preserve_resume_signature(monkeypatch): + manifest = {"complete": True, "signature": {"source": "unchanged-fixture"}} + monkeypatch.setattr(sys, "argv", ["run_40b.py"]) + expected = pipeline.train_signature(pipeline.arguments(), manifest) + for flags in (["--stop-at-step", "64000"], + ["--stop-at-step", "64000", "--max-steps", "1"], + ["--stop-at-step", "96000"], []): + monkeypatch.setattr(sys, "argv", ["run_40b.py", *flags]) + args = pipeline.arguments() + actual = pipeline.train_signature(args, manifest) + assert actual == expected + assert actual["settings"]["train_tokens"] == 40_000_000_000 + assert "stop_at_step" not in actual["settings"] + assert "max_steps" not in actual["settings"] + assert pipeline.check_resume_signature(expected, actual) is False + if "--max-steps" in flags: + assert args.max_steps == 1 and args.stop_at_step == 64000 + + +def _records(directory, name): + return [json.loads(line) for line in (directory / "run" / name).read_text().splitlines()] + + +@pytest.mark.parametrize(("world_size", "async_checkpoint"), [ + pytest.param(1, False, id="single-gpu-sync", + marks=pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")), + pytest.param(2, True, id="two-gpu-async", marks=TWO_GPUS), +]) +def test_absolute_stop_checkpoint_and_unlimited_resume_preserve_full_state( + tmp_path, monkeypatch, world_size, async_checkpoint): + (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) + # The original two-head tiny fixture showed a one-ULP step-two grad-norm + # difference after eager DDP resume, reproduced exactly with --max-steps + # alone. Its two-element parameter offsets may change foreach-norm alignment + # when DDP rebuilds buckets. Four heads make every parameter size divisible + # by four and retain strict uninterrupted-reference equality below, without + # changing training code or weakening tolerances. + config_path = tmp_path / "config.json" + config = WarpStateConfig(**json.loads(config_path.read_text())) + config.n_heads = 4 + config.validate() + config.to_json(config_path) + torch.manual_seed(133) + initial = WarpStateLM(config) + assert all(parameter.numel() % 4 == 0 for parameter in initial.parameters()) + torch.save({"config": vars(config), "model": initial.state_dict(), "tokens_seen": 100}, + tmp_path / "initial.pt") + # Four optimizer updates consume 32, 32, 32, then 9 tokens. Target three + # falls outside the ten-step evaluation/save cadence, after cosine decay + # begins. Replacing the full LR horizon with the pause target must fail. + # Two GPUs also exercise an entirely masked final rank on the last update. + common += ["--loss-mode", "full", "--bf16-residuals", "--no-activation-checkpointing", + "--eval-every", "10", "--save-every", "10", + "--train-tokens", "105", "--warmup-steps", "1"] + with monkeypatch.context() as context: + context.setattr(sys, "argv", common) + data_signature = pipeline.data_signature(pipeline.arguments()) + for directory in (continuous, resumed): + np.arange(105, dtype=" 0 + for microstep in range(8): + inputs, targets = _batch(500 + update * 8 + microstep) + torch.manual_seed(1000 + update * 8 + microstep) + with torch.autocast("cpu", dtype=torch.bfloat16): + _, expected_loss = reference(inputs, targets, return_logits=False) + (expected_loss / 8).backward() + torch.manual_seed(1000 + update * 8 + microstep) + with torch.autocast("cpu", dtype=torch.bfloat16): + _, actual_loss = experiment(inputs, targets, return_logits=False) + (actual_loss / 8).backward() + torch.testing.assert_close(actual_loss, expected_loss, rtol=0, atol=0) + for actual, expected in zip(experiment.parameters(), reference.parameters()): + assert actual.dtype == actual.grad.dtype == torch.float32 + torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0) + assert tuple(experiment.state_dict()) == original_keys + assert all(a is b for a, b in zip(original_parameters, experiment.parameters())) + assert not cache.active and cache.cached_tensor_count == cache.cached_bytes == 0 + for optimizer in optimizers: + optimizer.step() + for actual, expected in zip(experiment.parameters(), reference.parameters()): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + state = optimizers[1].state[actual] + assert state["exp_avg"].dtype == state["exp_avg_sq"].dtype == torch.float32 + + +def test_checkpoint_load_between_scopes_uses_fresh_casts_and_preserves_fp32_state(): + model = _model() + cache = apply_weight_cast_cache(model) + with cache.update(): + old = cache._views["wrapped.cores.0.in_proj.weight"].detach().clone() + checkpoint = {name: value.clone() for name, value in model.state_dict().items()} + checkpoint["cores.0.in_proj.weight"].add_(0.5) + model.load_state_dict(checkpoint, strict=True) + with cache.update(): + current = cache._views["wrapped.cores.0.in_proj.weight"] + assert not torch.equal(old, current) + torch.testing.assert_close(current, model.cores[0].in_proj.weight.bfloat16(), rtol=0, atol=0) + assert all(parameter.dtype == torch.float32 for parameter in model.parameters()) + + +def test_scope_failure_restores_parameters_and_releases_casts(): + model = _model() + masters = tuple(model.parameters()) + cache = apply_weight_cast_cache(model) + inputs, targets = _batch(40) + with pytest.raises(ValueError, match="must be divisible"): + with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16): + model(inputs[:, :-1], targets[:, :-1]) + assert not cache.active and cache.cached_bytes == 0 + assert all(a is b for a, b in zip(masters, model.parameters())) + with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16): + _, loss = model(inputs, targets, return_logits=False) + loss.backward() + + +@pytest.mark.parametrize("mutation", ["in_place", "load_state", "replace"]) +def test_parameter_mutation_inside_scope_is_detected_and_cleared(mutation): + model = _model() + cache = apply_weight_cast_cache(model) + with pytest.raises(RuntimeError, match="changed"): + with cache.update(): + if mutation == "in_place": + with torch.no_grad(): + model.cores[0].in_proj.weight.add_(0.01) + elif mutation == "load_state": + model.load_state_dict(model.state_dict()) + else: + model.cores[0].in_proj.weight = torch.nn.Parameter( + model.cores[0].in_proj.weight.detach().clone()) + assert not cache.active and cache.cached_bytes == 0 + + +def test_outside_scope_and_inference_are_unchanged_and_do_not_create_cache(): + reference = _model() + model = _model() + cache = apply_weight_cast_cache(model) + inputs, _ = _batch(71) + for candidate in [reference, model]: + candidate.eval() + with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16): + torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0) + assert not cache.active and cache.cached_bytes == 0 + with pytest.raises(RuntimeError, match="training with gradients"): + with cache.update(): + pass + + +def test_incompatible_modes_and_nested_scopes_fail_explicitly(): + model = _model() + model.cfg.logical_depth = 2 + with pytest.raises(ValueError, match="one use"): + apply_weight_cast_cache(model) + model.cfg.logical_depth = 1 + model.cfg.gradient_checkpointing = True + with pytest.raises(ValueError, match="checkpointing off"): + apply_weight_cast_cache(model) + model.cfg.gradient_checkpointing = False + cache = apply_weight_cast_cache(model) + with pytest.raises(ValueError, match="already been applied"): + apply_weight_cast_cache(model) + with cache.update(): + with pytest.raises(RuntimeError, match="cannot be nested"): + with cache.update(): + pass + with pytest.raises(RuntimeError, match="requires BF16 autocast"): + model(_batch(2)[0]) + + +def _ddp_worker(rank, directory): + torch.set_num_threads(1) + directory = Path(directory) + dist.init_process_group("gloo", init_method=(directory / "rendezvous").as_uri(), + rank=rank, world_size=2, timeout=timedelta(seconds=30)) + try: + reference = DistributedDataParallel(_model(), gradient_as_bucket_view=True) + raw = _model() + cache = apply_weight_cast_cache(raw) + experiment = DistributedDataParallel(raw, gradient_as_bucket_view=True) + optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False) + for model in [reference, experiment]] + for update in range(3): + for optimizer in optimizers: + optimizer.zero_grad(set_to_none=True) + with cache.update(): + for microstep in range(8): + inputs, targets = _batch(100 + rank * 100 + update * 8 + microstep) + for model in [reference, experiment]: + with model.no_sync() if microstep < 7 else nullcontext(): + with torch.autocast("cpu", dtype=torch.bfloat16): + _, loss = model(inputs, targets, return_logits=False) + (loss / 8).backward() + for actual, expected in zip(experiment.parameters(), reference.parameters()): + assert actual.grad.dtype == expected.grad.dtype == torch.float32 + torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0) + assert not cache.active + for optimizer in optimizers: + optimizer.step() + for actual, expected in zip(experiment.parameters(), reference.parameters()): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + (directory / f"rank-{rank}.json").write_text(json.dumps({"updates": 3, "equal": True})) + finally: + dist.destroy_process_group() + + +def test_two_rank_accumulation_preserves_ddp_leaf_hooks_exactly(tmp_path): + environment = os.environ.copy() + environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") + processes = [] + try: + for rank in range(2): + processes.append(subprocess.Popen( + [sys.executable, str(Path(__file__).resolve()), str(rank), str(tmp_path)], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + )) + for rank, process in enumerate(processes): + output, _ = process.communicate(timeout=90) + assert process.returncode == 0, f"Rank {rank} failed:\n{output}" + finally: + for process in processes: + if process.poll() is None: + process.kill() + for process in processes: + process.wait(timeout=10) + assert all(json.loads((tmp_path / f"rank-{rank}.json").read_text())["equal"] + for rank in range(2)) + + +@pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1", + reason="requires an explicitly reserved CUDA device") +def test_cuda_inductor_core_weight_cache_matches_bf16_updates(tmp_path): + from ops.loss_experiment import apply_loss_execution + + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False + cfg = WarpStateConfig(vocab_size=256, d_model=256, n_heads=4, n_cores=2, + logical_depth=2, ffn_hidden=896, chunk_size=128, + parallel_memory=True, gradient_checkpointing=False, + loss_chunk_size=256) + models = [] + for _ in range(2): + torch.manual_seed(314) + model = WarpStateLM(cfg).cuda().train() + for core in model.cores: + core.forward = torch.compile(core.forward, fullgraph=True, dynamic=False) + apply_loss_execution(model, "compiled-full") + apply_bf16_residuals(model) + models.append(model) + reference, experiment = models + cache = apply_weight_cast_cache(experiment) + optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, fused=True) + for model in models] + report = {"maximum_loss_difference": 0.0, "maximum_gradient_difference": 0.0, + "maximum_parameter_difference": 0.0, "updates": 3, "microsteps": 8} + for update in range(3): + for optimizer in optimizers: + optimizer.zero_grad(set_to_none=True) + with cache.update(): + for microstep in range(8): + generator = torch.Generator().manual_seed(42 + update * 8 + microstep) + inputs = torch.randint(256, (1, 256), generator=generator).cuda() + targets = torch.randint(256, (1, 256), generator=generator).cuda() + losses = [] + for model in models: + with torch.autocast("cuda", dtype=torch.bfloat16): + _, loss = model(inputs, targets, return_logits=False) + (loss / 8).backward() + losses.append(loss.detach()) + report["maximum_loss_difference"] = max(report["maximum_loss_difference"], + (losses[0] - losses[1]).abs().item()) + for actual, expected in zip(experiment.parameters(), reference.parameters()): + report["maximum_gradient_difference"] = max(report["maximum_gradient_difference"], + (actual.grad - expected.grad).abs().max().item()) + assert actual.grad.dtype == expected.grad.dtype == torch.float32 + for optimizer in optimizers: + optimizer.step() + for actual, expected in zip(experiment.parameters(), reference.parameters()): + report["maximum_parameter_difference"] = max(report["maximum_parameter_difference"], + (actual - expected).abs().max().item()) + (tmp_path / "cuda-weight-cast-comparison.json").write_text(json.dumps(report, indent=2) + "\n") + print("CUDA_WEIGHT_CAST_COMPARISON " + json.dumps(report), flush=True) + assert report["maximum_loss_difference"] == 0.0, report + assert report["maximum_gradient_difference"] == 0.0, report + assert report["maximum_parameter_difference"] == 0.0, report + + +if __name__ == "__main__": + _ddp_worker(int(sys.argv[1]), sys.argv[2]) diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..b42ed1b5fbb4925bef2258541f457a0424333834 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e +size 545080 diff --git a/warpstate/__init__.py b/warpstate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1d5764b66ed77b67fe866c880bd4876bc5518a8d --- /dev/null +++ b/warpstate/__init__.py @@ -0,0 +1,4 @@ +from .config import WarpStateConfig +from .model import WarpStateLM + +__all__ = ["WarpStateConfig", "WarpStateLM"] diff --git a/warpstate/config.py b/warpstate/config.py new file mode 100644 index 0000000000000000000000000000000000000000..d9ea90296384cb4a89c76b27b1b92584dfe88b01 --- /dev/null +++ b/warpstate/config.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +import json + + +@dataclass +class WarpStateConfig: + vocab_size: int = 32768 + d_model: int = 1280 + n_heads: int = 20 + n_cores: int = 4 + logical_depth: int = 16 + ffn_hidden: int = 4480 + chunk_size: int = 128 + dropout: float = 0.0 + rms_eps: float = 1e-5 + gradient_checkpointing: bool = True + parallel_memory: bool = False + loss_chunk_size: int = 128 + + def validate(self) -> None: + assert self.d_model % self.n_heads == 0 + assert self.logical_depth >= self.n_cores + assert self.chunk_size > 0 + assert self.head_dim % 2 == 0 + assert self.n_cores > 0 and self.loss_chunk_size > 0 + + @property + def head_dim(self) -> int: + return self.d_model // self.n_heads + + @classmethod + def from_json(cls, path: str | Path) -> "WarpStateConfig": + data = json.loads(Path(path).read_text(encoding="utf-8")) + cfg = cls(**data) + cfg.validate() + return cfg + + def to_json(self, path: str | Path) -> None: + Path(path).write_text(json.dumps(asdict(self), indent=2), encoding="utf-8") diff --git a/warpstate/model.py b/warpstate/model.py new file mode 100644 index 0000000000000000000000000000000000000000..7f0deb6f3e9b7ccbdd590b2374ca677f935f9894 --- /dev/null +++ b/warpstate/model.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import List + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint + +from .config import WarpStateConfig + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-5): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + self.use_fused = False + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.use_fused and x.device.type == "cuda": + return torch.ops.aten._fused_rms_norm(x, [x.shape[-1]], self.weight, self.eps)[0] + dtype = x.dtype + x32 = x.float() + x32 = x32 * torch.rsqrt(x32.pow(2).mean(dim=-1, keepdim=True) + self.eps) + return (x32.to(dtype) * self.weight.to(dtype)) + + +def rotate_half(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., ::2] + x2 = x[..., 1::2] + return torch.stack((-x2, x1), dim=-1).flatten(-2) + + +def rope_cos_sin(length: int, dim: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]: + pos = torch.arange(length, device=device, dtype=torch.float32) + inv = 1.0 / (10000 ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim)) + freqs = torch.outer(pos, inv) + emb = torch.repeat_interleave(freqs, 2, dim=-1) + return emb.cos().to(dtype), emb.sin().to(dtype) + + +def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + # x: [B, N, H, C, Dh], cos/sin: [C, Dh] + cos = cos.view(1, 1, 1, cos.shape[0], cos.shape[1]) + sin = sin.view(1, 1, 1, sin.shape[0], sin.shape[1]) + return x * cos + rotate_half(x) * sin + + +class SwiGLU(nn.Module): + def __init__(self, dim: int, hidden: int): + super().__init__() + self.up_gate = nn.Linear(dim, hidden * 2, bias=False) + self.down = nn.Linear(hidden, dim, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + up, gate = self.up_gate(x).chunk(2, dim=-1) + return self.down(F.silu(gate) * up) + + +@dataclass +class LayerCache: + fast_state: torch.Tensor + slow_state: torch.Tensor + k_cache: torch.Tensor + v_cache: torch.Tensor + write_accum: torch.Tensor + chunk_len: int = 0 + + +class WarpStateCore(nn.Module): + """ + GPU-first experimental block. + + The local path performs causal attention only inside a fixed-size tile. + The long-range path compresses completed tiles into two constant-size + tensor states (fast + slow). A single fused input projection produces + Q/K/V/gate/U for both paths. + """ + + def __init__(self, cfg: WarpStateConfig): + super().__init__() + self.cfg = cfg + d = cfg.d_model + h = cfg.n_heads + self.h = h + self.hd = cfg.head_dim + self.chunk = cfg.chunk_size + + self.norm1 = RMSNorm(d, cfg.rms_eps) + self.in_proj = nn.Linear(d, 5 * d, bias=False) # Q K V gate memory-U + self.out_proj = nn.Linear(d, d, bias=False) + + self.norm2 = RMSNorm(d, cfg.rms_eps) + self.ffn = SwiGLU(d, cfg.ffn_hidden) + + # Per-head timescales. Start with a quick and a slow memory. + self.fast_decay_logit = nn.Parameter(torch.full((h,), math.log(0.90 / 0.10))) + self.slow_decay_logit = nn.Parameter(torch.full((h,), math.log(0.99 / 0.01))) + self.memory_mix_logit = nn.Parameter(torch.zeros(h)) + cos, sin = rope_cos_sin(self.chunk, self.hd, torch.device("cpu"), torch.float32) + self.register_buffer("rope_cos", cos, persistent=False) + self.register_buffer("rope_sin", sin, persistent=False) + + def _split(self, z: torch.Tensor) -> tuple[torch.Tensor, ...]: + b, t, _ = z.shape + q, k, v, g, u = self.in_proj(z).chunk(5, dim=-1) + def heads(y: torch.Tensor) -> torch.Tensor: + return y.view(b, t, self.h, self.hd) + return heads(q), heads(k), heads(v), heads(g), heads(u) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + b, t, d = x.shape + if t % self.chunk != 0: + raise ValueError(f"sequence length {t} must be divisible by chunk_size={self.chunk}") + n_chunks = t // self.chunk + + z = self.norm1(x) + q, k, v, g, u = self._split(z) + + # [B, N, H, C, Dh] + def chunkify(y: torch.Tensor) -> torch.Tensor: + return y.view(b, n_chunks, self.chunk, self.h, self.hd).permute(0, 1, 3, 2, 4).contiguous() + + q = chunkify(q) + k = chunkify(k) + v = chunkify(v) + g = chunkify(g) + u = chunkify(u) + + cos, sin = self.rope_cos.to(q.dtype), self.rope_sin.to(q.dtype) + q = apply_rope(q, cos, sin) + k = apply_rope(k, cos, sin) + + # Local tile path: all tiles are batched together, so this is a single + # large SDPA workload rather than token-by-token attention. + q_local = q.reshape(b * n_chunks, self.h, self.chunk, self.hd) + k_local = k.reshape(b * n_chunks, self.h, self.chunk, self.hd) + v_local = v.reshape(b * n_chunks, self.h, self.chunk, self.hd) + local = F.scaled_dot_product_attention( + q_local, + k_local, + v_local, + dropout_p=self.cfg.dropout if self.training else 0.0, + is_causal=True, + ).view(b, n_chunks, self.h, self.chunk, self.hd) + + # Long-range tensor memory. Memory read uses only previous completed + # tiles; the current tile is written after its outputs are formed. + fast = torch.zeros((b, self.h, self.hd, self.hd), device=x.device, dtype=q.dtype) + slow = torch.zeros_like(fast) + fast_decay = torch.sigmoid(self.fast_decay_logit).to(q.dtype).view(1, self.h, 1, 1) + slow_decay = torch.sigmoid(self.slow_decay_logit).to(q.dtype).view(1, self.h, 1, 1) + memory_mix = torch.sigmoid(self.memory_mix_logit).to(q.dtype).view(1, self.h, 1, 1) + + if self.cfg.parallel_memory: + # A strictly lower-triangular decay matrix reads completed tiles only. + # Batch all writes and reads instead of launching kernels per tile. + writes = (torch.tanh(k).transpose(-2, -1) @ torch.tanh(u)) / self.chunk + index = torch.arange(n_chunks, device=x.device) + lag = index[:, None] - index[None, :] - 1 + mask = lag >= 0 + def states(logit): + decay = logit.float().sigmoid()[:, None, None] + weights = (1 - decay) * decay.pow(lag.clamp_min(0)[None]) * mask + values = writes.permute(0, 2, 1, 3, 4).flatten(-2) + result = torch.matmul(weights.to(values.dtype), values) + return result.view(b, self.h, n_chunks, self.hd, self.hd).transpose(1, 2) + mix = memory_mix.unsqueeze(1) + state = mix * states(self.fast_decay_logit) + (1 - mix) * states(self.slow_decay_logit) + memory = (q @ state) / math.sqrt(self.hd) + gate = torch.sigmoid(g) + y = gate * local + (1 - gate) * memory + y = y.permute(0, 1, 3, 2, 4).contiguous().view(b, t, d) + x = x + self.out_proj(y) + return x + self.ffn(self.norm2(x)) + + out_chunks: list[torch.Tensor] = [] + for i in range(n_chunks): + qi = q[:, i] + ki = k[:, i] + ui = u[:, i] + gi = torch.sigmoid(g[:, i]) + + read_fast = torch.matmul(qi, fast) / math.sqrt(self.hd) + read_slow = torch.matmul(qi, slow) / math.sqrt(self.hd) + memory = memory_mix * read_fast + (1.0 - memory_mix) * read_slow + + yi = gi * local[:, i] + (1.0 - gi) * memory + out_chunks.append(yi) + + # B,H,D,C @ B,H,C,D -> B,H,D,D. tanh bounds the write and makes + # early training considerably more stable than an unbounded outer product. + write = torch.matmul( + torch.tanh(ki).transpose(-2, -1), + torch.tanh(ui), + ) / float(self.chunk) + fast = fast_decay * fast + (1.0 - fast_decay) * write + slow = slow_decay * slow + (1.0 - slow_decay) * write + + y = torch.stack(out_chunks, dim=1) # [B,N,H,C,Dh] + y = y.permute(0, 1, 3, 2, 4).contiguous().view(b, t, d) + x = x + self.out_proj(y) + x = x + self.ffn(self.norm2(x)) + return x + + @torch.no_grad() + def init_cache(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> LayerCache: + state_shape = (batch_size, self.h, self.hd, self.hd) + kv_shape = (batch_size, self.h, self.chunk, self.hd) + return LayerCache( + fast_state=torch.zeros(state_shape, device=device, dtype=dtype), + slow_state=torch.zeros(state_shape, device=device, dtype=dtype), + k_cache=torch.zeros(kv_shape, device=device, dtype=dtype), + v_cache=torch.zeros(kv_shape, device=device, dtype=dtype), + write_accum=torch.zeros(state_shape, device=device, dtype=dtype), + chunk_len=0, + ) + + @torch.no_grad() + def step(self, x: torch.Tensor, cache: LayerCache) -> torch.Tensor: + # x: [B,1,D] + assert x.shape[1] == 1 + b = x.shape[0] + z = self.norm1(x) + q, k, v, g, u = self._split(z) + q = q.permute(0, 2, 1, 3) # B,H,1,Dh + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + g = g.permute(0, 2, 1, 3) + u = u.permute(0, 2, 1, 3) + + pos = cache.chunk_len + cos, sin = rope_cos_sin(pos + 1, self.hd, x.device, q.dtype) + c = cos[pos : pos + 1].view(1, 1, 1, self.hd) + s = sin[pos : pos + 1].view(1, 1, 1, self.hd) + q = q * c + rotate_half(q) * s + k = k * c + rotate_half(k) * s + + cache.k_cache[:, :, pos : pos + 1].copy_(k) + cache.v_cache[:, :, pos : pos + 1].copy_(v) + keys = cache.k_cache[:, :, : pos + 1] + vals = cache.v_cache[:, :, : pos + 1] + local = F.scaled_dot_product_attention(q, keys, vals, is_causal=False, dropout_p=0.0) + + fast_decay = torch.sigmoid(self.fast_decay_logit).to(q.dtype).view(1, self.h, 1, 1) + slow_decay = torch.sigmoid(self.slow_decay_logit).to(q.dtype).view(1, self.h, 1, 1) + memory_mix = torch.sigmoid(self.memory_mix_logit).to(q.dtype).view(1, self.h, 1, 1) + + read_fast = torch.matmul(q, cache.fast_state) / math.sqrt(self.hd) + read_slow = torch.matmul(q, cache.slow_state) / math.sqrt(self.hd) + memory = memory_mix * read_fast + (1.0 - memory_mix) * read_slow + gate = torch.sigmoid(g) + y = gate * local + (1.0 - gate) * memory + + cache.write_accum.add_(torch.matmul(torch.tanh(k).transpose(-2, -1), torch.tanh(u))) + + y = y.permute(0, 2, 1, 3).contiguous().view(b, 1, self.cfg.d_model) + x = x + self.out_proj(y) + x = x + self.ffn(self.norm2(x)) + + if pos + 1 == self.chunk: + write = cache.write_accum / float(self.chunk) + cache.fast_state.mul_(fast_decay).add_(write * (1.0 - fast_decay)) + cache.slow_state.mul_(slow_decay).add_(write * (1.0 - slow_decay)) + cache.k_cache.zero_() + cache.v_cache.zero_() + cache.write_accum.zero_() + cache.chunk_len = 0 + else: + cache.chunk_len = pos + 1 + return x + + +class WarpStateLM(nn.Module): + def __init__(self, cfg: WarpStateConfig): + super().__init__() + cfg.validate() + self.cfg = cfg + self.embed = nn.Embedding(cfg.vocab_size, cfg.d_model) + self.cores = nn.ModuleList([WarpStateCore(cfg) for _ in range(cfg.n_cores)]) + + # Tiny depth-specific modulation lets shared cores behave differently on + # each recurrent-depth pass without duplicating the large matrices. + self.depth_scale = nn.Parameter(torch.zeros(cfg.logical_depth, cfg.d_model)) + self.depth_bias = nn.Parameter(torch.zeros(cfg.logical_depth, cfg.d_model)) + self.final_norm = RMSNorm(cfg.d_model, cfg.rms_eps) + + self.apply(self._init_weights) + # Smaller residual projections are useful because the same cores are reused. + scale = 1.0 / math.sqrt(2.0 * cfg.logical_depth) + for core in self.cores: + nn.init.normal_(core.out_proj.weight, mean=0.0, std=0.02 * scale) + nn.init.normal_(core.ffn.down.weight, mean=0.0, std=0.02 * scale) + + @staticmethod + def _init_weights(module: nn.Module) -> None: + if isinstance(module, (nn.Linear, nn.Embedding)): + nn.init.normal_(module.weight, mean=0.0, std=0.02) + + def num_parameters(self) -> int: + return sum(p.numel() for p in self.parameters()) + + def forward(self, input_ids: torch.Tensor, targets: torch.Tensor | None = None, *, return_logits: bool = True): + x = self.embed(input_ids) + for depth in range(self.cfg.logical_depth): + x = x * (1.0 + self.depth_scale[depth]) + self.depth_bias[depth] + core = self.cores[depth % self.cfg.n_cores] + if self.training and self.cfg.gradient_checkpointing: + x = checkpoint(core, x, use_reentrant=False) + else: + x = core(x) + x = self.final_norm(x) + if targets is not None and not return_logits: + # Recompute each vocabulary projection during backward, so the full + # sequence-by-vocabulary logits and softmax are never retained. + hidden = x.reshape(-1, x.size(-1)) + labels = targets.reshape(-1) + def chunk_loss(h, w, y): + return F.cross_entropy(F.linear(h, w).float(), y, reduction="sum") + loss = x.new_zeros((), dtype=torch.float32) + loss_chunk_size = getattr(self, "loss_chunk_size", self.cfg.loss_chunk_size) + for start in range(0, labels.numel(), loss_chunk_size): + stop = start + loss_chunk_size + if self.training and torch.is_grad_enabled(): + value = checkpoint(chunk_loss, hidden[start:stop], self.embed.weight, + labels[start:stop], use_reentrant=False) + else: + value = chunk_loss(hidden[start:stop], self.embed.weight, labels[start:stop]) + loss = loss + value + # Empty ranks in a final distributed update contribute a finite zero loss. + return None, loss / labels.ne(-100).sum().clamp_min(1) + logits = F.linear(x, self.embed.weight) # tied LM head + if targets is None: + return logits + loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1)) + return logits, loss + + @torch.no_grad() + def init_generation_cache(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> List[LayerCache]: + caches: list[LayerCache] = [] + for depth in range(self.cfg.logical_depth): + core = self.cores[depth % self.cfg.n_cores] + caches.append(core.init_cache(batch_size, device, dtype)) + return caches + + @torch.no_grad() + def step(self, input_ids: torch.Tensor, caches: List[LayerCache]) -> torch.Tensor: + # input_ids [B,1], returns logits [B,1,V] + x = self.embed(input_ids) + for depth in range(self.cfg.logical_depth): + x = x * (1.0 + self.depth_scale[depth]) + self.depth_bias[depth] + core = self.cores[depth % self.cfg.n_cores] + x = core.step(x, caches[depth]) + x = self.final_norm(x) + return F.linear(x, self.embed.weight)