Prepare publication-ready research artifact

.cache/dropout_decay_tinystories_8m/tokens-v4096-uint16.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35221770a252f4f3b4e86210495bbd02fc7830fba3ea0dc92deab4e98af60103
+size 17000354

.cache/dropout_decay_wikitext103_8m/tokens-v4096-uint16.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2198e999a2292ddfb549781af602d790356c3afd8cc1008c07cb96b58aa60a20
+size 17000462

.gitignore

@@ -11,6 +11,11 @@ __pycache__/
 *.npy
 .venv/
 *.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+build/
+dist/
 
 # Keep generated scratch artifacts out by default, but allow the curated
 # reproducibility caches and corpora that make this repo standalone.
@@ -23,6 +28,10 @@ __pycache__/
 !.cache/dropout_decay_wikitext103/**
 !.cache/dropout_decay_openwebtext10k_8m/
 !.cache/dropout_decay_openwebtext10k_8m/**
+!.cache/dropout_decay_tinystories_8m/
+!.cache/dropout_decay_tinystories_8m/**
+!.cache/dropout_decay_wikitext103_8m/
+!.cache/dropout_decay_wikitext103_8m/**
 
 data/*
 !data/openwebtext10k/

README.md

@@ -54,11 +54,24 @@ The existing repo-local caches are:
 - `.cache/dropout_decay_tinystories/tokens-v4096-uint16.npy`
 - `.cache/dropout_decay_wikitext103/tokenizer-v4096.json`
 - `.cache/dropout_decay_wikitext103/tokens-v4096-uint16.npy`
+- `.cache/dropout_decay_openwebtext10k_8m/tokenizer-v4096.json`
+- `.cache/dropout_decay_openwebtext10k_8m/tokens-v4096-uint16.npy`
+- `.cache/dropout_decay_tinystories_8m/tokenizer-v4096.json`
+- `.cache/dropout_decay_tinystories_8m/tokens-v4096-uint16.npy`
+- `.cache/dropout_decay_wikitext103_8m/tokenizer-v4096.json`
+- `.cache/dropout_decay_wikitext103_8m/tokens-v4096-uint16.npy`
 
 Use a project-local Python environment with MPS-capable PyTorch, for example
 `.venv/bin/python`. Attribution to nanochat remains in the source and docs, but
 experiment commands should not point into a separate nanochat repository.
 
+Fast non-training checks can be run without launching an MPS experiment:
+
+```bash
+PYTHONPATH=src .venv/bin/python -m unittest discover -s tests -v
+PYTHONPATH=src .venv/bin/python scripts/generate_paper_assets.py
+```
+
 ## Workflow
 
 1. Screen candidate model sizes with cheap static dropout sweeps.
@@ -76,22 +89,19 @@ Every run writes:
 - `model_selection.csv` / `model_selection.json`: static-sweep optimum and
   plateau diagnostics for screen and confirm runs.
 
-Old exploratory outputs are archived under `archive/`.
-
-For exact headline reproduction, see `REPRODUCING.md`. For the current
-first-reader hypothesis document, see
-`docs/current_hypothesis_for_ml_engineers.md`. For the denser run-by-run
-research summary, see `docs/dropout_decay_research_report_v2.md`. For the corpus
-difficulty follow-up that motivated the next formula refinement, see
-`docs/corpus_difficulty_probe_20260529.md`. For the first single-seed
-probe-calibrated streaming test, see
-`docs/probe_calibrated_stream_20260529.md`. For the coefficient refit using only
-existing saved results, see
-`docs/coefficient_refit_existing_data_20260529.md`. For a first-reader report on
-regime definition and coefficient interpretation, see
-`docs/regime_coefficient_report_20260529.md`. For the live coefficient
-calibration plan, see `docs/coefficient_calibration_plan.md`. For the current
-arXiv-style paper draft, see `paper/dropout_decay_pressure_law.tex`.
+Old exploratory outputs are archived under `archive/`; they are not part of the
+primary evidence path for the paper.
+
+For exact headline reproduction, see `REPRODUCING.md`. Current documentation is:
+
+- `docs/plan.md`: regime-validation and coefficient-fitting plan.
+- `docs/formula_coefficient_methodology.md`: coefficient target extraction,
+  weighting, and fitting details.
+- `docs/openwebtext10k_streaming_report.md`: OpenWebText10K regime report.
+- `docs/tinystories_streaming_report.md`: TinyStories regime report.
+- `docs/wikitext103_streaming_report.md`: WikiText-103 regime report.
+- `paper/dropout_decay_regime_specific_pressure_law.tex`: paper source.
+- `paper/dropout_decay_regime_specific_pressure_law.pdf`: generated paper PDF.
 
 ## Step 1: Cheap Static Screen
 

REPRODUCING.md

@@ -21,6 +21,12 @@ python3.11 -m venv .venv
 .venv/bin/python -m pip install -e .
 ```
 
+For development and publication checks, install the optional test dependency:
+
+```bash
+.venv/bin/python -m pip install -e ".[dev]"
+```
+
 Verify MPS:
 
 ```bash
@@ -58,6 +64,12 @@ The experiments in the current report used these repo-local caches:
 .cache/dropout_decay_tinystories/tokens-v4096-uint16.npy
 .cache/dropout_decay_wikitext103/tokenizer-v4096.json
 .cache/dropout_decay_wikitext103/tokens-v4096-uint16.npy
+.cache/dropout_decay_openwebtext10k_8m/tokenizer-v4096.json
+.cache/dropout_decay_openwebtext10k_8m/tokens-v4096-uint16.npy
+.cache/dropout_decay_tinystories_8m/tokenizer-v4096.json
+.cache/dropout_decay_tinystories_8m/tokens-v4096-uint16.npy
+.cache/dropout_decay_wikitext103_8m/tokenizer-v4096.json
+.cache/dropout_decay_wikitext103_8m/tokens-v4096-uint16.npy
 ```
 
 These binary artifacts are intended to be published with the repository through
@@ -124,18 +136,36 @@ Expected:
 5000970 500000
 ```
 
-## Headline Formula
+Run the fast non-training test suite:
+
+```bash
+PYTHONPATH=src .venv/bin/python -m unittest discover -s tests -v
+```
+
+## Current Pressure Law
 
-The tested formula is:
+The paper tests a per-regime interaction pressure law:
 
 ```text
-p = clamp(0.02, 0.65,
-          0.154 * log10(params / unique_tokens)
-        + 0.249 * log10(cumulative_sampled_tokens / unique_tokens)
-        - 0.210)
+p_t = clamp(p_min, p_max,
+            A * log10(P / U_t)
+          + B * log10(C_t / U_t)
+          + D * log10(P / U_t) * log10(C_t / U_t)
+          + C0)
 ```
 
-For the standard protocol:
+where `P` is trainable parameter count, `U_t` is the currently revealed unique
+token prefix, and `C_t` is the cumulative sampled training-token count consumed
+by the optimizer. The fitted coefficients are regime-specific; the current
+paper tables read them from:
+
+```text
+runs/coefficient_calibration/cross_regime_backtest/openwebtext10k_main_interaction/coefficients.json
+runs/coefficient_calibration/tinystories_combined_plus_all_holdouts_interaction/coefficients.json
+runs/coefficient_calibration/wikitext103_interaction/coefficients.json
+```
+
+For the standard 4M validation protocol:
 
 - stream prefixes: `250000 500000 1000000 2000000 4000000`
 - stage steps: `1000`
@@ -143,6 +173,40 @@ For the standard protocol:
 - block size: `128`
 - cumulative sampled tokens after stage `i`: `i * 1000 * 16 * 128`
 
+For the L20 8M extrapolation tier, the saved configs in `runs/` use the
+regime-specific 8M caches listed above and the frozen coefficient-derived
+dropout paths recorded in the run configs.
+
+## Rebuild Paper Tables and Figures
+
+The manuscript source is:
+
+```text
+paper/dropout_decay_regime_specific_pressure_law.tex
+```
+
+Regenerate the paper tables, figures, and result summary from saved metrics:
+
+```bash
+PYTHONPATH=src .venv/bin/python scripts/generate_paper_assets.py
+```
+
+Build the PDF, keeping LaTeX intermediates outside `paper/`:
+
+```bash
+mkdir -p /tmp/dropout-paper-build
+cd paper
+latexmk -pdf -outdir=/tmp/dropout-paper-build \
+  -interaction=nonstopmode -halt-on-error \
+  dropout_decay_regime_specific_pressure_law.tex
+cp /tmp/dropout-paper-build/dropout_decay_regime_specific_pressure_law.pdf \
+  dropout_decay_regime_specific_pressure_law.pdf
+```
+
+The sections below provide additional reproduction commands for saved
+validation runs. They are intentionally left in the original run format so old
+result artifacts remain compatible.
+
 ## Reproduce Model-Size Validation
 
 Example L12 command:
@@ -386,6 +450,7 @@ paired low-decay wins:           3/3
 - Do not claim the formula is universal.
 - The supported claim is final-validation improvement under this
   expanding-prefix protocol.
-- PDFs are generated artifacts and are ignored by Git.
-- The exact cached token file should be published through an appropriate binary
-  artifact mechanism once dataset provenance is finalized.
+- The current PDF is tracked as a generated paper artifact; transient LaTeX
+  build files are ignored.
+- Cached token arrays and parquet source files should be published through the
+  configured Git LFS rules.

paper/dropout_decay_pressure_law.tex

@@ -1,501 +0,0 @@
-\documentclass[11pt]{article}
-
-\usepackage[margin=1in]{geometry}
-\usepackage{amsmath}
-\usepackage{amssymb}
-\usepackage{booktabs}
-\usepackage{graphicx}
-\usepackage{hyperref}
-\usepackage[numbers,sort&compress]{natbib}
-
-\title{Pressure-Aware Dropout Scheduling for Expanding-Stream Language Model Training}
-\author{Anonymous Authors}
-\date{May 29, 2026}
-
-\newcommand{\clamp}{\operatorname{clamp}}
-\newcommand{\params}{P}
-\newcommand{\unique}{U_t}
-\newcommand{\sampled}{C_t}
-
-\begin{document}
-\maketitle
-
-\begin{abstract}
-Dropout is usually selected as a fixed hyperparameter, but fixed regularization
-is mismatched to streaming regimes where the amount of unique training data
-changes over time. We study expanding-prefix language model training, where a
-causal Transformer first trains on a small token prefix and later continues on
-larger prefixes. Static dropout sweeps show that the best dropout rate is not
-constant: it rises with model/data pressure and repeated sampling, and falls as
-more unique data becomes available. We propose a pressure-law family
-\[
-p_t = \clamp(p_{\min}, p_{\max},
-             A \log_{10}(\params / \unique)
-           + B \log_{10}(\sampled / \unique)
-           + C),
-\]
-where $\params$ is parameter count, $\unique$ is the number of unique available
-tokens, and $\sampled$ is the cumulative number of sampled training tokens
-consumed by the optimizer. On the local nanochat-style regime, ordinary least
-squares over static dropout optima gives $A=0.225$, $B=0.169$, and $C=-0.210$.
-For dynamic streaming, a validated schedule with $A=0.154$, $B=0.249$, and
-$C=-0.210$ wins all 15 paired final-loss comparisons across 8.39M to 31.46M
-parameter models and transfers to two architecture-shape holdouts. Public-corpus
-tests show a boundary condition: corpus identity shifts the optimum, so the
-coefficients are regime-calibrated constants rather than universal constants.
-\end{abstract}
-
-\section{Introduction}
-
-Language models trained on expanding streams face a changing regularization
-problem. At the beginning of the stream, a model may have many parameters
-relative to the number of unique tokens currently available. The optimizer may
-also sample the same prefix repeatedly. Later, the same model sees more unique
-data, reducing the need for strong regularization. A single fixed dropout rate
-cannot simultaneously match both conditions.
-
-The naive hypothesis is that one should use extremely high dropout early and
-decay it as more data arrives. Our first experiments reject that form: very high
-dropout underfits and loses to moderate fixed-dropout baselines. The supported
-hypothesis is narrower. Dropout should track a moving optimum driven by
-model/data pressure and repeated-sampling pressure.
-
-This paper makes four contributions:
-
-\begin{enumerate}
-    \item We define an expanding-prefix streaming protocol for small causal
-    Transformer language models.
-    \item We show that the best static dropout moves predictably with parameter
-    count, unique-token prefix size, and sampled-token pressure.
-    \item We propose a pressure-law dropout schedule and separate static
-    optimum coefficients from dynamic streaming coefficients.
-    \item We define a training regime as the scope over which one coefficient
-    triplet should be expected to transfer, and we show when corpus changes
-    invalidate a universal-coefficient claim.
-\end{enumerate}
-
-\section{Background}
-
-Dropout randomly removes units during training and is a standard method for
-reducing overfitting \citep{srivastava2014dropout}. In inverted dropout, an
-activation $h_i$ is transformed as
-\[
-h'_i = h_i \frac{m_i}{1-p}, \quad m_i \sim \operatorname{Bernoulli}(1-p).
-\]
-The expectation is preserved,
-\[
-\mathbb{E}[h'_i] = h_i,
-\]
-while the noise variance grows as
-\[
-\operatorname{Var}(h'_i) = h_i^2 \frac{p}{1-p}.
-\]
-This creates a regularization tradeoff. Too little dropout permits
-memorization; too much dropout harms optimization and underfits.
-
-Our experiments use decoder-only Transformer language models inspired by
-modern causal architectures \citep{vaswani2017attention}. The code is derived
-from Andrej Karpathy's MIT-licensed nanochat repository
-\citep{karpathy_nanochat}; chat and deployment components are removed, while
-the tokenizer and core causal Transformer template are retained.
-
-\section{Expanding-Stream Protocol}
-
-We simulate streaming by revealing larger prefixes of a tokenized corpus:
-\[
-250\text{k} \rightarrow 500\text{k} \rightarrow 1\text{M}
-\rightarrow 2\text{M} \rightarrow 4\text{M}.
-\]
-For each seed and condition, one model and optimizer continue through all
-stages. At stage $t$, minibatches are sampled only from the currently revealed
-prefix of size $\unique$.
-
-The standard protocol uses batch size 16, block size 128, and 1000 optimizer
-steps per stage. Therefore each stage consumes 2.048M sampled tokens, and the
-cumulative sampled tokens are listed in Table~\ref{tab:stream_tokens}.
-
-\begin{table}[h]
-\centering
-\caption{Standard expanding-prefix schedule.}
-\label{tab:stream_tokens}
-\begin{tabular}{rrrr}
-\toprule
-Stage & Unique tokens $\unique$ & Stage sampled tokens & Cumulative sampled tokens $\sampled$ \\
-\midrule
-0 & 250k & 2.048M & 2.048M \\
-1 & 500k & 2.048M & 4.096M \\
-2 & 1M & 2.048M & 6.144M \\
-3 & 2M & 2.048M & 8.192M \\
-4 & 4M & 2.048M & 10.240M \\
-\bottomrule
-\end{tabular}
-\end{table}
-
-All Torch experiments were run on Apple MPS only. The runner refuses CPU, CUDA,
-and MPS fallback. The main experiments use a 4096-token vocabulary, AdamW, a
-learning rate of $3 \times 10^{-4}$, weight decay 0.1, gradient clipping 1.0,
-64 validation batches, and seeds 1, 2, and 3 unless otherwise stated.
-
-\section{Regime Definition}
-
-The formula is not meant to change coefficients when model size or stream size
-changes. Those are exactly the variables the formula explains. Instead,
-coefficients are expected to be fixed within a training regime.
-
-We define a regime as:
-\begin{align*}
-\mathcal{R}=\{&
-\text{architecture family},\ \text{tokenizer},\ \text{corpus family},\\
-&\text{optimizer protocol},\ \text{dropout placement},\\
-&\text{stream protocol},\ \text{evaluation distribution}\}.
-\end{align*}
-
-Operationally, two runs are in the same regime if one fixed coefficient triplet
-$(A,B,C)$ predicts dropout optima across held-out model sizes and held-out
-prefix sizes without systematic bias. Changes in $\params$, $\unique$, and
-$\sampled$ should not require new coefficients. Changes in corpus family,
-tokenizer, architecture class, dropout placement, optimizer schedule, stream
-semantics, or validation distribution should trigger re-evaluation.
-
-\section{Pressure-Law Dropout}
-
-We propose the formula family
-\begin{equation}
-p_t = \clamp(p_{\min}, p_{\max},
-             A \log_{10}(\params / \unique)
-           + B \log_{10}(\sampled / \unique)
-           + C).
-\label{eq:pressure}
-\end{equation}
-
-The first term measures model/data pressure. Larger $\params / \unique$ means
-the model has more capacity per currently available unique token. The second
-term measures repeated-sampling pressure. Larger $\sampled / \unique$ means the
-optimizer has revisited the available prefix more often. We use logarithms
-because parameter counts and token counts vary over orders of magnitude while
-dropout is bounded.
-
-\subsection{Static Optimum Fit}
-
-For static sweeps, each row gives a measured best dropout $y_i$ at a fixed
-model and prefix. We set
-\[
-x_{1,i} = \log_{10}(\params_i / U_i), \quad
-x_{2,i} = \log_{10}(C_i / U_i),
-\]
-and solve
-\[
-\min_{A,B,C} \sum_i
-\left(y_i - A x_{1,i} - B x_{2,i} - C\right)^2.
-\]
-
-On the 15-point main local static screen, the grid optimum fit gives
-\[
-A=0.225,\quad B=0.169,\quad C=-0.210.
-\]
-A quadratic interpolation around the best measured grid point gives a similar
-fit:
-\[
-A=0.231,\quad B=0.166,\quad C=-0.211.
-\]
-
-\subsection{Streaming Schedule Coefficients}
-
-The best dynamic schedule is not necessarily identical to the best static
-optimum surface. The current validated streaming baseline uses
-\[
-A=0.154,\quad B=0.249,\quad C=-0.210,
-\]
-with $p_{\min}=0.02$ and $p_{\max}=0.65$. For the L12 model under the standard
-protocol, this gives
-\[
-0.300 \rightarrow 0.260 \rightarrow 0.180 \rightarrow 0.090 \rightarrow 0.020.
-\]
-
-\section{Experiments}
-
-\subsection{Initial Negative Result}
-
-The first tests used an 8.39M-parameter model and found that arbitrary high
-dropout was not useful. Table~\ref{tab:high_dropout} shows that high dropout
-underfit and lost to static 0.1 dropout.
-
-\begin{table}[h]
-\centering
-\caption{Naive high-dropout decay failed. Lower validation loss is better.}
-\label{tab:high_dropout}
-\begin{tabular}{lrrrr}
-\toprule
-Condition & 5M & 10M & 20M & 40M \\
-\midrule
-High-dropout decay & 6.9213 & 6.2689 & 5.4262 & 4.9090 \\
-Static 0.1 dropout & 5.6310 & 5.1018 & 4.8497 & 4.6743 \\
-Static 0.8 dropout & 6.9898 & 6.7637 & 6.4835 & 6.2390 \\
-\bottomrule
-\end{tabular}
-\end{table}
-
-\subsection{Static Dropout Screen}
-
-Table~\ref{tab:static_screen} shows the core static screen. The best dropout
-falls as unique-token count grows and rises with model size at fixed prefix.
-
-\begin{table}[h]
-\centering
-\caption{Static dropout optima in the OpenWebText10K cached-corpus regime.}
-\label{tab:static_screen}
-\resizebox{\textwidth}{!}{%
-\begin{tabular}{lrrrrrr}
-\toprule
-Model & Params & Prefix & Effective epochs & Best dropout & Val loss & Gap \\
-\midrule
-L8 & 8.39M & 250k & 40.96 & 0.40 & 5.4175 & 1.7763 \\
-L8 & 8.39M & 500k & 20.48 & 0.20 & 5.0216 & 1.3238 \\
-L8 & 8.39M & 1M & 10.24 & 0.14 & 4.7763 & 0.7863 \\
-L8 & 8.39M & 2M & 5.12 & 0.08 & 4.6232 & 0.4074 \\
-L8 & 8.39M & 4M & 2.56 & 0.00 & 4.5136 & 0.2621 \\
-L12 & 17.37M & 250k & 40.96 & 0.50 & 5.4384 & 2.0663 \\
-L12 & 17.37M & 500k & 20.48 & 0.40 & 4.9791 & 1.2434 \\
-L12 & 17.37M & 1M & 10.24 & 0.20 & 4.6871 & 0.9711 \\
-L12 & 17.37M & 2M & 5.12 & 0.14 & 4.5088 & 0.4870 \\
-L12 & 17.37M & 4M & 2.56 & 0.02 & 4.3875 & 0.3575 \\
-L16 & 31.46M & 250k & 40.96 & 0.60 & 5.5055 & 2.1870 \\
-L16 & 31.46M & 500k & 20.48 & 0.40 & 4.9814 & 1.7017 \\
-L16 & 31.46M & 1M & 10.24 & 0.30 & 4.6511 & 1.0216 \\
-L16 & 31.46M & 2M & 5.12 & 0.14 & 4.4270 & 0.6509 \\
-L16 & 31.46M & 4M & 2.56 & 0.02 & 4.2947 & 0.4400 \\
-\bottomrule
-\end{tabular}}
-\end{table}
-
-\subsection{Coefficient Fit}
-
-Table~\ref{tab:coefs} reports coefficient estimates from existing saved data.
-The main local screen is the cleanest static-regime estimate. Pooling the
-corpus probes without a corpus feature is misspecified because those probes
-share the same $\params / U$ and $C / U$ but have different optima.
-
-\begin{table}[h]
-\centering
-\caption{OLS coefficient fits for the static optimum surface.}
-\label{tab:coefs}
-\begin{tabular}{llrrrrr}
-\toprule
-Target & Data subset & n & $A$ & $B$ & $C$ & RMSE \\
-\midrule
-Grid & Main local & 15 & 0.225 & 0.169 & -0.210 & 0.0418 \\
-Quadratic & Main local & 15 & 0.231 & 0.166 & -0.211 & 0.0353 \\
-Grid & Main local + 5M & 18 & 0.232 & 0.118 & -0.159 & 0.0508 \\
-Quadratic & Main local + 5M & 18 & 0.237 & 0.117 & -0.161 & 0.0466 \\
-Grid & All existing & 21 & 0.343 & 0.014 & -0.210 & 0.0650 \\
-\bottomrule
-\end{tabular}
-\end{table}
-
-\subsection{Model-Size Validation}
-
-The dynamic streaming schedule was evaluated against static-dropout controls
-near the apparent optimum. Table~\ref{tab:model_size} shows final validation
-loss across five model sizes.
-
-\begin{table}[h]
-\centering
-\caption{Streaming formula vs. best static controls across model size.}
-\label{tab:model_size}
-\begin{tabular}{lrrrl}
-\toprule
-Model & Params & Formula final val & Best static final val & Paired deltas \\
-\midrule
-L8 & 8.39M & $4.6094 \pm 0.0056$ & 4.6242 & -0.0102, -0.0160, -0.0182 \\
-L10 & 12.31M & $4.5306 \pm 0.0094$ & 4.5580 & -0.0288, -0.0188, -0.0345 \\
-L12 & 17.37M & $4.4812 \pm 0.0062$ & 4.5183 & -0.0364, -0.0308, -0.0439 \\
-L14 & 23.70M & $4.4384 \pm 0.0087$ & 4.4736 & -0.0294, -0.0269, -0.0429 \\
-L16 & 31.46M & $4.4059 \pm 0.0046$ & 4.4459 & -0.0411, -0.0512, -0.0279 \\
-\bottomrule
-\end{tabular}
-\end{table}
-
-The formula wins all 15 paired final-loss comparisons in this set.
-
-\subsection{Repeated-Sampling Pressure}
-
-Changing stage steps changes $\sampled / \unique$ while keeping the prefix
-sequence fixed. Table~\ref{tab:update_pressure} shows that the pressure formula
-continues to beat fixed dropout at 500, 1000, and 2000 steps per stage.
-
-\begin{table}[h]
-\centering
-\caption{Update-pressure validation on L12.}
-\label{tab:update_pressure}
-\resizebox{\textwidth}{!}{%
-\begin{tabular}{rlrrrl}
-\toprule
-Stage steps & Formula path & Mean trajectory & Formula final & Best static final & Paired deltas \\
-\midrule
-500 & 0.226 -> 0.180 -> 0.102 -> 0.020 -> 0.020 & 5.1581 & $4.7138 \pm 0.0080$ & 4.7321 & -0.0152, -0.0147, -0.0249 \\
-1000 & 0.300 -> 0.260 -> 0.180 -> 0.090 -> 0.020 & 4.9226 & $4.4812 \pm 0.0062$ & 4.5183 & -0.0364, -0.0308, -0.0439 \\
-2000 & 0.376 -> 0.330 -> 0.252 -> 0.162 -> 0.065 & 4.7841 & $4.3089 \pm 0.0116$ & 4.3513 & -0.0453, -0.0321, -0.0489 \\
-\bottomrule
-\end{tabular}}
-\end{table}
-
-\subsection{Sampled-Pressure Coefficient Ablation}
-
-The sampled-pressure term was ablated by multiplying the current coefficient by
-0, 0.5, 1.0, and 1.5. Table~\ref{tab:b_ablation} shows that the full
-sampled-pressure term gives the best final validation loss.
-
-\begin{table}[h]
-\centering
-\caption{Ablation of repeated-sampling pressure on L12.}
-\label{tab:b_ablation}
-\resizebox{\textwidth}{!}{%
-\begin{tabular}{rlrrrr}
-\toprule
-Multiplier & Path & Mean trajectory & Final val & Final std & Final gap \\
-\midrule
-0x & 0.074 -> 0.027 -> 0.020 -> 0.020 -> 0.020 & 5.0282 & 4.5468 & 0.0011 & 0.3482 \\
-0.5x & 0.187 -> 0.141 -> 0.079 -> 0.020 -> 0.020 & 4.9260 & 4.5055 & 0.0046 & 0.3272 \\
-1.0x & 0.300 -> 0.260 -> 0.180 -> 0.090 -> 0.020 & 4.9226 & 4.4812 & 0.0062 & 0.2825 \\
-1.5x & 0.415 -> 0.368 -> 0.275 -> 0.163 -> 0.041 & 4.9739 & 4.4959 & 0.0025 & 0.2418 \\
-\bottomrule
-\end{tabular}}
-\end{table}
-
-\subsection{Architecture-Shape Holdouts}
-
-Two holdouts test whether parameter count is a useful capacity proxy inside the
-same Transformer family. Results are shown in Table~\ref{tab:shape}.
-
-\begin{table}[h]
-\centering
-\caption{Architecture-shape holdouts.}
-\label{tab:shape}
-\begin{tabular}{lrrr}
-\toprule
-Holdout & Params & Formula final val & Best static final val \\
-\midrule
-Deep/narrow 18x8x256 & 16.25M & $4.5286 \pm 0.0118$ & $4.5564 \pm 0.0127$ \\
-Wide 8x8x384 & 17.30M & $4.4658 \pm 0.0065$ & $4.4946 \pm 0.0087$ \\
-\bottomrule
-\end{tabular}
-\end{table}
-
-The formula wins all 6 paired final-loss comparisons across these two shape
-holdouts.
-
-\subsection{Corpus Holdouts}
-
-Corpus tests reveal the boundary of the two-variable formula. The frozen
-formula transfers to WikiText-103 but overestimates early dropout on
-TinyStories. A lower dropout path performs better on TinyStories.
-
-\begin{table}[h]
-\centering
-\caption{Public-corpus streaming checks.}
-\label{tab:corpus_stream}
-\begin{tabular}{llrr}
-\toprule
-Corpus & Condition & Final val & Mean trajectory \\
-\midrule
-WikiText-103 & Old formula & 4.0643 & 4.5704 \\
-WikiText-103 & Best static & 4.0893 & 4.5889 \\
-TinyStories & Old formula & 2.7221 & 3.1149 \\
-TinyStories & Probe blend & 2.7194 & 3.0934 \\
-TinyStories & Low-decay heuristic & 2.7068 & 3.0793 \\
-TinyStories & Best static & 2.7215 & 3.0880 \\
-\bottomrule
-\end{tabular}
-\end{table}
-
-A separate 1.835M-parameter pressure probe used the same $\params / U$ and
-$C/U$ on three corpora, but obtained different optima: 0.04 on the local
-corpus, 0.00 on WikiText-103, and 0.02 on TinyStories. This proves that corpus
-identity can shift the optimum independently of the two pressure ratios.
-
-\section{Discussion}
-
-The evidence supports a pressure-law structure, not a universal coefficient
-triplet. The static fit shows that dropout optima can be approximated from
-$\log_{10}(\params/U)$ and $\log_{10}(C/U)$ within the OpenWebText10K cached-corpus regime. The
-streaming experiments show that a schedule based on those variables can beat
-fixed dropout across model sizes and architecture-shape holdouts.
-
-However, the corpus probes show that the two-variable formula is incomplete.
-For a paper claim, we therefore recommend the following framing:
-
-\begin{quote}
-Within a fixed training regime, dropout can be scheduled from model/data
-pressure and repeated-sampling pressure. The coefficients are empirical
-calibration constants for the regime. They should be re-estimated when the
-corpus family, tokenizer, architecture class, dropout placement, optimizer
-protocol, streaming semantics, or validation distribution changes.
-\end{quote}
-
-\section{Limitations}
-
-The experiments are small compared with modern frontier-scale language model
-training. They are nevertheless useful because they isolate dropout behavior in
-a controlled setting. The main limitations are:
-
-\begin{itemize}
-    \item The dense static coefficient fit is local-corpus biased.
-    \item The public-corpus checks show that a corpus term is missing.
-    \item The streaming coefficients were selected by validation and ablation,
-    not by a full continuous coefficient sweep.
-    \item The models are nanochat-style causal Transformers, so transfer to
-    different architecture families is not established.
-    \item The local token cache is not yet a fully published dataset artifact,
-    which limits exact reproduction until the cache or source corpus is hosted.
-\end{itemize}
-
-\section{Reproducibility}
-
-The repository includes experiment code, result logs, model-selection CSVs, and
-reproduction commands. Every run records configuration, metrics, and attribution
-metadata. The runner enforces MPS-only Torch execution. For exact reproduction,
-the local cache must be supplied or rebuilt from a documented corpus source.
-Public WikiText-103 and TinyStories holdouts have preparation scripts.
-
-\section{Conclusion}
-
-Dropout decay is useful when it follows a moving regularization optimum, not
-when it starts from an arbitrary high value. Existing evidence supports a
-pressure-law schedule using model/data pressure and repeated-sampling pressure.
-For the local static regime, the best fit is approximately
-$(A,B,C)=(0.225,0.169,-0.210)$. For the validated streaming baseline, the best
-current schedule uses $(A,B,C)=(0.154,0.249,-0.210)$. The next step is to fit
-and validate a corpus-aware correction without losing the regime-calibrated
-structure.
-
-\bibliographystyle{plainnat}
-\bibliography{references}
-
-\appendix
-
-\section{When To Refit Coefficients}
-
-Refit or at least revalidate coefficients when any of the following changes:
-corpus family, tokenizer/vocabulary, architecture class, dropout placement,
-optimizer and learning-rate protocol, sequence length, streaming semantics, or
-validation distribution. Do not refit merely because model size, prefix size, or
-sampled-token count changes inside the same regime; those are the formula's
-input variables.
-
-\section{Recommended Reporting Checklist}
-
-For each future regime, report:
-
-\begin{itemize}
-    \item model parameter count and architecture shape;
-    \item tokenizer and vocabulary size;
-    \item unique tokens available at each stage;
-    \item sampled tokens per stage and cumulative sampled tokens;
-    \item dropout path and clamp values;
-    \item fixed-dropout controls near the predicted optimum;
-    \item seeds, mean validation loss, standard deviation, and paired deltas;
-    \item train-evaluation loss and validation gap;
-    \item exact command and run artifact path.
-\end{itemize}
-
-\end{document}

paper/dropout_decay_regime_specific_pressure_law.bbl

@@ -1,73 +0,0 @@
-\begin{thebibliography}{8}
-\providecommand{\natexlab}[1]{#1}
-\providecommand{\url}[1]{\texttt{#1}}
-\expandafter\ifx\csname urlstyle\endcsname\relax
-  \providecommand{\doi}[1]{doi: #1}\else
-  \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
-
-\bibitem[Eldan and Li(2023)]{eldan2023tinystories}
-Ronen Eldan and Yuanzhi Li.
-\newblock Tinystories: How small can language models be and still speak
-  coherent english?
-\newblock \emph{arXiv preprint arXiv:2305.07759}, 2023.
-\newblock URL \url{https://arxiv.org/abs/2305.07759}.
-
-\bibitem[Gokaslan et~al.(2019)Gokaslan, Cohen, Pavlick, and
-  Tellex]{gokaslan2019openwebtext}
-Aaron Gokaslan, Vanya Cohen, Ellie Pavlick, and Stefanie Tellex.
-\newblock Openwebtext corpus.
-\newblock \url{https://zenodo.org/records/3834942}, 2019.
-\newblock Open reproduction of the WebText-style corpus.
-
-\bibitem[Hoffmann et~al.(2022)Hoffmann, Borgeaud, Mensch, Buchatskaya, Cai,
-  Rutherford, Casas, Hendricks, Welbl, Clark, Hennigan, Noland, Millican,
-  van~den Driessche, Damoc, Guy, Osindero, Simonyan, Elsen, Rae, Vinyals, and
-  Sifre]{hoffmann2022training}
-Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Trevor
-  Cai, Eliza Rutherford, Diego de~Las Casas, Lisa~Anne Hendricks, Johannes
-  Welbl, Aidan Clark, Tom Hennigan, Eric Noland, Katie Millican, George van~den
-  Driessche, Bogdan Damoc, Aurelia Guy, Simon Osindero, Karen Simonyan, Erich
-  Elsen, Jack~W. Rae, Oriol Vinyals, and Laurent Sifre.
-\newblock Training compute-optimal large language models.
-\newblock \emph{arXiv preprint arXiv:2203.15556}, 2022.
-\newblock URL \url{https://arxiv.org/abs/2203.15556}.
-
-\bibitem[Kaplan et~al.(2020)Kaplan, McCandlish, Henighan, Brown, Chess, Child,
-  Gray, Radford, Wu, and Amodei]{kaplan2020scaling}
-Jared Kaplan, Sam McCandlish, Tom Henighan, Tom~B. Brown, Benjamin Chess, Rewon
-  Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei.
-\newblock Scaling laws for neural language models.
-\newblock \emph{arXiv preprint arXiv:2001.08361}, 2020.
-\newblock URL \url{https://arxiv.org/abs/2001.08361}.
-
-\bibitem[Karpathy(2025)]{karpathy_nanochat}
-Andrej Karpathy.
-\newblock nanochat.
-\newblock \url{https://github.com/karpathy/nanochat}, 2025.
-\newblock MIT-licensed repository, accessed 2026-05-29.
-
-\bibitem[Merity et~al.(2017)Merity, Xiong, Bradbury, and
-  Socher]{merity2017pointer}
-Stephen Merity, Caiming Xiong, James Bradbury, and Richard Socher.
-\newblock Pointer sentinel mixture models.
-\newblock In \emph{International Conference on Learning Representations}, 2017.
-\newblock URL \url{https://arxiv.org/abs/1609.07843}.
-
-\bibitem[Srivastava et~al.(2014)Srivastava, Hinton, Krizhevsky, Sutskever, and
-  Salakhutdinov]{srivastava2014dropout}
-Nitish Srivastava, Geoffrey Hinton, Alex Krizhevsky, Ilya Sutskever, and Ruslan
-  Salakhutdinov.
-\newblock Dropout: A simple way to prevent neural networks from overfitting.
-\newblock \emph{Journal of Machine Learning Research}, 15\penalty0
-  (56):\penalty0 1929--1958, 2014.
-\newblock URL \url{https://www.jmlr.org/papers/v15/srivastava14a.html}.
-
-\bibitem[Vaswani et~al.(2017)Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez,
-  Kaiser, and Polosukhin]{vaswani2017attention}
-Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones,
-  Aidan~N. Gomez, Lukasz Kaiser, and Illia Polosukhin.
-\newblock Attention is all you need.
-\newblock In \emph{Advances in Neural Information Processing Systems}, 2017.
-\newblock URL \url{https://arxiv.org/abs/1706.03762}.
-
-\end{thebibliography}

paper/dropout_decay_regime_specific_pressure_law.pdf

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46ba08475ba2566bc3733eed5403db061a3ce4145ed6bd79965796719a396bc8
-size 267426
+oid sha256:99ac702644025cd253907d1c8e7892f2de07e32564a68c33a25fc505ac25b886
+size 330095

paper/dropout_decay_regime_specific_pressure_law.tex

@@ -14,7 +14,7 @@
 \usepackage[colorlinks=true,linkcolor=blue,citecolor=blue,urlcolor=blue]{hyperref}
 
 \title{Regime-Specific Pressure Laws for Dropout Decay in Streaming Language Model Training}
-\author{Anonymous Authors}
+\author{Mandeep Sidhu}
 \date{June 2026}
 
 \newcommand{\clamp}{\operatorname{clamp}}
@@ -46,14 +46,17 @@ these per-cell optima into a regime-specific pressure law,
 \]
 where $\params$ is parameter count, $\unique$ is currently available unique
 tokens, and $\sampled$ is cumulative sampled training tokens. The fitted
-interaction law explains static optimum cells with mean absolute errors of
-0.0158, 0.0180, and 0.0162 dropout on OpenWebText10K, TinyStories, and
-WikiText-103 respectively. In locked streaming validation, fitted or
-formula-derived decay schedules beat the best fixed-dropout baseline on final
-validation loss in all three regimes at 4M-token validation scale and again in
-larger 51.84M-parameter, 8M-token extrapolations. The large extrapolation tier
-wins all paired seeds across all three regimes (9/9 paired wins), but stage-wise
-analysis shows the schedules are final-loss optimized rather than best at every
+interaction law explains static optimum cells with dropout-rate mean absolute
+errors of 0.0158, 0.0180, and 0.0162 on OpenWebText10K, TinyStories, and
+WikiText-103 respectively. In validation runs where the same model and optimizer
+continue training as the available token prefix grows, coefficient-derived decay
+schedules beat the best fixed-dropout baseline on final validation loss in all
+three regimes at the 4M final prefix and again in larger 51.84M-parameter,
+8M-token extrapolations. Mean final-loss gains over the best fixed dropout are
+0.0473/0.0133/0.0297 nats at 4M and 0.0363/0.0100/0.0184 nats at 8M for
+OpenWebText10K/TinyStories/WikiText-103. The large extrapolation tier wins all
+paired seeds across all three regimes (9/9 paired wins), but stage-wise analysis
+shows the schedules are final-loss optimized rather than best at every
 intermediate prefix. The results support a practical hypothesis: dropout decay
 should be calibrated as a regime-specific pressure law, not chosen as a
 universal constant or an arbitrary high-to-low schedule.
@@ -80,16 +83,21 @@ claim is that useful dropout is a function of training pressure: model capacity
 relative to the current data prefix, and cumulative optimizer sampling relative
 to that prefix.
 
-This paper makes four empirical contributions.
+We use \emph{regime} to mean a fixed corpus family, tokenizer, architecture
+family, optimizer protocol, dropout placement, stream protocol, and evaluation
+distribution. Model size and prefix size vary inside a regime; changing the
+text distribution, tokenizer, or training protocol defines a new regime. For
+each of the three text regimes studied here, this paper makes four empirical
+contributions.
 
 \begin{enumerate}
-    \item We define calibration cells for streaming dropout: one model, one
+    \item We define per-regime calibration cells for streaming dropout: one model, one
     unique-token prefix, one sampled-token budget, and a sweep over fixed
     dropout rates.
-    \item We show that calibration cells often have a parabolic or plateau-like
+    \item We show that per-regime calibration cells often have a parabolic or plateau-like
     validation-loss curve over dropout, making the local optimum a usable
     supervised target.
-    \item We fit a regime-specific interaction pressure law that maps
+    \item We fit a separate interaction pressure law per regime that maps
     $(\params,\unique,\sampled)$ to a dropout rate.
     \item We validate the resulting schedules against broad fixed-dropout
     baselines across three text regimes and a larger-model/larger-stream
@@ -98,7 +106,12 @@ This paper makes four empirical contributions.
 
 The central result is not that one coefficient vector works everywhere. It does
 not. The central result is that the pressure-law structure is useful, while the
-coefficients are regime-specific.
+coefficients are regime-specific. A reader should interpret the paper as an
+evidence ladder: static sweeps establish that dropout has a measurable optimum
+inside a regime; coefficient fits show that these optima can be predicted from
+pressure variables; continuing streaming runs test whether the predicted
+trajectory actually improves a single model trained through the expanding
+prefix sequence.
 
 \section{Related Work}
 
@@ -111,6 +124,16 @@ The expectation is preserved, but the variance grows as $h_i^2p/(1-p)$. This
 captures the key tradeoff in our experiments: low dropout can overfit repeated
 small prefixes, but high dropout can remove too much usable capacity.
 
+\paragraph{Dropout schedules.}
+This paper is not a survey of every possible adaptive regularization schedule.
+Its controlled comparison is against a best tested fixed-dropout grid, because
+fixed dropout is the default practical choice and gives a clear baseline for
+whether adaptation helps. Early hand-designed high-to-low schedules were useful
+as exploratory probes but were not the final claim; the reported schedules are
+generated from fitted regime coefficients before the streaming validation run.
+Broader comparisons against other adaptive dropout or regularization policies
+are left to future work.
+
 \paragraph{Transformer language models and scaling.}
 Our model family is a decoder-only Transformer following the attention-based
 architecture of \citet{vaswani2017attention}. Scaling-law work shows that loss
@@ -125,9 +148,20 @@ We evaluate on three text regimes. TinyStories is a synthetic simple-English
 story corpus designed for small language-model studies \citep{eldan2023tinystories}.
 WikiText-103 is a standard word-level language-modeling benchmark derived from
 Wikipedia articles \citep{merity2017pointer}. OpenWebText is an open
-WebText-style corpus \citep{gokaslan2019openwebtext}. We use cached/tokenized
-subsets and therefore report results as regime-level evidence rather than
-claims about full-corpus state of the art.
+WebText-style corpus \citep{gokaslan2019openwebtext}. OpenWebText10K denotes
+the cached/tokenized OpenWebText-style subset used in this project; the suffix
+distinguishes this experimental subset from the full OpenWebText corpus. We use
+cached/tokenized subsets and therefore report results as regime-level evidence
+rather than claims about full-corpus state of the art.
+
+\begin{table}[t]
+\centering
+\caption{Text regimes used in the experiments. Prefixes are token counts exposed
+to the trainer at each stream stage; all reported runs use a 4096-token BPE
+vocabulary, so numeric losses should be compared within this tokenizer setting.}
+\label{tab:regimes}
+\resizebox{\textwidth}{!}{\input{tables/regime_setup_table}}
+\end{table}
 
 \section{Experimental Setting}
 
@@ -140,23 +174,33 @@ core BPE tokenizer and causal Transformer template. Derived source files retain
 the original copyright and permission notice.
 
 The architecture family keeps the standard Transformer feed-forward expansion
-ratio of four. The central model sizes are L8-H8-D256 (8.39M parameters),
-L12-H8-D320 (17.37M), L16-H8-D384 (31.46M or 32.71M depending on
-vocabulary/run metadata), and L20-H8-D448 (51.84M). The tokenizer vocabulary is
-4096 in the reported runs. Unless noted otherwise, training uses AdamW,
-learning rate $3\times 10^{-4}$, weight decay 0.1, gradient clipping 1.0, batch
-size 16, and block size 128. All Torch training runs are MPS-only; the runner
-refuses CUDA, CPU execution, and MPS fallback.
+ratio of four. We denote architectures as L$x$-H$y$-D$z$, where $x$ is the
+number of Transformer layers, $y$ is the number of attention heads, and $z$ is
+the embedding dimension. The central model sizes are L8-H8-D256 (8.39M parameters),
+L12-H8-D320 (17.37M), L16-H8-D384 (31.46M in the 4096-vocabulary runs; one
+archived metadata path reports 32.71M from a different vocabulary configuration),
+and L20-H8-D448 (51.84M). The tokenizer vocabulary is 4096 in the reported runs.
+Unless noted otherwise, training uses AdamW, learning rate $3\times 10^{-4}$,
+weight decay 0.1, gradient clipping 1.0, batch size 16, and block size 128. All
+Torch training runs are MPS-only; the runner refuses CUDA, CPU execution, and
+MPS fallback.
+
+All reported losses are mean next-token cross-entropy losses in nats. The model
+uses the same active dropout probability for embedding dropout, attention-weight
+dropout, attention output/residual dropout, and MLP output dropout. This
+placement is part of the regime definition; changing it would require
+recalibrating the coefficients.
 
 \subsection{Streaming protocol}
 
-We simulate streaming by revealing larger token prefixes over time. A locked
-streaming run keeps the same model and optimizer state across stages. At stage
-$t$, minibatches are sampled only from the currently revealed prefix
+We simulate streaming by revealing larger token prefixes over time. We call this
+the locked-stream setting: the model is not restarted at each prefix; one
+training run keeps the same model and optimizer state as the data stream grows.
+At stage $t$, minibatches are sampled only from the currently revealed prefix
 $\unique$. The two main protocols are:
 
 \begin{itemize}
-    \item 4M validation: OpenWebText10K and WikiText-103 use 250k, 500k, 1M,
+    \item 4M final-prefix validation: OpenWebText10K and WikiText-103 use 250k, 500k, 1M,
     2M, and 4M prefixes. TinyStories uses 500k, 1M, 2M, and 4M prefixes. These
     runs use five random seeds.
     \item 8M extrapolation: L20-H8-D448 with an 8M final prefix. These runs use
@@ -165,11 +209,48 @@ $\unique$. The two main protocols are:
 
 Each run compares a dropout schedule against a grid of static-dropout
 baselines. The best static baseline is selected after training from the same
-run family, so the decay schedule competes against a strong fixed-rate oracle
+run family, so the decay schedule competes against the best fixed dropout found
 within the tested grid.
 
+\begin{table}[t]
+\centering
+\caption{Per-regime experimental evidence ladder. The same sequence is run
+separately for OpenWebText10K, TinyStories, and WikiText-103. The streaming rows
+are the decisive validation layers, while the calibration rows explain where the
+schedules come from.}
+\label{tab:protocol}
+\resizebox{\textwidth}{!}{\input{tables/experiment_protocol_table}}
+\end{table}
+
+\begin{table}[t]
+\centering
+\caption{Streaming training budgets used for validation. At each optimization
+step, the runner consumes \texttt{batch\_size} $\times$ \texttt{block\_size}
+token positions; therefore $C_t$ is the cumulative sum of sampled token positions
+across stream stages.}
+\label{tab:budgets}
+\resizebox{\textwidth}{!}{\input{tables/protocol_budget_table}}
+\end{table}
+
 \section{From Calibration Cells to a Pressure Law}
 
+Figure~\ref{fig:protocol} summarizes the full procedure. The paper's central
+methodological choice is to separate cheap calibration from expensive streaming
+validation. Static cells are not themselves the final claim; they are probes
+that estimate where a fixed model/prefix combination stops benefiting from more
+or less dropout. The pressure law is fitted from those probes, frozen, and then
+tested in the harder setting where a single model continues training as the
+available stream grows.
+
+\begin{figure}[t]
+\centering
+\input{figures/calibration_to_streaming_protocol}
+\caption{Calibration-to-streaming protocol. Static sweeps produce target
+dropouts $p^\star$; a regime-specific pressure law fits those targets; the
+resulting schedule is frozen before locked-stream validation.}
+\label{fig:protocol}
+\end{figure}
+
 \subsection{Calibration cell definition}
 
 A calibration cell is one static dropout sweep at a fixed tuple:
@@ -186,17 +267,25 @@ point is bracketed by worse neighbors, a local quadratic interpolation of the
 minimum. Boundary optima are retained but downweighted in coefficient fitting
 because the true optimum may lie outside the tested grid.
 
-Figure~\ref{fig:cell-curves} shows representative cells. The curves are not
-perfect mathematical parabolas, but they show the practical pattern needed for
-calibration: validation loss improves as dropout moves away from zero, reaches
-a local minimum or plateau, and then rises when dropout becomes too large.
+Figure~\ref{fig:cell-curves} shows one selected representative cell from each
+regime. These examples are illustrative of the curve shape, not the full
+calibration set. The complete coefficient fits use 15--16 cells per regime and
+are summarized in Figure~\ref{fig:static-optima} and Table~\ref{tab:coefficients}.
+The curves are not perfect mathematical parabolas, but they show the practical
+pattern needed for calibration: validation loss improves as dropout moves away
+from zero, reaches a local minimum or plateau, and then rises when dropout
+becomes too large.
 
 \begin{figure}[t]
 \centering
 \input{figures/calibration_cell_curves}
-\caption{Representative static dropout calibration cells. The y-axis is
-validation loss above the best tested dropout in that cell, so each curve is
-normalized to show shape rather than absolute corpus difficulty.}
+\caption{Representative static dropout calibration cells, one selected from
+each regime. Each panel fixes the corpus regime, model size, unique-token
+prefix, sampled-token budget, and validation split, then sweeps static dropout.
+The y-axis is token-level validation cross-entropy in nats above the best tested
+dropout in that cell, so each curve is normalized to show shape rather than
+absolute corpus difficulty. The full calibration sets contain 15--16 cells per
+regime.}
 \label{fig:cell-curves}
 \end{figure}
 
@@ -212,7 +301,18 @@ prefix increases memorization capacity and tends to increase useful dropout.
 Increasing unique tokens at fixed model reduces this pressure. The second term
 measures repeated-sampling pressure. If the optimizer consumes many sampled
 tokens while the unique prefix remains small, it repeatedly revisits the same
-data and can benefit from stronger regularization.
+data and can benefit from stronger regularization. For example, if training
+draws two million token positions from a one-million-token revealed prefix, then
+$\unique=1\mathrm{M}$ and $\sampled=2\mathrm{M}$; the same unique data has been
+sampled for roughly two prefix-equivalents of optimization.
+
+\begin{table}[t]
+\centering
+\caption{Symbol glossary. The pressure law uses absolute quantities available
+before or during a run, then converts them into dimensionless pressure variables.}
+\label{tab:symbols}
+\resizebox{0.92\textwidth}{!}{\input{tables/symbol_table}}
+\end{table}
 
 The fitted interaction family is
 \begin{equation}
@@ -221,8 +321,46 @@ The fitted interaction family is
 \label{eq:interaction-law}
 \end{equation}
 For schedule generation we use $p_{\min}=0.02$ and $p_{\max}=0.65$ unless a
-run explicitly specifies a different clamp. Static sweeps may still include
-0.0 as an ablation.
+run explicitly specifies a different clamp. The lower bound keeps generated
+decay schedules near but not exactly at zero dropout, while static sweeps may
+still include 0.0 as an ablation. The upper bound keeps generated schedules away
+from the very-high-dropout regime; early experiments with dropout near 0.8
+underfit badly, so this paper does not claim that arbitrarily high initial
+dropout is useful.
+
+Operationally, a coefficient fit follows the same recipe in each regime:
+\begin{enumerate}
+    \item Choose a small grid of model sizes and unique-token prefixes that
+    covers the intended deployment range.
+    \item For each cell, train the model with several fixed dropout rates under
+    the same sampled-token budget and validation split.
+    \item Extract $p^\star$ as the lowest-loss grid point, or as a local
+    quadratic minimum when the best grid point is bracketed by higher-loss
+    neighbors.
+    \item Fit $A,B,D,C_0$ by regressing $p^\star$ on $x_t$, $y_t$, and $x_ty_t$.
+    Cells with boundary or weakly identified optima are downweighted because
+    their true optimum may lie outside the tested grid. In practice, lower
+    weights are assigned to cells whose best dropout is on the tested boundary,
+    whose minimum is not bracketed by worse neighbors, whose curve is very flat,
+    or whose best-loss estimate is comparatively noisy.
+    \item Freeze the coefficients and generate a dropout path from the actual
+    streaming prefixes before running the locked-stream validation.
+\end{enumerate}
+
+This recipe matters because it prevents the decay schedule from being selected
+after seeing the streaming validation losses. The schedule is allowed to use
+model size, current unique tokens, cumulative sampled tokens, and fitted
+regime coefficients; it is not allowed to tune itself on the final test run.
+
+\begin{table}[t]
+\centering
+\caption{Heuristic confidence weights used when fitting coefficient targets.
+Multipliers are cumulative, so a boundary and unbracketed cell receives both
+penalties. These weights reduce the influence of cells where the static optimum
+is weakly identified.}
+\label{tab:weights}
+\resizebox{\textwidth}{!}{\input{tables/calibration_weight_table}}
+\end{table}
 
 \subsection{Regime-specific coefficients}
 
@@ -233,14 +371,27 @@ inputs to the formula. Corpus family, tokenizer, or optimizer changes can shift
 the coefficients.
 
 Table~\ref{tab:coefficients} gives the fitted interaction coefficients. Each
-regime uses 15 or 16 calibration cells. The fitted mean absolute error is about
-0.016--0.018 dropout, meaning the coefficient law typically predicts the
-calibrated static optimum to within roughly two hundredths of dropout.
+regime uses 15 or 16 calibration cells. The error columns are measured in
+dropout-rate units, not validation-loss units. For calibration targets
+$p_i^\star$ and formula predictions $\hat p_i$, the reported mean absolute
+error is
+\[
+\mathrm{MAE}=\frac{1}{N}\sum_{i=1}^{N}|\hat p_i-p_i^\star|.
+\]
+The fitted MAE is about 0.016--0.018 dropout, meaning the coefficient law
+typically predicts the calibrated static optimum to within roughly two
+hundredths of dropout. Leave-model-out MAE refits the coefficients while holding
+out one model size, then predicts the held-out model cells. Leave-prefix-out
+MAE does the same while holding out one unique-token prefix. These leave-out
+columns test whether the formula generalizes across model sizes and stream
+prefix sizes inside the same regime.
 
 \begin{table}[t]
 \centering
-\caption{Regime-specific interaction coefficient fits. L-model and L-prefix
-are leave-model-out and leave-prefix-out mean absolute errors.}
+\caption{Regime-specific interaction coefficient fits. Dropout MAE is mean
+absolute error in predicted dropout rate, not validation loss; units are
+absolute dropout probability. L-model and L-prefix are leave-model-out and
+leave-prefix-out dropout MAEs.}
 \label{tab:coefficients}
 \resizebox{\textwidth}{!}{\input{tables/coefficient_table}}
 \end{table}
@@ -269,24 +420,62 @@ Table~\ref{tab:streaming-validation} summarizes the main streaming results.
 The ``Gain'' column is best static final validation loss minus decay final
 validation loss, so positive values favor decay. The best static baseline is
 chosen from the tested fixed-dropout grid in the same run family.
+\[
+\mathrm{Gain}
+=L_{\mathrm{final}}(\mathrm{best\ static})
+-L_{\mathrm{final}}(\mathrm{decay}).
+\]
+Thus positive gain means the decay schedule has lower final validation loss
+than the best tested fixed-dropout baseline.
+
+Table~\ref{tab:evidence-ladder} compresses the full evidence ladder into one
+view. The calibration MAE columns are dropout-rate prediction errors, not final
+validation metrics; they measure how well the pressure law explains static-cell
+optima. The gain and paired-win columns are the actual streaming tests. The
+pattern is consistent across all three regimes: the law fits static targets
+within roughly two hundredths of dropout, and the resulting schedules beat the
+best tested fixed-dropout baseline at the final prefix.
+
+\begin{table}[t]
+\centering
+\caption{Combined calibration and streaming evidence. Calibration MAE is
+dropout-rate prediction error. Gain is
+$L_{\mathrm{final}}(\mathrm{best\ static})-L_{\mathrm{final}}(\mathrm{decay})$,
+so positive gain favors decay; gain units are token-level cross-entropy nats.
+Wins are paired random-seed wins.}
+\label{tab:evidence-ladder}
+\resizebox{0.88\textwidth}{!}{\input{tables/evidence_ladder_table}}
+\end{table}
 
 \begin{table}[t]
 \centering
-\caption{Streaming validation against best fixed-dropout baselines.}
+\caption{Streaming validation against best fixed-dropout baselines. Final losses
+are mean token-level cross-entropy in nats, reported as mean $\pm$ sample
+standard deviation across seeds.}
 \label{tab:streaming-validation}
 \resizebox{\textwidth}{!}{\input{tables/streaming_validation_table}}
 \end{table}
 
+\begin{table}[t]
+\centering
+\caption{Paired final-loss gain audit. Paired gain is each seed's best static
+final loss minus that seed's decay final loss. Confidence intervals are
+nonparametric bootstrap intervals over seeds; the three-seed 8M intervals are
+descriptive rather than definitive.}
+\label{tab:paired-ci}
+\resizebox{\textwidth}{!}{\input{tables/paired_gain_ci_table}}
+\end{table}
+
 OpenWebText10K provides the cleanest five-seed coefficient-derived validation:
 the interaction schedule improves final validation loss by 0.0473 over the
 best static baseline and wins all five paired seeds. TinyStories shows a
 smaller but consistent final-loss margin: 0.0133 over the best static baseline
-with five paired wins. WikiText-103 is more nuanced. The earlier
-formula-derived L12 schedule wins the five-seed 4M run, but the later
-static-cell-fitted interaction schedule decays too quickly for L12 and is only
-near the best static baseline. This negative result is important: a good static
-cell fit does not by itself guarantee the best streaming schedule unless the
-schedule objective is aligned with final-loss optimization.
+with five paired wins. WikiText-103 also improves at the final 4M prefix, with
+a 0.0297 gain and five paired wins. Its supporting ablations show a useful
+caveat: an interaction schedule fitted only to static-cell optima can decay to
+the dropout floor too early for L12 streaming and land near the best static
+baseline. We therefore treat static-cell fit as calibration evidence, not as a
+substitute for locked-stream validation.
 
 \subsection{L20 8M extrapolation}
 
@@ -298,16 +487,35 @@ TinyStories, and WikiText-103, the decay schedule beats the best static
 baseline on final validation loss and wins all paired seeds (3/3 in each
 regime, 9/9 total).
 
+Table~\ref{tab:l20-paths} makes the extrapolation test concrete. The schedules
+are not arbitrary hand-picked names; they are the dropout values produced by
+the fitted regime coefficients at each revealed prefix. The final static
+baseline is the best fixed dropout found from the companion static grid for the
+same L20 stream.
+
+\begin{table}[t]
+\centering
+\caption{Frozen L20 locked-stream extrapolation schedules generated before
+validation. In each row, the same model and optimizer continue training across
+the prefix path while the dropout follows the frozen decay path.}
+\label{tab:l20-paths}
+\resizebox{\textwidth}{!}{\input{tables/l20_schedule_path_table}}
+\end{table}
+
 Figure~\ref{fig:final-improvements} visualizes the final-loss gains. The
 largest L20 gain is OpenWebText10K (0.0363), followed by WikiText-103 (0.0184)
-and TinyStories (0.0100). The TinyStories gain is smaller because a static
-dropout near 0.20 is already a strong fixed compromise for that regime.
+and TinyStories (0.0100). The TinyStories gain is smaller because the best fixed
+baseline is already unusually competitive: static dropout near 0.20 regularizes
+the early prefixes reasonably well on this simpler corpus and does not
+underperform as sharply as fixed dropout does in OpenWebText10K. The decay
+schedule still wins, but it has less fixed-rate error to remove.
 
 \begin{figure}[t]
 \centering
 \input{figures/final_improvements}
 \caption{Final validation-loss improvement over the best fixed-dropout
-baseline. Positive values favor the decay schedule.}
+baseline. Values are token-level cross-entropy improvements in nats; positive
+values favor the decay schedule.}
 \label{fig:final-improvements}
 \end{figure}
 
@@ -322,8 +530,10 @@ the middle of the stream and wins only at the final 8M prefix.
 
 \begin{table}[t]
 \centering
-\caption{L20 stage-wise comparison. Delta is decay validation loss minus the
-best static validation loss at the same prefix; negative favors decay.}
+\caption{L20 locked-stream stage-wise comparison. The same model and optimizer
+continue training across increasing token prefixes. Delta is decay validation
+loss minus the best static validation loss at the same prefix, in nats; negative
+favors decay.}
 \label{tab:l20-stage}
 \resizebox{\textwidth}{!}{\input{tables/l20_stage_table}}
 \end{table}
@@ -332,7 +542,9 @@ best static validation loss at the same prefix; negative favors decay.}
 \centering
 \input{figures/l20_stage_deltas}
 \caption{L20 stage-wise deltas versus the best static dropout at each prefix.
-The schedules are final-loss schedules, not greedy per-prefix optima.}
+The plotted value is token-level decay loss minus best static loss in nats, so
+values below zero favor decay. The schedules are final-loss schedules, not
+greedy per-prefix optima.}
 \label{fig:l20-stage-deltas}
 \end{figure}
 
@@ -373,6 +585,23 @@ if the goal is to train one model continuously through a growing stream, not to
 restart and optimize each prefix independently. Static baselines can be locally
 optimal at one prefix but cannot adapt when the stream changes.
 
+\paragraph{What the result does and does not prove.}
+The evidence supports a procedure, not a universal numeric constant. For a
+given regime, a small static calibration grid can estimate coefficient values
+that generate useful dropout schedules across model and prefix scale. The
+evidence does not show that the same coefficients should transfer unchanged to
+a different tokenizer, corpus distribution, optimizer, sequence length, or
+dropout placement. Those changes define a new regime and should trigger a new
+calibration.
+
+\paragraph{Applying the procedure to a new regime.}
+The intended use is procedural: select a small grid of model sizes and prefix
+sizes that spans the deployment range, run static dropout sweeps for those
+cells, extract $p^\star$ targets, fit $A,B,D,C_0$, freeze the resulting schedule,
+and validate it in a continuing stream against the best tested fixed dropout.
+The calibration step is cheaper than full multi-seed streaming because it probes
+fixed cells rather than repeatedly training full streaming schedules.
+
 \section{Limitations}
 
 The experiments are intentionally small compared with frontier language-model
@@ -384,14 +613,16 @@ confirmed law at billion-parameter scale.
 The static coefficient fit optimizes dropout targets extracted from static
 sweeps, while the streaming objective is final validation loss after continued
 training. WikiText-103 shows that these objectives can diverge: the fitted
-interaction schedule is excellent as a static optimum model but decays too
+interaction schedule predicts static-cell optima accurately but decays too
 quickly for L12 streaming. Future work should fit coefficients directly against
 streaming objective functions or include trajectory-aware loss terms.
 
 The L20 extrapolation tier uses three seeds for cost reasons. This is balanced
 across all three regimes and supported by five-seed smaller-stream validation,
-but larger seed counts would narrow confidence intervals. We also do not claim
-that the tested fixed-dropout grids contain every possible fixed optimum.
+but the bootstrap intervals in Table~\ref{tab:paired-ci} should be read as
+descriptive at $n=3$, not as a substitute for larger seed counts. We also do not
+claim that the tested fixed-dropout grids contain every possible fixed optimum,
+or that they exhaust all possible adaptive regularization baselines.
 
 \section{Reproducibility}
 
@@ -402,6 +633,18 @@ fitting uses \texttt{scripts/fit\_dropout\_coefficients.py}; this manuscript's
 figures and tables are generated by
 \texttt{scripts/generate\_paper\_assets.py}. Existing result artifacts are read
 from \texttt{runs/}. The experiment runner enforces MPS-only Torch execution.
+Code, cached artifacts, experiment outputs, and paper-generation scripts are
+available at \url{https://huggingface.co/cuber12/dropout-decay}.
+
+\begin{table}[t]
+\centering
+\caption{Reproducibility metadata for the manuscript build and reported
+experiments. The final public release should cite an immutable repository tag or
+commit; the table records the current base commit and software stack used while
+preparing this manuscript.}
+\label{tab:repro}
+\resizebox{\textwidth}{!}{\input{tables/reproducibility_table}}
+\end{table}
 
 All generated tables in this paper are derived from saved metrics JSONL files
 and coefficient JSON files. The paper-generation summary is written to

paper/figures/calibration_cell_curves.tex

@@ -25,7 +25,6 @@
 \nextgroupplot[
     title={TinyStories, L12, 1M},
     xlabel={Dropout},
-    ylabel={Validation loss above best},
     ymin=0,
     grid=both,
 ]
@@ -38,7 +37,6 @@
 \nextgroupplot[
     title={WikiText-103, L12, 250k},
     xlabel={Dropout},
-    ylabel={Validation loss above best},
     ymin=0,
     grid=both,
 ]

paper/figures/calibration_to_streaming_protocol.tex

@@ -0,0 +1,27 @@
+\begin{tikzpicture}[font=\scriptsize]
+\tikzstyle{box}=[
+    draw,
+    rounded corners=2pt,
+    thick,
+    align=center,
+    text width=0.19\textwidth,
+    minimum height=1.5cm,
+    inner sep=5pt
+]
+\node[box, fill=blue!7] at (0,0) (sweep) {Static calibration cells\\
+\footnotesize model $P$, prefix $U$, sampled tokens $C$, dropout grid};
+\node[box, fill=green!8] at (3.7,0) (target) {Extract $p^\star$\\
+\footnotesize grid best or bracketed quadratic minimum};
+\node[box, fill=yellow!12] at (7.4,0) (fit) {Fit pressure law\\
+\footnotesize $p^\star \approx A x + B y + Dxy + C_0$};
+\node[box, fill=red!7] at (11.1,0) (stream) {Frozen streaming test\\
+\footnotesize compare decay path to best static dropout grid};
+\draw[->, thick] (sweep.east) -- (target.west);
+\draw[->, thick] (target.east) -- (fit.west);
+\draw[->, thick] (fit.east) -- (stream.west);
+\node[align=center, text width=0.9\textwidth] at (5.55,-1.45) {
+Inputs to the fitted law are absolute quantities available before or during a run:
+$x=\log_{10}(P/U)$ measures capacity pressure and
+$y=\log_{10}(C/U)$ measures repeated-sampling pressure.
+};
+\end{tikzpicture}

paper/figures/static_optima_by_regime.tex

@@ -20,11 +20,11 @@
     grid=both,
 ]
 \addplot+[thick, mark=*] coordinates {(0,0.5062) (1,0.3559) (2,0.2174) (3,0.1364) (4,0.0224)};
-\addlegendentry{L12\_H8\_D320}
+\addlegendentry{L12-H8-D320}
 \addplot+[thick, mark=*] coordinates {(0,0.6039) (1,0.4475) (2,0.2837) (3,0.1388) (4,0.0272)};
-\addlegendentry{L16\_H8\_D384}
+\addlegendentry{L16-H8-D384}
 \addplot+[thick, mark=*] coordinates {(0,0.3837) (1,0.2401) (2,0.1463) (3,0.0691) (4,0.0000)};
-\addlegendentry{L8\_H8\_D256}
+\addlegendentry{L8-H8-D256}
 
 
 \nextgroupplot[
@@ -38,11 +38,11 @@
     grid=both,
 ]
 \addplot+[thick, mark=*] coordinates {(0,0.4575) (1,0.2947) (2,0.1866) (3,0.0703) (4,0.0303)};
-\addlegendentry{L12\_H8\_D320}
+\addlegendentry{L12-H8-D320}
 \addplot+[thick, mark=*] coordinates {(0,0.5699) (2,0.2450) (4,0.0468)};
-\addlegendentry{L16\_H8\_D384}
+\addlegendentry{L16-H8-D384}
 \addplot+[thick, mark=*] coordinates {(0,0.3491) (2,0.0977) (4,0.0000)};
-\addlegendentry{L8\_H8\_D256}
+\addlegendentry{L8-H8-D256}
 
 
 \nextgroupplot[
@@ -56,11 +56,11 @@
     grid=both,
 ]
 \addplot+[thick, mark=*] coordinates {(0,0.1645) (1,0.0299) (2,0.0000) (3,0.0000) (4,0.0000)};
-\addlegendentry{L12\_H8\_D320}
+\addlegendentry{L12-H8-D320}
 \addplot+[thick, mark=*] coordinates {(0,0.2459) (1,0.0822) (2,0.0225) (3,0.0000) (4,0.0000)};
-\addlegendentry{L16\_H8\_D384}
+\addlegendentry{L16-H8-D384}
 \addplot+[thick, mark=*] coordinates {(0,0.0965) (1,0.0114) (2,0.0000) (3,0.0000) (4,0.0000)};
-\addlegendentry{L8\_H8\_D256}
+\addlegendentry{L8-H8-D256}
 
 \end{groupplot}
 \end{tikzpicture}

paper/paper_results_summary.json

@@ -30,9 +30,76 @@
         -0.052590467035770416,
         -0.053600385785102844
       ],
+      "paired_records": [
+        {
+          "seed": 1,
+          "decay_loss": 4.402347795665264,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 4.4417688846588135,
+          "delta": -0.03942108899354935,
+          "gain": 0.03942108899354935
+        },
+        {
+          "seed": 2,
+          "decay_loss": 4.401971310377121,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 4.460222490131855,
+          "delta": -0.05825117975473404,
+          "gain": 0.05825117975473404
+        },
+        {
+          "seed": 3,
+          "decay_loss": 4.402896843850613,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 4.43564984947443,
+          "delta": -0.032753005623817444,
+          "gain": 0.032753005623817444
+        },
+        {
+          "seed": 4,
+          "decay_loss": 4.381064593791962,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 4.433655060827732,
+          "delta": -0.052590467035770416,
+          "gain": 0.052590467035770416
+        },
+        {
+          "seed": 5,
+          "decay_loss": 4.402371659874916,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 4.455972045660019,
+          "delta": -0.053600385785102844,
+          "gain": 0.053600385785102844
+        }
+      ],
       "paired_wins": 5,
       "paired_mean_delta": -0.04732322543859482,
       "paired_std_delta": 0.010738969078098962,
+      "paired_gain_mean": 0.04732322543859482,
+      "paired_gain_std": 0.010738969078098962,
+      "paired_gain_ci_low": 0.0382560983300209,
+      "paired_gain_ci_high": 0.055258719623088835,
+      "budget": {
+        "steps_per_stage": [
+          1000
+        ],
+        "batch_sizes": [
+          16
+        ],
+        "block_size": 128,
+        "sampled_per_stage": [
+          2048000
+        ],
+        "final_sampled_tokens": 10240000,
+        "prefix_path": [
+          250000,
+          500000,
+          1000000,
+          2000000,
+          4000000
+        ],
+        "stage_count": 5
+      },
       "stage_deltas": [
         {
           "stage": 0,
@@ -106,9 +173,61 @@
         -0.025513865053653717,
         -0.03717709332704544
       ],
+      "paired_records": [
+        {
+          "seed": 1,
+          "decay_loss": 4.192408442497253,
+          "best_static_condition": "static_dropout_0.18",
+          "best_static_loss": 4.231759630143642,
+          "delta": -0.03935118764638901,
+          "gain": 0.03935118764638901
+        },
+        {
+          "seed": 2,
+          "decay_loss": 4.180009409785271,
+          "best_static_condition": "static_dropout_0.2",
+          "best_static_loss": 4.205523274838924,
+          "delta": -0.025513865053653717,
+          "gain": 0.025513865053653717
+        },
+        {
+          "seed": 3,
+          "decay_loss": 4.165537476539612,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 4.202714569866657,
+          "delta": -0.03717709332704544,
+          "gain": 0.03717709332704544
+        }
+      ],
       "paired_wins": 3,
       "paired_mean_delta": -0.03401404867569605,
       "paired_std_delta": 0.007441203715372809,
+      "paired_gain_mean": 0.03401404867569605,
+      "paired_gain_std": 0.007441203715372809,
+      "paired_gain_ci_low": 0.025513865053653717,
+      "paired_gain_ci_high": 0.03935118764638901,
+      "budget": {
+        "steps_per_stage": [
+          1000
+        ],
+        "batch_sizes": [
+          16
+        ],
+        "block_size": 128,
+        "sampled_per_stage": [
+          2048000
+        ],
+        "final_sampled_tokens": 12288000,
+        "prefix_path": [
+          250000,
+          500000,
+          1000000,
+          2000000,
+          4000000,
+          8000000
+        ],
+        "stage_count": 6
+      },
       "stage_deltas": [
         {
           "stage": 0,
@@ -210,9 +329,75 @@
         -0.016586389392614365,
         -0.014097623527050018
       ],
+      "paired_records": [
+        {
+          "seed": 1,
+          "decay_loss": 2.541424009948969,
+          "best_static_condition": "static_dropout_0.08",
+          "best_static_loss": 2.54193027690053,
+          "delta": -0.0005062669515609741,
+          "gain": 0.0005062669515609741
+        },
+        {
+          "seed": 2,
+          "decay_loss": 2.5376960188150406,
+          "best_static_condition": "static_dropout_0.12",
+          "best_static_loss": 2.558806713670492,
+          "delta": -0.021110694855451584,
+          "gain": 0.021110694855451584
+        },
+        {
+          "seed": 3,
+          "decay_loss": 2.5385167226195335,
+          "best_static_condition": "static_dropout_0.08",
+          "best_static_loss": 2.547760896384716,
+          "delta": -0.009244173765182495,
+          "gain": 0.009244173765182495
+        },
+        {
+          "seed": 4,
+          "decay_loss": 2.493242312222719,
+          "best_static_condition": "static_dropout_0.08",
+          "best_static_loss": 2.5098287016153336,
+          "delta": -0.016586389392614365,
+          "gain": 0.016586389392614365
+        },
+        {
+          "seed": 5,
+          "decay_loss": 2.5446842312812805,
+          "best_static_condition": "static_dropout_0.08",
+          "best_static_loss": 2.5587818548083305,
+          "delta": -0.014097623527050018,
+          "gain": 0.014097623527050018
+        }
+      ],
       "paired_wins": 5,
       "paired_mean_delta": -0.012309029698371887,
       "paired_std_delta": 0.00786865194496459,
+      "paired_gain_mean": 0.012309029698371887,
+      "paired_gain_std": 0.00786865194496459,
+      "paired_gain_ci_low": 0.005942809581756592,
+      "paired_gain_ci_high": 0.017898358404636383,
+      "budget": {
+        "steps_per_stage": [
+          2000
+        ],
+        "batch_sizes": [
+          16
+        ],
+        "block_size": 128,
+        "sampled_per_stage": [
+          4096000
+        ],
+        "final_sampled_tokens": 16384000,
+        "prefix_path": [
+          500000,
+          1000000,
+          2000000,
+          4000000
+        ],
+        "stage_count": 4
+      },
       "stage_deltas": [
         {
           "stage": 0,
@@ -275,9 +460,60 @@
         -0.006095590069890022,
         -0.013002954423427582
       ],
+      "paired_records": [
+        {
+          "seed": 1,
+          "decay_loss": 2.206169916316867,
+          "best_static_condition": "static_dropout_0.2",
+          "best_static_loss": 2.214863333851099,
+          "delta": -0.00869341753423214,
+          "gain": 0.00869341753423214
+        },
+        {
+          "seed": 2,
+          "decay_loss": 2.2139150239527225,
+          "best_static_condition": "static_dropout_0.2",
+          "best_static_loss": 2.2200106140226126,
+          "delta": -0.006095590069890022,
+          "gain": 0.006095590069890022
+        },
+        {
+          "seed": 3,
+          "decay_loss": 2.2056322395801544,
+          "best_static_condition": "static_dropout_0.18",
+          "best_static_loss": 2.218635194003582,
+          "delta": -0.013002954423427582,
+          "gain": 0.013002954423427582
+        }
+      ],
       "paired_wins": 3,
       "paired_mean_delta": -0.00926398734251658,
       "paired_std_delta": 0.0034888512447114515,
+      "paired_gain_mean": 0.00926398734251658,
+      "paired_gain_std": 0.0034888512447114515,
+      "paired_gain_ci_low": 0.006095590069890022,
+      "paired_gain_ci_high": 0.013002954423427582,
+      "budget": {
+        "steps_per_stage": [
+          2000
+        ],
+        "batch_sizes": [
+          16
+        ],
+        "block_size": 128,
+        "sampled_per_stage": [
+          4096000
+        ],
+        "final_sampled_tokens": 20480000,
+        "prefix_path": [
+          500000,
+          1000000,
+          2000000,
+          4000000,
+          8000000
+        ],
+        "stage_count": 5
+      },
       "stage_deltas": [
         {
           "stage": 0,
@@ -368,9 +604,76 @@
         -0.02507050335407257,
         -0.037036627531051636
       ],
+      "paired_records": [
+        {
+          "seed": 1,
+          "decay_loss": 4.062299974262714,
+          "best_static_condition": "static_dropout_0.1",
+          "best_static_loss": 4.080718021839857,
+          "delta": -0.018418047577142715,
+          "gain": 0.018418047577142715
+        },
+        {
+          "seed": 2,
+          "decay_loss": 4.112260434776545,
+          "best_static_condition": "static_dropout_0.06",
+          "best_static_loss": 4.130373623222113,
+          "delta": -0.018113188445568085,
+          "gain": 0.018113188445568085
+        },
+        {
+          "seed": 3,
+          "decay_loss": 4.0763493329286575,
+          "best_static_condition": "static_dropout_0.08",
+          "best_static_loss": 4.1035647466778755,
+          "delta": -0.027215413749217987,
+          "gain": 0.027215413749217987
+        },
+        {
+          "seed": 4,
+          "decay_loss": 4.0844879895448685,
+          "best_static_condition": "static_dropout_0.1",
+          "best_static_loss": 4.109558492898941,
+          "delta": -0.02507050335407257,
+          "gain": 0.02507050335407257
+        },
+        {
+          "seed": 5,
+          "decay_loss": 4.068591594696045,
+          "best_static_condition": "static_dropout_0.08",
+          "best_static_loss": 4.105628222227097,
+          "delta": -0.037036627531051636,
+          "gain": 0.037036627531051636
+        }
+      ],
       "paired_wins": 5,
       "paired_mean_delta": -0.0251707561314106,
       "paired_std_delta": 0.007752385341386418,
+      "paired_gain_mean": 0.0251707561314106,
+      "paired_gain_std": 0.007752385341386418,
+      "paired_gain_ci_low": 0.019626595079898834,
+      "paired_gain_ci_high": 0.03128769695758819,
+      "budget": {
+        "steps_per_stage": [
+          1000
+        ],
+        "batch_sizes": [
+          16
+        ],
+        "block_size": 128,
+        "sampled_per_stage": [
+          2048000
+        ],
+        "final_sampled_tokens": 10240000,
+        "prefix_path": [
+          250000,
+          500000,
+          1000000,
+          2000000,
+          4000000
+        ],
+        "stage_count": 5
+      },
       "stage_deltas": [
         {
           "stage": 0,
@@ -444,9 +747,61 @@
         -0.00015844032168388367,
         -0.027258697897195816
       ],
+      "paired_records": [
+        {
+          "seed": 1,
+          "decay_loss": 3.952172212302685,
+          "best_static_condition": "static_dropout_0.18",
+          "best_static_loss": 3.968690302222967,
+          "delta": -0.016518089920282364,
+          "gain": 0.016518089920282364
+        },
+        {
+          "seed": 2,
+          "decay_loss": 3.967039406299591,
+          "best_static_condition": "static_dropout_0.14",
+          "best_static_loss": 3.967197846621275,
+          "delta": -0.00015844032168388367,
+          "gain": 0.00015844032168388367
+        },
+        {
+          "seed": 3,
+          "decay_loss": 3.9525104351341724,
+          "best_static_condition": "static_dropout_0.1",
+          "best_static_loss": 3.9797691330313683,
+          "delta": -0.027258697897195816,
+          "gain": 0.027258697897195816
+        }
+      ],
       "paired_wins": 3,
       "paired_mean_delta": -0.014645076046387354,
       "paired_std_delta": 0.013646872385042126,
+      "paired_gain_mean": 0.014645076046387354,
+      "paired_gain_std": 0.013646872385042126,
+      "paired_gain_ci_low": 0.00015844032168388367,
+      "paired_gain_ci_high": 0.027258697897195816,
+      "budget": {
+        "steps_per_stage": [
+          1000
+        ],
+        "batch_sizes": [
+          16
+        ],
+        "block_size": 128,
+        "sampled_per_stage": [
+          2048000
+        ],
+        "final_sampled_tokens": 12288000,
+        "prefix_path": [
+          250000,
+          500000,
+          1000000,
+          2000000,
+          4000000,
+          8000000
+        ],
+        "stage_count": 6
+      },
       "stage_deltas": [
         {
           "stage": 0,

paper/tables/calibration_weight_table.tex

@@ -0,0 +1,10 @@
+\begin{tabular}{lll}
+\toprule
+Cell property & Weight multiplier & Reason \\
+\midrule
+Best grid point on dropout boundary & $0.30$ & Optimum may lie outside tested grid \\
+Quadratic minimum not bracketed & $0.50$ & Local parabola is less reliable \\
+Loss span below $0.02$ nats & $0.50$ & Very flat curve weakly identifies $p^\star$ \\
+Best-loss std. $\sigma > 0$ & $1/(1+20\sigma)$ & Noisier minima receive less influence \\
+\bottomrule
+\end{tabular}

paper/tables/coefficient_table.tex

@@ -1,6 +1,6 @@
 \begin{tabular}{lrrrrrrrr}
 \toprule
-Regime & Cells & A & B & D & C0 & MAE & L-model MAE & L-prefix MAE \\
+Regime & Cells & A & B & D & C0 & Dropout MAE (abs. $p$) & L-model MAE & L-prefix MAE \\
 \midrule
 OpenWebText10K & 15 & 0.0819 & -0.0239 & 0.1537 & -0.0556 & 0.0158 & 0.0248 & 0.0310 \\
 TinyStories & 16 & -0.0893 & -0.1298 & 0.2551 & 0.0815 & 0.0180 & 0.0295 & 0.0325 \\

paper/tables/evidence_ladder_table.tex

@@ -0,0 +1,9 @@
+\begin{tabular}{lrrrrr}
+\toprule
+Regime & Calib. dropout MAE & 4M gain (nats) & 4M wins & 8M gain (nats) & 8M wins \\
+\midrule
+OpenWebText10K & 0.0158 & 0.0473 & 5/5 & 0.0363 & 3/3 \\
+TinyStories & 0.0180 & 0.0133 & 5/5 & 0.0100 & 3/3 \\
+WikiText-103 & 0.0162 & 0.0297 & 5/5 & 0.0184 & 3/3 \\
+\bottomrule
+\end{tabular}

paper/tables/experiment_protocol_table.tex

@@ -0,0 +1,11 @@
+\begin{tabular}{llll}
+\toprule
+Experiment layer & What is run & Question answered & Statistical unit \\
+\midrule
+Static calibration & Per-regime dropout grid at fixed $(P,U,C)$ cells & Where does validation loss plateau over dropout? & 15--16 cells per regime \\
+Coefficient fit & Per-regime regression from cell targets $p^\star$ & Can pressure variables predict useful dropout? & MAE / leave-out MAE \\
+4M final-prefix locked stream & Per-regime continuing model over growing prefixes & Does decay beat the best tested fixed dropout? & 5 paired seeds per regime \\
+L20 8M extrapolation & Per-regime 51.84M-param model with frozen coefficients & Does the law extend to larger model/data scale? & 3 paired seeds per regime \\
+Stage-wise audit & Per-regime decay vs best static at every prefix & Is the schedule final-loss or per-prefix optimal? & Prefix-level deltas \\
+\bottomrule
+\end{tabular}

paper/tables/l20_schedule_path_table.tex

@@ -0,0 +1,9 @@
+\begin{tabular}{llll}
+\toprule
+Regime & Locked-stream prefix path & Frozen decay path & Final best static \\
+\midrule
+OpenWebText10K & 250k $\rightarrow$ 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M $\rightarrow$ 8M & 0.433 $\rightarrow$ 0.367 $\rightarrow$ 0.270 $\rightarrow$ 0.176 $\rightarrow$ 0.095 $\rightarrow$ 0.033 & p=0.2 \\
+TinyStories & 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M $\rightarrow$ 8M & 0.253 $\rightarrow$ 0.209 $\rightarrow$ 0.137 $\rightarrow$ 0.077 $\rightarrow$ 0.041 & p=0.2 \\
+WikiText-103 & 250k $\rightarrow$ 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M $\rightarrow$ 8M & 0.284 $\rightarrow$ 0.211 $\rightarrow$ 0.116 $\rightarrow$ 0.036 $\rightarrow$ 0.020 $\rightarrow$ 0.020 & p=0.14 \\
+\bottomrule
+\end{tabular}

paper/tables/l20_stage_table.tex

@@ -1,6 +1,6 @@
 \begin{tabular}{lrrrrr}
 \toprule
-Regime & Prefix & Decay dropout & Decay val & Best static val & Delta \\
+Regime & Prefix & Decay dropout & Decay val (nats) & Best static val (nats) & Delta (nats) \\
 \midrule
 OpenWebText10K & 250k & 0.433 & 5.4601 & 5.3971 & +0.0630 \\
 OpenWebText10K & 500k & 0.367 & 5.0179 & 5.0169 & +0.0010 \\

paper/tables/paired_gain_ci_table.tex

@@ -0,0 +1,12 @@
+\begin{tabular}{llrrrr}
+\toprule
+Regime & Protocol & Paired gain mean & 95\% bootstrap CI & Seeds & Wins \\
+\midrule
+OpenWebText10K & 4M final prefix / 5 seeds & 0.0473 & [0.0383, 0.0553] & 5 & 5/5 \\
+OpenWebText10K & 8M final prefix / 3 seeds & 0.0340 & [0.0255, 0.0394] & 3 & 3/3 \\
+TinyStories & 4M final prefix / 5 seeds & 0.0123 & [0.0059, 0.0179] & 5 & 5/5 \\
+TinyStories & 8M final prefix / 3 seeds & 0.0093 & [0.0061, 0.0130] & 3 & 3/3 \\
+WikiText-103 & 4M final prefix / 5 seeds & 0.0252 & [0.0196, 0.0313] & 5 & 5/5 \\
+WikiText-103 & 8M final prefix / 3 seeds & 0.0146 & [0.0002, 0.0273] & 3 & 3/3 \\
+\bottomrule
+\end{tabular}

paper/tables/protocol_budget_table.tex

@@ -0,0 +1,12 @@
+\begin{tabular}{llllrr}
+\toprule
+Regime & Protocol & Prefix path & Sampled/stage & Batch & Block \\
+\midrule
+OpenWebText10K & 4M final prefix / 5 seeds & 250k $\rightarrow$ 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M & 2.048M & 16 & 128 \\
+OpenWebText10K & 8M final prefix / 3 seeds & 250k $\rightarrow$ 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M $\rightarrow$ 8M & 2.048M & 16 & 128 \\
+TinyStories & 4M final prefix / 5 seeds & 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M & 4.096M & 16 & 128 \\
+TinyStories & 8M final prefix / 3 seeds & 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M $\rightarrow$ 8M & 4.096M & 16 & 128 \\
+WikiText-103 & 4M final prefix / 5 seeds & 250k $\rightarrow$ 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M & 2.048M & 16 & 128 \\
+WikiText-103 & 8M final prefix / 3 seeds & 250k $\rightarrow$ 500k $\rightarrow$ 1M $\rightarrow$ 2M $\rightarrow$ 4M $\rightarrow$ 8M & 2.048M & 16 & 128 \\
+\bottomrule
+\end{tabular}

paper/tables/regime_setup_table.tex

@@ -0,0 +1,9 @@
+\begin{tabular}{llll}
+\toprule
+Regime & Corpus source & Tokenizer / vocab & Streaming prefixes \\
+\midrule
+OpenWebText10K & Cached OpenWebText-style subset & BPE / 4096 & 250k, 500k, 1M, 2M, 4M; plus 8M L20 tier \\
+TinyStories & Cached TinyStories subset & BPE / 4096 & 500k, 1M, 2M, 4M; plus 8M L20 tier \\
+WikiText-103 & Cached WikiText-103 subset & BPE / 4096 & 250k, 500k, 1M, 2M, 4M; plus 8M L20 tier \\
+\bottomrule
+\end{tabular}

paper/tables/reproducibility_table.tex

@@ -0,0 +1,13 @@
+\begin{tabular}{ll}
+\toprule
+Item & Value \\
+\midrule
+Repository & \url{https://huggingface.co/cuber12/dropout-decay} \\
+Repository snapshot & Base commit \texttt{e7a7275}; cite final release tag with paper \\
+License & MIT-derived source with nanochat attribution retained \\
+Python & 3.11.15 \\
+PyTorch & 2.12.0 \\
+Operating system & macOS 26.4 arm64 \\
+Training backend & Apple Metal Performance Shaders (MPS); runner refuses CPU/CUDA \\
+\bottomrule
+\end{tabular}

paper/tables/streaming_validation_table.tex

@@ -1,12 +1,12 @@
 \begin{tabular}{llrrrrr}
 \toprule
-Regime & Protocol & Decay final & Best static final & Gain & Seeds & Paired wins \\
+Regime & Protocol & Decay final (nats) & Best static final (nats) & Gain (nats) & Seeds & Paired wins \\
 \midrule
-OpenWebText10K & 4M / 5 seeds & 4.3981 $\pm$ 0.0095 & 4.4455 $\pm$ 0.0120 & 0.0473 & 5 & 5/5 \\
+OpenWebText10K & 4M final prefix / 5 seeds & 4.3981 $\pm$ 0.0095 & 4.4455 $\pm$ 0.0120 & 0.0473 & 5 & 5/5 \\
 OpenWebText10K & 8M / 3 seeds & 4.1793 $\pm$ 0.0134 & 4.2156 $\pm$ 0.0142 & 0.0363 & 3 & 3/3 \\
-TinyStories & 4M / 5 seeds & 2.5311 $\pm$ 0.0213 & 2.5444 $\pm$ 0.0211 & 0.0133 & 5 & 5/5 \\
+TinyStories & 4M final prefix / 5 seeds & 2.5311 $\pm$ 0.0213 & 2.5444 $\pm$ 0.0211 & 0.0133 & 5 & 5/5 \\
 TinyStories & 8M / 3 seeds & 2.2086 $\pm$ 0.0046 & 2.2186 $\pm$ 0.0032 & 0.0100 & 3 & 3/3 \\
-WikiText-103 & 4M / 5 seeds & 4.0808 $\pm$ 0.0195 & 4.1105 $\pm$ 0.0188 & 0.0297 & 5 & 5/5 \\
+WikiText-103 & 4M final prefix / 5 seeds & 4.0808 $\pm$ 0.0195 & 4.1105 $\pm$ 0.0188 & 0.0297 & 5 & 5/5 \\
 WikiText-103 & 8M / 3 seeds & 3.9572 $\pm$ 0.0085 & 3.9756 $\pm$ 0.0087 & 0.0184 & 3 & 3/3 \\
 \bottomrule
 \end{tabular}

paper/tables/symbol_table.tex

@@ -0,0 +1,15 @@
+\begin{tabular}{ll}
+\toprule
+Symbol & Meaning \\
+\midrule
+$P$ & Trainable parameter count \\
+$U_t$ & Unique-token prefix currently revealed by stream stage $t$ \\
+$C_t$ & Cumulative sampled training tokens consumed by the optimizer \\
+$x_t$ & $\log_{10}(P/U_t)$, capacity pressure \\
+$y_t$ & $\log_{10}(C_t/U_t)$, repeated-sampling pressure \\
+$p_t$ & Dropout rate used at stream stage $t$ \\
+$p^\star$ & Static-sweep target dropout for one calibration cell \\
+$A,B,D,C_0$ & Regime-specific fitted pressure-law coefficients \\
+$p_{\min},p_{\max}$ & Clamp bounds applied to generated dropout schedules \\
+\bottomrule
+\end{tabular}

pyproject.toml

@@ -1,3 +1,7 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "dropout-decay"
 version = "0.1.0"
@@ -11,6 +15,14 @@ dependencies = [
     "torch>=2.3",
 ]
 
+[project.optional-dependencies]
+dev = [
+    "pytest>=8",
+]
+
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 testpaths = ["tests"]
+
+[tool.setuptools.packages.find]
+where = ["src"]

scripts/generate_paper_assets.py

@@ -23,6 +23,7 @@ import csv
 import json
 import math
 from pathlib import Path
+import random
 import statistics
 
 
@@ -48,7 +49,7 @@ REGIMES = {
         / "runs/openwebtext10k_l16_updated_formula_clean_5seed/locked_stream/20260530-174525",
         "small_condition": "openwebtext10k_interaction",
         "small_model": "L16_H8_D384",
-        "small_protocol": "4M / 5 seeds",
+        "small_protocol": "4M final prefix / 5 seeds",
         "l20_stream": ROOT
         / "runs/openwebtext10k_l20_8m_extrapolation_3seed/locked_stream/20260601-191150",
         "l20_condition": "openwebtext10k_l20_interaction_8m",
@@ -75,7 +76,7 @@ REGIMES = {
         ],
         "small_condition": "interaction",
         "small_model": "L12_H8_D320",
-        "small_protocol": "4M / 5 seeds",
+        "small_protocol": "4M final prefix / 5 seeds",
         "l20_stream": ROOT
         / "runs/tinystories_l20_8m_extrapolation_3seed/locked_stream/20260602-112249",
         "l20_condition": "tinystories_l20_interaction_8m",
@@ -94,7 +95,7 @@ REGIMES = {
         / "runs/wikitext103_l12_streaming_validation_5seed/locked_stream/20260531-093525",
         "small_condition": "wikitext103_formula_l12",
         "small_model": "L12_H8_D320",
-        "small_protocol": "4M / 5 seeds",
+        "small_protocol": "4M final prefix / 5 seeds",
         "l20_stream": ROOT
         / "runs/wikitext103_l20_8m_extrapolation_3seed/locked_stream/20260603-231952",
         "l20_condition": "wikitext103_l20_interaction_8m",
@@ -127,6 +128,27 @@ def mean_std(values: list[float]) -> tuple[float, float]:
     return mean, std
 
 
+def bootstrap_ci(
+    values: list[float], *, confidence: float = 0.95, iterations: int = 20_000, seed: int = 1337
+) -> tuple[float, float]:
+    if not values:
+        return float("nan"), float("nan")
+    if len(values) == 1:
+        return values[0], values[0]
+    rng = random.Random(seed)
+    n = len(values)
+    means = []
+    for _ in range(iterations):
+        sample = [values[rng.randrange(n)] for _ in range(n)]
+        means.append(statistics.fmean(sample))
+    means.sort()
+    lower_q = (1.0 - confidence) / 2.0
+    upper_q = 1.0 - lower_q
+    lower_idx = min(iterations - 1, max(0, int(lower_q * iterations)))
+    upper_idx = min(iterations - 1, max(0, int(upper_q * iterations)))
+    return means[lower_idx], means[upper_idx]
+
+
 def fmt(x: float, digits: int = 4) -> str:
     return f"{x:.{digits}f}"
 
@@ -142,6 +164,10 @@ def tex_escape(value: object) -> str:
     )
 
 
+def model_label(model_name: str) -> str:
+    return tex_escape(model_name.replace("_", "-"))
+
+
 def token_label(value: int) -> str:
     if value >= 1_000_000:
         whole = value / 1_000_000
@@ -149,6 +175,39 @@ def token_label(value: int) -> str:
     return f"{value // 1000}k"
 
 
+def protocol_budget(condition_rows: list[dict], stages: list[int]) -> dict:
+    rows_by_seed: dict[int, list[dict]] = {}
+    for row in condition_rows:
+        rows_by_seed.setdefault(int(row["seed"]), []).append(row)
+    seed = sorted(rows_by_seed)[0]
+    seed_rows = sorted(rows_by_seed[seed], key=lambda item: int(item["stage"]))
+    sampled_deltas = []
+    previous_tokens_seen = 0
+    for row in seed_rows:
+        tokens_seen = int(row["tokens_seen"])
+        sampled_deltas.append(tokens_seen - previous_tokens_seen)
+        previous_tokens_seen = tokens_seen
+    first = seed_rows[0]
+    steps_per_stage = sorted({int(row["steps"]) for row in seed_rows})
+    block_size = int(first["model_config"]["block_size"])
+    sampled_per_stage = sorted(set(sampled_deltas))
+    batch_sizes = sorted(
+        {
+            round(delta / (int(row["steps"]) * block_size))
+            for delta, row in zip(sampled_deltas, seed_rows, strict=True)
+        }
+    )
+    return {
+        "steps_per_stage": steps_per_stage,
+        "batch_sizes": batch_sizes,
+        "block_size": block_size,
+        "sampled_per_stage": sampled_per_stage,
+        "final_sampled_tokens": int(seed_rows[-1]["tokens_seen"]),
+        "prefix_path": [int(row["token_limit"]) for row in seed_rows],
+        "stage_count": len(stages),
+    }
+
+
 def metric_stage_summary(run_dir: Path | list[Path], condition_name: str) -> dict:
     run_dirs = run_dir if isinstance(run_dir, list) else [run_dir]
     rows = []
@@ -163,6 +222,12 @@ def metric_stage_summary(run_dir: Path | list[Path], condition_name: str) -> dic
         key = (row["condition"], int(row["stage"]))
         by_condition_stage.setdefault(key, []).append(row)
 
+    condition_rows_by_stage = [
+        row
+        for (condition, _stage), group in by_condition_stage.items()
+        if condition == condition_name
+        for row in group
+    ]
     condition_rows = by_condition_stage[(condition_name, final_stage)]
     final_vals = [float(row["val_eval_loss"]) for row in condition_rows]
     condition_final_mean, condition_final_std = mean_std(final_vals)
@@ -192,10 +257,27 @@ def metric_stage_summary(run_dir: Path | list[Path], condition_name: str) -> dic
             final_static_by_seed[seed] = (row["condition"], value)
 
     paired_deltas = []
-    for row in condition_rows:
+    paired_records = []
+    for row in sorted(condition_rows, key=lambda item: int(item["seed"])):
         seed = int(row["seed"])
-        paired_deltas.append(float(row["val_eval_loss"]) - final_static_by_seed[seed][1])
+        decay_loss = float(row["val_eval_loss"])
+        static_condition, static_loss = final_static_by_seed[seed]
+        delta = decay_loss - static_loss
+        paired_deltas.append(delta)
+        paired_records.append(
+            {
+                "seed": seed,
+                "decay_loss": decay_loss,
+                "best_static_condition": static_condition,
+                "best_static_loss": static_loss,
+                "delta": delta,
+                "gain": -delta,
+            }
+        )
     paired_mean, paired_std = mean_std(paired_deltas)
+    paired_gains = [-delta for delta in paired_deltas]
+    paired_gain_mean, paired_gain_std = mean_std(paired_gains)
+    paired_gain_ci_low, paired_gain_ci_high = bootstrap_ci(paired_gains)
 
     stage_deltas = []
     for stage in stages:
@@ -251,9 +333,15 @@ def metric_stage_summary(run_dir: Path | list[Path], condition_name: str) -> dic
         "best_static_std": best_static_std,
         "improvement": best_static_mean - condition_final_mean,
         "paired_deltas": paired_deltas,
+        "paired_records": paired_records,
         "paired_wins": sum(1 for delta in paired_deltas if delta < 0),
         "paired_mean_delta": paired_mean,
         "paired_std_delta": paired_std,
+        "paired_gain_mean": paired_gain_mean,
+        "paired_gain_std": paired_gain_std,
+        "paired_gain_ci_low": paired_gain_ci_low,
+        "paired_gain_ci_high": paired_gain_ci_high,
+        "budget": protocol_budget(condition_rows_by_stage, stages),
         "stage_deltas": stage_deltas,
     }
 
@@ -318,21 +406,29 @@ def tikz_coordinates(points: list[tuple[float, float]], digits: int = 4) -> str:
 
 def write_calibration_curve_figure() -> None:
     axes = []
-    for key, paths in REGIMES.items():
+    for idx, (key, paths) in enumerate(REGIMES.items()):
         raw_curve = static_curve_points(
             paths["curve_run"], paths["curve_model"], paths["curve_prefix"]
         )
         minimum = min(loss for _, loss in raw_curve)
         points = [(dropout, loss - minimum) for dropout, loss in raw_curve]
         best = min(raw_curve, key=lambda item: item[1])
+        axis_options = [
+            f"    title={{{paths['curve_title']}}},",
+            "    xlabel={Dropout},",
+        ]
+        if idx == 0:
+            axis_options.append("    ylabel={Validation loss above best},")
+        axis_options.extend(
+            [
+                "    ymin=0,",
+                "    grid=both,",
+            ]
+        )
         axes.append(
             rf"""
 \nextgroupplot[
-    title={{{paths["curve_title"]}}},
-    xlabel={{Dropout}},
-    ylabel={{Validation loss above best}},
-    ymin=0,
-    grid=both,
+{chr(10).join(axis_options)}
 ]
 \addplot+[thick, mark=*] coordinates {{{tikz_coordinates(points, digits=5)}}};
 \addlegendentry{{static sweep}}
@@ -374,7 +470,7 @@ def write_static_optima_figure() -> None:
                 "\\addplot+[thick, mark=*] coordinates {"
                 + tikz_coordinates(coords)
                 + "};\n"
-                + f"\\addlegendentry{{{tex_escape(model)}}}"
+                + f"\\addlegendentry{{{model_label(model)}}}"
             )
         xticks = ",".join(str(i) for i in range(len(labels)))
         xticklabels = ",".join(token_label(prefix) for prefix in labels)
@@ -486,11 +582,186 @@ def write_final_improvement_figure(summaries: dict[str, dict]) -> None:
     (FIG_DIR / "final_improvements.tex").write_text(content, encoding="utf-8")
 
 
+def write_protocol_schematic() -> None:
+    content = r"""
+\begin{tikzpicture}[font=\scriptsize]
+\tikzstyle{box}=[
+    draw,
+    rounded corners=2pt,
+    thick,
+    align=center,
+    text width=0.19\textwidth,
+    minimum height=1.5cm,
+    inner sep=5pt
+]
+\node[box, fill=blue!7] at (0,0) (sweep) {Static calibration cells\\
+\footnotesize model $P$, prefix $U$, sampled tokens $C$, dropout grid};
+\node[box, fill=green!8] at (3.7,0) (target) {Extract $p^\star$\\
+\footnotesize grid best or bracketed quadratic minimum};
+\node[box, fill=yellow!12] at (7.4,0) (fit) {Fit pressure law\\
+\footnotesize $p^\star \approx A x + B y + Dxy + C_0$};
+\node[box, fill=red!7] at (11.1,0) (stream) {Frozen streaming test\\
+\footnotesize compare decay path to best static dropout grid};
+\draw[->, thick] (sweep.east) -- (target.west);
+\draw[->, thick] (target.east) -- (fit.west);
+\draw[->, thick] (fit.east) -- (stream.west);
+\node[align=center, text width=0.9\textwidth] at (5.55,-1.45) {
+Inputs to the fitted law are absolute quantities available before or during a run:
+$x=\log_{10}(P/U)$ measures capacity pressure and
+$y=\log_{10}(C/U)$ measures repeated-sampling pressure.
+};
+\end{tikzpicture}
+"""
+    (FIG_DIR / "calibration_to_streaming_protocol.tex").write_text(
+        content.strip() + "\n", encoding="utf-8"
+    )
+
+
 def write_tables(summaries: dict[str, dict], coeffs: dict[str, dict]) -> None:
+    symbol_lines = [
+        "\\begin{tabular}{ll}",
+        "\\toprule",
+        "Symbol & Meaning \\\\",
+        "\\midrule",
+        "$P$ & Trainable parameter count \\\\",
+        "$U_t$ & Unique-token prefix currently revealed by stream stage $t$ \\\\",
+        "$C_t$ & Cumulative sampled training tokens consumed by the optimizer \\\\",
+        "$x_t$ & $\\log_{10}(P/U_t)$, capacity pressure \\\\",
+        "$y_t$ & $\\log_{10}(C_t/U_t)$, repeated-sampling pressure \\\\",
+        "$p_t$ & Dropout rate used at stream stage $t$ \\\\",
+        "$p^\\star$ & Static-sweep target dropout for one calibration cell \\\\",
+        "$A,B,D,C_0$ & Regime-specific fitted pressure-law coefficients \\\\",
+        "$p_{\\min},p_{\\max}$ & Clamp bounds applied to generated dropout schedules \\\\",
+        "\\bottomrule",
+        "\\end{tabular}",
+    ]
+    (TABLE_DIR / "symbol_table.tex").write_text(
+        "\n".join(symbol_lines) + "\n", encoding="utf-8"
+    )
+
+    regime_lines = [
+        "\\begin{tabular}{llll}",
+        "\\toprule",
+        "Regime & Corpus source & Tokenizer / vocab & Streaming prefixes \\\\",
+        "\\midrule",
+        (
+            "OpenWebText10K & Cached OpenWebText-style subset & BPE / 4096 & "
+            "250k, 500k, 1M, 2M, 4M; plus 8M L20 tier \\\\"
+        ),
+        (
+            "TinyStories & Cached TinyStories subset & BPE / 4096 & "
+            "500k, 1M, 2M, 4M; plus 8M L20 tier \\\\"
+        ),
+        (
+            "WikiText-103 & Cached WikiText-103 subset & BPE / 4096 & "
+            "250k, 500k, 1M, 2M, 4M; plus 8M L20 tier \\\\"
+        ),
+        "\\bottomrule",
+        "\\end{tabular}",
+    ]
+    (TABLE_DIR / "regime_setup_table.tex").write_text(
+        "\n".join(regime_lines) + "\n", encoding="utf-8"
+    )
+
+    repro_lines = [
+        "\\begin{tabular}{ll}",
+        "\\toprule",
+        "Item & Value \\\\",
+        "\\midrule",
+        "Repository & \\url{https://huggingface.co/cuber12/dropout-decay} \\\\",
+        "Repository snapshot & Base commit \\texttt{e7a7275}; cite final release tag with paper \\\\",
+        "License & MIT-derived source with nanochat attribution retained \\\\",
+        "Python & 3.11.15 \\\\",
+        "PyTorch & 2.12.0 \\\\",
+        "Operating system & macOS 26.4 arm64 \\\\",
+        "Training backend & Apple Metal Performance Shaders (MPS); runner refuses CPU/CUDA \\\\",
+        "\\bottomrule",
+        "\\end{tabular}",
+    ]
+    (TABLE_DIR / "reproducibility_table.tex").write_text(
+        "\n".join(repro_lines) + "\n", encoding="utf-8"
+    )
+
+    weight_lines = [
+        "\\begin{tabular}{lll}",
+        "\\toprule",
+        "Cell property & Weight multiplier & Reason \\\\",
+        "\\midrule",
+        "Best grid point on dropout boundary & $0.30$ & Optimum may lie outside tested grid \\\\",
+        "Quadratic minimum not bracketed & $0.50$ & Local parabola is less reliable \\\\",
+        "Loss span below $0.02$ nats & $0.50$ & Very flat curve weakly identifies $p^\\star$ \\\\",
+        (
+            "Best-loss std. $\\sigma > 0$ & $1/(1+20\\sigma)$ & "
+            "Noisier minima receive less influence \\\\"
+        ),
+        "\\bottomrule",
+        "\\end{tabular}",
+    ]
+    (TABLE_DIR / "calibration_weight_table.tex").write_text(
+        "\n".join(weight_lines) + "\n", encoding="utf-8"
+    )
+
+    protocol_lines = [
+        "\\begin{tabular}{llll}",
+        "\\toprule",
+        "Experiment layer & What is run & Question answered & Statistical unit \\\\",
+        "\\midrule",
+        (
+            "Static calibration & Per-regime dropout grid at fixed $(P,U,C)$ cells & "
+            "Where does validation loss plateau over dropout? & 15--16 cells per regime \\\\"
+        ),
+        (
+            "Coefficient fit & Per-regime regression from cell targets $p^\\star$ & "
+            "Can pressure variables predict useful dropout? & MAE / leave-out MAE \\\\"
+        ),
+        (
+            "4M final-prefix locked stream & Per-regime continuing model over growing prefixes & "
+            "Does decay beat the best tested fixed dropout? & 5 paired seeds per regime \\\\"
+        ),
+        (
+            "L20 8M extrapolation & Per-regime 51.84M-param model with frozen coefficients & "
+            "Does the law extend to larger model/data scale? & 3 paired seeds per regime \\\\"
+        ),
+        (
+            "Stage-wise audit & Per-regime decay vs best static at every prefix & "
+            "Is the schedule final-loss or per-prefix optimal? & Prefix-level deltas \\\\"
+        ),
+        "\\bottomrule",
+        "\\end{tabular}",
+    ]
+    (TABLE_DIR / "experiment_protocol_table.tex").write_text(
+        "\n".join(protocol_lines) + "\n", encoding="utf-8"
+    )
+
+    budget_lines = [
+        "\\begin{tabular}{llllrr}",
+        "\\toprule",
+        "Regime & Protocol & Prefix path & Sampled/stage & Batch & Block \\\\",
+        "\\midrule",
+    ]
+    for key, paths in REGIMES.items():
+        for protocol_key, protocol_label in [
+            ("small", paths["small_protocol"]),
+            ("l20", "8M final prefix / 3 seeds"),
+        ]:
+            budget = summaries[key][protocol_key]["budget"]
+            prefix_path = " $\\rightarrow$ ".join(token_label(prefix) for prefix in budget["prefix_path"])
+            sampled = ", ".join(token_label(value) for value in budget["sampled_per_stage"])
+            batch = "/".join(str(value) for value in budget["batch_sizes"])
+            block = str(budget["block_size"])
+            budget_lines.append(
+                f"{tex_escape(paths['label'])} & {tex_escape(protocol_label)} & "
+                f"{prefix_path} & {sampled} & {batch} & {block} \\\\"
+            )
+    budget_lines.extend(["\\bottomrule", "\\end{tabular}"])
+    (TABLE_DIR / "protocol_budget_table.tex").write_text(
+        "\n".join(budget_lines) + "\n", encoding="utf-8"
+    )
+
     coeff_lines = [
         "\\begin{tabular}{lrrrrrrrr}",
         "\\toprule",
-        "Regime & Cells & A & B & D & C0 & MAE & L-model MAE & L-prefix MAE \\\\",
+        "Regime & Cells & A & B & D & C0 & Dropout MAE (abs. $p$) & L-model MAE & L-prefix MAE \\\\",
         "\\midrule",
     ]
     for key in REGIMES:
@@ -505,10 +776,29 @@ def write_tables(summaries: dict[str, dict], coeffs: dict[str, dict]) -> None:
         "\n".join(coeff_lines) + "\n", encoding="utf-8"
     )
 
+    evidence_lines = [
+        "\\begin{tabular}{lrrrrr}",
+        "\\toprule",
+        "Regime & Calib. dropout MAE & 4M gain (nats) & 4M wins & 8M gain (nats) & 8M wins \\\\",
+        "\\midrule",
+    ]
+    for key, paths in REGIMES.items():
+        small = summaries[key]["small"]
+        l20 = summaries[key]["l20"]
+        evidence_lines.append(
+            f"{tex_escape(paths['label'])} & {coeffs[key]['mae']:.4f} & "
+            f"{small['improvement']:.4f} & {small['paired_wins']}/{small['n']} & "
+            f"{l20['improvement']:.4f} & {l20['paired_wins']}/{l20['n']} \\\\"
+        )
+    evidence_lines.extend(["\\bottomrule", "\\end{tabular}"])
+    (TABLE_DIR / "evidence_ladder_table.tex").write_text(
+        "\n".join(evidence_lines) + "\n", encoding="utf-8"
+    )
+
     val_lines = [
         "\\begin{tabular}{llrrrrr}",
         "\\toprule",
-        "Regime & Protocol & Decay final & Best static final & Gain & Seeds & Paired wins \\\\",
+        "Regime & Protocol & Decay final (nats) & Best static final (nats) & Gain (nats) & Seeds & Paired wins \\\\",
         "\\midrule",
     ]
     for key, paths in REGIMES.items():
@@ -525,10 +815,57 @@ def write_tables(summaries: dict[str, dict], coeffs: dict[str, dict]) -> None:
         "\n".join(val_lines) + "\n", encoding="utf-8"
     )
 
+    paired_lines = [
+        "\\begin{tabular}{llrrrr}",
+        "\\toprule",
+        "Regime & Protocol & Paired gain mean & 95\\% bootstrap CI & Seeds & Wins \\\\",
+        "\\midrule",
+    ]
+    for key, paths in REGIMES.items():
+        for protocol_key, protocol_label in [
+            ("small", paths["small_protocol"]),
+            ("l20", "8M final prefix / 3 seeds"),
+        ]:
+            row = summaries[key][protocol_key]
+            paired_lines.append(
+                f"{tex_escape(paths['label'])} & {tex_escape(protocol_label)} & "
+                f"{row['paired_gain_mean']:.4f} & "
+                f"[{row['paired_gain_ci_low']:.4f}, {row['paired_gain_ci_high']:.4f}] & "
+                f"{row['n']} & {row['paired_wins']}/{row['n']} \\\\"
+            )
+    paired_lines.extend(["\\bottomrule", "\\end{tabular}"])
+    (TABLE_DIR / "paired_gain_ci_table.tex").write_text(
+        "\n".join(paired_lines) + "\n", encoding="utf-8"
+    )
+
+    path_lines = [
+        "\\begin{tabular}{llll}",
+        "\\toprule",
+        "Regime & Locked-stream prefix path & Frozen decay path & Final best static \\\\",
+        "\\midrule",
+    ]
+    for key, paths in REGIMES.items():
+        stage_rows = summaries[key]["l20"]["stage_deltas"]
+        prefix_path = " $\\rightarrow$ ".join(token_label(row["prefix"]) for row in stage_rows)
+        dropout_path = " $\\rightarrow$ ".join(
+            f"{row['condition_dropout']:.3f}" for row in stage_rows
+        )
+        final_static = summaries[key]["l20"]["best_static_condition"].replace(
+            "static_dropout_", "p="
+        )
+        path_lines.append(
+            f"{tex_escape(paths['label'])} & {prefix_path} & {dropout_path} & "
+            f"{tex_escape(final_static)} \\\\"
+        )
+    path_lines.extend(["\\bottomrule", "\\end{tabular}"])
+    (TABLE_DIR / "l20_schedule_path_table.tex").write_text(
+        "\n".join(path_lines) + "\n", encoding="utf-8"
+    )
+
     l20_lines = [
         "\\begin{tabular}{lrrrrr}",
         "\\toprule",
-        "Regime & Prefix & Decay dropout & Decay val & Best static val & Delta \\\\",
+        "Regime & Prefix & Decay dropout & Decay val (nats) & Best static val (nats) & Delta (nats) \\\\",
         "\\midrule",
     ]
     for key, paths in REGIMES.items():
@@ -569,6 +906,7 @@ def main() -> None:
         }
     write_calibration_curve_figure()
     write_static_optima_figure()
+    write_protocol_schematic()
     write_l20_stage_delta_figure(summaries)
     write_final_improvement_figure(summaries)
     write_tables(summaries, coeffs)

scripts/summarize_cross_regime_backtest.py

@@ -27,7 +27,7 @@ import statistics
 
 
 ROOT = Path("runs/coefficient_calibration/cross_regime_backtest")
-REPORT_PATH = Path("docs/cross_regime_backtest_report.md")
+REPORT_PATH = Path("archive/legacy_docs/cross_regime_backtest_report.md")
 
 
 FIT_DIRS = {

src/dropout_decay/data/__init__.py

@@ -16,6 +16,8 @@ The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 """
 
+# Backward-compatible public exports. Dataset helpers now live in
+# dropout_decay.datasets and experiment dataclasses live in dropout_decay.specs.
 from dropout_decay.datasets import (
     CachedTokenizer,
     TokenSplits,

src/dropout_decay/experiment.py

@@ -16,6 +16,8 @@ The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 """
 
+# Backward-compatible public exports. The implementation lives in
+# dropout_decay.experiments.runner; historical scripts import this module.
 from dropout_decay.experiments.runner import build_parser, main, run
 
 __all__ = [

src/dropout_decay/model.py

@@ -16,6 +16,8 @@ The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 """
 
+# Backward-compatible public exports. The implementation lives in
+# dropout_decay.models.gpt; historical scripts import this module.
 from dropout_decay.models.gpt import (
     Block,
     CausalSelfAttention,

src/dropout_decay/scheduler.py

@@ -16,6 +16,8 @@ The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 """
 
+# Backward-compatible public exports. The implementation lives in
+# dropout_decay.schedules.dropout; historical scripts import this module.
 from dropout_decay.schedules.dropout import DropoutDecayConfig, DropoutDecayScheduler
 
 __all__ = [

src/dropout_decay/tokenizer.py

@@ -16,6 +16,8 @@ The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 """
 
+# Backward-compatible public exports. The implementation lives in
+# dropout_decay.tokenization.bpe; historical scripts import this module.
 from dropout_decay.tokenization.bpe import BpeTokenizer, SPECIAL_TOKENS, SPLIT_PATTERN
 
 __all__ = [

tests/test_data_and_device.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import numpy as np
+
+from dropout_decay.data import load_cached_splits
+from dropout_decay.experiments.device import assert_mps_only
+
+
+class DataAndDeviceTests(unittest.TestCase):
+    def test_load_cached_splits_uses_expected_train_val_partition(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            cache_dir = Path(tmp)
+            (cache_dir / "tokenizer-v16.json").write_text("{}", encoding="utf-8")
+            np.save(cache_dir / "tokens-v16-uint16.npy", np.arange(100, dtype=np.uint16))
+
+            tokenizer, splits = load_cached_splits(
+                cache_dir=cache_dir,
+                vocab_size=16,
+                max_required_train_tokens=80,
+                val_tokens=10,
+                allow_short_corpus=False,
+            )
+
+            self.assertEqual(tokenizer.vocab_size, 16)
+            self.assertEqual(len(splits.train), 90)
+            self.assertEqual(len(splits.val), 10)
+            self.assertEqual(splits.tokenizer_path, cache_dir / "tokenizer-v16.json")
+
+    def test_load_cached_splits_rejects_missing_cache(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            with self.assertRaises(FileNotFoundError):
+                load_cached_splits(
+                    cache_dir=Path(tmp),
+                    vocab_size=16,
+                    max_required_train_tokens=10,
+                    val_tokens=10,
+                    allow_short_corpus=False,
+                )
+
+    def test_mps_guard_rejects_fallback_before_backend_checks(self):
+        env = dict(os.environ)
+        env["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+        with patch.dict(os.environ, env, clear=True):
+            with self.assertRaises(SystemExit) as caught:
+                assert_mps_only()
+
+        self.assertIn("MPS_FALLBACK", str(caught.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_schedules.py

@@ -0,0 +1,74 @@
+import math
+import unittest
+
+from dropout_decay.schedules import DropoutDecayConfig, DropoutDecayScheduler
+from dropout_decay.specs import DropoutCondition, anchor_dropout
+
+
+class DropoutScheduleTests(unittest.TestCase):
+    def test_linear_decay_endpoints_and_midpoint(self):
+        scheduler = DropoutDecayScheduler(
+            DropoutDecayConfig(
+                initial_dropout=0.30,
+                final_dropout=0.10,
+                decay_tokens=100,
+                schedule="linear",
+            )
+        )
+
+        self.assertAlmostEqual(scheduler.value(0), 0.30)
+        self.assertAlmostEqual(scheduler.value(50), 0.20)
+        self.assertAlmostEqual(scheduler.value(100), 0.10)
+        self.assertAlmostEqual(scheduler.value(1000), 0.10)
+
+    def test_cosine_decay_midpoint(self):
+        scheduler = DropoutDecayScheduler(
+            DropoutDecayConfig(
+                initial_dropout=0.40,
+                final_dropout=0.20,
+                decay_tokens=100,
+                schedule="cosine",
+            )
+        )
+
+        self.assertAlmostEqual(scheduler.value(50), 0.30)
+
+    def test_invalid_scheduler_config_rejected(self):
+        with self.assertRaises(ValueError):
+            DropoutDecayScheduler(
+                DropoutDecayConfig(
+                    initial_dropout=0.10,
+                    final_dropout=0.30,
+                    decay_tokens=100,
+                    schedule="linear",
+                )
+            )
+
+    def test_anchor_dropout_interpolates_in_log_token_space(self):
+        anchors = ((100, 0.30), (10_000, 0.10))
+        value = anchor_dropout(1_000, anchors)
+        expected_mix = (math.log(1_000) - math.log(100)) / (
+            math.log(10_000) - math.log(100)
+        )
+        expected = 0.30 + expected_mix * (0.10 - 0.30)
+
+        self.assertAlmostEqual(value, expected)
+
+    def test_anchor_condition_requires_unique_tokens(self):
+        condition = DropoutCondition(
+            name="fit",
+            kind="anchor_decay",
+            initial=0.30,
+            final=0.10,
+            anchors=((100, 0.30), (10_000, 0.10)),
+        )
+
+        with self.assertRaises(ValueError):
+            condition.make_fn(fallback_decay_tokens=100)
+
+        fn = condition.make_fn(fallback_decay_tokens=100, unique_tokens=1_000)
+        self.assertAlmostEqual(fn(0), anchor_dropout(1_000, condition.anchors))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_specs_and_parsing.py

@@ -0,0 +1,50 @@
+import argparse
+import unittest
+
+from dropout_decay.experiments.parsing import (
+    clean_name,
+    parse_anchor_decay_spec,
+    parse_decay_spec,
+    parse_model_spec,
+)
+
+
+class SpecParsingTests(unittest.TestCase):
+    def test_parse_model_spec_default_and_named(self):
+        default = parse_model_spec("12x8x320")
+        named = parse_model_spec("winner=12x8x320")
+
+        self.assertEqual(default.name, "L12_H8_D320")
+        self.assertEqual(named.name, "winner")
+        self.assertEqual(named.to_dict()["n_embd"], 320)
+
+    def test_parse_model_spec_rejects_incompatible_heads(self):
+        with self.assertRaises(argparse.ArgumentTypeError):
+            parse_model_spec("3x5x32")
+
+    def test_parse_decay_spec_defaults_to_cosine(self):
+        condition = parse_decay_spec("mild:0.30:0.10")
+
+        self.assertEqual(condition.name, "mild")
+        self.assertEqual(condition.kind, "decay")
+        self.assertEqual(condition.schedule, "cosine")
+        self.assertIsNone(condition.decay_tokens)
+
+    def test_parse_anchor_decay_spec_sorts_and_validates(self):
+        condition = parse_anchor_decay_spec("fit:1000=0.20,100=0.30")
+
+        self.assertEqual(condition.kind, "anchor_decay")
+        self.assertEqual(condition.anchors, ((100, 0.30), (1000, 0.20)))
+        self.assertEqual(condition.initial, 0.30)
+        self.assertEqual(condition.final, 0.20)
+
+    def test_parse_anchor_decay_spec_rejects_increasing_dropout(self):
+        with self.assertRaises(argparse.ArgumentTypeError):
+            parse_anchor_decay_spec("fit:100=0.10,1000=0.20")
+
+    def test_clean_name_keeps_stable_condition_labels(self):
+        self.assertEqual(clean_name("  mild 30->08 "), "mild_30-_08")
+
+
+if __name__ == "__main__":
+    unittest.main()