replace checkpoints with the full set of runs

README.md

@@ -11,47 +11,44 @@ tags:
 
 Trained weights from [carrybit](https://github.com/Supergoatscriptguy/carrybit),
 a small research project on tiny transformers learning exact integer
-arithmetic. The code, training configs, and result plots live in that repo;
-this repo only holds the final checkpoints so the analysis experiments can be
-run without retraining.
+arithmetic. The code, configs, figures and the full write-up live in that
+repo. This repo holds the final checkpoint of every run in the write-up so the
+analysis experiments can be run without retraining.
 
-Every checkpoint is a plain PyTorch `state_dict` for `carrybit.model.Transformer`.
-Each folder has the exact `config.json` it was trained with and the
-`metrics.csv` logged during training.
+Every checkpoint is a plain PyTorch `state_dict` for `carrybit.model.Transformer`
+(or `TwoHotMLP` for `modular_mlp`). Each folder has the exact `config.json` it
+was trained with and the `metrics.csv` logged during training. Folder names
+match `runs/` in the GitHub repo, so every experiment script there works on
+these files as they are.
 
 ```python
 import json, torch
 from carrybit.config import load_config
 from carrybit.model import Transformer
 
-cfg = load_config("configs/addition.yaml", ["task.zero_pad=true", "task.positions=coupled"])
+run = "addition_big/position_coupling_s1"
+raw = json.load(open(f"{run}/config.json"))
+cfg = load_config("configs/addition_big.yaml",
+                  [f"{s}.{k}={json.dumps(v)}" for s in ("task", "model")
+                   for k, v in raw[s].items() if k != "kind"])
 model = Transformer(16, cfg.model)
-model.load_state_dict(torch.load("ladder/position_coupling_s0/step_50000.pt"))
+model.load_state_dict(torch.load(f"{run}/step_60000.pt"))
 ```
 
 ## Contents
 
-`modular_add/`: one-layer transformer trained on a + b mod 113 (Nanda et al.
-setup), 40k full-batch steps. Groks around step 2000.
-
-`ladder/`: the length generalization ladder. 4-layer, width 256 decoder-only
-transformers (3.4M parameters) trained for 50k steps on addition with 1 to 20
-digit operands, three seeds per format. Exact match accuracy on operands of
-exactly n digits:
-
-| run | 20 | 25 | 30 | 40 | 50 |
-|---|---|---|---|---|---|
-| plain_s0 | 0.93 | 0 | 0 | 0 | 0 |
-| plain_s1 | 0.93 | 0 | 0 | 0 | 0 |
-| plain_s2 | 0.81 | 0 | 0 | 0 | 0 |
-| reversed_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 |
-| reversed_zero_pad_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 |
-| abacus_s0 | 1.00 | 0 | 0 | 0 | 0 |
-| abacus_s1 | 1.00 | 0 | 0 | 0 | 0 |
-| abacus_s2 | 1.00 | 1.00 | 0.93 | 0.75 | 0.25 |
-| position_coupling_s0 | 1.00 | 1.00 | 0.92 | 0.11 | 0 |
-| position_coupling_s1 | 1.00 | 0.88 | 0 | 0 | 0 |
-| position_coupling_s2 | 1.00 | 0 | 0 | 0 | 0 |
-| aligned_blankspace_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 |
-
-See the GitHub README for what the formats mean and a discussion of the results.
+| folder | what | runs |
+|---|---|---|
+| `modular_add`, `modular_add_wd0.1`, `modular_add_wd0` | one-layer transformer, a + b mod 113, weight decay 1 / 0.1 / 0 | 1 each |
+| `modular_mlp` | ReLU MLP on two-hot inputs, p = 97 (Swaroop 2026 setup) | 1 |
+| `addition` | the length ladder: 3.4M params, trained on 1 to 20 digits, nine formats | 3 seeds each, 6 for position coupling |
+| `addition_big` | 11M params, abacus and position coupling, trained on 1 to 30 digits | 2 seeds each |
+| `blankspace_big` | 11M params, fixed-width aligned blankspace, trained on 1 to 20 digits | 2 seeds |
+| `addition_constant_lr`, `addition_no_wd` | position coupling ablations | 3 and 6 seeds |
+| `addition_carry_heavy`, `addition_fixed_length` | position coupling with carry-heavy or single-length training data | 3 seeds each |
+| `subtraction` | position coupling and fixed blankspace on a - b | 3 seeds each |
+
+The one to try first is `addition_big/position_coupling_s1`. Trained on up to
+30 digits, it scores 0% exact match at 100 digits as is. Set
+`block.attn.scale = 2.0` on every block before decoding and it scores 100% at
+200. The GitHub README explains why.

addition/abacus_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder abacus seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "abacus",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition/abacus_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,40.4,0.04165126566593774,0.99609375,0.95703125,0.92578125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,77.8,0.007477885840074827,1.0,0.96875,0.9921875,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,114.0,0.0030776386032294453,1.0,1.0,0.99609375,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,150.1,0.0021365052922425946,1.0,0.9921875,0.95703125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,186.3,0.0014214514749505238,1.0,0.9765625,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,222.9,0.0010082435070390278,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,259.6,0.000492235087522748,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,296.1,0.0006661927837288326,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,332.9,0.00022146683685564763,0.99609375,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,369.7,0.00028362983838865844,1.0,0.96875,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,406.6,5.627922378600414e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,443.1,0.00010834653030878249,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,479.3,5.170799638041556e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,515.6,2.6278768232528415e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,551.9,1.3676077857973595e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,588.5,1.8227696000174497e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,625.2,2.0809437230908565e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,661.8,2.2113516591608795e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,698.7,2.246176344477924e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,735.4,2.244289403386741e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/abacus_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2e85eb9ef4ea4524bdd12b147cb5908492e7bc605af2a75b445a5edcb86faf3
+size 13738485

addition/abacus_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder abacus seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "abacus",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition/abacus_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,40.2,0.019421861825930153,0.9921875,0.984375,0.6640625,0.703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,76.9,0.00727983858716512,1.0,0.828125,0.86328125,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,113.2,0.008187808755326617,0.984375,0.98046875,0.94140625,0.76171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,149.5,0.0022380353683056874,1.0,0.9921875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,186.0,0.0017146116753800655,1.0,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,222.7,0.0010716926865862115,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,259.4,0.0011338222128818905,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,296.0,0.0006792790250233844,1.0,1.0,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,332.8,0.00046899516380649933,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,369.7,5.581149432375635e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,406.6,0.0003637340261300791,1.0,0.9921875,0.99609375,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,443.3,8.481105058334911e-05,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,479.6,5.903834703148728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,515.9,7.956959854725254e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,552.5,8.394848630662215e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,589.0,1.4101947604018792e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,626.0,1.7413494767162065e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,662.5,1.86887250719425e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,699.5,1.916155950283142e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,736.3,1.930204289234567e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/abacus_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71edef7f0633fc9562fff242174358eaa7c061316f51ca84feb02feb6879820d
+size 13738485

addition/abacus_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder abacus seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "abacus",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition/abacus_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,40.2,0.0025480398695461076,1.0,0.984375,0.96875,0.953125,0.91015625,0.6484375,0.36328125,0.0703125,0.0,0.0,0.0
+5000,76.4,0.000774010375750845,1.0,0.98828125,1.0,0.99609375,0.9921875,0.6875,0.39453125,0.13671875,0.0,0.0,0.0
+7500,112.8,0.000516119487618438,0.9921875,0.99609375,0.96484375,0.97265625,0.98828125,0.65625,0.140625,0.015625,0.0,0.0,0.0
+10000,149.2,0.0006419722169680194,1.0,0.99609375,0.99609375,0.9453125,0.9375,0.83984375,0.2734375,0.05859375,0.0,0.0,0.0
+12500,185.7,7.209724002654492e-08,1.0,1.0,1.0,1.0,1.0,0.953125,0.8203125,0.39453125,0.05078125,0.0,0.0
+15000,222.4,1.0956978970981467e-08,1.0,1.0,1.0,1.0,1.0,0.99609375,0.875,0.70703125,0.296875,0.01171875,0.0
+17500,259.1,0.0001840910556279454,1.0,1.0,1.0,1.0,0.9921875,0.9140625,0.2734375,0.078125,0.0,0.0,0.0
+20000,295.7,2.3715848697307856e-08,1.0,1.0,1.0,1.0,0.99609375,0.96875,0.64453125,0.35546875,0.015625,0.0,0.0
+22500,332.8,2.134711718499249e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.71484375,0.41796875,0.09765625,0.0,0.0
+25000,369.7,2.3915868879389415e-08,1.0,1.0,1.0,1.0,1.0,0.98046875,0.5703125,0.05078125,0.0,0.0,0.0
+27500,406.4,2.525873054866377e-08,1.0,1.0,1.0,1.0,1.0,0.90625,0.0234375,0.0,0.0,0.0,0.0
+30000,442.9,2.6034143760423384e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.86328125,0.38671875,0.0078125,0.0,0.0
+32500,479.2,1.2933618968363646e-06,1.0,1.0,1.0,1.0,1.0,0.80078125,0.5,0.046875,0.0,0.0,0.0
+35000,515.5,8.196465929258812e-05,1.0,1.0,1.0,1.0,0.9921875,0.9765625,0.0625,0.00390625,0.0,0.0,0.0
+37500,552.3,2.5679873355496745e-08,1.0,1.0,1.0,1.0,1.0,0.75,0.15234375,0.00390625,0.0,0.0,0.0
+40000,589.0,2.668754422022952e-09,1.0,1.0,1.0,1.0,1.0,0.89453125,0.48046875,0.07421875,0.0,0.0,0.0
+42500,625.9,1.6834652089024804e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.76171875,0.24609375,0.015625,0.0,0.0
+45000,662.5,1.7842648507998153e-08,1.0,1.0,1.0,1.0,1.0,0.9296875,0.75390625,0.2890625,0.015625,0.0,0.0
+47500,699.0,1.9654443694697242e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.74609375,0.26171875,0.0078125,0.0,0.0
+50000,736.0,2.0090057245062915e-08,1.0,1.0,1.0,1.0,1.0,0.93359375,0.75,0.25390625,0.00390625,0.0,0.0

addition/abacus_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1c41cdf7af8f6f213462f7998d95e2a43d4edc22c0d441b0f69fd50cc2dea1
+size 13738485

addition/blankspace_fixed_relative_s0/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace fixed + relative seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "relative",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition/blankspace_fixed_relative_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,22.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,226.3,0.20703520938244796,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,430.3,0.2627567995265704,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,634.5,0.4574976999808654,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,838.7,0.2651583137941563,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,1044.1,0.10680842386954857,0.09765625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,1248.2,0.12207524907863034,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1452.5,0.1762504720602938,0.08203125,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1656.8,0.14041061588127907,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1862.4,0.1001222274342093,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,2066.6,0.2697289037550494,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,2270.6,0.6783406467437754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,2474.8,0.6374371794799828,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2680.1,0.6329233304221513,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2884.3,0.6234669802913972,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,3088.3,0.6205317425213412,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,3292.4,0.6017276442331264,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,3497.7,0.7258058367817909,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,3701.7,0.6323961474374555,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,3906.0,0.6066491667535895,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,4110.2,0.593319668324286,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/blankspace_fixed_relative_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e9ef8a03a4264db75ff7349ff6fe4ac8f8d3c53e127ec36215b926c21fa2ee3
+size 12756597

addition/blankspace_fixed_relative_s1/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace fixed + relative seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "relative",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition/blankspace_fixed_relative_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,22.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,226.3,0.6826302532060446,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,430.5,0.09242761518813643,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,634.7,0.18230011434451712,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,838.9,0.6496547036008226,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,1044.0,0.6364924383871152,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,1248.2,0.11263084836659418,0.0390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1452.4,0.08725647997445593,0.0546875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1656.6,0.0927207347182343,0.1015625,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1861.8,0.12975181270519814,0.01953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,2066.1,0.08782499054882002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,2270.4,0.2652910798088735,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,2474.5,0.04182807987974134,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2679.8,0.02653962984752767,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2884.1,0.06692170471305299,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,3088.4,0.06239397073924014,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,3292.4,0.040825869940407496,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,3497.7,0.04419280434274837,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,3701.8,0.04129039797399679,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,3905.8,0.03914603508183192,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,4110.0,0.03725589552857548,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/blankspace_fixed_relative_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe3fc7a5e4ba9b793a726ecab888554396c70796e09bee42b1168b5c9bedba5c
+size 12756597

addition/blankspace_fixed_relative_s2/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace fixed + relative seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "relative",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition/blankspace_fixed_relative_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,22.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,226.4,0.14129522232535732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,430.6,0.15880897545836423,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,634.7,0.10007927374338119,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,839.1,0.14276802959215126,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,1044.5,0.23306011773899402,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,1248.6,0.04054757494554516,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1452.8,0.06636011190511847,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1657.2,0.42169396162704836,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1862.6,0.2511044359851245,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,2066.8,0.02148349786548707,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,2271.0,0.0108850836969352,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,2475.2,0.06255321251730356,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2680.6,0.02701192433966193,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2884.9,0.016707464822334345,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,3088.9,0.010058896048647946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,3293.2,0.008355475885859501,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,3498.7,0.007721445332368437,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,3702.9,0.005973847859699946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,3907.0,0.0052722387621380465,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,4111.0,0.004958737693459962,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/blankspace_fixed_relative_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99804beb8fa954954f3ae0432c670baf93b338e07c765ae34beb6087e78f2f45
+size 12756597

addition/blankspace_fixed_s0/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace fixed seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition/blankspace_fixed_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,20.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,196.9,0.0005506453078795269,0.8203125,0.8671875,0.73828125,0.6328125,0.51171875,0.3984375,0.11328125,0.015625,0.0,0.0,0.0
+5000,373.3,0.00012206451032746232,0.98046875,0.92578125,0.88671875,0.83984375,0.75390625,0.60546875,0.265625,0.07421875,0.0078125,0.0,0.0
+7500,549.6,9.21903775479684e-05,0.98828125,1.0,0.97265625,0.953125,0.890625,0.8984375,0.4296875,0.13671875,0.01171875,0.0,0.0
+10000,725.8,3.749034867578125e-05,1.0,0.98828125,0.984375,0.9140625,0.8828125,0.74609375,0.61328125,0.2421875,0.08203125,0.0,0.0
+12500,903.5,3.472558525563995e-05,1.0,0.99609375,0.99609375,0.94921875,0.94921875,0.859375,0.36328125,0.0859375,0.02734375,0.0,0.0
+15000,1080.8,1.7849055845936695e-05,0.984375,0.98828125,0.95703125,0.875,0.68359375,0.4921875,0.26171875,0.05859375,0.0,0.0,0.0
+17500,1257.1,1.8422598429383474e-05,0.99609375,0.9921875,0.9921875,0.9609375,0.87109375,0.7578125,0.25390625,0.08203125,0.00390625,0.0,0.0
+20000,1433.2,4.3433618352920054e-07,1.0,1.0,0.99609375,0.99609375,0.9921875,0.9609375,0.83203125,0.3984375,0.0859375,0.0,0.0
+22500,1610.6,1.3758578947589133e-05,1.0,1.0,0.98828125,0.92578125,0.890625,0.61328125,0.32421875,0.07421875,0.0,0.0,0.0
+25000,1786.8,3.7128141571484146e-07,0.99609375,1.0,0.9921875,0.9921875,0.984375,0.93359375,0.5625,0.0859375,0.0,0.0,0.0
+27500,1963.3,6.213710957201114e-07,0.98828125,1.0,0.98046875,0.93359375,0.7890625,0.625,0.16796875,0.0,0.0,0.0,0.0
+30000,2139.5,3.5937933895871672e-09,1.0,1.0,1.0,0.9921875,0.98046875,0.890625,0.4453125,0.0,0.0,0.0,0.0
+32500,2317.3,2.1753980313264555e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.72265625,0.1328125,0.0,0.0,0.0
+35000,2493.6,2.2173957132333224e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.5859375,0.03125,0.0,0.0,0.0
+37500,2669.8,5.5568823425201534e-09,1.0,1.0,1.0,1.0,0.984375,0.9453125,0.59765625,0.078125,0.0,0.0,0.0
+40000,2846.0,2.607981333519621e-09,1.0,1.0,1.0,1.0,1.0,0.9765625,0.76171875,0.1171875,0.0,0.0,0.0
+42500,3023.7,2.8111192429016504e-09,1.0,1.0,1.0,1.0,1.0,0.96875,0.71484375,0.0859375,0.0,0.0,0.0
+45000,3200.0,3.023483413213966e-09,1.0,1.0,1.0,1.0,1.0,0.97265625,0.71875,0.10546875,0.0,0.0,0.0
+47500,3376.3,2.967679517038681e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.68359375,0.08984375,0.0,0.0,0.0
+50000,3552.5,3.0245826619401742e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.68359375,0.0859375,0.0,0.0,0.0

addition/blankspace_fixed_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4733e444551e358ae460cdac12828231d3e07297a1162bdbe6afb422e6e301a8
+size 13738485

addition/blankspace_fixed_s1/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace fixed seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition/blankspace_fixed_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,20.3,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,196.7,0.0026082363351385583,0.1015625,0.0546875,0.046875,0.0546875,0.04296875,0.0234375,0.00390625,0.0,0.0,0.0,0.0
+5000,373.0,0.0003393877649152727,0.97265625,0.5,0.93359375,0.859375,0.6953125,0.35546875,0.1171875,0.00390625,0.0,0.0,0.0
+7500,549.2,0.00011643498715221173,0.90625,0.3828125,0.86328125,0.61328125,0.58984375,0.5078125,0.484375,0.18359375,0.046875,0.0,0.0
+10000,725.4,7.968605815850177e-05,0.91796875,0.7109375,0.96484375,0.83203125,0.70703125,0.3671875,0.12109375,0.01171875,0.0,0.0,0.0
+12500,903.1,2.1121372331914974e-05,0.9921875,1.0,0.99609375,0.9609375,0.9375,0.88671875,0.5234375,0.1953125,0.03515625,0.00390625,0.0
+15000,1080.6,8.796565490238032e-06,1.0,1.0,0.99609375,0.9921875,0.9375,0.84375,0.484375,0.09375,0.015625,0.0,0.0
+17500,1257.0,2.82035677643842e-05,0.9921875,0.96875,0.9609375,0.9296875,0.765625,0.58203125,0.3125,0.08203125,0.0,0.0,0.0
+20000,1433.0,2.394601428167586e-09,1.0,1.0,1.0,0.99609375,0.984375,0.96484375,0.73046875,0.24609375,0.04296875,0.0,0.0
+22500,1610.7,3.2071281316063323e-06,0.9921875,0.99609375,1.0,0.9921875,0.9765625,0.875,0.56640625,0.05859375,0.00390625,0.0,0.0
+25000,1786.8,4.094170965169126e-05,1.0,1.0,0.9765625,0.96875,0.86328125,0.68359375,0.328125,0.01953125,0.0,0.0,0.0
+27500,1963.2,2.391484625870849e-06,1.0,1.0,0.953125,0.890625,0.7421875,0.6171875,0.29296875,0.03515625,0.00390625,0.0,0.0
+30000,2139.4,4.713165529820018e-08,1.0,1.0,0.99609375,1.0,0.9609375,0.75,0.3359375,0.02734375,0.0,0.0,0.0
+32500,2315.8,2.4914220951256115e-09,1.0,1.0,1.0,1.0,0.98828125,0.8984375,0.578125,0.0859375,0.01171875,0.0,0.0
+35000,2492.0,2.8948093464735e-09,1.0,1.0,1.0,0.99609375,0.96484375,0.87109375,0.59375,0.1328125,0.0078125,0.0,0.0
+37500,2668.3,2.5616916061582564e-09,1.0,1.0,1.0,1.0,0.984375,0.8828125,0.5,0.0625,0.0,0.0,0.0
+40000,2844.6,2.8025650015356648e-09,1.0,1.0,1.0,1.0,0.98046875,0.8828125,0.4453125,0.03125,0.0,0.0,0.0
+42500,3022.3,2.862597413463905e-09,1.0,1.0,1.0,1.0,0.9765625,0.875,0.43359375,0.03515625,0.0,0.0,0.0
+45000,3198.5,3.084677307498172e-09,1.0,1.0,1.0,1.0,0.98046875,0.87109375,0.40625,0.02734375,0.0,0.0,0.0
+47500,3377.1,3.0422148427183186e-09,1.0,1.0,1.0,1.0,0.984375,0.875,0.3984375,0.02734375,0.0,0.0,0.0
+50000,3554.0,3.001311497162208e-09,1.0,1.0,1.0,1.0,0.98046875,0.8671875,0.3984375,0.02734375,0.0,0.0,0.0

addition/blankspace_fixed_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:974e486e24ae91e70c3ab03bcbb6ab593b358a92a19889bb0d17c2604e7f8773
+size 13738485

addition/blankspace_fixed_s2/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace fixed seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition/blankspace_fixed_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,20.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,197.2,0.0004957558742136849,0.9140625,0.82421875,0.70703125,0.546875,0.2265625,0.0625,0.00390625,0.0,0.0,0.0,0.0
+5000,373.9,0.00012956480872600203,0.99609375,0.99609375,0.96875,0.765625,0.52734375,0.16796875,0.015625,0.0,0.0,0.0,0.0
+7500,550.1,9.812221885565382e-05,0.96875,0.953125,0.8984375,0.6484375,0.3984375,0.16796875,0.01171875,0.0078125,0.0,0.0,0.0
+10000,726.2,1.7760837309919371e-07,1.0,1.0,1.0,0.98828125,0.91015625,0.625,0.14453125,0.00390625,0.0,0.0,0.0
+12500,903.9,6.116268558052304e-05,0.98828125,0.92578125,0.5625,0.2734375,0.11328125,0.015625,0.0,0.0,0.0,0.0,0.0
+15000,1081.4,6.285491913409828e-05,0.99609375,0.99609375,0.94140625,0.85546875,0.62890625,0.4296875,0.03515625,0.0,0.0,0.0,0.0
+17500,1257.6,4.916129410842028e-05,1.0,1.0,0.96875,0.75390625,0.6640625,0.47265625,0.0859375,0.015625,0.0,0.0,0.0
+20000,1434.0,1.533371229892242e-08,1.0,1.0,0.99609375,0.89453125,0.75390625,0.609375,0.16015625,0.0390625,0.0,0.0,0.0
+22500,1611.6,2.292736706764027e-05,1.0,1.0,0.97265625,0.859375,0.69140625,0.45703125,0.08203125,0.00390625,0.0,0.0,0.0
+25000,1787.8,2.0849592988569546e-05,0.99609375,0.99609375,0.984375,0.9375,0.77734375,0.52734375,0.0390625,0.0,0.0,0.0,0.0
+27500,1964.2,3.0127465835718325e-09,1.0,1.0,0.9921875,0.984375,0.953125,0.75,0.27734375,0.078125,0.0,0.0,0.0
+30000,2140.5,2.0596147923065815e-06,1.0,0.99609375,0.99609375,0.9609375,0.921875,0.6484375,0.0546875,0.0078125,0.0,0.0,0.0
+32500,2316.8,3.6242308188774898e-09,1.0,1.0,1.0,0.96875,0.90625,0.578125,0.10546875,0.01953125,0.0,0.0,0.0
+35000,2493.2,3.184096991594267e-09,1.0,0.99609375,0.99609375,0.9453125,0.921875,0.69140625,0.12109375,0.015625,0.0,0.0,0.0
+37500,2669.7,4.223212647507518e-09,1.0,1.0,1.0,0.98046875,0.91796875,0.70703125,0.1328125,0.01171875,0.0,0.0,0.0
+40000,2845.9,4.627020220290975e-09,1.0,1.0,1.0,0.984375,0.8984375,0.66796875,0.12109375,0.01171875,0.0,0.0,0.0
+42500,3023.6,4.912223292150009e-09,1.0,1.0,1.0,0.984375,0.8984375,0.6328125,0.11328125,0.01171875,0.0,0.0,0.0
+45000,3199.9,4.980930373748483e-09,1.0,1.0,1.0,0.98046875,0.87890625,0.6171875,0.109375,0.01171875,0.0,0.0,0.0
+47500,3376.4,5.028134725070195e-09,1.0,1.0,1.0,0.98046875,0.87890625,0.60546875,0.1015625,0.01171875,0.0,0.0,0.0
+50000,3552.6,5.002201414504762e-09,1.0,1.0,1.0,0.98046875,0.87890625,0.61328125,0.09765625,0.0078125,0.0,0.0,0.0

addition/blankspace_fixed_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc25ea67fc887e8cadd6dc2fa0283952bd8fbd9dfa4da23d651378ef8584290d
+size 13738485

addition/blankspace_var_s0/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace var seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": false,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition/blankspace_var_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,162.3,0.6186378883550033,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,320.9,0.19051331784140904,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,479.7,0.09004364974590386,0.8359375,0.44921875,0.2890625,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,638.5,0.14656466887716998,0.015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,798.6,0.006658630277786855,0.93359375,0.3125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,957.3,0.005940055596195082,1.0,1.0,0.9921875,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1116.1,0.01754279880689707,1.0,0.984375,0.9296875,0.296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1276.3,0.0034275892633903672,1.0,0.99609375,1.0,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1436.2,0.0047208392852198374,1.0,1.0,0.9921875,0.890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,1595.0,0.0018665735886661443,0.984375,1.0,0.9375,0.58984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,1753.9,0.0013890370618582118,0.9921875,1.0,0.7265625,0.3515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,1912.8,0.0009846590920853096,1.0,1.0,0.98828125,0.9140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2073.0,0.0032889325377649515,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2232.0,0.00019508725264521203,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,2390.8,0.00045013484071009506,1.0,1.0,1.0,0.9140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,2549.6,4.6959747138616026e-05,1.0,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,2709.7,5.6767481562301165e-05,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,2868.5,7.050720125570499e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,3027.3,1.1120138110482406e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,3186.0,7.488752917851346e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/blankspace_var_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9d617bb8100d1a131146043d65fa491df62c4f9d4dbdb12e8f93c0e6d1bcd27
+size 13738485

addition/blankspace_var_s1/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace var seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": false,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition/blankspace_var_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,162.2,0.6694373498431923,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,320.9,0.3174608366777573,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,479.6,0.088214992116239,0.63671875,0.57421875,0.4609375,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,638.4,0.033645382226527756,0.921875,0.98046875,0.91796875,0.234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,798.3,0.01124123413962232,0.61328125,0.97265625,0.91015625,0.3046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,957.2,0.03232797719789472,0.98828125,0.9609375,0.65234375,0.02734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1116.0,0.00718463512902601,0.9921875,1.0,1.0,0.86328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1274.9,0.014129337214051867,0.96875,1.0,0.98828125,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1434.9,0.0036740895380866687,1.0,1.0,0.99609375,0.87890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,1593.6,0.007062510357253116,1.0,0.9921875,0.98046875,0.59375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,1752.5,0.008447402562925583,1.0,1.0,0.9921875,0.671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,1911.2,0.0025287677941350072,1.0,0.9375,0.671875,0.66796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2071.3,0.0018959840800858183,0.99609375,1.0,0.98046875,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2230.0,0.0003868506276501107,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,2388.9,0.00031602887532225857,1.0,1.0,1.0,0.9140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,2547.6,7.280061912272834e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,2707.7,2.988267797327319e-05,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,2866.4,9.73891918357995e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,3025.3,1.6834128019312507e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,3184.1,1.622445492837834e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/blankspace_var_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:563dddf1a15890c379f95a0880119fa2b564efd22e9275924ec5c46dacfe2e5c
+size 13738485

addition/blankspace_var_s2/config.json

@@ -0,0 +1,53 @@
+{
+  "name": "addition blankspace var seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 121,
+    "blanks_fixed": false,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition/blankspace_var_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,162.2,0.5832304521701059,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,321.1,0.04690088438062398,0.68359375,0.60546875,0.3046875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,479.9,0.08495105978154338,0.67578125,0.76953125,0.04296875,0.01953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,638.7,0.012313809212109967,0.98046875,1.0,0.99609375,0.48046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,798.8,0.009126608279989946,1.0,1.0,1.0,0.609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,957.7,0.034043930974871496,0.98046875,0.93359375,0.66796875,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1116.6,0.0034535789100757666,0.9921875,1.0,0.80078125,0.3046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1276.5,0.0021677381241184076,1.0,1.0,1.0,0.9375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1436.6,0.004518133543420206,1.0,1.0,0.93359375,0.328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,1595.4,0.023756263721328504,0.01171875,0.140625,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,1754.3,0.0041635120231823105,1.0,0.99609375,0.87890625,0.51171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,1913.1,0.0067360217149997036,0.6015625,0.9921875,0.66796875,0.07421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2073.3,0.0009128844293960459,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2232.1,0.0003308118617656198,1.0,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,2391.2,0.000812493642517381,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,2550.0,9.513217252444159e-05,1.0,1.0,1.0,0.546875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0
+42500,2710.2,1.910369615979087e-05,1.0,1.0,1.0,1.0,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0
+45000,2869.0,9.98705935296285e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,3027.7,1.0420948403523156e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,3186.6,6.497759708227786e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/blankspace_var_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:766d1f5b22f994f58bf66af29e61968331d18956aba158e8f0fcdf3630dd3416
+size 13738485

addition/plain_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder plain seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": false,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition/plain_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,45.8,1.1552440607522254,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,85.3,1.0496538571563998,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,121.6,0.22190272788699789,0.625,0.30078125,0.13671875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,156.8,0.1361104787168321,0.62890625,0.265625,0.1328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,194.1,0.1299326336596537,0.83984375,0.82421875,0.66015625,0.26953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,230.1,0.08724782346629964,0.87890625,0.72265625,0.6171875,0.37109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,265.7,0.03294547912425976,0.8828125,0.80078125,0.7109375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,301.2,0.03188540525085875,0.9296875,0.8125,0.75390625,0.60546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,338.5,0.024272466284350352,0.94921875,0.84765625,0.76953125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,377.9,0.02135416777169092,0.95703125,0.84765625,0.77734375,0.62109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,413.0,0.017336725187138323,0.95703125,0.81640625,0.72265625,0.59375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,450.0,0.015351897706221264,0.95703125,0.90625,0.8359375,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,485.1,0.011902291315282323,0.9375,0.921875,0.78515625,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,520.3,0.010157805927521121,0.98046875,0.9453125,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,555.7,0.008457286916127297,0.98828125,0.9453125,0.89453125,0.84765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,591.0,0.006343905665149985,0.984375,0.95703125,0.90234375,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,626.2,0.004807067632394225,0.984375,0.94921875,0.93359375,0.90625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,661.3,0.0039473881130650396,0.9765625,0.96484375,0.93359375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,696.4,0.003158288052806785,0.98828125,0.96875,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,731.5,0.0031530544255801425,0.98828125,0.96484375,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/plain_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3273a0e6726cde985810bea69fe181a1c622e441242ac728493e1349c94dc96b
+size 13738485

addition/plain_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder plain seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": false,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition/plain_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,39.9,1.0926829596295016,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,75.2,0.34977536998246683,0.578125,0.17578125,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,110.7,0.13636325329445406,0.80859375,0.59765625,0.27734375,0.0859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,147.0,0.10624530700855975,0.4921875,0.5703125,0.13671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,182.0,0.04835331203503549,0.890625,0.7109375,0.71875,0.3828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,217.1,0.13925495517488404,0.71484375,0.64453125,0.20703125,0.06640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,252.1,0.09695242193667337,0.90234375,0.5859375,0.6015625,0.1796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,287.1,0.030026417599016593,0.9296875,0.7421875,0.63671875,0.11328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,321.9,0.02373246684440765,0.94921875,0.828125,0.7265625,0.625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,356.6,0.027057840476159962,0.9453125,0.8515625,0.7578125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,391.2,0.020241652287542256,0.95703125,0.8203125,0.796875,0.609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,425.8,0.0187081657161948,0.95703125,0.66796875,0.7890625,0.59765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,461.1,0.013485327495607533,0.96484375,0.89453125,0.78125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,496.0,0.012634533764863868,0.9609375,0.91015625,0.83984375,0.70703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,530.9,0.009974493826907564,0.98046875,0.890625,0.87890625,0.8125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,566.1,0.007701182666293666,0.9765625,0.93359375,0.875,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,601.0,0.005880211502039975,0.98046875,0.94140625,0.9453125,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,635.9,0.0047997324281110814,0.9921875,0.97265625,0.95703125,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,670.7,0.004150005655083656,0.9921875,0.9609375,0.94921875,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,705.4,0.0038938612596869458,0.99609375,0.98046875,0.94921875,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/plain_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43cd4ea777cd8b8fd33034d97522b28ab6a20afddf953275a7028985a20bc71
+size 13738485

addition/plain_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder plain seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": false,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition/plain_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,38.3,1.2351460234888572,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,72.8,0.8908147140877666,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,107.4,0.3303994115917782,0.5625,0.23046875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,142.0,0.121963283541433,0.78125,0.5234375,0.328125,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,176.7,0.12380598067946852,0.81640625,0.58203125,0.20703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,211.4,0.06437959507165898,0.875,0.75390625,0.5546875,0.18359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,246.3,0.09695373926330297,0.90234375,0.7734375,0.640625,0.24609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,281.0,0.03774574688353087,0.87890625,0.74609375,0.54296875,0.140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,315.9,0.04852251501855399,0.90234375,0.76953125,0.65625,0.30078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,350.4,0.03372801105576202,0.91796875,0.84765625,0.734375,0.61328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,384.8,0.02787428909461228,0.97265625,0.85546875,0.765625,0.55078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,419.3,0.022389781303720466,0.9375,0.84765625,0.62109375,0.52734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,454.0,0.018736086882025,0.9609375,0.87109375,0.71875,0.62890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,488.7,0.015919025027907152,0.953125,0.88671875,0.82421875,0.71875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,523.5,0.012297407657478424,0.953125,0.890625,0.8671875,0.75,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,558.3,0.01050108779963775,0.984375,0.921875,0.85546875,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,593.0,0.008854387478807818,0.98046875,0.9140625,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,627.8,0.0075194296041089384,0.98046875,0.94921875,0.890625,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,662.6,0.006260404597938438,0.98046875,0.94921875,0.90234375,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,697.1,0.006391584459112504,0.984375,0.95703125,0.890625,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0

addition/plain_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af439c0424fe765d90897cfcd165a09545200a90ed9ddaedda956c1f419fa0bf
+size 13738485

addition/position_coupling_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder position coupling seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition/position_coupling_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,41.7,0.05849933046576848,0.9765625,0.99609375,0.96484375,0.89453125,0.640625,0.25390625,0.0,0.0,0.0,0.0,0.0
+5000,79.5,0.0011903617309327293,0.9453125,0.94921875,0.9453125,0.91015625,0.6953125,0.4140625,0.01953125,0.0,0.0,0.0,0.0
+7500,117.6,0.0008857806100816371,0.984375,1.0,0.9921875,0.96875,0.46484375,0.140625,0.01171875,0.0,0.0,0.0,0.0
+10000,155.7,0.0005239002259623858,1.0,0.9921875,1.0,0.98828125,0.80859375,0.65625,0.16015625,0.0,0.0,0.0,0.0
+12500,193.9,0.00031071801167808805,1.0,1.0,1.0,1.0,0.9140625,0.6875,0.3046875,0.04296875,0.0,0.0,0.0
+15000,232.4,9.674870884781087e-05,1.0,1.0,1.0,1.0,0.984375,0.890625,0.4296875,0.046875,0.0,0.0,0.0
+17500,270.8,1.1628271768843432e-07,1.0,1.0,1.0,1.0,0.99609375,0.953125,0.62890625,0.13671875,0.0078125,0.0,0.0
+20000,309.3,9.673758767857765e-09,1.0,1.0,1.0,1.0,1.0,0.9765625,0.41796875,0.03125,0.0,0.0,0.0
+22500,347.8,0.00010900244251551255,1.0,1.0,0.99609375,0.99609375,0.6328125,0.33203125,0.04296875,0.0,0.0,0.0,0.0
+25000,386.3,2.2218434021451218e-05,1.0,1.0,1.0,1.0,0.984375,0.90625,0.5234375,0.0234375,0.0,0.0,0.0
+27500,424.4,0.00024115022228942076,1.0,1.0,1.0,0.99609375,0.92578125,0.60546875,0.26171875,0.00390625,0.0,0.0,0.0
+30000,462.2,1.261457133340515e-08,1.0,1.0,1.0,1.0,0.99609375,0.89453125,0.484375,0.015625,0.0,0.0,0.0
+32500,500.0,2.513377858377487e-06,1.0,1.0,1.0,1.0,0.96484375,0.74609375,0.234375,0.0,0.0,0.0,0.0
+35000,538.0,1.1292256512601291e-07,1.0,1.0,1.0,1.0,0.97265625,0.85546875,0.34765625,0.0078125,0.0,0.0,0.0
+37500,576.4,1.731580459432587e-08,1.0,1.0,1.0,1.0,0.98828125,0.890625,0.1640625,0.00390625,0.0,0.0,0.0
+40000,614.8,1.9072867737526124e-08,1.0,1.0,1.0,1.0,0.99609375,0.9296875,0.140625,0.0,0.0,0.0,0.0
+42500,653.2,2.0661178407917825e-08,1.0,1.0,1.0,1.0,0.99609375,0.94140625,0.12890625,0.0,0.0,0.0,0.0
+45000,691.5,1.976445106136551e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0
+47500,730.0,1.9826637076144926e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0
+50000,768.0,2.006347927688552e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0

addition/position_coupling_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5367b75422a22700d5f4e0cf6b5f8277aba3934897b128c8810ac33d5b7d2dd0
+size 13738485

addition/position_coupling_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder position coupling seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}