add ladder and modular addition checkpoints

README.md

@@ -0,0 +1,57 @@
+---
+license: mit
+tags:
+  - arithmetic
+  - length-generalization
+  - grokking
+  - interpretability
+---
+
+# carrybit checkpoints
+
+Trained weights from [carrybit](https://github.com/Supergoatscriptguy/carrybit),
+a small research project on tiny transformers learning exact integer
+arithmetic. The code, training configs, and result plots live in that repo;
+this repo only holds the final checkpoints so the analysis experiments can be
+run without retraining.
+
+Every checkpoint is a plain PyTorch `state_dict` for `carrybit.model.Transformer`.
+Each folder has the exact `config.json` it was trained with and the
+`metrics.csv` logged during training.
+
+```python
+import json, torch
+from carrybit.config import load_config
+from carrybit.model import Transformer
+
+cfg = load_config("configs/addition.yaml", ["task.zero_pad=true", "task.positions=coupled"])
+model = Transformer(16, cfg.model)
+model.load_state_dict(torch.load("ladder/position_coupling_s0/step_50000.pt"))
+```
+
+## Contents
+
+`modular_add/`: one-layer transformer trained on a + b mod 113 (Nanda et al.
+setup), 40k full-batch steps. Groks around step 2000.
+
+`ladder/`: the length generalization ladder. 4-layer, width 256 decoder-only
+transformers (3.4M parameters) trained for 50k steps on addition with 1 to 20
+digit operands, three seeds per format. Exact match accuracy on operands of
+exactly n digits:
+
+| run | 20 | 25 | 30 | 40 | 50 |
+|---|---|---|---|---|---|
+| plain_s0 | 0.93 | 0 | 0 | 0 | 0 |
+| plain_s1 | 0.93 | 0 | 0 | 0 | 0 |
+| plain_s2 | 0.81 | 0 | 0 | 0 | 0 |
+| reversed_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 |
+| reversed_zero_pad_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 |
+| abacus_s0 | 1.00 | 0 | 0 | 0 | 0 |
+| abacus_s1 | 1.00 | 0 | 0 | 0 | 0 |
+| abacus_s2 | 1.00 | 1.00 | 0.93 | 0.75 | 0.25 |
+| position_coupling_s0 | 1.00 | 1.00 | 0.92 | 0.11 | 0 |
+| position_coupling_s1 | 1.00 | 0.88 | 0 | 0 | 0 |
+| position_coupling_s2 | 1.00 | 0 | 0 | 0 | 0 |
+| aligned_blankspace_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 |
+
+See the GitHub README for what the formats mean and a discussion of the results.

ladder/abacus_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder abacus seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "abacus",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

ladder/abacus_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,40.4,0.04165126566593774,0.99609375,0.95703125,0.92578125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,77.8,0.007477885840074827,1.0,0.96875,0.9921875,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,114.0,0.0030776386032294453,1.0,1.0,0.99609375,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,150.1,0.0021365052922425946,1.0,0.9921875,0.95703125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,186.3,0.0014214514749505238,1.0,0.9765625,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,222.9,0.0010082435070390278,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,259.6,0.000492235087522748,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,296.1,0.0006661927837288326,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,332.9,0.00022146683685564763,0.99609375,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,369.7,0.00028362983838865844,1.0,0.96875,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,406.6,5.627922378600414e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,443.1,0.00010834653030878249,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,479.3,5.170799638041556e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,515.6,2.6278768232528415e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,551.9,1.3676077857973595e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,588.5,1.8227696000174497e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,625.2,2.0809437230908565e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,661.8,2.2113516591608795e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,698.7,2.246176344477924e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,735.4,2.244289403386741e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/abacus_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2e85eb9ef4ea4524bdd12b147cb5908492e7bc605af2a75b445a5edcb86faf3
+size 13738485

ladder/abacus_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder abacus seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "abacus",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

ladder/abacus_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,40.2,0.019421861825930153,0.9921875,0.984375,0.6640625,0.703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,76.9,0.00727983858716512,1.0,0.828125,0.86328125,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,113.2,0.008187808755326617,0.984375,0.98046875,0.94140625,0.76171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,149.5,0.0022380353683056874,1.0,0.9921875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,186.0,0.0017146116753800655,1.0,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,222.7,0.0010716926865862115,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,259.4,0.0011338222128818905,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,296.0,0.0006792790250233844,1.0,1.0,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,332.8,0.00046899516380649933,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,369.7,5.581149432375635e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,406.6,0.0003637340261300791,1.0,0.9921875,0.99609375,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,443.3,8.481105058334911e-05,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,479.6,5.903834703148728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,515.9,7.956959854725254e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,552.5,8.394848630662215e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,589.0,1.4101947604018792e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,626.0,1.7413494767162065e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,662.5,1.86887250719425e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,699.5,1.916155950283142e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,736.3,1.930204289234567e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/abacus_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71edef7f0633fc9562fff242174358eaa7c061316f51ca84feb02feb6879820d
+size 13738485

ladder/abacus_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder abacus seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "abacus",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

ladder/abacus_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,40.2,0.0025480398695461076,1.0,0.984375,0.96875,0.953125,0.91015625,0.6484375,0.36328125,0.0703125,0.0,0.0,0.0
+5000,76.4,0.000774010375750845,1.0,0.98828125,1.0,0.99609375,0.9921875,0.6875,0.39453125,0.13671875,0.0,0.0,0.0
+7500,112.8,0.000516119487618438,0.9921875,0.99609375,0.96484375,0.97265625,0.98828125,0.65625,0.140625,0.015625,0.0,0.0,0.0
+10000,149.2,0.0006419722169680194,1.0,0.99609375,0.99609375,0.9453125,0.9375,0.83984375,0.2734375,0.05859375,0.0,0.0,0.0
+12500,185.7,7.209724002654492e-08,1.0,1.0,1.0,1.0,1.0,0.953125,0.8203125,0.39453125,0.05078125,0.0,0.0
+15000,222.4,1.0956978970981467e-08,1.0,1.0,1.0,1.0,1.0,0.99609375,0.875,0.70703125,0.296875,0.01171875,0.0
+17500,259.1,0.0001840910556279454,1.0,1.0,1.0,1.0,0.9921875,0.9140625,0.2734375,0.078125,0.0,0.0,0.0
+20000,295.7,2.3715848697307856e-08,1.0,1.0,1.0,1.0,0.99609375,0.96875,0.64453125,0.35546875,0.015625,0.0,0.0
+22500,332.8,2.134711718499249e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.71484375,0.41796875,0.09765625,0.0,0.0
+25000,369.7,2.3915868879389415e-08,1.0,1.0,1.0,1.0,1.0,0.98046875,0.5703125,0.05078125,0.0,0.0,0.0
+27500,406.4,2.525873054866377e-08,1.0,1.0,1.0,1.0,1.0,0.90625,0.0234375,0.0,0.0,0.0,0.0
+30000,442.9,2.6034143760423384e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.86328125,0.38671875,0.0078125,0.0,0.0
+32500,479.2,1.2933618968363646e-06,1.0,1.0,1.0,1.0,1.0,0.80078125,0.5,0.046875,0.0,0.0,0.0
+35000,515.5,8.196465929258812e-05,1.0,1.0,1.0,1.0,0.9921875,0.9765625,0.0625,0.00390625,0.0,0.0,0.0
+37500,552.3,2.5679873355496745e-08,1.0,1.0,1.0,1.0,1.0,0.75,0.15234375,0.00390625,0.0,0.0,0.0
+40000,589.0,2.668754422022952e-09,1.0,1.0,1.0,1.0,1.0,0.89453125,0.48046875,0.07421875,0.0,0.0,0.0
+42500,625.9,1.6834652089024804e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.76171875,0.24609375,0.015625,0.0,0.0
+45000,662.5,1.7842648507998153e-08,1.0,1.0,1.0,1.0,1.0,0.9296875,0.75390625,0.2890625,0.015625,0.0,0.0
+47500,699.0,1.9654443694697242e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.74609375,0.26171875,0.0078125,0.0,0.0
+50000,736.0,2.0090057245062915e-08,1.0,1.0,1.0,1.0,1.0,0.93359375,0.75,0.25390625,0.00390625,0.0,0.0

ladder/abacus_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1c41cdf7af8f6f213462f7998d95e2a43d4edc22c0d441b0f69fd50cc2dea1
+size 13738485

ladder/aligned_blankspace_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder aligned blankspace seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 100,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

ladder/aligned_blankspace_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,160.6,0.7917892214369971,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,318.7,0.5569781793359214,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,476.5,0.34066541885250246,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,634.1,0.18926071675040285,0.0,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,791.7,0.062253484789649455,0.18359375,0.046875,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,949.3,0.03320162610751076,0.70703125,0.8046875,0.7578125,0.48046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1106.9,0.035588766036489326,0.484375,0.71484375,0.3359375,0.046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1264.5,0.0038275444310702242,0.75,0.96484375,0.921875,0.83203125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1422.0,0.003006601163685774,0.81640625,0.87890625,0.921875,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,1579.6,0.011074456668347736,0.84375,0.84765625,0.91015625,0.40625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,1737.1,0.001762914059545628,0.92578125,0.88671875,0.640625,0.703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,1894.6,0.0014152395520202114,0.93359375,0.96875,0.99609375,0.953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2052.1,0.0010265571908707196,0.6796875,0.8671875,0.92578125,0.921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2209.7,0.0011848017017817018,0.92578125,0.96484375,0.9765625,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,2367.3,0.0006486929476190831,0.99609375,0.99609375,1.0,0.984375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0
+40000,2525.0,0.00040265640954147873,0.98828125,0.98046875,0.98828125,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,2682.7,0.0002718847481531166,1.0,1.0,0.99609375,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,2840.3,0.00017955203406809373,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,2997.9,0.00010366824067182914,1.0,1.0,1.0,1.0,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0
+50000,3155.3,9.943649635785177e-05,1.0,1.0,1.0,1.0,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0

ladder/aligned_blankspace_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:836b2be468fe485ee87d265e3aecb211d89e3daf8de32876d0ea099b804200d0
+size 13738485

ladder/aligned_blankspace_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder aligned blankspace seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 100,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

ladder/aligned_blankspace_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,159.6,0.6841813967027067,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,317.1,0.5332419979059715,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,474.5,0.5848314746810976,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,632.0,0.06758648690330075,0.27734375,0.66796875,0.28125,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,789.4,0.06667672275362742,0.81640625,0.66796875,0.5859375,0.09375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,946.9,0.08104756263283373,0.80078125,0.375,0.5703125,0.1796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1104.4,0.009508176230786156,0.94140625,0.9921875,0.9765625,0.8515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1262.0,0.005313269208058787,0.59765625,0.97265625,0.984375,0.5078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1419.3,0.020851006702746026,0.9921875,0.97265625,0.9140625,0.66015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,1579.0,0.030252543132024315,0.8984375,0.671875,0.140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,1736.3,0.0017718396566087776,1.0,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,1894.0,0.0020576261933826483,0.99609375,0.9921875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2051.6,0.001282006106958442,1.0,1.0,0.99609375,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2208.9,0.0011698463187407417,1.0,1.0,0.99609375,0.734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,2366.4,0.0017852982986684327,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,2524.0,0.0005796184832130665,1.0,0.99609375,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,2681.6,0.0002655833977485612,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,2838.9,0.00017084999995307094,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,2996.5,0.00010716800060294189,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,3153.8,0.0001084555846907145,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/aligned_blankspace_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d655456e7085744ef93a6acf14ba4765773ae0e312f8269e9a05cde21b235972
+size 13738485

ladder/aligned_blankspace_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder aligned blankspace seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 100,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

ladder/aligned_blankspace_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,159.7,0.6849955003274822,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,317.1,0.40563453485916695,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,474.7,0.08712828624975155,0.01171875,0.078125,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,632.3,0.03410129050206946,0.62109375,0.8984375,0.734375,0.21875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0
+12500,789.8,0.019461113056267913,0.796875,0.984375,0.91015625,0.359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,947.4,0.013094285961964889,0.859375,0.99609375,0.93359375,0.62109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,1104.9,0.005254189050186441,0.9609375,1.0,0.90625,0.47265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,1262.5,0.00643620762702321,0.98828125,0.96875,0.97265625,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,1420.0,0.012707637149868941,0.96875,0.99609375,0.921875,0.8359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,1577.5,0.004140390980604492,0.6953125,1.0,0.98828125,0.89453125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,1734.9,0.01499487356419211,1.0,0.984375,0.06640625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,1892.5,0.010458501131076558,0.68359375,0.85546875,0.94921875,0.3125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,2050.1,0.0009105636122346806,0.8046875,1.0,0.98828125,0.578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,2207.7,0.0015334090907651347,0.92578125,1.0,1.0,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,2365.2,0.0015833325072135197,0.98828125,1.0,0.9921875,0.5859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,2522.7,0.00013386686397942181,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,2680.4,7.658184016056906e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,2837.9,4.006964043643779e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,2994.2,3.0022523610616597e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,3151.8,2.813782367823345e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/aligned_blankspace_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7135c51ac830f86bf849804d8aabef148097bb7f7a347d8791fee97c333878a9
+size 13738485

ladder/plain_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder plain seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": false,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

ladder/plain_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,45.8,1.1552440607522254,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,85.3,1.0496538571563998,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,121.6,0.22190272788699789,0.625,0.30078125,0.13671875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,156.8,0.1361104787168321,0.62890625,0.265625,0.1328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,194.1,0.1299326336596537,0.83984375,0.82421875,0.66015625,0.26953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,230.1,0.08724782346629964,0.87890625,0.72265625,0.6171875,0.37109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,265.7,0.03294547912425976,0.8828125,0.80078125,0.7109375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,301.2,0.03188540525085875,0.9296875,0.8125,0.75390625,0.60546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,338.5,0.024272466284350352,0.94921875,0.84765625,0.76953125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,377.9,0.02135416777169092,0.95703125,0.84765625,0.77734375,0.62109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,413.0,0.017336725187138323,0.95703125,0.81640625,0.72265625,0.59375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,450.0,0.015351897706221264,0.95703125,0.90625,0.8359375,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,485.1,0.011902291315282323,0.9375,0.921875,0.78515625,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,520.3,0.010157805927521121,0.98046875,0.9453125,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,555.7,0.008457286916127297,0.98828125,0.9453125,0.89453125,0.84765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,591.0,0.006343905665149985,0.984375,0.95703125,0.90234375,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,626.2,0.004807067632394225,0.984375,0.94921875,0.93359375,0.90625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,661.3,0.0039473881130650396,0.9765625,0.96484375,0.93359375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,696.4,0.003158288052806785,0.98828125,0.96875,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,731.5,0.0031530544255801425,0.98828125,0.96484375,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/plain_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3273a0e6726cde985810bea69fe181a1c622e441242ac728493e1349c94dc96b
+size 13738485

ladder/plain_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder plain seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": false,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

ladder/plain_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,39.9,1.0926829596295016,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,75.2,0.34977536998246683,0.578125,0.17578125,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,110.7,0.13636325329445406,0.80859375,0.59765625,0.27734375,0.0859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,147.0,0.10624530700855975,0.4921875,0.5703125,0.13671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,182.0,0.04835331203503549,0.890625,0.7109375,0.71875,0.3828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,217.1,0.13925495517488404,0.71484375,0.64453125,0.20703125,0.06640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,252.1,0.09695242193667337,0.90234375,0.5859375,0.6015625,0.1796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,287.1,0.030026417599016593,0.9296875,0.7421875,0.63671875,0.11328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,321.9,0.02373246684440765,0.94921875,0.828125,0.7265625,0.625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,356.6,0.027057840476159962,0.9453125,0.8515625,0.7578125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,391.2,0.020241652287542256,0.95703125,0.8203125,0.796875,0.609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,425.8,0.0187081657161948,0.95703125,0.66796875,0.7890625,0.59765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,461.1,0.013485327495607533,0.96484375,0.89453125,0.78125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,496.0,0.012634533764863868,0.9609375,0.91015625,0.83984375,0.70703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,530.9,0.009974493826907564,0.98046875,0.890625,0.87890625,0.8125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,566.1,0.007701182666293666,0.9765625,0.93359375,0.875,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,601.0,0.005880211502039975,0.98046875,0.94140625,0.9453125,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,635.9,0.0047997324281110814,0.9921875,0.97265625,0.95703125,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,670.7,0.004150005655083656,0.9921875,0.9609375,0.94921875,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,705.4,0.0038938612596869458,0.99609375,0.98046875,0.94921875,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/plain_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43cd4ea777cd8b8fd33034d97522b28ab6a20afddf953275a7028985a20bc71
+size 13738485

ladder/plain_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder plain seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": false,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

ladder/plain_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,38.3,1.2351460234888572,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,72.8,0.8908147140877666,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,107.4,0.3303994115917782,0.5625,0.23046875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,142.0,0.121963283541433,0.78125,0.5234375,0.328125,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,176.7,0.12380598067946852,0.81640625,0.58203125,0.20703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,211.4,0.06437959507165898,0.875,0.75390625,0.5546875,0.18359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,246.3,0.09695373926330297,0.90234375,0.7734375,0.640625,0.24609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,281.0,0.03774574688353087,0.87890625,0.74609375,0.54296875,0.140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,315.9,0.04852251501855399,0.90234375,0.76953125,0.65625,0.30078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,350.4,0.03372801105576202,0.91796875,0.84765625,0.734375,0.61328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,384.8,0.02787428909461228,0.97265625,0.85546875,0.765625,0.55078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,419.3,0.022389781303720466,0.9375,0.84765625,0.62109375,0.52734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,454.0,0.018736086882025,0.9609375,0.87109375,0.71875,0.62890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,488.7,0.015919025027907152,0.953125,0.88671875,0.82421875,0.71875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,523.5,0.012297407657478424,0.953125,0.890625,0.8671875,0.75,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,558.3,0.01050108779963775,0.984375,0.921875,0.85546875,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,593.0,0.008854387478807818,0.98046875,0.9140625,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,627.8,0.0075194296041089384,0.98046875,0.94921875,0.890625,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,662.6,0.006260404597938438,0.98046875,0.94921875,0.90234375,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,697.1,0.006391584459112504,0.984375,0.95703125,0.890625,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/plain_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af439c0424fe765d90897cfcd165a09545200a90ed9ddaedda956c1f419fa0bf
+size 13738485

ladder/position_coupling_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder position coupling seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

ladder/position_coupling_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,41.7,0.05849933046576848,0.9765625,0.99609375,0.96484375,0.89453125,0.640625,0.25390625,0.0,0.0,0.0,0.0,0.0
+5000,79.5,0.0011903617309327293,0.9453125,0.94921875,0.9453125,0.91015625,0.6953125,0.4140625,0.01953125,0.0,0.0,0.0,0.0
+7500,117.6,0.0008857806100816371,0.984375,1.0,0.9921875,0.96875,0.46484375,0.140625,0.01171875,0.0,0.0,0.0,0.0
+10000,155.7,0.0005239002259623858,1.0,0.9921875,1.0,0.98828125,0.80859375,0.65625,0.16015625,0.0,0.0,0.0,0.0
+12500,193.9,0.00031071801167808805,1.0,1.0,1.0,1.0,0.9140625,0.6875,0.3046875,0.04296875,0.0,0.0,0.0
+15000,232.4,9.674870884781087e-05,1.0,1.0,1.0,1.0,0.984375,0.890625,0.4296875,0.046875,0.0,0.0,0.0
+17500,270.8,1.1628271768843432e-07,1.0,1.0,1.0,1.0,0.99609375,0.953125,0.62890625,0.13671875,0.0078125,0.0,0.0
+20000,309.3,9.673758767857765e-09,1.0,1.0,1.0,1.0,1.0,0.9765625,0.41796875,0.03125,0.0,0.0,0.0
+22500,347.8,0.00010900244251551255,1.0,1.0,0.99609375,0.99609375,0.6328125,0.33203125,0.04296875,0.0,0.0,0.0,0.0
+25000,386.3,2.2218434021451218e-05,1.0,1.0,1.0,1.0,0.984375,0.90625,0.5234375,0.0234375,0.0,0.0,0.0
+27500,424.4,0.00024115022228942076,1.0,1.0,1.0,0.99609375,0.92578125,0.60546875,0.26171875,0.00390625,0.0,0.0,0.0
+30000,462.2,1.261457133340515e-08,1.0,1.0,1.0,1.0,0.99609375,0.89453125,0.484375,0.015625,0.0,0.0,0.0
+32500,500.0,2.513377858377487e-06,1.0,1.0,1.0,1.0,0.96484375,0.74609375,0.234375,0.0,0.0,0.0,0.0
+35000,538.0,1.1292256512601291e-07,1.0,1.0,1.0,1.0,0.97265625,0.85546875,0.34765625,0.0078125,0.0,0.0,0.0
+37500,576.4,1.731580459432587e-08,1.0,1.0,1.0,1.0,0.98828125,0.890625,0.1640625,0.00390625,0.0,0.0,0.0
+40000,614.8,1.9072867737526124e-08,1.0,1.0,1.0,1.0,0.99609375,0.9296875,0.140625,0.0,0.0,0.0,0.0
+42500,653.2,2.0661178407917825e-08,1.0,1.0,1.0,1.0,0.99609375,0.94140625,0.12890625,0.0,0.0,0.0,0.0
+45000,691.5,1.976445106136551e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0
+47500,730.0,1.9826637076144926e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0
+50000,768.0,2.006347927688552e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0

ladder/position_coupling_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5367b75422a22700d5f4e0cf6b5f8277aba3934897b128c8810ac33d5b7d2dd0
+size 13738485

ladder/position_coupling_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder position coupling seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

ladder/position_coupling_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,41.1,0.05145847376760608,0.98046875,0.96875,0.7421875,0.43359375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0
+5000,78.7,0.15403296481562503,0.0859375,0.09765625,0.0625,0.125,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0
+7500,116.5,0.003376625910415864,1.0,0.98828125,0.96875,0.9375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0
+10000,154.7,0.0017005495388153179,0.9921875,0.98828125,0.9921875,0.921875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0
+12500,193.1,0.0008433014665956887,1.0,1.0,0.9921875,0.98046875,0.7109375,0.0,0.0,0.0,0.0,0.0,0.0
+15000,231.4,0.0011252931659162543,0.87890625,0.98046875,0.96875,0.875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0
+17500,270.0,0.0006694048880327893,1.0,1.0,0.984375,0.984375,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0
+20000,308.4,0.0005087599628814538,1.0,1.0,1.0,0.98828125,0.859375,0.0078125,0.0,0.0,0.0,0.0,0.0
+22500,346.8,0.0003347884842192211,1.0,1.0,1.0,0.984375,0.79296875,0.01171875,0.0,0.0,0.0,0.0,0.0
+25000,385.2,0.00019936315146112815,1.0,1.0,1.0,0.984375,0.8125,0.0,0.0,0.0,0.0,0.0,0.0
+27500,423.6,0.00023418337315626932,1.0,0.99609375,0.9921875,0.9921875,0.78125,0.0,0.0,0.0,0.0,0.0,0.0
+30000,462.1,3.036543236018947e-05,1.0,1.0,0.99609375,0.9921875,0.8984375,0.0,0.0,0.0,0.0,0.0,0.0
+32500,500.8,1.1332860303378958e-06,1.0,1.0,1.0,1.0,0.97265625,0.05078125,0.0,0.0,0.0,0.0,0.0
+35000,539.6,2.2576315286650695e-05,1.0,1.0,1.0,1.0,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0
+37500,578.3,6.027202978377045e-07,1.0,1.0,1.0,1.0,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0
+40000,617.2,2.0994051087808977e-09,1.0,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0
+42500,656.0,1.8263265294455678e-09,1.0,1.0,1.0,1.0,0.88671875,0.0,0.0,0.0,0.0,0.0,0.0
+45000,694.7,2.2382413536553586e-09,1.0,1.0,1.0,1.0,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0
+47500,733.4,2.342642124859866e-09,1.0,1.0,1.0,1.0,0.87890625,0.0,0.0,0.0,0.0,0.0,0.0
+50000,771.8,2.4091175711973268e-09,1.0,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0

ladder/position_coupling_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17b948384e3edf48c94cd35c43b46e6654c612166a81c1d6db1d14267c066855
+size 13738485

ladder/position_coupling_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder position coupling seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

ladder/position_coupling_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.8,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,43.0,0.020700893404300407,0.9375,0.921875,0.859375,0.33984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,81.0,0.05859560856595267,0.0625,0.08984375,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,118.9,0.0043224874084095,0.98046875,0.9765625,0.93359375,0.94140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,156.6,0.003302708169984759,1.0,1.0,1.0,0.87109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,194.5,0.0011700204703183339,1.0,1.0,0.99609375,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,232.7,0.0006573165836842692,1.0,0.9921875,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,270.6,0.0006703957555014628,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,308.5,0.0009429449397675106,1.0,0.9921875,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,346.4,0.00032470079907343934,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,384.3,0.00014316028949393518,1.0,1.0,1.0,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,422.4,0.000150102722229327,1.0,0.99609375,0.9921875,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,460.3,0.0002283228692740234,0.9921875,0.9921875,0.98828125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,498.5,7.47707316082816e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,536.4,1.676158338888255e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,574.4,2.1676778716729662e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,612.3,2.5858003174553218e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,650.1,2.7781036704315324e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,688.0,2.876706517140481e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,726.0,2.90941245793862e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,764.0,2.9081300761590858e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/position_coupling_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b5c91acc0a25734fbff87150e36eaf49784404ad6bc8474c501daca2b8a6471
+size 13738485

ladder/reversed_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder reversed seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

ladder/reversed_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,37.5,0.6899808970337034,0.02734375,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,71.5,0.11962956568376582,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,105.7,0.09103466037908249,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,139.9,0.06821191482090103,1.0,0.96484375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,174.4,0.0001272123632397109,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,208.7,0.008226176047167542,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,243.2,7.887461830654756e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,277.7,3.3091967498198034e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,312.1,0.001415348713319323,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,346.4,3.8733289335832e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,380.6,2.2778572673513522e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,414.7,0.000505518517989372,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,448.9,5.484113087084111e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,483.2,5.711551111572616e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,517.6,4.2846015616914855e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,552.2,3.535017908514598e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,586.8,3.446250352525007e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,621.4,3.434506841888134e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,655.9,3.450636705566721e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,690.3,3.442937016086712e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/reversed_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9619da3b81ad150b590b8672b21247528f88ff9aec3490680f116e296b5717a0
+size 13738485

ladder/reversed_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder reversed seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

ladder/reversed_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,37.6,0.6935743615820404,0.0703125,0.0625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,71.8,0.5975450208887972,0.078125,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,105.9,0.4572495186125052,0.11328125,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,140.1,0.012656225491346918,1.0,0.99609375,0.90234375,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,174.6,0.005384431403670865,0.90625,1.0,0.83203125,0.22265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,209.0,0.006117246615816689,1.0,1.0,1.0,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,243.3,0.0034891020506632574,1.0,1.0,0.99609375,0.73828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,277.7,0.0057696616780318006,1.0,0.69140625,0.984375,0.828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,312.2,0.0009214650922157513,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,346.8,0.0010159356764143335,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,381.3,0.0016378494398793593,0.9921875,0.99609375,1.0,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,415.4,0.0006733740390607537,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,449.6,0.00015172600247770622,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,483.9,7.84259512676389e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,518.3,2.6318664865720264e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,552.8,4.334490324539598e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,587.2,6.986863678033721e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,621.7,8.482358137454979e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,656.1,9.151078649171663e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,690.6,9.153665366514486e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/reversed_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:052fce71ee826422eb6682b771eb88f0ae59411d5bd0ae5666077e95d82f760c
+size 13738485

ladder/reversed_s2/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder reversed seed 2",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": false,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

ladder/reversed_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,37.8,1.1030993474196946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,72.0,0.049267443183339234,1.0,1.0,1.0,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,106.2,0.025880768940122825,0.9921875,1.0,0.99609375,0.81640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,140.1,0.033528285323752245,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,174.4,0.0039047176543949573,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,208.8,0.002880435906337188,1.0,0.95703125,1.0,0.9453125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,243.4,0.0002447265964478629,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,277.8,0.0003152005626777964,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,312.4,0.001328313205134429,0.99609375,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,346.9,0.00035081833219575097,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,381.3,0.0006951169616613335,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,415.5,0.00025632233376511165,1.0,0.92578125,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,449.7,9.637635914961821e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,483.9,1.794882481248124e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,518.2,2.02643950102753e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,552.5,2.5137038381641247e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,586.9,2.6688879258399354e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,621.2,2.7669927692196918e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,655.7,2.8053395570058626e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,690.2,2.82486893320944e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/reversed_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d41daa6813cccf16a4743b6bceb7a032d21e309a1819e1dc53625caaa181d0
+size 13738485

ladder/reversed_zero_pad_s0/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder reversed + zero pad seed 0",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

ladder/reversed_zero_pad_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,39.1,0.06619868960771663,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,74.5,0.0017159350450853426,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+7500,109.7,0.007220026861467229,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+10000,145.0,4.138583902967584e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+12500,180.4,0.0017232142162460764,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+15000,216.0,5.582981389100563e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+17500,251.7,0.0028660412605447704,0.98828125,1.0,0.98828125,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+20000,287.4,0.0016964663427645665,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+22500,323.1,4.693074184738278e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+25000,358.9,1.5369728299991919e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+27500,394.6,1.354610709535728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+30000,430.3,7.381509885401397e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+32500,465.8,2.8535493048394157e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+35000,501.3,5.019004728542278e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+37500,536.7,3.140849492873736e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+40000,572.2,3.371324708507817e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+42500,607.9,3.425680808133598e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+45000,643.6,3.454033180070662e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+47500,679.2,3.45309485176879e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+50000,715.0,3.4386437766012695e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

ladder/reversed_zero_pad_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c83cdd60823c2fbe39e932b8ea1532146777366e317f8fcd44aafd019cf36489
+size 13738485

ladder/reversed_zero_pad_s1/config.json

@@ -0,0 +1,52 @@
+{
+  "name": "ladder reversed + zero pad seed 1",
+  "task": {
+    "max_digits": 20,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "sequential",
+    "blanks": 0,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "positional": true,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}