diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8b05fd80283b7f68135b42f2325002a15b3443a2 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +--- +license: mit +tags: + - arithmetic + - length-generalization + - grokking + - interpretability +--- + +# carrybit checkpoints + +Trained weights from [carrybit](https://github.com/Supergoatscriptguy/carrybit), +a small research project on tiny transformers learning exact integer +arithmetic. The code, training configs, and result plots live in that repo; +this repo only holds the final checkpoints so the analysis experiments can be +run without retraining. + +Every checkpoint is a plain PyTorch `state_dict` for `carrybit.model.Transformer`. +Each folder has the exact `config.json` it was trained with and the +`metrics.csv` logged during training. + +```python +import json, torch +from carrybit.config import load_config +from carrybit.model import Transformer + +cfg = load_config("configs/addition.yaml", ["task.zero_pad=true", "task.positions=coupled"]) +model = Transformer(16, cfg.model) +model.load_state_dict(torch.load("ladder/position_coupling_s0/step_50000.pt")) +``` + +## Contents + +`modular_add/`: one-layer transformer trained on a + b mod 113 (Nanda et al. +setup), 40k full-batch steps. Groks around step 2000. + +`ladder/`: the length generalization ladder. 4-layer, width 256 decoder-only +transformers (3.4M parameters) trained for 50k steps on addition with 1 to 20 +digit operands, three seeds per format. Exact match accuracy on operands of +exactly n digits: + +| run | 20 | 25 | 30 | 40 | 50 | +|---|---|---|---|---|---| +| plain_s0 | 0.93 | 0 | 0 | 0 | 0 | +| plain_s1 | 0.93 | 0 | 0 | 0 | 0 | +| plain_s2 | 0.81 | 0 | 0 | 0 | 0 | +| reversed_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 | +| reversed_zero_pad_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 | +| abacus_s0 | 1.00 | 0 | 0 | 0 | 0 | +| abacus_s1 | 1.00 | 0 | 0 | 0 | 0 | +| abacus_s2 | 1.00 | 1.00 | 0.93 | 0.75 | 0.25 | +| position_coupling_s0 | 1.00 | 1.00 | 0.92 | 0.11 | 0 | +| position_coupling_s1 | 1.00 | 0.88 | 0 | 0 | 0 | +| position_coupling_s2 | 1.00 | 0 | 0 | 0 | 0 | +| aligned_blankspace_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 | + +See the GitHub README for what the formats mean and a discussion of the results. diff --git a/ladder/abacus_s0/config.json b/ladder/abacus_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d240ff4f7d2578efc3640d9c1d70acef85098d8 --- /dev/null +++ b/ladder/abacus_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder abacus seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/abacus_s0/metrics.csv b/ladder/abacus_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..1a87ea11530975a6462ff8602d0a894766ae4bc2 --- /dev/null +++ b/ladder/abacus_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,40.4,0.04165126566593774,0.99609375,0.95703125,0.92578125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,77.8,0.007477885840074827,1.0,0.96875,0.9921875,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,114.0,0.0030776386032294453,1.0,1.0,0.99609375,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,150.1,0.0021365052922425946,1.0,0.9921875,0.95703125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,186.3,0.0014214514749505238,1.0,0.9765625,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,222.9,0.0010082435070390278,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,259.6,0.000492235087522748,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,296.1,0.0006661927837288326,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,332.9,0.00022146683685564763,0.99609375,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,369.7,0.00028362983838865844,1.0,0.96875,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,406.6,5.627922378600414e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,443.1,0.00010834653030878249,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,479.3,5.170799638041556e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,515.6,2.6278768232528415e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,551.9,1.3676077857973595e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,588.5,1.8227696000174497e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,625.2,2.0809437230908565e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,661.8,2.2113516591608795e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,698.7,2.246176344477924e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,735.4,2.244289403386741e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/abacus_s0/step_50000.pt b/ladder/abacus_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..38cd762e132256fe611c850ceb62f68ad29adaa5 --- /dev/null +++ b/ladder/abacus_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e85eb9ef4ea4524bdd12b147cb5908492e7bc605af2a75b445a5edcb86faf3 +size 13738485 diff --git a/ladder/abacus_s1/config.json b/ladder/abacus_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3211056fbf89072bd8c296d872fd73895bc1752b --- /dev/null +++ b/ladder/abacus_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder abacus seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/abacus_s1/metrics.csv b/ladder/abacus_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..5749f6ae6505eda3ad1d00825d55bce8e9d1c3c6 --- /dev/null +++ b/ladder/abacus_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,40.2,0.019421861825930153,0.9921875,0.984375,0.6640625,0.703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,76.9,0.00727983858716512,1.0,0.828125,0.86328125,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,113.2,0.008187808755326617,0.984375,0.98046875,0.94140625,0.76171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,149.5,0.0022380353683056874,1.0,0.9921875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,186.0,0.0017146116753800655,1.0,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,222.7,0.0010716926865862115,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,259.4,0.0011338222128818905,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,296.0,0.0006792790250233844,1.0,1.0,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,332.8,0.00046899516380649933,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,369.7,5.581149432375635e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,406.6,0.0003637340261300791,1.0,0.9921875,0.99609375,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,443.3,8.481105058334911e-05,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,479.6,5.903834703148728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,515.9,7.956959854725254e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,552.5,8.394848630662215e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,589.0,1.4101947604018792e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,626.0,1.7413494767162065e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,662.5,1.86887250719425e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,699.5,1.916155950283142e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,736.3,1.930204289234567e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/abacus_s1/step_50000.pt b/ladder/abacus_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..31daa8992368f604f4f142ff1a500aa6bedb95fc --- /dev/null +++ b/ladder/abacus_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71edef7f0633fc9562fff242174358eaa7c061316f51ca84feb02feb6879820d +size 13738485 diff --git a/ladder/abacus_s2/config.json b/ladder/abacus_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..60f9ed00d918c70bad182af60fba51848e199181 --- /dev/null +++ b/ladder/abacus_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder abacus seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/abacus_s2/metrics.csv b/ladder/abacus_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..13628ad9da5cd1a15aaecfd7b4a0308c848c45d1 --- /dev/null +++ b/ladder/abacus_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,40.2,0.0025480398695461076,1.0,0.984375,0.96875,0.953125,0.91015625,0.6484375,0.36328125,0.0703125,0.0,0.0,0.0 +5000,76.4,0.000774010375750845,1.0,0.98828125,1.0,0.99609375,0.9921875,0.6875,0.39453125,0.13671875,0.0,0.0,0.0 +7500,112.8,0.000516119487618438,0.9921875,0.99609375,0.96484375,0.97265625,0.98828125,0.65625,0.140625,0.015625,0.0,0.0,0.0 +10000,149.2,0.0006419722169680194,1.0,0.99609375,0.99609375,0.9453125,0.9375,0.83984375,0.2734375,0.05859375,0.0,0.0,0.0 +12500,185.7,7.209724002654492e-08,1.0,1.0,1.0,1.0,1.0,0.953125,0.8203125,0.39453125,0.05078125,0.0,0.0 +15000,222.4,1.0956978970981467e-08,1.0,1.0,1.0,1.0,1.0,0.99609375,0.875,0.70703125,0.296875,0.01171875,0.0 +17500,259.1,0.0001840910556279454,1.0,1.0,1.0,1.0,0.9921875,0.9140625,0.2734375,0.078125,0.0,0.0,0.0 +20000,295.7,2.3715848697307856e-08,1.0,1.0,1.0,1.0,0.99609375,0.96875,0.64453125,0.35546875,0.015625,0.0,0.0 +22500,332.8,2.134711718499249e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.71484375,0.41796875,0.09765625,0.0,0.0 +25000,369.7,2.3915868879389415e-08,1.0,1.0,1.0,1.0,1.0,0.98046875,0.5703125,0.05078125,0.0,0.0,0.0 +27500,406.4,2.525873054866377e-08,1.0,1.0,1.0,1.0,1.0,0.90625,0.0234375,0.0,0.0,0.0,0.0 +30000,442.9,2.6034143760423384e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.86328125,0.38671875,0.0078125,0.0,0.0 +32500,479.2,1.2933618968363646e-06,1.0,1.0,1.0,1.0,1.0,0.80078125,0.5,0.046875,0.0,0.0,0.0 +35000,515.5,8.196465929258812e-05,1.0,1.0,1.0,1.0,0.9921875,0.9765625,0.0625,0.00390625,0.0,0.0,0.0 +37500,552.3,2.5679873355496745e-08,1.0,1.0,1.0,1.0,1.0,0.75,0.15234375,0.00390625,0.0,0.0,0.0 +40000,589.0,2.668754422022952e-09,1.0,1.0,1.0,1.0,1.0,0.89453125,0.48046875,0.07421875,0.0,0.0,0.0 +42500,625.9,1.6834652089024804e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.76171875,0.24609375,0.015625,0.0,0.0 +45000,662.5,1.7842648507998153e-08,1.0,1.0,1.0,1.0,1.0,0.9296875,0.75390625,0.2890625,0.015625,0.0,0.0 +47500,699.0,1.9654443694697242e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.74609375,0.26171875,0.0078125,0.0,0.0 +50000,736.0,2.0090057245062915e-08,1.0,1.0,1.0,1.0,1.0,0.93359375,0.75,0.25390625,0.00390625,0.0,0.0 diff --git a/ladder/abacus_s2/step_50000.pt b/ladder/abacus_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e29873292ceeeccaca26a7a2418eebdaed958a0b --- /dev/null +++ b/ladder/abacus_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb1c41cdf7af8f6f213462f7998d95e2a43d4edc22c0d441b0f69fd50cc2dea1 +size 13738485 diff --git a/ladder/aligned_blankspace_s0/config.json b/ladder/aligned_blankspace_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..906eaa9e27a244893ede7a53350f454962092843 --- /dev/null +++ b/ladder/aligned_blankspace_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder aligned blankspace seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 100, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/aligned_blankspace_s0/metrics.csv b/ladder/aligned_blankspace_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4aedb127929dbf22fa516970cc717a93ac9f407 --- /dev/null +++ b/ladder/aligned_blankspace_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,160.6,0.7917892214369971,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,318.7,0.5569781793359214,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,476.5,0.34066541885250246,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,634.1,0.18926071675040285,0.0,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,791.7,0.062253484789649455,0.18359375,0.046875,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,949.3,0.03320162610751076,0.70703125,0.8046875,0.7578125,0.48046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1106.9,0.035588766036489326,0.484375,0.71484375,0.3359375,0.046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1264.5,0.0038275444310702242,0.75,0.96484375,0.921875,0.83203125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1422.0,0.003006601163685774,0.81640625,0.87890625,0.921875,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1579.6,0.011074456668347736,0.84375,0.84765625,0.91015625,0.40625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1737.1,0.001762914059545628,0.92578125,0.88671875,0.640625,0.703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1894.6,0.0014152395520202114,0.93359375,0.96875,0.99609375,0.953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2052.1,0.0010265571908707196,0.6796875,0.8671875,0.92578125,0.921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2209.7,0.0011848017017817018,0.92578125,0.96484375,0.9765625,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,2367.3,0.0006486929476190831,0.99609375,0.99609375,1.0,0.984375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +40000,2525.0,0.00040265640954147873,0.98828125,0.98046875,0.98828125,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,2682.7,0.0002718847481531166,1.0,1.0,0.99609375,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,2840.3,0.00017955203406809373,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,2997.9,0.00010366824067182914,1.0,1.0,1.0,1.0,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3155.3,9.943649635785177e-05,1.0,1.0,1.0,1.0,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/aligned_blankspace_s0/step_50000.pt b/ladder/aligned_blankspace_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d02f2886495bcaa20089d4120dbca59191a082f --- /dev/null +++ b/ladder/aligned_blankspace_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:836b2be468fe485ee87d265e3aecb211d89e3daf8de32876d0ea099b804200d0 +size 13738485 diff --git a/ladder/aligned_blankspace_s1/config.json b/ladder/aligned_blankspace_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b2515b3bb480c181d9e5449a48032209cb2c4d3f --- /dev/null +++ b/ladder/aligned_blankspace_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder aligned blankspace seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 100, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/aligned_blankspace_s1/metrics.csv b/ladder/aligned_blankspace_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..d377666c93b55fe8795349fb60be1678c2cf8c1a --- /dev/null +++ b/ladder/aligned_blankspace_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,159.6,0.6841813967027067,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,317.1,0.5332419979059715,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,474.5,0.5848314746810976,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,632.0,0.06758648690330075,0.27734375,0.66796875,0.28125,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,789.4,0.06667672275362742,0.81640625,0.66796875,0.5859375,0.09375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,946.9,0.08104756263283373,0.80078125,0.375,0.5703125,0.1796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1104.4,0.009508176230786156,0.94140625,0.9921875,0.9765625,0.8515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1262.0,0.005313269208058787,0.59765625,0.97265625,0.984375,0.5078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1419.3,0.020851006702746026,0.9921875,0.97265625,0.9140625,0.66015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1579.0,0.030252543132024315,0.8984375,0.671875,0.140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1736.3,0.0017718396566087776,1.0,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1894.0,0.0020576261933826483,0.99609375,0.9921875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2051.6,0.001282006106958442,1.0,1.0,0.99609375,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2208.9,0.0011698463187407417,1.0,1.0,0.99609375,0.734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,2366.4,0.0017852982986684327,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,2524.0,0.0005796184832130665,1.0,0.99609375,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,2681.6,0.0002655833977485612,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,2838.9,0.00017084999995307094,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,2996.5,0.00010716800060294189,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3153.8,0.0001084555846907145,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/aligned_blankspace_s1/step_50000.pt b/ladder/aligned_blankspace_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..93670568dc561620692ba2b8fa2ae9cd238607b4 --- /dev/null +++ b/ladder/aligned_blankspace_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d655456e7085744ef93a6acf14ba4765773ae0e312f8269e9a05cde21b235972 +size 13738485 diff --git a/ladder/aligned_blankspace_s2/config.json b/ladder/aligned_blankspace_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1f8294ecd5deab4ad3fa2587fa411f756a358c6e --- /dev/null +++ b/ladder/aligned_blankspace_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder aligned blankspace seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 100, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/aligned_blankspace_s2/metrics.csv b/ladder/aligned_blankspace_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..9e9234e0404fa35de6e788a7ed27143ea7f20025 --- /dev/null +++ b/ladder/aligned_blankspace_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,159.7,0.6849955003274822,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,317.1,0.40563453485916695,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,474.7,0.08712828624975155,0.01171875,0.078125,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,632.3,0.03410129050206946,0.62109375,0.8984375,0.734375,0.21875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +12500,789.8,0.019461113056267913,0.796875,0.984375,0.91015625,0.359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,947.4,0.013094285961964889,0.859375,0.99609375,0.93359375,0.62109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1104.9,0.005254189050186441,0.9609375,1.0,0.90625,0.47265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1262.5,0.00643620762702321,0.98828125,0.96875,0.97265625,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1420.0,0.012707637149868941,0.96875,0.99609375,0.921875,0.8359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1577.5,0.004140390980604492,0.6953125,1.0,0.98828125,0.89453125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1734.9,0.01499487356419211,1.0,0.984375,0.06640625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1892.5,0.010458501131076558,0.68359375,0.85546875,0.94921875,0.3125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2050.1,0.0009105636122346806,0.8046875,1.0,0.98828125,0.578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2207.7,0.0015334090907651347,0.92578125,1.0,1.0,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,2365.2,0.0015833325072135197,0.98828125,1.0,0.9921875,0.5859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,2522.7,0.00013386686397942181,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,2680.4,7.658184016056906e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,2837.9,4.006964043643779e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,2994.2,3.0022523610616597e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3151.8,2.813782367823345e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/aligned_blankspace_s2/step_50000.pt b/ladder/aligned_blankspace_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..10ef031827a76a82517506cb3af7e9f796b99586 --- /dev/null +++ b/ladder/aligned_blankspace_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7135c51ac830f86bf849804d8aabef148097bb7f7a347d8791fee97c333878a9 +size 13738485 diff --git a/ladder/plain_s0/config.json b/ladder/plain_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..619378a7b866249cfedd234c51493186b0ac0aeb --- /dev/null +++ b/ladder/plain_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder plain seed 0", + "task": { + "max_digits": 20, + "reverse": false, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/plain_s0/metrics.csv b/ladder/plain_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..3a6261b06c4834baaecca4ddbcf72b57c825577d --- /dev/null +++ b/ladder/plain_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,45.8,1.1552440607522254,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,85.3,1.0496538571563998,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,121.6,0.22190272788699789,0.625,0.30078125,0.13671875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,156.8,0.1361104787168321,0.62890625,0.265625,0.1328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,194.1,0.1299326336596537,0.83984375,0.82421875,0.66015625,0.26953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,230.1,0.08724782346629964,0.87890625,0.72265625,0.6171875,0.37109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,265.7,0.03294547912425976,0.8828125,0.80078125,0.7109375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,301.2,0.03188540525085875,0.9296875,0.8125,0.75390625,0.60546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,338.5,0.024272466284350352,0.94921875,0.84765625,0.76953125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,377.9,0.02135416777169092,0.95703125,0.84765625,0.77734375,0.62109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,413.0,0.017336725187138323,0.95703125,0.81640625,0.72265625,0.59375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,450.0,0.015351897706221264,0.95703125,0.90625,0.8359375,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,485.1,0.011902291315282323,0.9375,0.921875,0.78515625,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,520.3,0.010157805927521121,0.98046875,0.9453125,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,555.7,0.008457286916127297,0.98828125,0.9453125,0.89453125,0.84765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,591.0,0.006343905665149985,0.984375,0.95703125,0.90234375,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,626.2,0.004807067632394225,0.984375,0.94921875,0.93359375,0.90625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,661.3,0.0039473881130650396,0.9765625,0.96484375,0.93359375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,696.4,0.003158288052806785,0.98828125,0.96875,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,731.5,0.0031530544255801425,0.98828125,0.96484375,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/plain_s0/step_50000.pt b/ladder/plain_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..abd4a6eb21dde96263ed5c035e2ff2f5a9a25d2b --- /dev/null +++ b/ladder/plain_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3273a0e6726cde985810bea69fe181a1c622e441242ac728493e1349c94dc96b +size 13738485 diff --git a/ladder/plain_s1/config.json b/ladder/plain_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0e56e34daff33cd0b9d29a1ef57d1e01731119e7 --- /dev/null +++ b/ladder/plain_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder plain seed 1", + "task": { + "max_digits": 20, + "reverse": false, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/plain_s1/metrics.csv b/ladder/plain_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ee21298d516bebd70470d92f60f0361f011ff54 --- /dev/null +++ b/ladder/plain_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.9,1.0926829596295016,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,75.2,0.34977536998246683,0.578125,0.17578125,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,110.7,0.13636325329445406,0.80859375,0.59765625,0.27734375,0.0859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,147.0,0.10624530700855975,0.4921875,0.5703125,0.13671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,182.0,0.04835331203503549,0.890625,0.7109375,0.71875,0.3828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,217.1,0.13925495517488404,0.71484375,0.64453125,0.20703125,0.06640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,252.1,0.09695242193667337,0.90234375,0.5859375,0.6015625,0.1796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,287.1,0.030026417599016593,0.9296875,0.7421875,0.63671875,0.11328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,321.9,0.02373246684440765,0.94921875,0.828125,0.7265625,0.625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,356.6,0.027057840476159962,0.9453125,0.8515625,0.7578125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,391.2,0.020241652287542256,0.95703125,0.8203125,0.796875,0.609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,425.8,0.0187081657161948,0.95703125,0.66796875,0.7890625,0.59765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,461.1,0.013485327495607533,0.96484375,0.89453125,0.78125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,496.0,0.012634533764863868,0.9609375,0.91015625,0.83984375,0.70703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,530.9,0.009974493826907564,0.98046875,0.890625,0.87890625,0.8125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,566.1,0.007701182666293666,0.9765625,0.93359375,0.875,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,601.0,0.005880211502039975,0.98046875,0.94140625,0.9453125,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,635.9,0.0047997324281110814,0.9921875,0.97265625,0.95703125,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,670.7,0.004150005655083656,0.9921875,0.9609375,0.94921875,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,705.4,0.0038938612596869458,0.99609375,0.98046875,0.94921875,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/plain_s1/step_50000.pt b/ladder/plain_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6ba4a51cd1d6bc331dc9ec963892995fa641f87 --- /dev/null +++ b/ladder/plain_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b43cd4ea777cd8b8fd33034d97522b28ab6a20afddf953275a7028985a20bc71 +size 13738485 diff --git a/ladder/plain_s2/config.json b/ladder/plain_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5920014f9fa12602994afc7cb150c2d2f0e3baaf --- /dev/null +++ b/ladder/plain_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder plain seed 2", + "task": { + "max_digits": 20, + "reverse": false, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/plain_s2/metrics.csv b/ladder/plain_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..51559ca71ecfbbd4f00a1fe8ed11a1cfaec1f547 --- /dev/null +++ b/ladder/plain_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,38.3,1.2351460234888572,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,72.8,0.8908147140877666,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,107.4,0.3303994115917782,0.5625,0.23046875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,142.0,0.121963283541433,0.78125,0.5234375,0.328125,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,176.7,0.12380598067946852,0.81640625,0.58203125,0.20703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,211.4,0.06437959507165898,0.875,0.75390625,0.5546875,0.18359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,246.3,0.09695373926330297,0.90234375,0.7734375,0.640625,0.24609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,281.0,0.03774574688353087,0.87890625,0.74609375,0.54296875,0.140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,315.9,0.04852251501855399,0.90234375,0.76953125,0.65625,0.30078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,350.4,0.03372801105576202,0.91796875,0.84765625,0.734375,0.61328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,384.8,0.02787428909461228,0.97265625,0.85546875,0.765625,0.55078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,419.3,0.022389781303720466,0.9375,0.84765625,0.62109375,0.52734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,454.0,0.018736086882025,0.9609375,0.87109375,0.71875,0.62890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,488.7,0.015919025027907152,0.953125,0.88671875,0.82421875,0.71875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,523.5,0.012297407657478424,0.953125,0.890625,0.8671875,0.75,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,558.3,0.01050108779963775,0.984375,0.921875,0.85546875,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,593.0,0.008854387478807818,0.98046875,0.9140625,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,627.8,0.0075194296041089384,0.98046875,0.94921875,0.890625,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,662.6,0.006260404597938438,0.98046875,0.94921875,0.90234375,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,697.1,0.006391584459112504,0.984375,0.95703125,0.890625,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/plain_s2/step_50000.pt b/ladder/plain_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebdd1e0bfee78da680b0622abe3eb0a76398ac71 --- /dev/null +++ b/ladder/plain_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af439c0424fe765d90897cfcd165a09545200a90ed9ddaedda956c1f419fa0bf +size 13738485 diff --git a/ladder/position_coupling_s0/config.json b/ladder/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..353a1866e4b5063c2f2e98e5e39a00220aa0ebde --- /dev/null +++ b/ladder/position_coupling_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder position coupling seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/position_coupling_s0/metrics.csv b/ladder/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..c89ccd784bed7da2ad5c6092ce9fbe0e21c59699 --- /dev/null +++ b/ladder/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,41.7,0.05849933046576848,0.9765625,0.99609375,0.96484375,0.89453125,0.640625,0.25390625,0.0,0.0,0.0,0.0,0.0 +5000,79.5,0.0011903617309327293,0.9453125,0.94921875,0.9453125,0.91015625,0.6953125,0.4140625,0.01953125,0.0,0.0,0.0,0.0 +7500,117.6,0.0008857806100816371,0.984375,1.0,0.9921875,0.96875,0.46484375,0.140625,0.01171875,0.0,0.0,0.0,0.0 +10000,155.7,0.0005239002259623858,1.0,0.9921875,1.0,0.98828125,0.80859375,0.65625,0.16015625,0.0,0.0,0.0,0.0 +12500,193.9,0.00031071801167808805,1.0,1.0,1.0,1.0,0.9140625,0.6875,0.3046875,0.04296875,0.0,0.0,0.0 +15000,232.4,9.674870884781087e-05,1.0,1.0,1.0,1.0,0.984375,0.890625,0.4296875,0.046875,0.0,0.0,0.0 +17500,270.8,1.1628271768843432e-07,1.0,1.0,1.0,1.0,0.99609375,0.953125,0.62890625,0.13671875,0.0078125,0.0,0.0 +20000,309.3,9.673758767857765e-09,1.0,1.0,1.0,1.0,1.0,0.9765625,0.41796875,0.03125,0.0,0.0,0.0 +22500,347.8,0.00010900244251551255,1.0,1.0,0.99609375,0.99609375,0.6328125,0.33203125,0.04296875,0.0,0.0,0.0,0.0 +25000,386.3,2.2218434021451218e-05,1.0,1.0,1.0,1.0,0.984375,0.90625,0.5234375,0.0234375,0.0,0.0,0.0 +27500,424.4,0.00024115022228942076,1.0,1.0,1.0,0.99609375,0.92578125,0.60546875,0.26171875,0.00390625,0.0,0.0,0.0 +30000,462.2,1.261457133340515e-08,1.0,1.0,1.0,1.0,0.99609375,0.89453125,0.484375,0.015625,0.0,0.0,0.0 +32500,500.0,2.513377858377487e-06,1.0,1.0,1.0,1.0,0.96484375,0.74609375,0.234375,0.0,0.0,0.0,0.0 +35000,538.0,1.1292256512601291e-07,1.0,1.0,1.0,1.0,0.97265625,0.85546875,0.34765625,0.0078125,0.0,0.0,0.0 +37500,576.4,1.731580459432587e-08,1.0,1.0,1.0,1.0,0.98828125,0.890625,0.1640625,0.00390625,0.0,0.0,0.0 +40000,614.8,1.9072867737526124e-08,1.0,1.0,1.0,1.0,0.99609375,0.9296875,0.140625,0.0,0.0,0.0,0.0 +42500,653.2,2.0661178407917825e-08,1.0,1.0,1.0,1.0,0.99609375,0.94140625,0.12890625,0.0,0.0,0.0,0.0 +45000,691.5,1.976445106136551e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0 +47500,730.0,1.9826637076144926e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0 +50000,768.0,2.006347927688552e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0 diff --git a/ladder/position_coupling_s0/step_50000.pt b/ladder/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bbc0719fda1d3ce7df7d90bbb6e04c45e119561 --- /dev/null +++ b/ladder/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5367b75422a22700d5f4e0cf6b5f8277aba3934897b128c8810ac33d5b7d2dd0 +size 13738485 diff --git a/ladder/position_coupling_s1/config.json b/ladder/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fdc52abf81ea0342e7b296c745d7475adf3ef236 --- /dev/null +++ b/ladder/position_coupling_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder position coupling seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/position_coupling_s1/metrics.csv b/ladder/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..0ab2fe24c90e33eaf1f18fc4bab8c373bc2f0566 --- /dev/null +++ b/ladder/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,41.1,0.05145847376760608,0.98046875,0.96875,0.7421875,0.43359375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +5000,78.7,0.15403296481562503,0.0859375,0.09765625,0.0625,0.125,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +7500,116.5,0.003376625910415864,1.0,0.98828125,0.96875,0.9375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0 +10000,154.7,0.0017005495388153179,0.9921875,0.98828125,0.9921875,0.921875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0 +12500,193.1,0.0008433014665956887,1.0,1.0,0.9921875,0.98046875,0.7109375,0.0,0.0,0.0,0.0,0.0,0.0 +15000,231.4,0.0011252931659162543,0.87890625,0.98046875,0.96875,0.875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0 +17500,270.0,0.0006694048880327893,1.0,1.0,0.984375,0.984375,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0 +20000,308.4,0.0005087599628814538,1.0,1.0,1.0,0.98828125,0.859375,0.0078125,0.0,0.0,0.0,0.0,0.0 +22500,346.8,0.0003347884842192211,1.0,1.0,1.0,0.984375,0.79296875,0.01171875,0.0,0.0,0.0,0.0,0.0 +25000,385.2,0.00019936315146112815,1.0,1.0,1.0,0.984375,0.8125,0.0,0.0,0.0,0.0,0.0,0.0 +27500,423.6,0.00023418337315626932,1.0,0.99609375,0.9921875,0.9921875,0.78125,0.0,0.0,0.0,0.0,0.0,0.0 +30000,462.1,3.036543236018947e-05,1.0,1.0,0.99609375,0.9921875,0.8984375,0.0,0.0,0.0,0.0,0.0,0.0 +32500,500.8,1.1332860303378958e-06,1.0,1.0,1.0,1.0,0.97265625,0.05078125,0.0,0.0,0.0,0.0,0.0 +35000,539.6,2.2576315286650695e-05,1.0,1.0,1.0,1.0,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0 +37500,578.3,6.027202978377045e-07,1.0,1.0,1.0,1.0,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0 +40000,617.2,2.0994051087808977e-09,1.0,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0 +42500,656.0,1.8263265294455678e-09,1.0,1.0,1.0,1.0,0.88671875,0.0,0.0,0.0,0.0,0.0,0.0 +45000,694.7,2.2382413536553586e-09,1.0,1.0,1.0,1.0,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0 +47500,733.4,2.342642124859866e-09,1.0,1.0,1.0,1.0,0.87890625,0.0,0.0,0.0,0.0,0.0,0.0 +50000,771.8,2.4091175711973268e-09,1.0,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/position_coupling_s1/step_50000.pt b/ladder/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ea6b8f36e51f5387becf823eb55830cd81f9bf4 --- /dev/null +++ b/ladder/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b948384e3edf48c94cd35c43b46e6654c612166a81c1d6db1d14267c066855 +size 13738485 diff --git a/ladder/position_coupling_s2/config.json b/ladder/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..32d8dc6475ab43fd33efe97aa677de4be7a2042d --- /dev/null +++ b/ladder/position_coupling_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder position coupling seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/position_coupling_s2/metrics.csv b/ladder/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..a40ddc873b2c6984259bbb60183fe7d0a7bbcb0b --- /dev/null +++ b/ladder/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.8,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.0,0.020700893404300407,0.9375,0.921875,0.859375,0.33984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,81.0,0.05859560856595267,0.0625,0.08984375,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,118.9,0.0043224874084095,0.98046875,0.9765625,0.93359375,0.94140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,156.6,0.003302708169984759,1.0,1.0,1.0,0.87109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,194.5,0.0011700204703183339,1.0,1.0,0.99609375,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,232.7,0.0006573165836842692,1.0,0.9921875,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,270.6,0.0006703957555014628,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,308.5,0.0009429449397675106,1.0,0.9921875,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,346.4,0.00032470079907343934,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,384.3,0.00014316028949393518,1.0,1.0,1.0,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,422.4,0.000150102722229327,1.0,0.99609375,0.9921875,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,460.3,0.0002283228692740234,0.9921875,0.9921875,0.98828125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,498.5,7.47707316082816e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,536.4,1.676158338888255e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,574.4,2.1676778716729662e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,612.3,2.5858003174553218e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,650.1,2.7781036704315324e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,688.0,2.876706517140481e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,726.0,2.90941245793862e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,764.0,2.9081300761590858e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/position_coupling_s2/step_50000.pt b/ladder/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1d0ddd95c03638e4576983740d9a03066aea520 --- /dev/null +++ b/ladder/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5c91acc0a25734fbff87150e36eaf49784404ad6bc8474c501daca2b8a6471 +size 13738485 diff --git a/ladder/reversed_s0/config.json b/ladder/reversed_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e17304043a2e5b2e7e57dba229394c4e985bdb --- /dev/null +++ b/ladder/reversed_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/reversed_s0/metrics.csv b/ladder/reversed_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..1583968d4c74b8afa2ab94843f9123b30a2ee742 --- /dev/null +++ b/ladder/reversed_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,37.5,0.6899808970337034,0.02734375,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,71.5,0.11962956568376582,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,105.7,0.09103466037908249,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,139.9,0.06821191482090103,1.0,0.96484375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,174.4,0.0001272123632397109,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,208.7,0.008226176047167542,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,243.2,7.887461830654756e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,277.7,3.3091967498198034e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,312.1,0.001415348713319323,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,346.4,3.8733289335832e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,380.6,2.2778572673513522e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,414.7,0.000505518517989372,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,448.9,5.484113087084111e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,483.2,5.711551111572616e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,517.6,4.2846015616914855e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,552.2,3.535017908514598e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,586.8,3.446250352525007e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,621.4,3.434506841888134e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,655.9,3.450636705566721e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,690.3,3.442937016086712e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/reversed_s0/step_50000.pt b/ladder/reversed_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e02d2d2c9f0295241fa41f7a067bb560a7ab9c2 --- /dev/null +++ b/ladder/reversed_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9619da3b81ad150b590b8672b21247528f88ff9aec3490680f116e296b5717a0 +size 13738485 diff --git a/ladder/reversed_s1/config.json b/ladder/reversed_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6f72fa98e13e184a54a74e4524d6d9299556ad --- /dev/null +++ b/ladder/reversed_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/reversed_s1/metrics.csv b/ladder/reversed_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..58855eb871b80464f2d89650ef577548fd86de24 --- /dev/null +++ b/ladder/reversed_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,37.6,0.6935743615820404,0.0703125,0.0625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,71.8,0.5975450208887972,0.078125,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,105.9,0.4572495186125052,0.11328125,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,140.1,0.012656225491346918,1.0,0.99609375,0.90234375,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,174.6,0.005384431403670865,0.90625,1.0,0.83203125,0.22265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,209.0,0.006117246615816689,1.0,1.0,1.0,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,243.3,0.0034891020506632574,1.0,1.0,0.99609375,0.73828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,277.7,0.0057696616780318006,1.0,0.69140625,0.984375,0.828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,312.2,0.0009214650922157513,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,346.8,0.0010159356764143335,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,381.3,0.0016378494398793593,0.9921875,0.99609375,1.0,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,415.4,0.0006733740390607537,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,449.6,0.00015172600247770622,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,483.9,7.84259512676389e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,518.3,2.6318664865720264e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,552.8,4.334490324539598e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,587.2,6.986863678033721e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,621.7,8.482358137454979e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,656.1,9.151078649171663e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,690.6,9.153665366514486e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/reversed_s1/step_50000.pt b/ladder/reversed_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..206e6351dd3db9b553ffb247befae2b69ab0c871 --- /dev/null +++ b/ladder/reversed_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052fce71ee826422eb6682b771eb88f0ae59411d5bd0ae5666077e95d82f760c +size 13738485 diff --git a/ladder/reversed_s2/config.json b/ladder/reversed_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e73856e3d6ec24585136212769833b26ddac4f09 --- /dev/null +++ b/ladder/reversed_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/reversed_s2/metrics.csv b/ladder/reversed_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..487782772516f6250be1c1f3779f5bc7439311b1 --- /dev/null +++ b/ladder/reversed_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,37.8,1.1030993474196946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,72.0,0.049267443183339234,1.0,1.0,1.0,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,106.2,0.025880768940122825,0.9921875,1.0,0.99609375,0.81640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,140.1,0.033528285323752245,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,174.4,0.0039047176543949573,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,208.8,0.002880435906337188,1.0,0.95703125,1.0,0.9453125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,243.4,0.0002447265964478629,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,277.8,0.0003152005626777964,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,312.4,0.001328313205134429,0.99609375,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,346.9,0.00035081833219575097,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,381.3,0.0006951169616613335,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,415.5,0.00025632233376511165,1.0,0.92578125,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,449.7,9.637635914961821e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,483.9,1.794882481248124e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,518.2,2.02643950102753e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,552.5,2.5137038381641247e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,586.9,2.6688879258399354e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,621.2,2.7669927692196918e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,655.7,2.8053395570058626e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,690.2,2.82486893320944e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/reversed_s2/step_50000.pt b/ladder/reversed_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..96f3caa40dcc544f806ebb46613344f784ed38f2 --- /dev/null +++ b/ladder/reversed_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d41daa6813cccf16a4743b6bceb7a032d21e309a1819e1dc53625caaa181d0 +size 13738485 diff --git a/ladder/reversed_zero_pad_s0/config.json b/ladder/reversed_zero_pad_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..11c9eddf6fa00bafcc85f309c98304723e50da67 --- /dev/null +++ b/ladder/reversed_zero_pad_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed + zero pad seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/reversed_zero_pad_s0/metrics.csv b/ladder/reversed_zero_pad_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..e1efe9dc85ce035e860c6d95b7c8d3ef207d71d0 --- /dev/null +++ b/ladder/reversed_zero_pad_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.1,0.06619868960771663,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,74.5,0.0017159350450853426,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,109.7,0.007220026861467229,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,145.0,4.138583902967584e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,180.4,0.0017232142162460764,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,216.0,5.582981389100563e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,251.7,0.0028660412605447704,0.98828125,1.0,0.98828125,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,287.4,0.0016964663427645665,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,323.1,4.693074184738278e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,358.9,1.5369728299991919e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,394.6,1.354610709535728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,430.3,7.381509885401397e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,465.8,2.8535493048394157e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,501.3,5.019004728542278e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,536.7,3.140849492873736e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,572.2,3.371324708507817e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,607.9,3.425680808133598e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,643.6,3.454033180070662e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,679.2,3.45309485176879e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,715.0,3.4386437766012695e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/reversed_zero_pad_s0/step_50000.pt b/ladder/reversed_zero_pad_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf64988ed33fe42efa0fe2d93099db7eb7c3715a --- /dev/null +++ b/ladder/reversed_zero_pad_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c83cdd60823c2fbe39e932b8ea1532146777366e317f8fcd44aafd019cf36489 +size 13738485 diff --git a/ladder/reversed_zero_pad_s1/config.json b/ladder/reversed_zero_pad_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..900f60fe2cf2a8e2a6999d005c213e0efac1b33e --- /dev/null +++ b/ladder/reversed_zero_pad_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed + zero pad seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/reversed_zero_pad_s1/metrics.csv b/ladder/reversed_zero_pad_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..9269de5ef4321188cacf7dc7cf6b07cbfedef771 --- /dev/null +++ b/ladder/reversed_zero_pad_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.1,0.505330329683363,0.88671875,0.87109375,0.83203125,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,74.7,0.013944046411834343,0.97265625,0.9765625,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,110.2,0.0054805784648486325,0.99609375,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,145.7,0.002925935737893005,0.99609375,0.99609375,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,181.1,0.00786076482663685,0.9453125,0.99609375,1.0,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,216.6,0.002306089214874477,0.42578125,0.99609375,0.9921875,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,252.2,9.141079915561852e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,287.8,7.74360808163545e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,323.5,0.0009985137306061157,1.0,1.0,0.99609375,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,359.3,3.4804709802909685e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,395.0,1.5507826306689775e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,430.6,7.562815979003721e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,466.1,1.9148897382522122e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,501.7,2.456168471434791e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,537.1,3.037901886569355e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,572.7,3.132561227860439e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,608.4,3.180539363786996e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,644.0,3.206723244783721e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,679.7,3.2186608966719705e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,715.4,3.228653236357116e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/reversed_zero_pad_s1/step_50000.pt b/ladder/reversed_zero_pad_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ee6ee0b25cfe2d88a39f6e819a9d496dbea6866 --- /dev/null +++ b/ladder/reversed_zero_pad_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11770eda2d4f3f239ecf488e96b38ebeff5bc17d78babf2c8f40e4d8a5c95d25 +size 13738485 diff --git a/ladder/reversed_zero_pad_s2/config.json b/ladder/reversed_zero_pad_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b099555a13167eb0e9c0290334b63726c2c1b81e --- /dev/null +++ b/ladder/reversed_zero_pad_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed + zero pad seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/ladder/reversed_zero_pad_s2/metrics.csv b/ladder/reversed_zero_pad_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..afce8cde7c6f8ae45abf8dbee81806df86bdebae --- /dev/null +++ b/ladder/reversed_zero_pad_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.1,0.1500092168730642,0.9765625,0.953125,0.85546875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,74.8,0.003555504089126276,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,110.3,0.0009906474932091412,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,145.7,0.0010326231041283284,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,181.1,2.2762600781179806e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,216.7,4.521779913216281e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,252.3,5.024763246116733e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,288.1,6.7219712028696075e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,323.8,0.0005750678458532328,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,359.6,1.8484896897509966e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,395.4,2.342360363074533e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,431.1,0.0001828320891425703,0.99609375,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,466.6,2.327544670335616e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,502.0,1.1036670111638973e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,537.5,2.9005017205428075e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,573.0,2.9550467564524477e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,608.6,3.156387802874369e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,644.3,3.198746761356656e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,680.0,3.1998500999506706e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,715.7,3.182416284321898e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/ladder/reversed_zero_pad_s2/step_50000.pt b/ladder/reversed_zero_pad_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce3d04e9f4da94c12f2fcacfe3c2b2f026786f69 --- /dev/null +++ b/ladder/reversed_zero_pad_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ac5cd371214ccc1107553d40bbb95a29e789ed8337b19d2225be0d2184b2dda +size 13738485 diff --git a/modular_add/config.json b/modular_add/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1c9129465d5de2e22be4d6fba64a04db44615bd0 --- /dev/null +++ b/modular_add/config.json @@ -0,0 +1,32 @@ +{ + "name": "modular_add", + "task": { + "p": 113, + "train_frac": 0.3, + "kind": "modular" + }, + "model": { + "d_model": 128, + "n_layers": 1, + "n_heads": 4, + "d_mlp": 512, + "max_positions": 3, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 40000, + "batch_size": 0, + "lr": 0.001, + "weight_decay": 1.0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 0, + "eval_every": 100, + "checkpoint_every": 500, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/modular_add/metrics.csv b/modular_add/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..9170da50bb153321611401210438b54fe7523409 --- /dev/null +++ b/modular_add/metrics.csv @@ -0,0 +1,402 @@ +step,elapsed,train_loss,train_acc,test_loss,test_acc +0,0.0,4.7616143226623535,0.009660574607551098,4.762050151824951,0.00827833078801632 +100,0.8,0.1685766726732254,1.0,4.590503215789795,0.24465824663639069 +200,1.3,0.029104363173246384,1.0,4.530285358428955,0.2740798890590668 +300,1.8,0.0216338150203228,1.0,4.445437431335449,0.2886228859424591 +400,2.3,0.01258074026554823,1.0,4.299746036529541,0.2962300181388855 +500,2.8,0.006273516453802586,1.0,4.312851905822754,0.29857927560806274 +600,3.2,0.012299809604883194,1.0,3.9216959476470947,0.31010180711746216 +700,3.7,0.009999701753258705,1.0,3.7636139392852783,0.314129114151001 +800,4.2,0.00509463157504797,1.0,3.692552089691162,0.3180445432662964 +900,4.7,0.002197114983573556,1.0,3.6361052989959717,0.3267703354358673 +1000,5.2,0.0009332005283795297,1.0,3.568415880203247,0.33482491970062256 +1100,5.7,0.0003910401137545705,1.0,3.492629289627075,0.34713056683540344 +1200,6.1,0.0001666255120653659,1.0,3.41487717628479,0.3554089069366455 +1300,6.6,0.004765611607581377,1.0,2.6445014476776123,0.4068687856197357 +1400,7.1,0.005297891329973936,1.0,2.382455825805664,0.4258865714073181 +1500,7.6,0.014239107258617878,1.0,1.7212132215499878,0.5430137515068054 +1600,8.1,0.012043540365993977,1.0,1.5408352613449097,0.6118134260177612 +1700,8.5,0.006985262967646122,1.0,1.275530219078064,0.691240668296814 +1800,9.0,0.0030147237703204155,1.0,0.9464532732963562,0.782637894153595 +1900,9.5,0.0012752332258969545,1.0,0.6555971503257751,0.8591565489768982 +2000,10.0,0.0005410491139627993,1.0,0.43341580033302307,0.9168810844421387 +2100,10.4,0.00023161059652920812,1.0,0.27861881256103516,0.9488757252693176 +2200,10.9,9.997520101023838e-05,1.0,0.17426809668540955,0.9718089699745178 +2300,11.4,4.384289786685258e-05,1.0,0.10947011411190033,0.984785795211792 +2400,11.9,1.956131927727256e-05,1.0,0.06694776564836502,0.9917216897010803 +2500,12.4,0.002068975707516074,1.0,0.10334945470094681,0.9874706864356995 +2600,12.9,0.0018950446974486113,1.0,0.056251391768455505,0.9972032904624939 +2700,13.3,0.038790877908468246,1.0,0.18772314488887787,0.9916098117828369 +2800,13.8,0.010595878586173058,1.0,0.05027606338262558,0.9995525479316711 +2900,14.3,0.02573176845908165,1.0,0.06594014912843704,1.0 +3000,14.8,0.012696159072220325,1.0,0.031211240217089653,1.0 +3100,15.3,0.0075059980154037476,1.0,0.01827137917280197,1.0 +3200,15.7,0.003237507538869977,1.0,0.00778270373120904,1.0 +3300,16.2,0.001360321301035583,1.0,0.003161293687298894,1.0 +3400,16.7,0.000579379906412214,1.0,0.0013025131775066257,1.0 +3500,17.2,0.00024824557476677,1.0,0.0005462610861286521,1.0 +3600,17.7,0.00010757338895928115,1.0,0.00023506060824729502,1.0 +3700,18.1,4.710876601166092e-05,1.0,0.00010424039646750316,1.0 +3800,18.6,2.0959334506187588e-05,1.0,4.726535553345457e-05,1.0 +3900,19.1,9.678623428044375e-06,1.0,2.2248801542446017e-05,1.0 +4000,19.6,4.671124770538881e-06,1.0,1.0892742466239724e-05,1.0 +4100,20.0,0.0005926468293182552,1.0,0.0016443049535155296,1.0 +4200,20.5,0.0007911568973213434,1.0,0.0015760519308969378,1.0 +4300,21.0,0.0008353309822268784,1.0,0.0014344416558742523,1.0 +4400,21.5,0.0004670524795074016,1.0,0.0008043177658692002,1.0 +4500,22.0,0.00020260130986571312,1.0,0.0003375782398506999,1.0 +4600,22.5,9.218222839990631e-05,1.0,0.0001534665934741497,1.0 +4700,22.9,4.0846422052709386e-05,1.0,6.81631063343957e-05,1.0 +4800,23.4,1.8287297280039638e-05,1.0,3.060691597056575e-05,1.0 +4900,23.9,8.496562259097118e-06,1.0,1.4256095710152294e-05,1.0 +5000,24.4,4.143618298257934e-06,1.0,6.9562624958052766e-06,1.0 +5100,24.8,2.1658092919096816e-06,1.0,3.65603204954823e-06,1.0 +5200,25.3,1.2839423106925096e-06,1.0,2.15263230529672e-06,1.0 +5300,25.8,0.00046268405276350677,1.0,0.0011544801527634263,1.0 +5400,26.3,0.000684943690430373,1.0,0.0011853005271404982,1.0 +5500,26.8,0.0006766197038814425,1.0,0.0010354813421145082,1.0 +5600,27.2,0.00036385294515639544,1.0,0.0005645775236189365,1.0 +5700,27.7,0.00016452914860565215,1.0,0.00026286710635758936,1.0 +5800,28.2,7.277911208802834e-05,1.0,0.00012045382027281448,1.0 +5900,28.7,3.2475418265676126e-05,1.0,5.6583845434943214e-05,1.0 +6000,29.2,1.4532953173329588e-05,1.0,2.6840240025194362e-05,1.0 +6100,29.6,6.677251349174185e-06,1.0,1.2953828445461113e-05,1.0 +6200,30.1,3.3104015528806485e-06,1.0,7.027241281321039e-06,1.0 +6300,30.6,1.8011789961747127e-06,1.0,4.159307536610868e-06,1.0 +6400,31.1,1.1144660447826027e-06,1.0,2.710921080506523e-06,1.0 +6500,31.6,0.0003923251060768962,1.0,0.0016450249822810292,1.0 +6600,32.1,0.0004951342707499862,1.0,0.0011818903731182218,1.0 +6700,32.6,0.000585910223890096,1.0,0.001049555605277419,1.0 +6800,33.0,0.0003535434661898762,1.0,0.0005965897580608726,1.0 +6900,33.5,0.00015776019426994026,1.0,0.0002642474137246609,1.0 +7000,34.0,6.944742926862091e-05,1.0,0.0001130931923398748,1.0 +7100,34.5,3.1228795705828816e-05,1.0,5.1809616707032546e-05,1.0 +7200,34.9,1.4134619050309993e-05,1.0,2.4156523068086244e-05,1.0 +7300,35.4,6.611764092667727e-06,1.0,1.178601542051183e-05,1.0 +7400,35.9,3.278747726653819e-06,1.0,6.146764917502878e-06,1.0 +7500,36.4,1.7743492435329244e-06,1.0,3.4604609027155675e-06,1.0 +7600,36.9,1.0832164889507112e-06,1.0,2.189318365708459e-06,1.0 +7700,37.4,7.534148949162045e-07,1.0,1.5395169157272903e-06,1.0 +7800,37.8,0.002883616369217634,1.0,0.011213692836463451,0.9995525479316711 +7900,38.3,0.0005080680712126195,1.0,0.0012760566314682364,1.0 +8000,38.8,0.0007631474873051047,1.0,0.001352960243821144,1.0 +8100,39.3,0.000707934785168618,1.0,0.0011232757242396474,1.0 +8200,39.8,0.00036372695467434824,1.0,0.0005772125441581011,1.0 +8300,40.2,0.00016437875456176698,1.0,0.00026198080740869045,1.0 +8400,40.7,6.996525917202234e-05,1.0,0.00011064098362112418,1.0 +8500,41.2,3.145332811982371e-05,1.0,4.96091743116267e-05,1.0 +8600,41.7,1.4321126400318462e-05,1.0,2.3140304620028473e-05,1.0 +8700,42.2,6.698598554066848e-06,1.0,1.1170396646775771e-05,1.0 +8800,42.7,3.3381645607732935e-06,1.0,5.754519861511653e-06,1.0 +8900,43.1,1.787639348549419e-06,1.0,3.157364972139476e-06,1.0 +9000,43.6,1.11860572360456e-06,1.0,2.029515599133447e-06,1.0 +9100,44.1,0.0005313060246407986,1.0,0.0017576608806848526,1.0 +9200,44.6,0.0006873324746266007,1.0,0.0014380940701812506,1.0 +9300,45.1,0.0006630882853642106,1.0,0.001120651955716312,1.0 +9400,45.5,0.0003749238094314933,1.0,0.0005917057860642672,1.0 +9500,46.0,0.0001821612095227465,1.0,0.00028026034124195576,1.0 +9600,46.5,8.261817129096016e-05,1.0,0.00012737742508761585,1.0 +9700,47.0,3.694035694934428e-05,1.0,5.73440884181764e-05,1.0 +9800,47.4,1.6698806575732306e-05,1.0,2.6144236471736804e-05,1.0 +9900,47.9,7.772502613079268e-06,1.0,1.2236708244017791e-05,1.0 +10000,48.4,3.825707153737312e-06,1.0,6.032919372955803e-06,1.0 +10100,48.9,2.033123109868029e-06,1.0,3.1950660286383936e-06,1.0 +10200,49.4,1.2412076557666296e-06,1.0,1.9395922663534293e-06,1.0 +10300,49.8,0.0045984783209860325,1.0,0.0172222051769495,0.9984338879585266 +10400,50.3,0.0006751334294676781,1.0,0.002352476352825761,1.0 +10500,50.8,0.0009377398528158665,1.0,0.0017300304025411606,1.0 +10600,51.3,0.006734296213835478,1.0,0.013921461999416351,1.0 +10700,51.8,0.007099259179085493,1.0,0.010820728726685047,1.0 +10800,52.2,0.00506747467443347,1.0,0.00701179401949048,1.0 +10900,52.7,0.0024894042871892452,1.0,0.003410044824704528,1.0 +11000,53.2,0.0011040373938158154,1.0,0.0015170699916779995,1.0 +11100,53.7,0.00048132092342711985,1.0,0.000666449370328337,1.0 +11200,54.2,0.0002104034647345543,1.0,0.00029545140569098294,1.0 +11300,54.6,9.230555588146672e-05,1.0,0.00013231727643869817,1.0 +11400,55.1,4.1027604311238974e-05,1.0,6.0474558267742395e-05,1.0 +11500,55.6,1.839345713960938e-05,1.0,2.80937510979129e-05,1.0 +11600,56.1,8.76154808793217e-06,1.0,1.3968598977953661e-05,1.0 +11700,56.5,4.319814706832403e-06,1.0,7.144550181692466e-06,1.0 +11800,57.0,2.298090748809045e-06,1.0,3.905594439856941e-06,1.0 +11900,57.5,0.0002798094064928591,1.0,0.001120972796343267,0.999776303768158 +12000,58.0,0.0006983376224525273,1.0,0.0020941284019500017,1.0 +12100,58.5,0.0009680902003310621,1.0,0.00177027378231287,1.0 +12200,58.9,0.01199363637715578,1.0,0.03490154817700386,0.9966439604759216 +12300,59.4,0.009156409651041031,1.0,0.015013160184025764,1.0 +12400,59.9,0.008246197365224361,1.0,0.010931829921901226,1.0 +12500,60.4,0.004167424514889717,1.0,0.005325275007635355,1.0 +12600,60.9,0.0018630772829055786,1.0,0.0023749680258333683,1.0 +12700,61.3,0.0008114685188047588,1.0,0.0010484744561836123,1.0 +12800,61.8,0.0003523377235978842,1.0,0.00046581617789343,1.0 +12900,62.3,0.0001532813475932926,1.0,0.00020870123989880085,1.0 +13000,62.8,6.704378756694496e-05,1.0,9.471135854255408e-05,1.0 +13100,63.2,2.9554024877143092e-05,1.0,4.3497933802427724e-05,1.0 +13200,63.7,1.3568059330282267e-05,1.0,2.0952393242623657e-05,1.0 +13300,64.2,6.275299710978288e-06,1.0,1.0177901458519045e-05,1.0 +13400,64.7,3.2025839118432486e-06,1.0,5.438662810774986e-06,1.0 +13500,65.2,0.0004917937912978232,1.0,0.001359581365250051,1.0 +13600,65.6,0.0007334965048357844,1.0,0.001335139269940555,1.0 +13700,66.1,0.00086883147014305,1.0,0.001314869150519371,1.0 +13800,66.6,0.00048447874723933637,1.0,0.0007079103379510343,1.0 +13900,67.1,0.00021739391377195716,1.0,0.0003028364444617182,1.0 +14000,67.5,9.879273420665413e-05,1.0,0.0001387533120578155,1.0 +14100,68.0,4.4082029489800334e-05,1.0,6.28748894087039e-05,1.0 +14200,68.5,1.9763076124945655e-05,1.0,2.8721509806928225e-05,1.0 +14300,69.0,9.088336810236797e-06,1.0,1.3511278666555882e-05,1.0 +14400,69.5,4.471613010537112e-06,1.0,6.8229696807975415e-06,1.0 +14500,69.9,2.299989318998996e-06,1.0,3.592873554225662e-06,1.0 +14600,70.4,1.3905769264965784e-06,1.0,2.219497901023715e-06,1.0 +14700,70.9,8.971818488134886e-07,1.0,1.452447691008274e-06,1.0 +14800,71.4,0.0005548478220589459,1.0,0.0034676529467105865,0.9993287920951843 +14900,71.9,0.0008203514153137803,1.0,0.0024806626606732607,0.9998881816864014 +15000,72.3,0.3301088511943817,0.987728476524353,0.4432120621204376,0.9799754023551941 +15100,72.8,0.007329842075705528,1.0,0.01197031233459711,1.0 +15200,73.3,0.009141476824879646,1.0,0.012009644880890846,1.0 +15300,73.8,0.005374308675527573,1.0,0.006819399539381266,1.0 +15400,74.3,0.0024337302893400192,1.0,0.003086224664002657,1.0 +15500,74.7,0.0010649461764842272,1.0,0.0013580908998847008,1.0 +15600,75.2,0.0004643466090783477,1.0,0.0005991535144858062,1.0 +15700,75.7,0.000202906274353154,1.0,0.0002665232168510556,1.0 +15800,76.2,8.931693446356803e-05,1.0,0.00011989210906904191,1.0 +15900,76.6,3.9617236325284466e-05,1.0,5.457377847051248e-05,1.0 +16000,77.1,1.811216316127684e-05,1.0,2.5691480914247222e-05,1.0 +16100,77.6,8.414041985815857e-06,1.0,1.2315945241425652e-05,1.0 +16200,78.1,4.2350616240582895e-06,1.0,6.390140697476454e-06,1.0 +16300,78.6,2.2418164462578716e-06,1.0,3.444311687417212e-06,1.0 +16400,79.0,0.002211393555626273,1.0,0.0070736343041062355,1.0 +16500,79.5,0.0005224687047302723,1.0,0.0012233888264745474,1.0 +16600,80.0,0.0006988064851611853,1.0,0.0011281550396233797,1.0 +16700,80.5,0.0005407471326179802,1.0,0.0007378423470072448,1.0 +16800,81.0,0.00027808875893242657,1.0,0.0003747000591829419,1.0 +16900,81.4,0.00012549659004434943,1.0,0.00017047798610292375,1.0 +17000,81.9,5.5600510677322745e-05,1.0,7.655193621758372e-05,1.0 +17100,82.4,2.4854642106220126e-05,1.0,3.476999336271547e-05,1.0 +17200,82.9,1.1300125152047258e-05,1.0,1.6135316400323063e-05,1.0 +17300,83.4,5.367700850911206e-06,1.0,7.836620170564856e-06,1.0 +17400,83.8,2.7145131298311753e-06,1.0,4.063080268679187e-06,1.0 +17500,84.3,1.5220492741718772e-06,1.0,2.3105817490431946e-06,1.0 +17600,84.8,0.0009477337589487433,1.0,0.002827127231284976,1.0 +17700,85.3,0.0008222770411521196,1.0,0.001656360225751996,1.0 +17800,85.8,0.001036999048665166,1.0,0.0016034099971875548,1.0 +17900,86.3,0.0007643711869604886,1.0,0.0010936452308669686,1.0 +18000,86.7,0.00037399015855044127,1.0,0.0005370200960896909,1.0 +18100,87.2,0.0001713020319584757,1.0,0.00024987285723909736,1.0 +18200,87.7,7.729998469585553e-05,1.0,0.00011575953976716846,1.0 +18300,88.2,3.466704220045358e-05,1.0,5.377516936277971e-05,1.0 +18400,88.6,1.573052395542618e-05,1.0,2.5520244889776222e-05,1.0 +18500,89.1,7.533525604230817e-06,1.0,1.2918816537421662e-05,1.0 +18600,89.6,3.672043021651916e-06,1.0,6.705344276269898e-06,1.0 +18700,90.1,0.0012368567986413836,1.0,0.004544305149465799,1.0 +18800,90.5,0.0016762643354013562,1.0,0.0035588534083217382,1.0 +18900,91.0,0.0015084183542057872,1.0,0.0024383217096328735,1.0 +19000,91.5,0.0009229227434843779,1.0,0.0013539334759116173,1.0 +19100,92.0,0.0004541188245639205,1.0,0.0006523402407765388,1.0 +19200,92.5,0.00020567994215525687,1.0,0.00029652126249857247,1.0 +19300,93.0,9.11656316020526e-05,1.0,0.0001330472732661292,1.0 +19400,93.4,4.0502523916075006e-05,1.0,6.035158730810508e-05,1.0 +19500,93.9,1.8319322407478467e-05,1.0,2.8081642085453495e-05,1.0 +19600,94.4,8.53126221045386e-06,1.0,1.3558656064560637e-05,1.0 +19700,94.9,4.109100245841546e-06,1.0,6.817590929131256e-06,1.0 +19800,95.3,5.634999524772866e-06,1.0,9.927807695930824e-06,1.0 +19900,95.8,0.0006442063604481518,1.0,0.0014361984794959426,1.0 +20000,96.3,0.0010698649566620588,1.0,0.0017319556791335344,1.0 +20100,96.8,0.0010934072779491544,1.0,0.0016132511664181948,1.0 +20200,97.3,0.0005623938050121069,1.0,0.0008251647232100368,1.0 +20300,97.7,0.00024058629060164094,1.0,0.0003518572193570435,1.0 +20400,98.2,0.00011082698620157316,1.0,0.00015999747847672552,1.0 +20500,98.7,4.964464096701704e-05,1.0,7.281923899427056e-05,1.0 +20600,99.2,2.2400703528546728e-05,1.0,3.364321673871018e-05,1.0 +20700,99.7,1.0269361155224033e-05,1.0,1.590874308021739e-05,1.0 +20800,100.1,4.949162303091725e-06,1.0,7.948260645207483e-06,1.0 +20900,100.6,2.567789124441333e-06,1.0,4.265462848707102e-06,1.0 +21000,101.1,0.0007666641031391919,1.0,0.0022306563332676888,1.0 +21100,101.6,0.0011683569755405188,1.0,0.002229501958936453,1.0 +21200,102.1,0.0013404921628534794,1.0,0.002009003423154354,1.0 +21300,102.5,0.000792286591604352,1.0,0.0011332312133163214,1.0 +21400,103.0,0.0003797024837695062,1.0,0.0005285012302920222,1.0 +21500,103.5,0.00017398636555299163,1.0,0.00024352679611183703,1.0 +21600,104.0,7.820414612069726e-05,1.0,0.00011106150486739352,1.0 +21700,104.5,3.514664058457129e-05,1.0,5.106910612084903e-05,1.0 +21800,104.9,1.601676194695756e-05,1.0,2.3989705368876457e-05,1.0 +21900,105.4,7.434328381350497e-06,1.0,1.1531759810168296e-05,1.0 +22000,105.9,3.7165825688134646e-06,1.0,5.938298727414804e-06,1.0 +22100,106.4,0.0006315343780443072,1.0,0.001404088456183672,1.0 +22200,106.9,0.001079067005775869,1.0,0.0017374465242028236,1.0 +22300,107.3,0.0012993748532608151,1.0,0.0018054975662380457,1.0 +22400,107.8,0.0007591251633130014,1.0,0.001039925729855895,1.0 +22500,108.3,0.0003302383702248335,1.0,0.0004537054046522826,1.0 +22600,108.8,0.00014331044803839177,1.0,0.00019349677313584834,1.0 +22700,109.2,6.444998143706471e-05,1.0,8.844323019729927e-05,1.0 +22800,109.7,2.8845068300142884e-05,1.0,4.0656072087585926e-05,1.0 +22900,110.2,1.3121822121320292e-05,1.0,1.9057350073126145e-05,1.0 +23000,110.7,6.243992174859159e-06,1.0,9.385763405589387e-06,1.0 +23100,111.2,3.138124839097145e-06,1.0,4.879350399278337e-06,1.0 +23200,111.7,0.0008675913559272885,1.0,0.0021766365971416235,1.0 +23300,112.1,0.0012546065263450146,1.0,0.002146842423826456,1.0 +23400,112.6,0.001075842184945941,1.0,0.0016078235348686576,1.0 +23500,113.1,0.0005450936150737107,1.0,0.0007974550826475024,1.0 +23600,113.6,0.0002592594246380031,1.0,0.00036537094274535775,1.0 +23700,114.0,0.00011879096564371139,1.0,0.00016849397798068821,1.0 +23800,114.5,5.316842725733295e-05,1.0,7.649810868315399e-05,1.0 +23900,115.0,2.391943780821748e-05,1.0,3.509407542878762e-05,1.0 +24000,115.5,1.0971905794576742e-05,1.0,1.6509722627233714e-05,1.0 +24100,116.0,5.332030468707671e-06,1.0,8.246697689173743e-06,1.0 +24200,116.4,2.7073226647189585e-06,1.0,4.287960109650157e-06,1.0 +24300,116.9,0.0005011926405131817,1.0,0.001250040833838284,1.0 +24400,117.4,0.0008104377775453031,1.0,0.0013459640322253108,1.0 +24500,117.9,0.0008461957913823426,1.0,0.0012210853165015578,1.0 +24600,118.4,0.00045067392056807876,1.0,0.000645310094114393,1.0 +24700,118.8,0.00020247681823093444,1.0,0.00029190111672505736,1.0 +24800,119.3,8.665033965371549e-05,1.0,0.0001227188331540674,1.0 +24900,119.8,3.932224353775382e-05,1.0,5.608615174423903e-05,1.0 +25000,120.3,1.7729635146679357e-05,1.0,2.5863559130812064e-05,1.0 +25100,120.8,8.225777492043562e-06,1.0,1.2277150744921528e-05,1.0 +25200,121.2,4.0310706026502885e-06,1.0,6.160464181448333e-06,1.0 +25300,121.7,2.2735330276191235e-06,1.0,3.565121460269438e-06,1.0 +25400,122.2,0.0008401934173889458,1.0,0.002033665543422103,1.0 +25500,122.7,0.0011953682405874133,1.0,0.002068485366180539,1.0 +25600,123.1,0.001043739845044911,1.0,0.0015325590502470732,1.0 +25700,123.6,0.0006046230555512011,1.0,0.0008391322917304933,1.0 +25800,124.1,0.00028595898766070604,1.0,0.0003957389562856406,1.0 +25900,124.6,0.00012899840658064932,1.0,0.00018002078286372125,1.0 +26000,125.1,5.779427374363877e-05,1.0,8.188448555301875e-05,1.0 +26100,125.5,2.6070012609125115e-05,1.0,3.759337778319605e-05,1.0 +26200,126.0,1.198822519654641e-05,1.0,1.7650083464104682e-05,1.0 +26300,126.5,5.716984105674783e-06,1.0,8.592494850745425e-06,1.0 +26400,127.0,2.8887816370115615e-06,1.0,4.4412954594008625e-06,1.0 +26500,127.5,1.6091994439193513e-06,1.0,2.5128726974799065e-06,1.0 +26600,127.9,0.0005350273568183184,1.0,0.0015316092176362872,1.0 +26700,128.4,0.000811939244158566,1.0,0.0015084597980603576,1.0 +26800,128.9,0.0008564452291466296,1.0,0.001253992086276412,1.0 +26900,129.4,0.0005484418943524361,1.0,0.0007419759058393538,1.0 +27000,129.9,0.0002705449587665498,1.0,0.0003624797100201249,1.0 +27100,130.3,0.0001254620001418516,1.0,0.00016959146887529641,1.0 +27200,130.8,5.6672532082302496e-05,1.0,7.774309779051691e-05,1.0 +27300,131.3,2.555980609031394e-05,1.0,3.5749162634601817e-05,1.0 +27400,131.8,1.1790896678576246e-05,1.0,1.6909993064473383e-05,1.0 +27500,132.2,0.001641804468818009,1.0,0.0027698760386556387,1.0 +27600,132.7,0.0009138115565292537,1.0,0.0019570442382246256,1.0 +27700,133.2,0.001334909349679947,1.0,0.0021001307759433985,1.0 +27800,133.7,0.0009531270479783416,1.0,0.0013473944272845984,1.0 +27900,134.2,0.00046325515722855926,1.0,0.0006553237326443195,1.0 +28000,134.7,0.00020622966985683888,1.0,0.0002952086506411433,1.0 +28100,135.1,9.030741784954444e-05,1.0,0.00013165657583158463,1.0 +28200,135.6,3.951612234232016e-05,1.0,5.88691582379397e-05,1.0 +28300,136.1,1.7443135220673867e-05,1.0,2.6509944291319698e-05,1.0 +28400,136.6,8.065297151915729e-06,1.0,1.2552874977700412e-05,1.0 +28500,137.0,3.9795581869839225e-06,1.0,6.426243544410681e-06,1.0 +28600,137.5,2.0957777451258153e-06,1.0,3.453393674135441e-06,1.0 +28700,138.0,0.0021518534049391747,1.0,0.006964025087654591,1.0 +28800,138.5,0.0004392733972053975,1.0,0.0009647279512137175,1.0 +28900,139.0,0.0005488572060130537,1.0,0.0008706892258487642,1.0 +29000,139.5,0.00039429281605407596,1.0,0.0005920868134126067,1.0 +29100,139.9,0.0001876351743703708,1.0,0.0002827202551998198,1.0 +29200,140.4,8.402076491620392e-05,1.0,0.00012784327554982156,1.0 +29300,140.9,3.7774087104480714e-05,1.0,5.8987036027247086e-05,1.0 +29400,141.4,1.695680111879483e-05,1.0,2.7346872229827568e-05,1.0 +29500,141.8,7.796059435349889e-06,1.0,1.2951684766449034e-05,1.0 +29600,142.3,3.79333641831181e-06,1.0,6.4234632191073615e-06,1.0 +29700,142.8,2.0576806036842754e-06,1.0,3.5342072806088254e-06,1.0 +29800,143.3,0.0008360974607057869,1.0,0.0027455349918454885,1.0 +29900,143.8,0.0008648824295960367,1.0,0.0017791681457310915,1.0 +30000,144.2,0.0010286164470016956,1.0,0.0016454393044114113,1.0 +30100,144.7,0.0006325239082798362,1.0,0.000938022043555975,1.0 +30200,145.2,0.0002889255410991609,1.0,0.0004310100630391389,1.0 +30300,145.7,0.00011984517914243042,1.0,0.00017704602214507759,1.0 +30400,146.2,5.4625757911708206e-05,1.0,7.893858855823055e-05,1.0 +30500,146.6,2.4656776076881215e-05,1.0,3.629387356340885e-05,1.0 +30600,147.1,1.1336015631968621e-05,1.0,1.710975448077079e-05,1.0 +30700,147.6,5.4365809774026275e-06,1.0,8.428564797213767e-06,1.0 +30800,148.1,2.794908823489095e-06,1.0,4.442589670361485e-06,1.0 +30900,148.6,1.5707912552898051e-06,1.0,2.557774450906436e-06,1.0 +31000,149.0,0.0007001275080256164,1.0,0.0024113506078720093,1.0 +31100,149.5,0.0009740054956637323,1.0,0.001965248491615057,1.0 +31200,150.0,0.001069405348971486,1.0,0.0016485531814396381,1.0 +31300,150.5,0.0006994553841650486,1.0,0.001050001592375338,1.0 +31400,150.9,0.00032921673846431077,1.0,0.0005019352538511157,1.0 +31500,151.4,0.00014730791735928506,1.0,0.00022832096146885306,1.0 +31600,151.9,6.348235183395445e-05,1.0,9.799366671359167e-05,1.0 +31700,152.4,2.8240490792086348e-05,1.0,4.231529601383954e-05,1.0 +31800,152.9,1.29397658383823e-05,1.0,1.9640385289676487e-05,1.0 +31900,153.3,6.155253231554525e-06,1.0,9.531427167530637e-06,1.0 +32000,153.8,3.1504810067417566e-06,1.0,4.969643669028301e-06,1.0 +32100,154.3,1.7138419252660242e-06,1.0,2.752743512246525e-06,1.0 +32200,154.8,0.00022070510021876544,1.0,0.0008070211042650044,1.0 +32300,155.3,0.00027943463646806777,1.0,0.0005424200790002942,1.0 +32400,155.7,0.00028485391521826386,1.0,0.00043507941882126033,1.0 +32500,156.2,0.00016252649948000908,1.0,0.00023372238501906395,1.0 +32600,156.7,7.713441300438717e-05,1.0,0.00010992003080900759,1.0 +32700,157.2,3.4897690056823194e-05,1.0,5.0287035264773294e-05,1.0 +32800,157.7,1.5900010112090968e-05,1.0,2.3308613890549168e-05,1.0 +32900,158.1,7.409609679598361e-06,1.0,1.1084300240327138e-05,1.0 +33000,158.6,3.666687916847877e-06,1.0,5.584740847552894e-06,1.0 +33100,159.1,1.9714018435479375e-06,1.0,3.0527055514539825e-06,1.0 +33200,159.6,1.1731369795597857e-06,1.0,1.8321851484870422e-06,1.0 +33300,160.1,0.0009818350663408637,1.0,0.0029433723539114,1.0 +33400,160.5,0.0014791685389354825,1.0,0.002775584114715457,1.0 +33500,161.0,0.0016155204502865672,1.0,0.0023542670533061028,1.0 +33600,161.5,0.0010357400169596076,1.0,0.0014561228454113007,1.0 +33700,162.0,0.0004846363444812596,1.0,0.0006905843038111925,1.0 +33800,162.4,0.00021705051767639816,1.0,0.0003163564542774111,1.0 +33900,162.9,9.650861466070637e-05,1.0,0.00014477560762315989,1.0 +34000,163.4,4.2776362533913925e-05,1.0,6.646296969847754e-05,1.0 +34100,163.9,1.911480831040535e-05,1.0,3.0784074624534696e-05,1.0 +34200,164.4,8.642875400255434e-06,1.0,1.4390564501809422e-05,1.0 +34300,164.8,4.27748636866454e-06,1.0,7.372902473434806e-06,1.0 +34400,165.3,0.0009354072972200811,1.0,0.00280961487442255,1.0 +34500,165.8,0.0012345538707450032,1.0,0.002343927277252078,1.0 +34600,166.3,0.0011400593211874366,1.0,0.0018375474028289318,1.0 +34700,166.8,0.000670194101985544,1.0,0.0010461966739967465,1.0 +34800,167.3,0.00029445503605529666,1.0,0.0004501276707742363,1.0 +34900,167.7,0.00013363173638936132,1.0,0.00019922188948839903,1.0 +35000,168.2,5.979478737572208e-05,1.0,9.130416583502665e-05,1.0 +35100,168.7,2.6781257474794984e-05,1.0,4.2262006900273263e-05,1.0 +35200,169.2,1.2195582712593023e-05,1.0,1.9951867216150276e-05,1.0 +35300,169.6,5.7930542425310705e-06,1.0,9.852305083768442e-06,1.0 +35400,170.1,0.061183176934719086,0.9934725761413574,0.09982078522443771,0.9855688810348511 +35500,170.6,0.0012622798094525933,1.0,0.003038577036932111,1.0 +35600,171.1,0.0015832859789952636,1.0,0.0027024103328585625,1.0 +35700,171.6,0.0012877220287919044,1.0,0.0019188629230484366,1.0 +35800,172.0,0.0006764077697880566,1.0,0.0010040531633421779,1.0 +35900,172.5,0.0003113254497293383,1.0,0.0004679745761677623,1.0 +36000,173.0,0.00013970227155368775,1.0,0.00021467183250933886,1.0 +36100,173.5,6.235668115550652e-05,1.0,9.875012619886547e-05,1.0 +36200,174.0,2.7956088160863146e-05,1.0,4.602251647156663e-05,1.0 +36300,174.4,1.2738426448777318e-05,1.0,2.190931081713643e-05,1.0 +36400,174.9,6.319158273981884e-06,1.0,1.1396232366678305e-05,1.0 +36500,175.4,0.0014799929922446609,1.0,0.0037643529940396547,1.0 +36600,175.9,0.001289251376874745,1.0,0.0024075815454125404,1.0 +36700,176.4,0.0015515796840190887,1.0,0.0023720725439488888,1.0 +36800,176.8,0.0010513741290196776,1.0,0.001600598101504147,1.0 +36900,177.3,0.00047702935989946127,1.0,0.0007402586634270847,1.0 +37000,177.8,0.00020715770369861275,1.0,0.000321362167596817,1.0 +37100,178.3,9.075689013116062e-05,1.0,0.00013647315790876746,1.0 +37200,178.8,4.0768518374534324e-05,1.0,6.283259426709265e-05,1.0 +37300,179.2,1.8368564269621857e-05,1.0,2.9320297471713275e-05,1.0 +37400,179.7,8.551747669116594e-06,1.0,1.4220803677744698e-05,1.0 +37500,180.2,4.459163847059244e-06,1.0,7.776514394208789e-06,1.0 +37600,180.7,0.0011325523955747485,1.0,0.0027160332538187504,1.0 +37700,181.2,0.0017164598684757948,1.0,0.003048005048185587,1.0 +37800,181.6,0.0013948684791103005,1.0,0.002149956999346614,1.0 +37900,182.1,0.000742921547498554,1.0,0.0011513573117554188,1.0 +38000,182.6,0.00033838761737570167,1.0,0.0005411971360445023,1.0 +38100,183.1,0.00015059787256177515,1.0,0.00025130578433163464,1.0 +38200,183.5,6.665930413873866e-05,1.0,0.00011760069173760712,1.0 +38300,184.0,2.9790433472953737e-05,1.0,5.642922769766301e-05,1.0 +38400,184.5,1.3483185284712818e-05,1.0,2.8042521080351435e-05,1.0 +38500,185.0,6.286852567427559e-06,1.0,1.469523886044044e-05,1.0 +38600,185.5,0.001793139730580151,1.0,0.007782443426549435,0.9995525479316711 +38700,185.9,0.0007951961597427726,1.0,0.0020863846875727177,1.0 +38800,186.4,0.0008166965562850237,1.0,0.0014718691818416119,1.0 +38900,186.9,0.0006048438372090459,1.0,0.0010123709216713905,1.0 +39000,187.4,0.0002926464076153934,1.0,0.0004923052038066089,1.0 +39100,187.9,0.00013198429951444268,1.0,0.00022661946422886103,1.0 +39200,188.3,5.855754716321826e-05,1.0,0.00010410530376248062,1.0 +39300,188.8,2.603120265121106e-05,1.0,4.856671512243338e-05,1.0 +39400,189.3,1.1819845894933678e-05,1.0,2.347218105569482e-05,1.0 +39500,189.8,5.816086741106119e-06,1.0,1.2543257071229164e-05,1.0 +39600,190.3,2.9020720830885693e-06,1.0,6.801554718549596e-06,1.0 +39700,190.7,0.0014797316398471594,1.0,0.0038965384010225534,1.0 +39800,191.2,0.0020074632484465837,1.0,0.003919458948075771,1.0 +39900,191.7,0.0016954356105998158,1.0,0.003080500988289714,1.0 +40000,192.2,0.0008347955299541354,1.0,0.0013898900942876935,1.0 diff --git a/modular_add/step_40000.pt b/modular_add/step_40000.pt new file mode 100644 index 0000000000000000000000000000000000000000..942af3b4f4e3d11b64f79e3194434ad2155a5f15 --- /dev/null +++ b/modular_add/step_40000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f606ebd6a560e5d1363d673fdd05e72f74c41a0febf8308dcb41e9155b5e912 +size 918965