diff --git a/README.md b/README.md index 8b05fd80283b7f68135b42f2325002a15b3443a2..875db63858c4c2732a408b2d23de97b974dc7b69 100644 --- a/README.md +++ b/README.md @@ -11,47 +11,44 @@ tags: Trained weights from [carrybit](https://github.com/Supergoatscriptguy/carrybit), a small research project on tiny transformers learning exact integer -arithmetic. The code, training configs, and result plots live in that repo; -this repo only holds the final checkpoints so the analysis experiments can be -run without retraining. +arithmetic. The code, configs, figures and the full write-up live in that +repo. This repo holds the final checkpoint of every run in the write-up so the +analysis experiments can be run without retraining. -Every checkpoint is a plain PyTorch `state_dict` for `carrybit.model.Transformer`. -Each folder has the exact `config.json` it was trained with and the -`metrics.csv` logged during training. +Every checkpoint is a plain PyTorch `state_dict` for `carrybit.model.Transformer` +(or `TwoHotMLP` for `modular_mlp`). Each folder has the exact `config.json` it +was trained with and the `metrics.csv` logged during training. Folder names +match `runs/` in the GitHub repo, so every experiment script there works on +these files as they are. ```python import json, torch from carrybit.config import load_config from carrybit.model import Transformer -cfg = load_config("configs/addition.yaml", ["task.zero_pad=true", "task.positions=coupled"]) +run = "addition_big/position_coupling_s1" +raw = json.load(open(f"{run}/config.json")) +cfg = load_config("configs/addition_big.yaml", + [f"{s}.{k}={json.dumps(v)}" for s in ("task", "model") + for k, v in raw[s].items() if k != "kind"]) model = Transformer(16, cfg.model) -model.load_state_dict(torch.load("ladder/position_coupling_s0/step_50000.pt")) +model.load_state_dict(torch.load(f"{run}/step_60000.pt")) ``` ## Contents -`modular_add/`: one-layer transformer trained on a + b mod 113 (Nanda et al. -setup), 40k full-batch steps. Groks around step 2000. - -`ladder/`: the length generalization ladder. 4-layer, width 256 decoder-only -transformers (3.4M parameters) trained for 50k steps on addition with 1 to 20 -digit operands, three seeds per format. Exact match accuracy on operands of -exactly n digits: - -| run | 20 | 25 | 30 | 40 | 50 | -|---|---|---|---|---|---| -| plain_s0 | 0.93 | 0 | 0 | 0 | 0 | -| plain_s1 | 0.93 | 0 | 0 | 0 | 0 | -| plain_s2 | 0.81 | 0 | 0 | 0 | 0 | -| reversed_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 | -| reversed_zero_pad_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 | -| abacus_s0 | 1.00 | 0 | 0 | 0 | 0 | -| abacus_s1 | 1.00 | 0 | 0 | 0 | 0 | -| abacus_s2 | 1.00 | 1.00 | 0.93 | 0.75 | 0.25 | -| position_coupling_s0 | 1.00 | 1.00 | 0.92 | 0.11 | 0 | -| position_coupling_s1 | 1.00 | 0.88 | 0 | 0 | 0 | -| position_coupling_s2 | 1.00 | 0 | 0 | 0 | 0 | -| aligned_blankspace_s0, s1, s2 | 1.00 | 0 | 0 | 0 | 0 | - -See the GitHub README for what the formats mean and a discussion of the results. +| folder | what | runs | +|---|---|---| +| `modular_add`, `modular_add_wd0.1`, `modular_add_wd0` | one-layer transformer, a + b mod 113, weight decay 1 / 0.1 / 0 | 1 each | +| `modular_mlp` | ReLU MLP on two-hot inputs, p = 97 (Swaroop 2026 setup) | 1 | +| `addition` | the length ladder: 3.4M params, trained on 1 to 20 digits, nine formats | 3 seeds each, 6 for position coupling | +| `addition_big` | 11M params, abacus and position coupling, trained on 1 to 30 digits | 2 seeds each | +| `blankspace_big` | 11M params, fixed-width aligned blankspace, trained on 1 to 20 digits | 2 seeds | +| `addition_constant_lr`, `addition_no_wd` | position coupling ablations | 3 and 6 seeds | +| `addition_carry_heavy`, `addition_fixed_length` | position coupling with carry-heavy or single-length training data | 3 seeds each | +| `subtraction` | position coupling and fixed blankspace on a - b | 3 seeds each | + +The one to try first is `addition_big/position_coupling_s1`. Trained on up to +30 digits, it scores 0% exact match at 100 digits as is. Set +`block.attn.scale = 2.0` on every block before decoding and it scores 100% at +200. The GitHub README explains why. diff --git a/addition/abacus_s0/config.json b/addition/abacus_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d240ff4f7d2578efc3640d9c1d70acef85098d8 --- /dev/null +++ b/addition/abacus_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder abacus seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/abacus_s0/metrics.csv b/addition/abacus_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..1a87ea11530975a6462ff8602d0a894766ae4bc2 --- /dev/null +++ b/addition/abacus_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,40.4,0.04165126566593774,0.99609375,0.95703125,0.92578125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,77.8,0.007477885840074827,1.0,0.96875,0.9921875,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,114.0,0.0030776386032294453,1.0,1.0,0.99609375,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,150.1,0.0021365052922425946,1.0,0.9921875,0.95703125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,186.3,0.0014214514749505238,1.0,0.9765625,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,222.9,0.0010082435070390278,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,259.6,0.000492235087522748,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,296.1,0.0006661927837288326,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,332.9,0.00022146683685564763,0.99609375,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,369.7,0.00028362983838865844,1.0,0.96875,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,406.6,5.627922378600414e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,443.1,0.00010834653030878249,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,479.3,5.170799638041556e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,515.6,2.6278768232528415e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,551.9,1.3676077857973595e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,588.5,1.8227696000174497e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,625.2,2.0809437230908565e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,661.8,2.2113516591608795e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,698.7,2.246176344477924e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,735.4,2.244289403386741e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/abacus_s0/step_50000.pt b/addition/abacus_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..38cd762e132256fe611c850ceb62f68ad29adaa5 --- /dev/null +++ b/addition/abacus_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e85eb9ef4ea4524bdd12b147cb5908492e7bc605af2a75b445a5edcb86faf3 +size 13738485 diff --git a/addition/abacus_s1/config.json b/addition/abacus_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3211056fbf89072bd8c296d872fd73895bc1752b --- /dev/null +++ b/addition/abacus_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder abacus seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/abacus_s1/metrics.csv b/addition/abacus_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..5749f6ae6505eda3ad1d00825d55bce8e9d1c3c6 --- /dev/null +++ b/addition/abacus_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,40.2,0.019421861825930153,0.9921875,0.984375,0.6640625,0.703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,76.9,0.00727983858716512,1.0,0.828125,0.86328125,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,113.2,0.008187808755326617,0.984375,0.98046875,0.94140625,0.76171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,149.5,0.0022380353683056874,1.0,0.9921875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,186.0,0.0017146116753800655,1.0,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,222.7,0.0010716926865862115,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,259.4,0.0011338222128818905,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,296.0,0.0006792790250233844,1.0,1.0,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,332.8,0.00046899516380649933,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,369.7,5.581149432375635e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,406.6,0.0003637340261300791,1.0,0.9921875,0.99609375,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,443.3,8.481105058334911e-05,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,479.6,5.903834703148728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,515.9,7.956959854725254e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,552.5,8.394848630662215e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,589.0,1.4101947604018792e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,626.0,1.7413494767162065e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,662.5,1.86887250719425e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,699.5,1.916155950283142e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,736.3,1.930204289234567e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/abacus_s1/step_50000.pt b/addition/abacus_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..31daa8992368f604f4f142ff1a500aa6bedb95fc --- /dev/null +++ b/addition/abacus_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71edef7f0633fc9562fff242174358eaa7c061316f51ca84feb02feb6879820d +size 13738485 diff --git a/addition/abacus_s2/config.json b/addition/abacus_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..60f9ed00d918c70bad182af60fba51848e199181 --- /dev/null +++ b/addition/abacus_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder abacus seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/abacus_s2/metrics.csv b/addition/abacus_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..13628ad9da5cd1a15aaecfd7b4a0308c848c45d1 --- /dev/null +++ b/addition/abacus_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,40.2,0.0025480398695461076,1.0,0.984375,0.96875,0.953125,0.91015625,0.6484375,0.36328125,0.0703125,0.0,0.0,0.0 +5000,76.4,0.000774010375750845,1.0,0.98828125,1.0,0.99609375,0.9921875,0.6875,0.39453125,0.13671875,0.0,0.0,0.0 +7500,112.8,0.000516119487618438,0.9921875,0.99609375,0.96484375,0.97265625,0.98828125,0.65625,0.140625,0.015625,0.0,0.0,0.0 +10000,149.2,0.0006419722169680194,1.0,0.99609375,0.99609375,0.9453125,0.9375,0.83984375,0.2734375,0.05859375,0.0,0.0,0.0 +12500,185.7,7.209724002654492e-08,1.0,1.0,1.0,1.0,1.0,0.953125,0.8203125,0.39453125,0.05078125,0.0,0.0 +15000,222.4,1.0956978970981467e-08,1.0,1.0,1.0,1.0,1.0,0.99609375,0.875,0.70703125,0.296875,0.01171875,0.0 +17500,259.1,0.0001840910556279454,1.0,1.0,1.0,1.0,0.9921875,0.9140625,0.2734375,0.078125,0.0,0.0,0.0 +20000,295.7,2.3715848697307856e-08,1.0,1.0,1.0,1.0,0.99609375,0.96875,0.64453125,0.35546875,0.015625,0.0,0.0 +22500,332.8,2.134711718499249e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.71484375,0.41796875,0.09765625,0.0,0.0 +25000,369.7,2.3915868879389415e-08,1.0,1.0,1.0,1.0,1.0,0.98046875,0.5703125,0.05078125,0.0,0.0,0.0 +27500,406.4,2.525873054866377e-08,1.0,1.0,1.0,1.0,1.0,0.90625,0.0234375,0.0,0.0,0.0,0.0 +30000,442.9,2.6034143760423384e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.86328125,0.38671875,0.0078125,0.0,0.0 +32500,479.2,1.2933618968363646e-06,1.0,1.0,1.0,1.0,1.0,0.80078125,0.5,0.046875,0.0,0.0,0.0 +35000,515.5,8.196465929258812e-05,1.0,1.0,1.0,1.0,0.9921875,0.9765625,0.0625,0.00390625,0.0,0.0,0.0 +37500,552.3,2.5679873355496745e-08,1.0,1.0,1.0,1.0,1.0,0.75,0.15234375,0.00390625,0.0,0.0,0.0 +40000,589.0,2.668754422022952e-09,1.0,1.0,1.0,1.0,1.0,0.89453125,0.48046875,0.07421875,0.0,0.0,0.0 +42500,625.9,1.6834652089024804e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.76171875,0.24609375,0.015625,0.0,0.0 +45000,662.5,1.7842648507998153e-08,1.0,1.0,1.0,1.0,1.0,0.9296875,0.75390625,0.2890625,0.015625,0.0,0.0 +47500,699.0,1.9654443694697242e-08,1.0,1.0,1.0,1.0,1.0,0.92578125,0.74609375,0.26171875,0.0078125,0.0,0.0 +50000,736.0,2.0090057245062915e-08,1.0,1.0,1.0,1.0,1.0,0.93359375,0.75,0.25390625,0.00390625,0.0,0.0 diff --git a/addition/abacus_s2/step_50000.pt b/addition/abacus_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e29873292ceeeccaca26a7a2418eebdaed958a0b --- /dev/null +++ b/addition/abacus_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb1c41cdf7af8f6f213462f7998d95e2a43d4edc22c0d441b0f69fd50cc2dea1 +size 13738485 diff --git a/addition/blankspace_fixed_relative_s0/config.json b/addition/blankspace_fixed_relative_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6db119269df8fccbf8b132ef2bdeeeaf800f1513 --- /dev/null +++ b/addition/blankspace_fixed_relative_s0/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace fixed + relative seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "relative", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_fixed_relative_s0/metrics.csv b/addition/blankspace_fixed_relative_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..b80df850d994037f3c0bfe5fa2faa880352054a7 --- /dev/null +++ b/addition/blankspace_fixed_relative_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,22.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,226.3,0.20703520938244796,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,430.3,0.2627567995265704,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,634.5,0.4574976999808654,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,838.7,0.2651583137941563,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,1044.1,0.10680842386954857,0.09765625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,1248.2,0.12207524907863034,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1452.5,0.1762504720602938,0.08203125,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1656.8,0.14041061588127907,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1862.4,0.1001222274342093,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,2066.6,0.2697289037550494,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,2270.6,0.6783406467437754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,2474.8,0.6374371794799828,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2680.1,0.6329233304221513,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2884.3,0.6234669802913972,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,3088.3,0.6205317425213412,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,3292.4,0.6017276442331264,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,3497.7,0.7258058367817909,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,3701.7,0.6323961474374555,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,3906.0,0.6066491667535895,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,4110.2,0.593319668324286,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/blankspace_fixed_relative_s0/step_50000.pt b/addition/blankspace_fixed_relative_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d5e276fe99872b3fe27ae65afe6d9aefccb135b --- /dev/null +++ b/addition/blankspace_fixed_relative_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e9ef8a03a4264db75ff7349ff6fe4ac8f8d3c53e127ec36215b926c21fa2ee3 +size 12756597 diff --git a/addition/blankspace_fixed_relative_s1/config.json b/addition/blankspace_fixed_relative_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..517152a7c9d9ce537e7913ff1ceec9d090ca55e0 --- /dev/null +++ b/addition/blankspace_fixed_relative_s1/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace fixed + relative seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "relative", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_fixed_relative_s1/metrics.csv b/addition/blankspace_fixed_relative_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..cb3ed3e1ad50500a92990c8845c7573b68290505 --- /dev/null +++ b/addition/blankspace_fixed_relative_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,22.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,226.3,0.6826302532060446,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,430.5,0.09242761518813643,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,634.7,0.18230011434451712,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,838.9,0.6496547036008226,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,1044.0,0.6364924383871152,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,1248.2,0.11263084836659418,0.0390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1452.4,0.08725647997445593,0.0546875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1656.6,0.0927207347182343,0.1015625,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1861.8,0.12975181270519814,0.01953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,2066.1,0.08782499054882002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,2270.4,0.2652910798088735,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,2474.5,0.04182807987974134,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2679.8,0.02653962984752767,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2884.1,0.06692170471305299,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,3088.4,0.06239397073924014,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,3292.4,0.040825869940407496,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,3497.7,0.04419280434274837,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,3701.8,0.04129039797399679,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,3905.8,0.03914603508183192,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,4110.0,0.03725589552857548,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/blankspace_fixed_relative_s1/step_50000.pt b/addition/blankspace_fixed_relative_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ade60fe295e94884a8aaa80b5061e9487d7f85f9 --- /dev/null +++ b/addition/blankspace_fixed_relative_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe3fc7a5e4ba9b793a726ecab888554396c70796e09bee42b1168b5c9bedba5c +size 12756597 diff --git a/addition/blankspace_fixed_relative_s2/config.json b/addition/blankspace_fixed_relative_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eec2c27c42c1a8909b63fd241a3aa828915fb008 --- /dev/null +++ b/addition/blankspace_fixed_relative_s2/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace fixed + relative seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "relative", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_fixed_relative_s2/metrics.csv b/addition/blankspace_fixed_relative_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..7604e3d8bdfb4b9d1f38981f88fa538a869fc857 --- /dev/null +++ b/addition/blankspace_fixed_relative_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,22.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,226.4,0.14129522232535732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,430.6,0.15880897545836423,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,634.7,0.10007927374338119,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,839.1,0.14276802959215126,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,1044.5,0.23306011773899402,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,1248.6,0.04054757494554516,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1452.8,0.06636011190511847,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1657.2,0.42169396162704836,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1862.6,0.2511044359851245,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,2066.8,0.02148349786548707,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,2271.0,0.0108850836969352,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,2475.2,0.06255321251730356,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2680.6,0.02701192433966193,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2884.9,0.016707464822334345,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,3088.9,0.010058896048647946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,3293.2,0.008355475885859501,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,3498.7,0.007721445332368437,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,3702.9,0.005973847859699946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,3907.0,0.0052722387621380465,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,4111.0,0.004958737693459962,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/blankspace_fixed_relative_s2/step_50000.pt b/addition/blankspace_fixed_relative_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..1031ab894ef2a11e8a52ceaf87aacaf0f58c1543 --- /dev/null +++ b/addition/blankspace_fixed_relative_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99804beb8fa954954f3ae0432c670baf93b338e07c765ae34beb6087e78f2f45 +size 12756597 diff --git a/addition/blankspace_fixed_s0/config.json b/addition/blankspace_fixed_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..970b66a868cedb3598d58ec150a5a072174d9609 --- /dev/null +++ b/addition/blankspace_fixed_s0/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace fixed seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_fixed_s0/metrics.csv b/addition/blankspace_fixed_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..6eb03ec83fe22e7b9d1b66476c5c7441fbd9cec4 --- /dev/null +++ b/addition/blankspace_fixed_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,20.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,196.9,0.0005506453078795269,0.8203125,0.8671875,0.73828125,0.6328125,0.51171875,0.3984375,0.11328125,0.015625,0.0,0.0,0.0 +5000,373.3,0.00012206451032746232,0.98046875,0.92578125,0.88671875,0.83984375,0.75390625,0.60546875,0.265625,0.07421875,0.0078125,0.0,0.0 +7500,549.6,9.21903775479684e-05,0.98828125,1.0,0.97265625,0.953125,0.890625,0.8984375,0.4296875,0.13671875,0.01171875,0.0,0.0 +10000,725.8,3.749034867578125e-05,1.0,0.98828125,0.984375,0.9140625,0.8828125,0.74609375,0.61328125,0.2421875,0.08203125,0.0,0.0 +12500,903.5,3.472558525563995e-05,1.0,0.99609375,0.99609375,0.94921875,0.94921875,0.859375,0.36328125,0.0859375,0.02734375,0.0,0.0 +15000,1080.8,1.7849055845936695e-05,0.984375,0.98828125,0.95703125,0.875,0.68359375,0.4921875,0.26171875,0.05859375,0.0,0.0,0.0 +17500,1257.1,1.8422598429383474e-05,0.99609375,0.9921875,0.9921875,0.9609375,0.87109375,0.7578125,0.25390625,0.08203125,0.00390625,0.0,0.0 +20000,1433.2,4.3433618352920054e-07,1.0,1.0,0.99609375,0.99609375,0.9921875,0.9609375,0.83203125,0.3984375,0.0859375,0.0,0.0 +22500,1610.6,1.3758578947589133e-05,1.0,1.0,0.98828125,0.92578125,0.890625,0.61328125,0.32421875,0.07421875,0.0,0.0,0.0 +25000,1786.8,3.7128141571484146e-07,0.99609375,1.0,0.9921875,0.9921875,0.984375,0.93359375,0.5625,0.0859375,0.0,0.0,0.0 +27500,1963.3,6.213710957201114e-07,0.98828125,1.0,0.98046875,0.93359375,0.7890625,0.625,0.16796875,0.0,0.0,0.0,0.0 +30000,2139.5,3.5937933895871672e-09,1.0,1.0,1.0,0.9921875,0.98046875,0.890625,0.4453125,0.0,0.0,0.0,0.0 +32500,2317.3,2.1753980313264555e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.72265625,0.1328125,0.0,0.0,0.0 +35000,2493.6,2.2173957132333224e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.5859375,0.03125,0.0,0.0,0.0 +37500,2669.8,5.5568823425201534e-09,1.0,1.0,1.0,1.0,0.984375,0.9453125,0.59765625,0.078125,0.0,0.0,0.0 +40000,2846.0,2.607981333519621e-09,1.0,1.0,1.0,1.0,1.0,0.9765625,0.76171875,0.1171875,0.0,0.0,0.0 +42500,3023.7,2.8111192429016504e-09,1.0,1.0,1.0,1.0,1.0,0.96875,0.71484375,0.0859375,0.0,0.0,0.0 +45000,3200.0,3.023483413213966e-09,1.0,1.0,1.0,1.0,1.0,0.97265625,0.71875,0.10546875,0.0,0.0,0.0 +47500,3376.3,2.967679517038681e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.68359375,0.08984375,0.0,0.0,0.0 +50000,3552.5,3.0245826619401742e-09,1.0,1.0,1.0,1.0,1.0,0.96484375,0.68359375,0.0859375,0.0,0.0,0.0 diff --git a/addition/blankspace_fixed_s0/step_50000.pt b/addition/blankspace_fixed_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..62873f2178be0b4c9fcb576a8430e9c7b2f0248b --- /dev/null +++ b/addition/blankspace_fixed_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4733e444551e358ae460cdac12828231d3e07297a1162bdbe6afb422e6e301a8 +size 13738485 diff --git a/addition/blankspace_fixed_s1/config.json b/addition/blankspace_fixed_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e39791114c44b370e9c1c487d91557af6558b300 --- /dev/null +++ b/addition/blankspace_fixed_s1/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace fixed seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_fixed_s1/metrics.csv b/addition/blankspace_fixed_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..36c14b40bc3ffa9920d443f4166c3f009a6e6eb4 --- /dev/null +++ b/addition/blankspace_fixed_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,20.3,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,196.7,0.0026082363351385583,0.1015625,0.0546875,0.046875,0.0546875,0.04296875,0.0234375,0.00390625,0.0,0.0,0.0,0.0 +5000,373.0,0.0003393877649152727,0.97265625,0.5,0.93359375,0.859375,0.6953125,0.35546875,0.1171875,0.00390625,0.0,0.0,0.0 +7500,549.2,0.00011643498715221173,0.90625,0.3828125,0.86328125,0.61328125,0.58984375,0.5078125,0.484375,0.18359375,0.046875,0.0,0.0 +10000,725.4,7.968605815850177e-05,0.91796875,0.7109375,0.96484375,0.83203125,0.70703125,0.3671875,0.12109375,0.01171875,0.0,0.0,0.0 +12500,903.1,2.1121372331914974e-05,0.9921875,1.0,0.99609375,0.9609375,0.9375,0.88671875,0.5234375,0.1953125,0.03515625,0.00390625,0.0 +15000,1080.6,8.796565490238032e-06,1.0,1.0,0.99609375,0.9921875,0.9375,0.84375,0.484375,0.09375,0.015625,0.0,0.0 +17500,1257.0,2.82035677643842e-05,0.9921875,0.96875,0.9609375,0.9296875,0.765625,0.58203125,0.3125,0.08203125,0.0,0.0,0.0 +20000,1433.0,2.394601428167586e-09,1.0,1.0,1.0,0.99609375,0.984375,0.96484375,0.73046875,0.24609375,0.04296875,0.0,0.0 +22500,1610.7,3.2071281316063323e-06,0.9921875,0.99609375,1.0,0.9921875,0.9765625,0.875,0.56640625,0.05859375,0.00390625,0.0,0.0 +25000,1786.8,4.094170965169126e-05,1.0,1.0,0.9765625,0.96875,0.86328125,0.68359375,0.328125,0.01953125,0.0,0.0,0.0 +27500,1963.2,2.391484625870849e-06,1.0,1.0,0.953125,0.890625,0.7421875,0.6171875,0.29296875,0.03515625,0.00390625,0.0,0.0 +30000,2139.4,4.713165529820018e-08,1.0,1.0,0.99609375,1.0,0.9609375,0.75,0.3359375,0.02734375,0.0,0.0,0.0 +32500,2315.8,2.4914220951256115e-09,1.0,1.0,1.0,1.0,0.98828125,0.8984375,0.578125,0.0859375,0.01171875,0.0,0.0 +35000,2492.0,2.8948093464735e-09,1.0,1.0,1.0,0.99609375,0.96484375,0.87109375,0.59375,0.1328125,0.0078125,0.0,0.0 +37500,2668.3,2.5616916061582564e-09,1.0,1.0,1.0,1.0,0.984375,0.8828125,0.5,0.0625,0.0,0.0,0.0 +40000,2844.6,2.8025650015356648e-09,1.0,1.0,1.0,1.0,0.98046875,0.8828125,0.4453125,0.03125,0.0,0.0,0.0 +42500,3022.3,2.862597413463905e-09,1.0,1.0,1.0,1.0,0.9765625,0.875,0.43359375,0.03515625,0.0,0.0,0.0 +45000,3198.5,3.084677307498172e-09,1.0,1.0,1.0,1.0,0.98046875,0.87109375,0.40625,0.02734375,0.0,0.0,0.0 +47500,3377.1,3.0422148427183186e-09,1.0,1.0,1.0,1.0,0.984375,0.875,0.3984375,0.02734375,0.0,0.0,0.0 +50000,3554.0,3.001311497162208e-09,1.0,1.0,1.0,1.0,0.98046875,0.8671875,0.3984375,0.02734375,0.0,0.0,0.0 diff --git a/addition/blankspace_fixed_s1/step_50000.pt b/addition/blankspace_fixed_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e18ff3a49c3dbde28d44f025e368ca6cc2452c6 --- /dev/null +++ b/addition/blankspace_fixed_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:974e486e24ae91e70c3ab03bcbb6ab593b358a92a19889bb0d17c2604e7f8773 +size 13738485 diff --git a/addition/blankspace_fixed_s2/config.json b/addition/blankspace_fixed_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1747c8329f80cf0c16c013080cbed70a2f3e9301 --- /dev/null +++ b/addition/blankspace_fixed_s2/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace fixed seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_fixed_s2/metrics.csv b/addition/blankspace_fixed_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..473149ffe7f5c6cf73bb60ee9828c48a6303a0f7 --- /dev/null +++ b/addition/blankspace_fixed_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,20.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,197.2,0.0004957558742136849,0.9140625,0.82421875,0.70703125,0.546875,0.2265625,0.0625,0.00390625,0.0,0.0,0.0,0.0 +5000,373.9,0.00012956480872600203,0.99609375,0.99609375,0.96875,0.765625,0.52734375,0.16796875,0.015625,0.0,0.0,0.0,0.0 +7500,550.1,9.812221885565382e-05,0.96875,0.953125,0.8984375,0.6484375,0.3984375,0.16796875,0.01171875,0.0078125,0.0,0.0,0.0 +10000,726.2,1.7760837309919371e-07,1.0,1.0,1.0,0.98828125,0.91015625,0.625,0.14453125,0.00390625,0.0,0.0,0.0 +12500,903.9,6.116268558052304e-05,0.98828125,0.92578125,0.5625,0.2734375,0.11328125,0.015625,0.0,0.0,0.0,0.0,0.0 +15000,1081.4,6.285491913409828e-05,0.99609375,0.99609375,0.94140625,0.85546875,0.62890625,0.4296875,0.03515625,0.0,0.0,0.0,0.0 +17500,1257.6,4.916129410842028e-05,1.0,1.0,0.96875,0.75390625,0.6640625,0.47265625,0.0859375,0.015625,0.0,0.0,0.0 +20000,1434.0,1.533371229892242e-08,1.0,1.0,0.99609375,0.89453125,0.75390625,0.609375,0.16015625,0.0390625,0.0,0.0,0.0 +22500,1611.6,2.292736706764027e-05,1.0,1.0,0.97265625,0.859375,0.69140625,0.45703125,0.08203125,0.00390625,0.0,0.0,0.0 +25000,1787.8,2.0849592988569546e-05,0.99609375,0.99609375,0.984375,0.9375,0.77734375,0.52734375,0.0390625,0.0,0.0,0.0,0.0 +27500,1964.2,3.0127465835718325e-09,1.0,1.0,0.9921875,0.984375,0.953125,0.75,0.27734375,0.078125,0.0,0.0,0.0 +30000,2140.5,2.0596147923065815e-06,1.0,0.99609375,0.99609375,0.9609375,0.921875,0.6484375,0.0546875,0.0078125,0.0,0.0,0.0 +32500,2316.8,3.6242308188774898e-09,1.0,1.0,1.0,0.96875,0.90625,0.578125,0.10546875,0.01953125,0.0,0.0,0.0 +35000,2493.2,3.184096991594267e-09,1.0,0.99609375,0.99609375,0.9453125,0.921875,0.69140625,0.12109375,0.015625,0.0,0.0,0.0 +37500,2669.7,4.223212647507518e-09,1.0,1.0,1.0,0.98046875,0.91796875,0.70703125,0.1328125,0.01171875,0.0,0.0,0.0 +40000,2845.9,4.627020220290975e-09,1.0,1.0,1.0,0.984375,0.8984375,0.66796875,0.12109375,0.01171875,0.0,0.0,0.0 +42500,3023.6,4.912223292150009e-09,1.0,1.0,1.0,0.984375,0.8984375,0.6328125,0.11328125,0.01171875,0.0,0.0,0.0 +45000,3199.9,4.980930373748483e-09,1.0,1.0,1.0,0.98046875,0.87890625,0.6171875,0.109375,0.01171875,0.0,0.0,0.0 +47500,3376.4,5.028134725070195e-09,1.0,1.0,1.0,0.98046875,0.87890625,0.60546875,0.1015625,0.01171875,0.0,0.0,0.0 +50000,3552.6,5.002201414504762e-09,1.0,1.0,1.0,0.98046875,0.87890625,0.61328125,0.09765625,0.0078125,0.0,0.0,0.0 diff --git a/addition/blankspace_fixed_s2/step_50000.pt b/addition/blankspace_fixed_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..229611073c93f8d53847045855fccf51d89decbc --- /dev/null +++ b/addition/blankspace_fixed_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc25ea67fc887e8cadd6dc2fa0283952bd8fbd9dfa4da23d651378ef8584290d +size 13738485 diff --git a/addition/blankspace_var_s0/config.json b/addition/blankspace_var_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..577e580989ea72baa9e659f0f5448087313f1f63 --- /dev/null +++ b/addition/blankspace_var_s0/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace var seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": false, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_var_s0/metrics.csv b/addition/blankspace_var_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..ffcb57a7acbbba4979cf82e55dc766caf8a18046 --- /dev/null +++ b/addition/blankspace_var_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,162.3,0.6186378883550033,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,320.9,0.19051331784140904,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,479.7,0.09004364974590386,0.8359375,0.44921875,0.2890625,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,638.5,0.14656466887716998,0.015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,798.6,0.006658630277786855,0.93359375,0.3125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,957.3,0.005940055596195082,1.0,1.0,0.9921875,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1116.1,0.01754279880689707,1.0,0.984375,0.9296875,0.296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1276.3,0.0034275892633903672,1.0,0.99609375,1.0,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1436.2,0.0047208392852198374,1.0,1.0,0.9921875,0.890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1595.0,0.0018665735886661443,0.984375,1.0,0.9375,0.58984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1753.9,0.0013890370618582118,0.9921875,1.0,0.7265625,0.3515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1912.8,0.0009846590920853096,1.0,1.0,0.98828125,0.9140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2073.0,0.0032889325377649515,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2232.0,0.00019508725264521203,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,2390.8,0.00045013484071009506,1.0,1.0,1.0,0.9140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,2549.6,4.6959747138616026e-05,1.0,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,2709.7,5.6767481562301165e-05,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,2868.5,7.050720125570499e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,3027.3,1.1120138110482406e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3186.0,7.488752917851346e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/blankspace_var_s0/step_50000.pt b/addition/blankspace_var_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..51f59610f00ed6c4fcbda303143c2ed2cadb6ccd --- /dev/null +++ b/addition/blankspace_var_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d617bb8100d1a131146043d65fa491df62c4f9d4dbdb12e8f93c0e6d1bcd27 +size 13738485 diff --git a/addition/blankspace_var_s1/config.json b/addition/blankspace_var_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..27d0694fcc736ee375698c16309f782ffc3f05d0 --- /dev/null +++ b/addition/blankspace_var_s1/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace var seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": false, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_var_s1/metrics.csv b/addition/blankspace_var_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..abc087341eda0c5bf7ec55586f118f4abbe01aac --- /dev/null +++ b/addition/blankspace_var_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,162.2,0.6694373498431923,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,320.9,0.3174608366777573,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,479.6,0.088214992116239,0.63671875,0.57421875,0.4609375,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,638.4,0.033645382226527756,0.921875,0.98046875,0.91796875,0.234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,798.3,0.01124123413962232,0.61328125,0.97265625,0.91015625,0.3046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,957.2,0.03232797719789472,0.98828125,0.9609375,0.65234375,0.02734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1116.0,0.00718463512902601,0.9921875,1.0,1.0,0.86328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1274.9,0.014129337214051867,0.96875,1.0,0.98828125,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1434.9,0.0036740895380866687,1.0,1.0,0.99609375,0.87890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1593.6,0.007062510357253116,1.0,0.9921875,0.98046875,0.59375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1752.5,0.008447402562925583,1.0,1.0,0.9921875,0.671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1911.2,0.0025287677941350072,1.0,0.9375,0.671875,0.66796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2071.3,0.0018959840800858183,0.99609375,1.0,0.98046875,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2230.0,0.0003868506276501107,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,2388.9,0.00031602887532225857,1.0,1.0,1.0,0.9140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,2547.6,7.280061912272834e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,2707.7,2.988267797327319e-05,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,2866.4,9.73891918357995e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,3025.3,1.6834128019312507e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3184.1,1.622445492837834e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/blankspace_var_s1/step_50000.pt b/addition/blankspace_var_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb5212ee6213a0b87d618b93fdad32a62d713b7c --- /dev/null +++ b/addition/blankspace_var_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:563dddf1a15890c379f95a0880119fa2b564efd22e9275924ec5c46dacfe2e5c +size 13738485 diff --git a/addition/blankspace_var_s2/config.json b/addition/blankspace_var_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2b86edef944617cc41fcc665015fac943ba867dd --- /dev/null +++ b/addition/blankspace_var_s2/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition blankspace var seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": false, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/blankspace_var_s2/metrics.csv b/addition/blankspace_var_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..fb7364f4eb48473df6270915b20dd77f10f8aa45 --- /dev/null +++ b/addition/blankspace_var_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,162.2,0.5832304521701059,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,321.1,0.04690088438062398,0.68359375,0.60546875,0.3046875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,479.9,0.08495105978154338,0.67578125,0.76953125,0.04296875,0.01953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,638.7,0.012313809212109967,0.98046875,1.0,0.99609375,0.48046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,798.8,0.009126608279989946,1.0,1.0,1.0,0.609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,957.7,0.034043930974871496,0.98046875,0.93359375,0.66796875,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,1116.6,0.0034535789100757666,0.9921875,1.0,0.80078125,0.3046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1276.5,0.0021677381241184076,1.0,1.0,1.0,0.9375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,1436.6,0.004518133543420206,1.0,1.0,0.93359375,0.328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1595.4,0.023756263721328504,0.01171875,0.140625,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1754.3,0.0041635120231823105,1.0,0.99609375,0.87890625,0.51171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1913.1,0.0067360217149997036,0.6015625,0.9921875,0.66796875,0.07421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,2073.3,0.0009128844293960459,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2232.1,0.0003308118617656198,1.0,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,2391.2,0.000812493642517381,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,2550.0,9.513217252444159e-05,1.0,1.0,1.0,0.546875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +42500,2710.2,1.910369615979087e-05,1.0,1.0,1.0,1.0,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +45000,2869.0,9.98705935296285e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,3027.7,1.0420948403523156e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3186.6,6.497759708227786e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/blankspace_var_s2/step_50000.pt b/addition/blankspace_var_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f7de83b20bc6a70b8c4241be79d7c100afc2603 --- /dev/null +++ b/addition/blankspace_var_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:766d1f5b22f994f58bf66af29e61968331d18956aba158e8f0fcdf3630dd3416 +size 13738485 diff --git a/addition/plain_s0/config.json b/addition/plain_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..619378a7b866249cfedd234c51493186b0ac0aeb --- /dev/null +++ b/addition/plain_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder plain seed 0", + "task": { + "max_digits": 20, + "reverse": false, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/plain_s0/metrics.csv b/addition/plain_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..3a6261b06c4834baaecca4ddbcf72b57c825577d --- /dev/null +++ b/addition/plain_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,45.8,1.1552440607522254,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,85.3,1.0496538571563998,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,121.6,0.22190272788699789,0.625,0.30078125,0.13671875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,156.8,0.1361104787168321,0.62890625,0.265625,0.1328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,194.1,0.1299326336596537,0.83984375,0.82421875,0.66015625,0.26953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,230.1,0.08724782346629964,0.87890625,0.72265625,0.6171875,0.37109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,265.7,0.03294547912425976,0.8828125,0.80078125,0.7109375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,301.2,0.03188540525085875,0.9296875,0.8125,0.75390625,0.60546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,338.5,0.024272466284350352,0.94921875,0.84765625,0.76953125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,377.9,0.02135416777169092,0.95703125,0.84765625,0.77734375,0.62109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,413.0,0.017336725187138323,0.95703125,0.81640625,0.72265625,0.59375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,450.0,0.015351897706221264,0.95703125,0.90625,0.8359375,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,485.1,0.011902291315282323,0.9375,0.921875,0.78515625,0.71484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,520.3,0.010157805927521121,0.98046875,0.9453125,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,555.7,0.008457286916127297,0.98828125,0.9453125,0.89453125,0.84765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,591.0,0.006343905665149985,0.984375,0.95703125,0.90234375,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,626.2,0.004807067632394225,0.984375,0.94921875,0.93359375,0.90625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,661.3,0.0039473881130650396,0.9765625,0.96484375,0.93359375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,696.4,0.003158288052806785,0.98828125,0.96875,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,731.5,0.0031530544255801425,0.98828125,0.96484375,0.95703125,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/plain_s0/step_50000.pt b/addition/plain_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..abd4a6eb21dde96263ed5c035e2ff2f5a9a25d2b --- /dev/null +++ b/addition/plain_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3273a0e6726cde985810bea69fe181a1c622e441242ac728493e1349c94dc96b +size 13738485 diff --git a/addition/plain_s1/config.json b/addition/plain_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0e56e34daff33cd0b9d29a1ef57d1e01731119e7 --- /dev/null +++ b/addition/plain_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder plain seed 1", + "task": { + "max_digits": 20, + "reverse": false, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/plain_s1/metrics.csv b/addition/plain_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ee21298d516bebd70470d92f60f0361f011ff54 --- /dev/null +++ b/addition/plain_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.9,1.0926829596295016,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,75.2,0.34977536998246683,0.578125,0.17578125,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,110.7,0.13636325329445406,0.80859375,0.59765625,0.27734375,0.0859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,147.0,0.10624530700855975,0.4921875,0.5703125,0.13671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,182.0,0.04835331203503549,0.890625,0.7109375,0.71875,0.3828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,217.1,0.13925495517488404,0.71484375,0.64453125,0.20703125,0.06640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,252.1,0.09695242193667337,0.90234375,0.5859375,0.6015625,0.1796875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,287.1,0.030026417599016593,0.9296875,0.7421875,0.63671875,0.11328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,321.9,0.02373246684440765,0.94921875,0.828125,0.7265625,0.625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,356.6,0.027057840476159962,0.9453125,0.8515625,0.7578125,0.57421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,391.2,0.020241652287542256,0.95703125,0.8203125,0.796875,0.609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,425.8,0.0187081657161948,0.95703125,0.66796875,0.7890625,0.59765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,461.1,0.013485327495607533,0.96484375,0.89453125,0.78125,0.69140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,496.0,0.012634533764863868,0.9609375,0.91015625,0.83984375,0.70703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,530.9,0.009974493826907564,0.98046875,0.890625,0.87890625,0.8125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,566.1,0.007701182666293666,0.9765625,0.93359375,0.875,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,601.0,0.005880211502039975,0.98046875,0.94140625,0.9453125,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,635.9,0.0047997324281110814,0.9921875,0.97265625,0.95703125,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,670.7,0.004150005655083656,0.9921875,0.9609375,0.94921875,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,705.4,0.0038938612596869458,0.99609375,0.98046875,0.94921875,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/plain_s1/step_50000.pt b/addition/plain_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6ba4a51cd1d6bc331dc9ec963892995fa641f87 --- /dev/null +++ b/addition/plain_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b43cd4ea777cd8b8fd33034d97522b28ab6a20afddf953275a7028985a20bc71 +size 13738485 diff --git a/addition/plain_s2/config.json b/addition/plain_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5920014f9fa12602994afc7cb150c2d2f0e3baaf --- /dev/null +++ b/addition/plain_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder plain seed 2", + "task": { + "max_digits": 20, + "reverse": false, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/plain_s2/metrics.csv b/addition/plain_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..51559ca71ecfbbd4f00a1fe8ed11a1cfaec1f547 --- /dev/null +++ b/addition/plain_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,38.3,1.2351460234888572,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,72.8,0.8908147140877666,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,107.4,0.3303994115917782,0.5625,0.23046875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,142.0,0.121963283541433,0.78125,0.5234375,0.328125,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,176.7,0.12380598067946852,0.81640625,0.58203125,0.20703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,211.4,0.06437959507165898,0.875,0.75390625,0.5546875,0.18359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,246.3,0.09695373926330297,0.90234375,0.7734375,0.640625,0.24609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,281.0,0.03774574688353087,0.87890625,0.74609375,0.54296875,0.140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,315.9,0.04852251501855399,0.90234375,0.76953125,0.65625,0.30078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,350.4,0.03372801105576202,0.91796875,0.84765625,0.734375,0.61328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,384.8,0.02787428909461228,0.97265625,0.85546875,0.765625,0.55078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,419.3,0.022389781303720466,0.9375,0.84765625,0.62109375,0.52734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,454.0,0.018736086882025,0.9609375,0.87109375,0.71875,0.62890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,488.7,0.015919025027907152,0.953125,0.88671875,0.82421875,0.71875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,523.5,0.012297407657478424,0.953125,0.890625,0.8671875,0.75,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,558.3,0.01050108779963775,0.984375,0.921875,0.85546875,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,593.0,0.008854387478807818,0.98046875,0.9140625,0.88671875,0.78515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,627.8,0.0075194296041089384,0.98046875,0.94921875,0.890625,0.765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,662.6,0.006260404597938438,0.98046875,0.94921875,0.90234375,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,697.1,0.006391584459112504,0.984375,0.95703125,0.890625,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/plain_s2/step_50000.pt b/addition/plain_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebdd1e0bfee78da680b0622abe3eb0a76398ac71 --- /dev/null +++ b/addition/plain_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af439c0424fe765d90897cfcd165a09545200a90ed9ddaedda956c1f419fa0bf +size 13738485 diff --git a/addition/position_coupling_s0/config.json b/addition/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..353a1866e4b5063c2f2e98e5e39a00220aa0ebde --- /dev/null +++ b/addition/position_coupling_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder position coupling seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/position_coupling_s0/metrics.csv b/addition/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..c89ccd784bed7da2ad5c6092ce9fbe0e21c59699 --- /dev/null +++ b/addition/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,41.7,0.05849933046576848,0.9765625,0.99609375,0.96484375,0.89453125,0.640625,0.25390625,0.0,0.0,0.0,0.0,0.0 +5000,79.5,0.0011903617309327293,0.9453125,0.94921875,0.9453125,0.91015625,0.6953125,0.4140625,0.01953125,0.0,0.0,0.0,0.0 +7500,117.6,0.0008857806100816371,0.984375,1.0,0.9921875,0.96875,0.46484375,0.140625,0.01171875,0.0,0.0,0.0,0.0 +10000,155.7,0.0005239002259623858,1.0,0.9921875,1.0,0.98828125,0.80859375,0.65625,0.16015625,0.0,0.0,0.0,0.0 +12500,193.9,0.00031071801167808805,1.0,1.0,1.0,1.0,0.9140625,0.6875,0.3046875,0.04296875,0.0,0.0,0.0 +15000,232.4,9.674870884781087e-05,1.0,1.0,1.0,1.0,0.984375,0.890625,0.4296875,0.046875,0.0,0.0,0.0 +17500,270.8,1.1628271768843432e-07,1.0,1.0,1.0,1.0,0.99609375,0.953125,0.62890625,0.13671875,0.0078125,0.0,0.0 +20000,309.3,9.673758767857765e-09,1.0,1.0,1.0,1.0,1.0,0.9765625,0.41796875,0.03125,0.0,0.0,0.0 +22500,347.8,0.00010900244251551255,1.0,1.0,0.99609375,0.99609375,0.6328125,0.33203125,0.04296875,0.0,0.0,0.0,0.0 +25000,386.3,2.2218434021451218e-05,1.0,1.0,1.0,1.0,0.984375,0.90625,0.5234375,0.0234375,0.0,0.0,0.0 +27500,424.4,0.00024115022228942076,1.0,1.0,1.0,0.99609375,0.92578125,0.60546875,0.26171875,0.00390625,0.0,0.0,0.0 +30000,462.2,1.261457133340515e-08,1.0,1.0,1.0,1.0,0.99609375,0.89453125,0.484375,0.015625,0.0,0.0,0.0 +32500,500.0,2.513377858377487e-06,1.0,1.0,1.0,1.0,0.96484375,0.74609375,0.234375,0.0,0.0,0.0,0.0 +35000,538.0,1.1292256512601291e-07,1.0,1.0,1.0,1.0,0.97265625,0.85546875,0.34765625,0.0078125,0.0,0.0,0.0 +37500,576.4,1.731580459432587e-08,1.0,1.0,1.0,1.0,0.98828125,0.890625,0.1640625,0.00390625,0.0,0.0,0.0 +40000,614.8,1.9072867737526124e-08,1.0,1.0,1.0,1.0,0.99609375,0.9296875,0.140625,0.0,0.0,0.0,0.0 +42500,653.2,2.0661178407917825e-08,1.0,1.0,1.0,1.0,0.99609375,0.94140625,0.12890625,0.0,0.0,0.0,0.0 +45000,691.5,1.976445106136551e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0 +47500,730.0,1.9826637076144926e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0 +50000,768.0,2.006347927688552e-08,1.0,1.0,1.0,1.0,0.99609375,0.921875,0.10546875,0.0,0.0,0.0,0.0 diff --git a/addition/position_coupling_s0/step_50000.pt b/addition/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bbc0719fda1d3ce7df7d90bbb6e04c45e119561 --- /dev/null +++ b/addition/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5367b75422a22700d5f4e0cf6b5f8277aba3934897b128c8810ac33d5b7d2dd0 +size 13738485 diff --git a/addition/position_coupling_s1/config.json b/addition/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fdc52abf81ea0342e7b296c745d7475adf3ef236 --- /dev/null +++ b/addition/position_coupling_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder position coupling seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/position_coupling_s1/metrics.csv b/addition/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..0ab2fe24c90e33eaf1f18fc4bab8c373bc2f0566 --- /dev/null +++ b/addition/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,41.1,0.05145847376760608,0.98046875,0.96875,0.7421875,0.43359375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +5000,78.7,0.15403296481562503,0.0859375,0.09765625,0.0625,0.125,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +7500,116.5,0.003376625910415864,1.0,0.98828125,0.96875,0.9375,0.50390625,0.0,0.0,0.0,0.0,0.0,0.0 +10000,154.7,0.0017005495388153179,0.9921875,0.98828125,0.9921875,0.921875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0 +12500,193.1,0.0008433014665956887,1.0,1.0,0.9921875,0.98046875,0.7109375,0.0,0.0,0.0,0.0,0.0,0.0 +15000,231.4,0.0011252931659162543,0.87890625,0.98046875,0.96875,0.875,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0 +17500,270.0,0.0006694048880327893,1.0,1.0,0.984375,0.984375,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0 +20000,308.4,0.0005087599628814538,1.0,1.0,1.0,0.98828125,0.859375,0.0078125,0.0,0.0,0.0,0.0,0.0 +22500,346.8,0.0003347884842192211,1.0,1.0,1.0,0.984375,0.79296875,0.01171875,0.0,0.0,0.0,0.0,0.0 +25000,385.2,0.00019936315146112815,1.0,1.0,1.0,0.984375,0.8125,0.0,0.0,0.0,0.0,0.0,0.0 +27500,423.6,0.00023418337315626932,1.0,0.99609375,0.9921875,0.9921875,0.78125,0.0,0.0,0.0,0.0,0.0,0.0 +30000,462.1,3.036543236018947e-05,1.0,1.0,0.99609375,0.9921875,0.8984375,0.0,0.0,0.0,0.0,0.0,0.0 +32500,500.8,1.1332860303378958e-06,1.0,1.0,1.0,1.0,0.97265625,0.05078125,0.0,0.0,0.0,0.0,0.0 +35000,539.6,2.2576315286650695e-05,1.0,1.0,1.0,1.0,0.8046875,0.0,0.0,0.0,0.0,0.0,0.0 +37500,578.3,6.027202978377045e-07,1.0,1.0,1.0,1.0,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0 +40000,617.2,2.0994051087808977e-09,1.0,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0 +42500,656.0,1.8263265294455678e-09,1.0,1.0,1.0,1.0,0.88671875,0.0,0.0,0.0,0.0,0.0,0.0 +45000,694.7,2.2382413536553586e-09,1.0,1.0,1.0,1.0,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0 +47500,733.4,2.342642124859866e-09,1.0,1.0,1.0,1.0,0.87890625,0.0,0.0,0.0,0.0,0.0,0.0 +50000,771.8,2.4091175711973268e-09,1.0,1.0,1.0,1.0,0.875,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/position_coupling_s1/step_50000.pt b/addition/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ea6b8f36e51f5387becf823eb55830cd81f9bf4 --- /dev/null +++ b/addition/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b948384e3edf48c94cd35c43b46e6654c612166a81c1d6db1d14267c066855 +size 13738485 diff --git a/addition/position_coupling_s2/config.json b/addition/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..32d8dc6475ab43fd33efe97aa677de4be7a2042d --- /dev/null +++ b/addition/position_coupling_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder position coupling seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/position_coupling_s2/metrics.csv b/addition/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..a40ddc873b2c6984259bbb60183fe7d0a7bbcb0b --- /dev/null +++ b/addition/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.8,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.0,0.020700893404300407,0.9375,0.921875,0.859375,0.33984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,81.0,0.05859560856595267,0.0625,0.08984375,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,118.9,0.0043224874084095,0.98046875,0.9765625,0.93359375,0.94140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,156.6,0.003302708169984759,1.0,1.0,1.0,0.87109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,194.5,0.0011700204703183339,1.0,1.0,0.99609375,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,232.7,0.0006573165836842692,1.0,0.9921875,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,270.6,0.0006703957555014628,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,308.5,0.0009429449397675106,1.0,0.9921875,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,346.4,0.00032470079907343934,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,384.3,0.00014316028949393518,1.0,1.0,1.0,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,422.4,0.000150102722229327,1.0,0.99609375,0.9921875,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,460.3,0.0002283228692740234,0.9921875,0.9921875,0.98828125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,498.5,7.47707316082816e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,536.4,1.676158338888255e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,574.4,2.1676778716729662e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,612.3,2.5858003174553218e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,650.1,2.7781036704315324e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,688.0,2.876706517140481e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,726.0,2.90941245793862e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,764.0,2.9081300761590858e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/position_coupling_s2/step_50000.pt b/addition/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1d0ddd95c03638e4576983740d9a03066aea520 --- /dev/null +++ b/addition/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5c91acc0a25734fbff87150e36eaf49784404ad6bc8474c501daca2b8a6471 +size 13738485 diff --git a/addition/position_coupling_s3/config.json b/addition/position_coupling_s3/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3026d729ec1ab4b5f01eef46f03700075bafcbef --- /dev/null +++ b/addition/position_coupling_s3/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition position coupling seed 3", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 3, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/position_coupling_s3/metrics.csv b/addition/position_coupling_s3/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..8bba2de2e990539c802d81de1cfaeb12e8211c00 --- /dev/null +++ b/addition/position_coupling_s3/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,44.1,0.004850631115049649,0.99609375,0.96484375,0.81640625,0.703125,0.02734375,0.0,0.0,0.0,0.0,0.0,0.0 +5000,83.4,0.003373821544012892,1.0,0.94921875,0.95703125,0.86328125,0.39453125,0.01171875,0.0,0.0,0.0,0.0,0.0 +7500,122.5,0.0020431649009441523,1.0,1.0,1.0,0.91796875,0.203125,0.0,0.0,0.0,0.0,0.0,0.0 +10000,161.8,0.0012933910039262292,1.0,0.99609375,0.984375,0.9609375,0.7265625,0.16796875,0.0,0.0,0.0,0.0,0.0 +12500,201.2,0.0011156448547994355,1.0,0.99609375,1.0,0.9609375,0.640625,0.06640625,0.0,0.0,0.0,0.0,0.0 +15000,240.2,0.001028409785681,1.0,1.0,0.9765625,0.94140625,0.80078125,0.09375,0.0,0.0,0.0,0.0,0.0 +17500,279.2,0.0004935246820961545,0.99609375,0.9921875,0.98828125,0.9765625,0.796875,0.2109375,0.0,0.0,0.0,0.0,0.0 +20000,318.6,0.0003446493452945265,1.0,1.0,0.99609375,0.9921875,0.8984375,0.34765625,0.0,0.0,0.0,0.0,0.0 +22500,357.9,0.00035101331179737916,1.0,1.0,0.98828125,0.984375,0.83203125,0.33203125,0.0,0.0,0.0,0.0,0.0 +25000,396.9,0.00015035623130311442,1.0,1.0,0.99609375,1.0,0.86328125,0.25390625,0.0,0.0,0.0,0.0,0.0 +27500,436.1,9.998802317709618e-05,0.99609375,0.99609375,1.0,1.0,0.8203125,0.25,0.0,0.0,0.0,0.0,0.0 +30000,475.6,9.292084105669427e-06,1.0,1.0,1.0,1.0,0.86328125,0.34765625,0.0,0.0,0.0,0.0,0.0 +32500,514.8,2.1721334429399254e-05,1.0,1.0,1.0,0.99609375,0.89453125,0.3125,0.0,0.0,0.0,0.0,0.0 +35000,553.8,2.0419899765934713e-06,1.0,1.0,1.0,1.0,0.95703125,0.484375,0.0,0.0,0.0,0.0,0.0 +37500,593.3,1.9998270895216435e-08,1.0,1.0,1.0,1.0,0.94140625,0.51171875,0.0,0.0,0.0,0.0,0.0 +40000,632.7,2.327768390220642e-09,1.0,1.0,1.0,1.0,0.96875,0.4296875,0.0,0.0,0.0,0.0,0.0 +42500,672.0,3.7369679962519325e-09,1.0,1.0,1.0,1.0,0.97265625,0.3984375,0.0,0.0,0.0,0.0,0.0 +45000,711.1,5.400074163988241e-09,1.0,1.0,1.0,1.0,0.97265625,0.3828125,0.0,0.0,0.0,0.0,0.0 +47500,750.4,5.933824072106011e-09,1.0,1.0,1.0,1.0,0.9765625,0.38671875,0.0,0.0,0.0,0.0,0.0 +50000,789.8,6.284888925721629e-09,1.0,1.0,1.0,1.0,0.9765625,0.390625,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/position_coupling_s3/step_50000.pt b/addition/position_coupling_s3/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bc923cb6b21a16a797a70b04670dbb37512e7e3 --- /dev/null +++ b/addition/position_coupling_s3/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:802ab5e7efb3a7a359c38d3c226df80935f5deedb617ac43ee4fdd7728b1e2f4 +size 13738485 diff --git a/addition/position_coupling_s4/config.json b/addition/position_coupling_s4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a8b3a622b34a5300de8e288e83408abe6e0d3341 --- /dev/null +++ b/addition/position_coupling_s4/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition position coupling seed 4", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 4, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/position_coupling_s4/metrics.csv b/addition/position_coupling_s4/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..4147c5875026af74be35cfd6fc1c9d614b450173 --- /dev/null +++ b/addition/position_coupling_s4/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.9,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.0,0.22518860345102612,0.921875,0.99609375,0.96875,0.46875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,82.3,0.04724631871449029,0.8828125,0.97265625,0.96875,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,121.5,0.0410524533403662,1.0,0.9921875,0.99609375,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,160.8,0.0030762999872309136,0.94140625,1.0,1.0,0.93359375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,199.7,0.0028580032175879178,0.8828125,0.98828125,0.984375,0.921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,239.2,0.001117561123859787,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,278.5,0.0024052515046375264,1.0,1.0,0.9921875,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,318.0,0.0008923539224610124,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,357.3,0.0006949997509023186,1.0,0.99609375,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,396.8,0.0004422316466865576,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,435.9,0.00022831150932432664,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,475.1,4.3577316504452787e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,514.4,6.471081701567521e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,553.8,6.525275389128426e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,592.9,8.21924206633759e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,632.1,1.3325394863318211e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,671.4,1.6287152584307998e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,710.7,1.7920361956161393e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,749.9,1.8642637299402013e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,789.3,1.87153014685498e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/position_coupling_s4/step_50000.pt b/addition/position_coupling_s4/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad3d944329bde4048fe52f23a549a1e9fbae4078 --- /dev/null +++ b/addition/position_coupling_s4/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73ff277da2d3cb713abd5342f9aeb0c6b1d320f1931fec85f181bbd8149022ff +size 13738485 diff --git a/addition/position_coupling_s5/config.json b/addition/position_coupling_s5/config.json new file mode 100644 index 0000000000000000000000000000000000000000..68875f5722120ced9e6bab81834dfb2246ecdd8e --- /dev/null +++ b/addition/position_coupling_s5/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition position coupling seed 5", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 5, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/position_coupling_s5/metrics.csv b/addition/position_coupling_s5/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..e09789ae0316b619a11c67b9985bda815dfaf036 --- /dev/null +++ b/addition/position_coupling_s5/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.3,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.3,0.08161965796299628,0.79296875,0.921875,0.83984375,0.39453125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,82.5,0.018591034643309738,0.01953125,0.01171875,0.01953125,0.02734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,122.0,0.03994800915542356,1.0,1.0,0.9921875,0.94921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,161.0,0.0029848094861992403,1.0,1.0,0.984375,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,200.0,0.0016353042876490662,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,239.1,0.0015699884993066752,1.0,1.0,1.0,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,278.4,0.0011652278045353836,0.87890625,0.98828125,0.9609375,0.9609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,317.5,0.00016850536301121156,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,356.6,0.0008801457530294086,1.0,1.0,1.0,0.70703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,395.7,7.339332692703692e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,435.0,0.0004002674427832871,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,474.0,1.704690900100181e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,513.0,2.604397256955884e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,551.9,3.2321118283948744e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,591.4,3.4022926796971225e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,630.5,3.463443880508178e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,669.5,3.515947748682975e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,708.6,3.5563348357768966e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,747.8,3.5131199542851266e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,786.8,3.4826679801799004e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/position_coupling_s5/step_50000.pt b/addition/position_coupling_s5/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b9112eee0c67584e0bb3637c8f7b3d633f5de43 --- /dev/null +++ b/addition/position_coupling_s5/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc32ed5138e2f36907e9d46170a2230d8a564e10019d380231367c23f7f446b2 +size 13738485 diff --git a/addition/reversed_s0/config.json b/addition/reversed_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e17304043a2e5b2e7e57dba229394c4e985bdb --- /dev/null +++ b/addition/reversed_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/reversed_s0/metrics.csv b/addition/reversed_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..1583968d4c74b8afa2ab94843f9123b30a2ee742 --- /dev/null +++ b/addition/reversed_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,37.5,0.6899808970337034,0.02734375,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,71.5,0.11962956568376582,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,105.7,0.09103466037908249,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,139.9,0.06821191482090103,1.0,0.96484375,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,174.4,0.0001272123632397109,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,208.7,0.008226176047167542,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,243.2,7.887461830654756e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,277.7,3.3091967498198034e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,312.1,0.001415348713319323,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,346.4,3.8733289335832e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,380.6,2.2778572673513522e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,414.7,0.000505518517989372,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,448.9,5.484113087084111e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,483.2,5.711551111572616e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,517.6,4.2846015616914855e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,552.2,3.535017908514598e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,586.8,3.446250352525007e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,621.4,3.434506841888134e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,655.9,3.450636705566721e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,690.3,3.442937016086712e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/reversed_s0/step_50000.pt b/addition/reversed_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e02d2d2c9f0295241fa41f7a067bb560a7ab9c2 --- /dev/null +++ b/addition/reversed_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9619da3b81ad150b590b8672b21247528f88ff9aec3490680f116e296b5717a0 +size 13738485 diff --git a/addition/reversed_s1/config.json b/addition/reversed_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6f72fa98e13e184a54a74e4524d6d9299556ad --- /dev/null +++ b/addition/reversed_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/reversed_s1/metrics.csv b/addition/reversed_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..58855eb871b80464f2d89650ef577548fd86de24 --- /dev/null +++ b/addition/reversed_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,37.6,0.6935743615820404,0.0703125,0.0625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,71.8,0.5975450208887972,0.078125,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,105.9,0.4572495186125052,0.11328125,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,140.1,0.012656225491346918,1.0,0.99609375,0.90234375,0.03125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,174.6,0.005384431403670865,0.90625,1.0,0.83203125,0.22265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,209.0,0.006117246615816689,1.0,1.0,1.0,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,243.3,0.0034891020506632574,1.0,1.0,0.99609375,0.73828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,277.7,0.0057696616780318006,1.0,0.69140625,0.984375,0.828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,312.2,0.0009214650922157513,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,346.8,0.0010159356764143335,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,381.3,0.0016378494398793593,0.9921875,0.99609375,1.0,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,415.4,0.0006733740390607537,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,449.6,0.00015172600247770622,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,483.9,7.84259512676389e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,518.3,2.6318664865720264e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,552.8,4.334490324539598e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,587.2,6.986863678033721e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,621.7,8.482358137454979e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,656.1,9.151078649171663e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,690.6,9.153665366514486e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/reversed_s1/step_50000.pt b/addition/reversed_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..206e6351dd3db9b553ffb247befae2b69ab0c871 --- /dev/null +++ b/addition/reversed_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052fce71ee826422eb6682b771eb88f0ae59411d5bd0ae5666077e95d82f760c +size 13738485 diff --git a/addition/reversed_s2/config.json b/addition/reversed_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e73856e3d6ec24585136212769833b26ddac4f09 --- /dev/null +++ b/addition/reversed_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": false, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/reversed_s2/metrics.csv b/addition/reversed_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..487782772516f6250be1c1f3779f5bc7439311b1 --- /dev/null +++ b/addition/reversed_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,37.8,1.1030993474196946,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,72.0,0.049267443183339234,1.0,1.0,1.0,0.79296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,106.2,0.025880768940122825,0.9921875,1.0,0.99609375,0.81640625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,140.1,0.033528285323752245,1.0,1.0,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,174.4,0.0039047176543949573,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,208.8,0.002880435906337188,1.0,0.95703125,1.0,0.9453125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,243.4,0.0002447265964478629,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,277.8,0.0003152005626777964,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,312.4,0.001328313205134429,0.99609375,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,346.9,0.00035081833219575097,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,381.3,0.0006951169616613335,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,415.5,0.00025632233376511165,1.0,0.92578125,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,449.7,9.637635914961821e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,483.9,1.794882481248124e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,518.2,2.02643950102753e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,552.5,2.5137038381641247e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,586.9,2.6688879258399354e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,621.2,2.7669927692196918e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,655.7,2.8053395570058626e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,690.2,2.82486893320944e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/reversed_s2/step_50000.pt b/addition/reversed_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..96f3caa40dcc544f806ebb46613344f784ed38f2 --- /dev/null +++ b/addition/reversed_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d41daa6813cccf16a4743b6bceb7a032d21e309a1819e1dc53625caaa181d0 +size 13738485 diff --git a/addition/reversed_zero_pad_s0/config.json b/addition/reversed_zero_pad_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..11c9eddf6fa00bafcc85f309c98304723e50da67 --- /dev/null +++ b/addition/reversed_zero_pad_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed + zero pad seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/reversed_zero_pad_s0/metrics.csv b/addition/reversed_zero_pad_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..e1efe9dc85ce035e860c6d95b7c8d3ef207d71d0 --- /dev/null +++ b/addition/reversed_zero_pad_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.1,0.06619868960771663,1.0,1.0,1.0,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,74.5,0.0017159350450853426,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,109.7,0.007220026861467229,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,145.0,4.138583902967584e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,180.4,0.0017232142162460764,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,216.0,5.582981389100563e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,251.7,0.0028660412605447704,0.98828125,1.0,0.98828125,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,287.4,0.0016964663427645665,1.0,1.0,0.99609375,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,323.1,4.693074184738278e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,358.9,1.5369728299991919e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,394.6,1.354610709535728e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,430.3,7.381509885401397e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,465.8,2.8535493048394157e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,501.3,5.019004728542278e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,536.7,3.140849492873736e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,572.2,3.371324708507817e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,607.9,3.425680808133598e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,643.6,3.454033180070662e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,679.2,3.45309485176879e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,715.0,3.4386437766012695e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/reversed_zero_pad_s0/step_50000.pt b/addition/reversed_zero_pad_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf64988ed33fe42efa0fe2d93099db7eb7c3715a --- /dev/null +++ b/addition/reversed_zero_pad_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c83cdd60823c2fbe39e932b8ea1532146777366e317f8fcd44aafd019cf36489 +size 13738485 diff --git a/addition/reversed_zero_pad_s1/config.json b/addition/reversed_zero_pad_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..900f60fe2cf2a8e2a6999d005c213e0efac1b33e --- /dev/null +++ b/addition/reversed_zero_pad_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed + zero pad seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/reversed_zero_pad_s1/metrics.csv b/addition/reversed_zero_pad_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..9269de5ef4321188cacf7dc7cf6b07cbfedef771 --- /dev/null +++ b/addition/reversed_zero_pad_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.1,0.505330329683363,0.88671875,0.87109375,0.83203125,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,74.7,0.013944046411834343,0.97265625,0.9765625,0.99609375,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,110.2,0.0054805784648486325,0.99609375,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,145.7,0.002925935737893005,0.99609375,0.99609375,1.0,0.97265625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,181.1,0.00786076482663685,0.9453125,0.99609375,1.0,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,216.6,0.002306089214874477,0.42578125,0.99609375,0.9921875,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,252.2,9.141079915561852e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,287.8,7.74360808163545e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,323.5,0.0009985137306061157,1.0,1.0,0.99609375,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,359.3,3.4804709802909685e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,395.0,1.5507826306689775e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,430.6,7.562815979003721e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,466.1,1.9148897382522122e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,501.7,2.456168471434791e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,537.1,3.037901886569355e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,572.7,3.132561227860439e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,608.4,3.180539363786996e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,644.0,3.206723244783721e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,679.7,3.2186608966719705e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,715.4,3.228653236357116e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/reversed_zero_pad_s1/step_50000.pt b/addition/reversed_zero_pad_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ee6ee0b25cfe2d88a39f6e819a9d496dbea6866 --- /dev/null +++ b/addition/reversed_zero_pad_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11770eda2d4f3f239ecf488e96b38ebeff5bc17d78babf2c8f40e4d8a5c95d25 +size 13738485 diff --git a/addition/reversed_zero_pad_s2/config.json b/addition/reversed_zero_pad_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b099555a13167eb0e9c0290334b63726c2c1b81e --- /dev/null +++ b/addition/reversed_zero_pad_s2/config.json @@ -0,0 +1,52 @@ +{ + "name": "ladder reversed + zero pad seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "positional": true, + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/reversed_zero_pad_s2/metrics.csv b/addition/reversed_zero_pad_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..afce8cde7c6f8ae45abf8dbee81806df86bdebae --- /dev/null +++ b/addition/reversed_zero_pad_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,39.1,0.1500092168730642,0.9765625,0.953125,0.85546875,0.04296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,74.8,0.003555504089126276,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,110.3,0.0009906474932091412,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,145.7,0.0010326231041283284,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,181.1,2.2762600781179806e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,216.7,4.521779913216281e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,252.3,5.024763246116733e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,288.1,6.7219712028696075e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,323.8,0.0005750678458532328,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,359.6,1.8484896897509966e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,395.4,2.342360363074533e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,431.1,0.0001828320891425703,0.99609375,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,466.6,2.327544670335616e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,502.0,1.1036670111638973e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,537.5,2.9005017205428075e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,573.0,2.9550467564524477e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,608.6,3.156387802874369e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,644.3,3.198746761356656e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,680.0,3.1998500999506706e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,715.7,3.182416284321898e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/reversed_zero_pad_s2/step_50000.pt b/addition/reversed_zero_pad_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce3d04e9f4da94c12f2fcacfe3c2b2f026786f69 --- /dev/null +++ b/addition/reversed_zero_pad_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ac5cd371214ccc1107553d40bbb95a29e789ed8337b19d2225be0d2184b2dda +size 13738485 diff --git a/addition/zero_pad_relative_s0/config.json b/addition/zero_pad_relative_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3f794d1fc2a0eb7f13f41c348991c729d9c57907 --- /dev/null +++ b/addition/zero_pad_relative_s0/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition zero pad + relative seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "relative", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/zero_pad_relative_s0/metrics.csv b/addition/zero_pad_relative_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..321f41e0eb3dd1bafe98ae64570cecc60c3d4768 --- /dev/null +++ b/addition/zero_pad_relative_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.6,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,44.3,1.5913414430910537,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,85.0,1.090214231854473,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,125.8,0.7170236909072951,0.9375,0.53125,0.34765625,0.05078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,166.4,0.03833440840576273,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,207.1,0.009139366658671717,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,247.9,0.07064036116655499,0.99609375,0.95703125,0.99609375,0.921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,288.9,0.0007408642528555264,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,329.6,5.852601976878764e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,370.3,0.01632626176781026,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,410.7,0.0008980597834440186,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,451.2,4.5746371402686947e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,491.5,0.022908393495129485,1.0,0.8984375,0.921875,0.59765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,532.4,1.5905145100652976e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,573.1,0.00029099377585137807,1.0,1.0,0.9921875,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,613.9,4.992979753726521e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,654.5,0.00010090279990380237,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,695.3,6.804399840174375e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,735.8,8.115773612697506e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,776.4,8.92655252125933e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,817.0,9.064875258489247e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/zero_pad_relative_s0/step_50000.pt b/addition/zero_pad_relative_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..474f3401b79a4abdd3ef51d96fb5b0bf794f9531 --- /dev/null +++ b/addition/zero_pad_relative_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af2d15e016ef7dbf6ceb5c7ee4b4dad90cbfa74a98e7f78098e4e5ea30085064 +size 12756597 diff --git a/addition/zero_pad_relative_s1/config.json b/addition/zero_pad_relative_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..600be9b525e06ade756f5dbe3495cde052d2a1d7 --- /dev/null +++ b/addition/zero_pad_relative_s1/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition zero pad + relative seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "relative", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/zero_pad_relative_s1/metrics.csv b/addition/zero_pad_relative_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..8620b32cb8927b1dff372153e6bf36e43ef47ceb --- /dev/null +++ b/addition/zero_pad_relative_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.6,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,44.6,1.2340895072843698,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,85.1,0.4038156209362173,0.0078125,0.00390625,0.015625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,125.7,0.036453592387947126,0.9921875,0.9921875,0.98828125,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,166.4,8.53868517858002e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,207.1,0.0016759939967765048,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,247.9,0.0030648796563260017,0.98828125,0.98046875,0.97265625,0.90625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,288.5,0.0005731670398736295,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,329.1,0.014302217634793246,0.9921875,0.9765625,0.9921875,0.953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,369.6,0.0011272604608261428,1.0,0.99609375,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,410.0,0.0077736075512846465,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,450.7,5.051744327402895e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,491.7,4.292800075773495e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,532.4,0.0002819249736445851,1.0,0.98828125,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,573.2,1.4115901554110353e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,613.6,0.00024883499709011205,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,654.2,1.0098377355404627e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,694.9,1.4159944669224052e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,735.6,1.6079427964892577e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,776.3,1.6740952209750877e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,816.9,1.6735705897858667e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/zero_pad_relative_s1/step_50000.pt b/addition/zero_pad_relative_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f985e19f493fd1445f4fc360e09af6c6af4edf4e --- /dev/null +++ b/addition/zero_pad_relative_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6a458ae3b37b26d5998c94366312101e3d416470862d599d3c2ec18d912c26d +size 12756597 diff --git a/addition/zero_pad_relative_s2/config.json b/addition/zero_pad_relative_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f34c28463113e794e793ce2e07e22e475d39aa32 --- /dev/null +++ b/addition/zero_pad_relative_s2/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition zero pad + relative seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "relative", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition/zero_pad_relative_s2/metrics.csv b/addition/zero_pad_relative_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..64d11dc2c90b06d0ffadd57f3ff3f3198219df90 --- /dev/null +++ b/addition/zero_pad_relative_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.6,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,44.3,1.6930036471402574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,85.1,0.5565985109605073,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,125.7,0.008675566044231541,1.0,0.9921875,0.99609375,0.875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,166.3,0.027710440318138168,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,206.8,0.0030650454623728555,1.0,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,247.4,0.00039695932170701704,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,288.1,0.001020204009082941,1.0,0.99609375,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,328.9,1.0322389896251214e-05,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,369.6,0.010915066739945745,1.0,0.98828125,0.98828125,0.83984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,410.5,0.00025989802697349586,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,451.3,6.106662808232821e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,492.1,0.0008971799019560514,0.97265625,0.9765625,0.98046875,0.92578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,532.8,7.879301155842676e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,573.4,1.2646747469807025e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,614.1,0.00016088087315540043,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,654.8,3.740238021555492e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,695.6,1.609775147054656e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,736.4,1.7636598377101813e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,777.2,1.807492042023307e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,818.0,1.8732512350554365e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition/zero_pad_relative_s2/step_50000.pt b/addition/zero_pad_relative_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..40ee4d84ffcde6e35efa83a98ebdb365ccb827a5 --- /dev/null +++ b/addition/zero_pad_relative_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8fd2e647084571e8add864bfdbbac9d575472e0062de8bee5a54f4cac35bab7 +size 12756597 diff --git a/addition_big/abacus_s0/config.json b/addition_big/abacus_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85ec73d89b69e09a10b306bb353ab4194e898f5f --- /dev/null +++ b/addition_big/abacus_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "addition_big abacus seed 0", + "task": { + "max_digits": 30, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 200, + "test_digits": [ + 10, + 20, + 30, + 40, + 50, + 60, + 80, + 100, + 150, + 200 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 384, + "n_layers": 6, + "n_heads": 6, + "d_mlp": 1536, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 60000, + "batch_size": 384, + "lr": 0.0002, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 2000, + "cosine": true, + "amp": true, + "eval_every": 5000, + "checkpoint_every": 20000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_big/abacus_s0/metrics.csv b/addition_big/abacus_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..7fbaa507284ddc04468c26e45500475b0e4372d1 --- /dev/null +++ b/addition_big/abacus_s0/metrics.csv @@ -0,0 +1,14 @@ +step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200 +0,392.9,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,724.0,0.015931887146874192,0.92578125,0.8671875,0.1875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,1054.7,0.003162519941155048,0.9921875,0.9921875,0.87109375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,1385.1,0.0018523827192165538,1.0,1.0,0.96875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1715.5,0.0008092147650410454,1.0,0.984375,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,2047.0,0.0008986434196728957,1.0,0.99609375,0.890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,2377.4,0.00036964330554701815,1.0,0.9921875,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2708.1,0.00015256808633070315,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,3039.7,2.1969347841899055e-05,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,3371.3,3.3908628197133478e-09,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3701.5,7.988891602315237e-09,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +55000,4031.6,1.0144761406196326e-08,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +60000,4361.4,1.047814085527202e-08,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_big/abacus_s0/step_60000.pt b/addition_big/abacus_s0/step_60000.pt new file mode 100644 index 0000000000000000000000000000000000000000..95d36c5539e63c6ef401dd370acb8ae9579eabc3 --- /dev/null +++ b/addition_big/abacus_s0/step_60000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab7c1b24a9dae24afd657ae1aa104d6c85c01c1533c267aace83944907d106c +size 44239477 diff --git a/addition_big/abacus_s1/config.json b/addition_big/abacus_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4311ed03533767b0827cd943754d17e6e54e4e72 --- /dev/null +++ b/addition_big/abacus_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "addition_big abacus seed 1", + "task": { + "max_digits": 30, + "reverse": true, + "zero_pad": false, + "positions": "abacus", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 200, + "test_digits": [ + 10, + 20, + 30, + 40, + 50, + 60, + 80, + 100, + 150, + 200 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 384, + "n_layers": 6, + "n_heads": 6, + "d_mlp": 1536, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 60000, + "batch_size": 384, + "lr": 0.0002, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 2000, + "cosine": true, + "amp": true, + "eval_every": 5000, + "checkpoint_every": 20000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_big/abacus_s1/metrics.csv b/addition_big/abacus_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..b1e60d1d45fca6361c8da4aae47edbeec76b0d6b --- /dev/null +++ b/addition_big/abacus_s1/metrics.csv @@ -0,0 +1,14 @@ +step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200 +0,373.0,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,704.7,0.020500686424957563,0.99609375,0.953125,0.55078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,1034.9,0.00286406126701662,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,1364.8,0.0017564902773053162,1.0,0.98828125,0.90234375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,1695.3,0.0005959988102150061,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,2026.9,0.0003559402355249067,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,2357.2,0.00028341319483821614,1.0,1.0,0.96875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,2687.7,0.00010348523441575124,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,3019.2,5.050371341448431e-05,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,3350.8,1.2222725262004333e-07,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,3681.5,5.2916664659985944e-09,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +55000,4011.7,7.305537222767434e-09,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +60000,4341.8,7.792950942513793e-09,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_big/abacus_s1/step_60000.pt b/addition_big/abacus_s1/step_60000.pt new file mode 100644 index 0000000000000000000000000000000000000000..20ddc628e59215f5e2af3e23a73c7c0b76289aee --- /dev/null +++ b/addition_big/abacus_s1/step_60000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68c89a14dbae04171e9cef6533c84fb82d6a1240d9ae48e5ee26900193c4af3a +size 44239477 diff --git a/addition_big/position_coupling_s0/config.json b/addition_big/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..11b1b74f177c6ebcd77660ee57c5340df825d361 --- /dev/null +++ b/addition_big/position_coupling_s0/config.json @@ -0,0 +1,52 @@ +{ + "name": "addition_big position coupling seed 0", + "task": { + "max_digits": 30, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 200, + "test_digits": [ + 10, + 20, + 30, + 40, + 50, + 60, + 80, + 100, + 150, + 200 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 384, + "n_layers": 6, + "n_heads": 6, + "d_mlp": 1536, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 60000, + "batch_size": 384, + "lr": 0.0002, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 2000, + "cosine": true, + "amp": true, + "eval_every": 5000, + "checkpoint_every": 20000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_big/position_coupling_s0/metrics.csv b/addition_big/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..5061c643cf7d565e803fce64363cc39a73752d8e --- /dev/null +++ b/addition_big/position_coupling_s0/metrics.csv @@ -0,0 +1,14 @@ +step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200 +0,369.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,707.7,0.0021335504189133126,1.0,0.9765625,0.88671875,0.08984375,0.0,0.0,0.0,0.0,0.0,0.0 +10000,1045.9,0.0005988225522752769,1.0,1.0,0.97265625,0.46875,0.02734375,0.0,0.0,0.0,0.0,0.0 +15000,1383.6,0.0003596282952293136,1.0,1.0,0.9921875,0.77734375,0.125,0.0,0.0,0.0,0.0,0.0 +20000,1721.2,0.0001952215258099061,1.0,1.0,0.9921875,0.83203125,0.16015625,0.0,0.0,0.0,0.0,0.0 +25000,2060.2,0.00013690077699715804,1.0,1.0,1.0,0.84375,0.0546875,0.0,0.0,0.0,0.0,0.0 +30000,2398.0,5.8673442737359475e-05,1.0,1.0,1.0,0.97265625,0.6015625,0.046875,0.0,0.0,0.0,0.0 +35000,2735.8,2.2766635645277487e-05,1.0,0.99609375,1.0,0.97265625,0.23046875,0.0,0.0,0.0,0.0,0.0 +40000,3074.7,2.1722178719549003e-06,1.0,1.0,1.0,1.0,0.73046875,0.078125,0.0,0.0,0.0,0.0 +45000,3413.9,4.8574869786049955e-06,1.0,1.0,1.0,0.9921875,0.64453125,0.01171875,0.0,0.0,0.0,0.0 +50000,3751.8,1.7049233275240674e-09,1.0,1.0,1.0,0.9921875,0.52734375,0.0,0.0,0.0,0.0,0.0 +55000,4090.0,1.8582465040273468e-09,1.0,1.0,1.0,0.9921875,0.41796875,0.0,0.0,0.0,0.0,0.0 +60000,4428.0,2.0082123903360907e-09,1.0,1.0,1.0,0.98828125,0.40625,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_big/position_coupling_s0/step_60000.pt b/addition_big/position_coupling_s0/step_60000.pt new file mode 100644 index 0000000000000000000000000000000000000000..883a8265c184c93e407160916d3a5bf991cc24f1 --- /dev/null +++ b/addition_big/position_coupling_s0/step_60000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b233e94f3a6bb43c1ee1cd9194526317b32d2814d22e4d4da72c9e1a6ce55db9 +size 44239477 diff --git a/addition_big/position_coupling_s1/config.json b/addition_big/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..588212283d6fe07dd78f546b369f60059f6674d8 --- /dev/null +++ b/addition_big/position_coupling_s1/config.json @@ -0,0 +1,52 @@ +{ + "name": "addition_big position coupling seed 1", + "task": { + "max_digits": 30, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 200, + "test_digits": [ + 10, + 20, + 30, + 40, + 50, + 60, + 80, + 100, + 150, + 200 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 384, + "n_layers": 6, + "n_heads": 6, + "d_mlp": 1536, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 60000, + "batch_size": 384, + "lr": 0.0002, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 2000, + "cosine": true, + "amp": true, + "eval_every": 5000, + "checkpoint_every": 20000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_big/position_coupling_s1/metrics.csv b/addition_big/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..cf88679163e070ae134a316f00f5773ddd4a90cc --- /dev/null +++ b/addition_big/position_coupling_s1/metrics.csv @@ -0,0 +1,14 @@ +step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200 +0,373.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,712.9,0.0010000877147622843,1.0,0.97265625,0.91015625,0.3828125,0.0078125,0.0,0.0,0.0,0.0,0.0 +10000,1051.2,0.00017556309426417974,1.0,1.0,1.0,0.984375,0.9375,0.671875,0.171875,0.0234375,0.0,0.0 +15000,1389.2,1.581491136558231e-08,1.0,1.0,1.0,1.0,0.984375,0.84765625,0.47265625,0.10546875,0.0,0.0 +20000,1727.7,0.00015942439237103236,1.0,1.0,1.0,0.89453125,0.86328125,0.7578125,0.24609375,0.02734375,0.0,0.0 +25000,2067.4,5.512708368853023e-05,1.0,1.0,1.0,0.97265625,0.8671875,0.7890625,0.21875,0.015625,0.0,0.0 +30000,2405.4,5.754868245976826e-06,1.0,1.0,1.0,0.94140625,0.8203125,0.49609375,0.0703125,0.00390625,0.0,0.0 +35000,2744.1,3.2070465901703698e-09,1.0,1.0,1.0,0.9921875,0.94921875,0.75,0.12890625,0.0,0.0,0.0 +40000,3082.2,2.4690369814382186e-08,1.0,1.0,1.0,1.0,0.98046875,0.80859375,0.09375,0.0,0.0,0.0 +45000,3421.5,2.8582572837779553e-08,1.0,1.0,1.0,1.0,0.96484375,0.73046875,0.03125,0.0,0.0,0.0 +50000,3759.4,3.0093426596659054e-08,1.0,1.0,1.0,1.0,0.96875,0.734375,0.0234375,0.0,0.0,0.0 +55000,4097.4,3.0713027076380115e-08,1.0,1.0,1.0,1.0,0.9765625,0.7421875,0.01953125,0.0,0.0,0.0 +60000,4435.6,3.072274956838485e-08,1.0,1.0,1.0,1.0,0.97265625,0.74609375,0.015625,0.0,0.0,0.0 diff --git a/addition_big/position_coupling_s1/step_60000.pt b/addition_big/position_coupling_s1/step_60000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5025b44185a78e12771ca3a3fb0053ed0eac415c --- /dev/null +++ b/addition_big/position_coupling_s1/step_60000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf63aeced97689e3474204d9ee9e2ef06a8f3632c27a4d578fcf3d060fd5ae42 +size 44239477 diff --git a/addition_carry_heavy/position_coupling_s0/config.json b/addition_carry_heavy/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ebb3220cc4519882a1410cb86e51b9b165e6d100 --- /dev/null +++ b/addition_carry_heavy/position_coupling_s0/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_carry_heavy position coupling seed 0", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.5, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_carry_heavy/position_coupling_s0/metrics.csv b/addition_carry_heavy/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..5c66116535a0056999498c769b1b54cc3252e095 --- /dev/null +++ b/addition_carry_heavy/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.2,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,50.2,0.010676809231595598,0.9765625,0.97265625,0.90625,0.87890625,0.515625,0.3046875,0.0078125,0.0,0.0,0.0,0.0 +5000,96.9,0.002054925492752434,0.99609375,1.0,0.9765625,0.9453125,0.703125,0.48828125,0.046875,0.0,0.0,0.0,0.0 +7500,143.5,0.0014871896823870972,0.98046875,0.97265625,1.0,0.9453125,0.828125,0.63671875,0.21484375,0.015625,0.00390625,0.0,0.0 +10000,190.8,0.0009028255073633367,0.99609375,0.99609375,0.9921875,0.9921875,0.8984375,0.84765625,0.23828125,0.00390625,0.0,0.0,0.0 +12500,238.9,0.0005924529243804312,1.0,0.99609375,0.98046875,0.9765625,0.875,0.703125,0.1328125,0.0078125,0.0,0.0,0.0 +15000,286.4,5.893205959863709e-05,1.0,1.0,1.0,1.0,1.0,0.984375,0.71484375,0.2265625,0.046875,0.0,0.0 +17500,332.7,0.0002964308436559661,1.0,1.0,1.0,1.0,0.9765625,0.93359375,0.6484375,0.109375,0.00390625,0.0,0.0 +20000,379.5,0.00011092086269267702,1.0,1.0,1.0,1.0,0.95703125,0.8515625,0.2109375,0.0078125,0.0,0.0,0.0 +22500,425.8,2.284288949690613e-05,1.0,1.0,1.0,1.0,0.98828125,0.95703125,0.546875,0.05859375,0.0,0.0,0.0 +25000,471.8,1.2733576755018532e-05,1.0,1.0,1.0,1.0,0.98828125,0.95703125,0.5703125,0.04296875,0.0,0.0,0.0 +27500,519.2,9.310291505739755e-05,1.0,1.0,1.0,0.9921875,0.95703125,0.8515625,0.4296875,0.0703125,0.0,0.0,0.0 +30000,564.9,1.756804747224651e-05,1.0,1.0,1.0,1.0,0.97265625,0.90625,0.24609375,0.00390625,0.0,0.0,0.0 +32500,610.7,9.117580115125206e-06,1.0,1.0,1.0,1.0,0.9453125,0.84375,0.171875,0.0,0.0,0.0,0.0 +35000,656.5,4.8601968237810115e-06,1.0,1.0,1.0,1.0,0.921875,0.8203125,0.1796875,0.0,0.0,0.0,0.0 +37500,703.9,2.6929957575639504e-09,1.0,1.0,1.0,1.0,0.9921875,0.94140625,0.48828125,0.0234375,0.0,0.0,0.0 +40000,750.6,2.911435900174519e-09,1.0,1.0,1.0,1.0,0.99609375,0.96875,0.50390625,0.01953125,0.0,0.0,0.0 +42500,797.6,1.1904759367759617e-08,1.0,1.0,1.0,1.0,0.99609375,0.9921875,0.6171875,0.0390625,0.0,0.0,0.0 +45000,844.2,8.022760837244727e-09,1.0,1.0,1.0,1.0,0.99609375,0.98046875,0.46484375,0.0234375,0.0,0.0,0.0 +47500,891.5,8.786156917115539e-09,1.0,1.0,1.0,1.0,0.99609375,0.97265625,0.43359375,0.01171875,0.0,0.0,0.0 +50000,938.4,8.843949006739112e-09,1.0,1.0,1.0,1.0,0.99609375,0.97265625,0.4296875,0.0078125,0.0,0.0,0.0 diff --git a/addition_carry_heavy/position_coupling_s0/step_50000.pt b/addition_carry_heavy/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c488059e12f55c889899985ad5a8497688142d0a --- /dev/null +++ b/addition_carry_heavy/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:435bfddb2374e79543739445955fc866e7dab71629a6c798861caf47fd194f53 +size 13738485 diff --git a/addition_carry_heavy/position_coupling_s1/config.json b/addition_carry_heavy/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2f39403f63dc68b6eae8ce6073635f04d37414cf --- /dev/null +++ b/addition_carry_heavy/position_coupling_s1/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_carry_heavy position coupling seed 1", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.5, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_carry_heavy/position_coupling_s1/metrics.csv b/addition_carry_heavy/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..ba0d302ed13f919d2e06a0f5ad264e3bc3785c89 --- /dev/null +++ b/addition_carry_heavy/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,49.5,0.004940971778743328,0.9609375,0.99609375,0.9921875,0.98828125,0.83984375,0.0078125,0.0,0.0,0.0,0.0,0.0 +5000,94.4,0.0008512423491482025,0.9921875,0.9921875,0.9765625,0.95703125,0.42578125,0.015625,0.0,0.0,0.0,0.0,0.0 +7500,139.3,0.0007970085632052872,1.0,0.9921875,1.0,1.0,0.77734375,0.02734375,0.0,0.0,0.0,0.0,0.0 +10000,184.7,3.2021044885313054e-05,1.0,1.0,1.0,1.0,0.9921875,0.0703125,0.0,0.0,0.0,0.0,0.0 +12500,232.0,0.0004865186449694645,1.0,1.0,1.0,0.9765625,0.91796875,0.09765625,0.0,0.0,0.0,0.0,0.0 +15000,279.1,0.00025899607932744,1.0,1.0,1.0,0.99609375,0.97265625,0.10546875,0.0,0.0,0.0,0.0,0.0 +17500,325.8,0.00024072623703026857,1.0,1.0,1.0,1.0,0.94921875,0.17578125,0.0,0.0,0.0,0.0,0.0 +20000,371.0,0.00017856434226779574,1.0,1.0,1.0,1.0,0.7890625,0.0,0.0,0.0,0.0,0.0,0.0 +22500,416.4,6.660732057094627e-09,1.0,1.0,1.0,1.0,1.0,0.3203125,0.0,0.0,0.0,0.0,0.0 +25000,460.3,3.828245408557209e-07,1.0,1.0,1.0,1.0,1.0,0.12109375,0.0,0.0,0.0,0.0,0.0 +27500,506.5,5.220396505231354e-07,1.0,1.0,1.0,1.0,0.99609375,0.0546875,0.0,0.0,0.0,0.0,0.0 +30000,552.4,6.096137793533933e-09,1.0,1.0,1.0,1.0,1.0,0.24609375,0.0,0.0,0.0,0.0,0.0 +32500,598.2,9.71640746216191e-09,1.0,1.0,1.0,1.0,1.0,0.1953125,0.0,0.0,0.0,0.0,0.0 +35000,643.7,1.3610566318048316e-08,1.0,1.0,1.0,1.0,1.0,0.2265625,0.0,0.0,0.0,0.0,0.0 +37500,688.0,1.603926524274291e-08,1.0,1.0,1.0,1.0,1.0,0.21484375,0.0,0.0,0.0,0.0,0.0 +40000,730.9,1.7971939851950004e-08,1.0,1.0,1.0,1.0,1.0,0.23828125,0.0,0.0,0.0,0.0,0.0 +42500,773.8,1.8856745614216587e-08,1.0,1.0,1.0,1.0,1.0,0.25390625,0.0,0.0,0.0,0.0,0.0 +45000,820.7,1.946772628472e-08,1.0,1.0,1.0,1.0,1.0,0.265625,0.0,0.0,0.0,0.0,0.0 +47500,867.2,1.9701182743147328e-08,1.0,1.0,1.0,1.0,1.0,0.26171875,0.0,0.0,0.0,0.0,0.0 +50000,916.7,1.974120911727862e-08,1.0,1.0,1.0,1.0,1.0,0.25,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_carry_heavy/position_coupling_s1/step_50000.pt b/addition_carry_heavy/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b39e7515cfbf67128009183a9d9190a82993108 --- /dev/null +++ b/addition_carry_heavy/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6f14004163422d8bf6b124f55eba6634e072ae5afb1901d452891c992e0028e +size 13738485 diff --git a/addition_carry_heavy/position_coupling_s2/config.json b/addition_carry_heavy/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a79764882d0237645f0cc416ff0dda4f3f54d06e --- /dev/null +++ b/addition_carry_heavy/position_coupling_s2/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_carry_heavy position coupling seed 2", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.5, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_carry_heavy/position_coupling_s2/metrics.csv b/addition_carry_heavy/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..2fe03054cbbbb64d11201ea53db9e458fd9e401c --- /dev/null +++ b/addition_carry_heavy/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,53.0,0.10720635405619515,0.890625,0.953125,0.671875,0.32421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,100.9,0.03279938525706554,1.0,0.95703125,0.98828125,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,149.9,0.02124077235206988,0.984375,0.99609375,0.984375,0.86328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,195.2,0.002723707521341975,1.0,1.0,1.0,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,241.8,0.0022220749355433144,0.96484375,0.99609375,1.0,0.8984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,289.4,0.0016490826073179317,1.0,1.0,1.0,0.91015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,337.2,0.0011255368215346137,1.0,0.984375,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,385.2,0.0005864900667511477,1.0,1.0,0.9765625,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,433.1,0.0006622003492464368,1.0,1.0,1.0,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,482.5,0.0003470492127613328,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,531.4,0.00033570730871460187,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,580.2,1.408639631390746e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,629.8,1.801094581104906e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,678.6,2.622694843946881e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,727.7,3.0085523359748305e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,776.1,3.216073666486856e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,824.8,3.311186141852077e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,872.3,3.360088634220868e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,920.8,3.363193316175736e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,969.6,3.380890640942023e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_carry_heavy/position_coupling_s2/step_50000.pt b/addition_carry_heavy/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b392884a5792b57779d3e81f84c11d19d579b899 --- /dev/null +++ b/addition_carry_heavy/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222a2f3605053f1521dce1331413dc1db203c7688ea94e034b4863712098c1ad +size 13738485 diff --git a/addition_constant_lr/position_coupling_s0/config.json b/addition_constant_lr/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4d014e7787b829b138ee2c0fb435d77099f927cf --- /dev/null +++ b/addition_constant_lr/position_coupling_s0/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition_constant_lr position coupling seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": false, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_constant_lr/position_coupling_s0/metrics.csv b/addition_constant_lr/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..b8fa42a665794c6c1e4edcd9eedebb50dfe41567 --- /dev/null +++ b/addition_constant_lr/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.6,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,42.7,0.0866226251033078,0.94921875,0.953125,0.9375,0.8515625,0.25,0.078125,0.0,0.0,0.0,0.0,0.0 +5000,337.8,0.0038365160204502824,0.9296875,0.96875,0.9609375,0.9296875,0.6015625,0.3046875,0.05859375,0.0,0.0,0.0,0.0 +7500,376.5,0.0014801951787938872,0.9765625,0.96875,0.98046875,0.8984375,0.8359375,0.29296875,0.015625,0.0,0.0,0.0,0.0 +10000,415.3,0.001696138756094547,0.9453125,0.9921875,0.99609375,0.984375,0.734375,0.4140625,0.0390625,0.0,0.0,0.0,0.0 +12500,453.7,0.0009663887346714611,1.0,1.0,1.0,0.98828125,0.921875,0.71484375,0.09765625,0.00390625,0.0,0.0,0.0 +15000,492.1,0.00043938949121746107,1.0,1.0,0.99609375,1.0,0.93359375,0.8203125,0.1171875,0.00390625,0.0,0.0,0.0 +17500,530.4,0.000586944214780278,1.0,1.0,0.99609375,1.0,0.9296875,0.8671875,0.3828125,0.02734375,0.0,0.0,0.0 +20000,568.7,7.161856276887645e-05,1.0,1.0,1.0,1.0,0.9921875,0.9453125,0.70703125,0.328125,0.03515625,0.0,0.0 +22500,607.0,0.0006918341357976951,1.0,1.0,1.0,0.99609375,0.9375,0.76953125,0.15234375,0.0078125,0.0,0.0,0.0 +25000,645.1,0.0003768360854437187,0.99609375,0.99609375,0.98828125,0.99609375,0.9375,0.78125,0.3515625,0.03125,0.0,0.0,0.0 +27500,683.5,0.00019313198963186466,1.0,1.0,1.0,0.99609375,0.8984375,0.8515625,0.5390625,0.0625,0.0078125,0.0,0.0 +30000,721.6,0.0007078119866688751,1.0,1.0,0.99609375,0.98828125,0.80859375,0.38671875,0.046875,0.0,0.0,0.0,0.0 +32500,760.4,0.0003037646260176355,1.0,1.0,1.0,1.0,0.83984375,0.66015625,0.109375,0.00390625,0.0,0.0,0.0 +35000,799.3,0.00021819930541144212,1.0,1.0,1.0,1.0,0.86328125,0.609375,0.109375,0.0,0.0,0.0,0.0 +37500,837.7,3.162388637853274e-05,1.0,1.0,1.0,1.0,0.80859375,0.69921875,0.23046875,0.00390625,0.0,0.0,0.0 +40000,876.2,1.4809822309760198e-05,1.0,1.0,1.0,0.99609375,0.87109375,0.6953125,0.30078125,0.00390625,0.0,0.0,0.0 +42500,914.7,0.00019152396719774754,1.0,1.0,0.99609375,1.0,0.71484375,0.609375,0.2265625,0.00390625,0.0,0.0,0.0 +45000,953.2,0.0010502359310161798,1.0,1.0,1.0,1.0,0.75,0.50390625,0.2421875,0.0234375,0.0,0.0,0.0 +47500,991.5,3.7581023784715195e-06,1.0,1.0,1.0,1.0,0.77734375,0.52734375,0.18359375,0.00390625,0.0,0.0,0.0 +50000,1029.6,0.0002920383882665511,1.0,1.0,1.0,0.9921875,0.60546875,0.125,0.015625,0.0,0.0,0.0,0.0 diff --git a/addition_constant_lr/position_coupling_s0/step_50000.pt b/addition_constant_lr/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc34bf3e8953aee94c5503c6a03623b70203aeda --- /dev/null +++ b/addition_constant_lr/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:307776ad13c13f96bcba6b025056cd969e45e276ce24fd8d2934fb7b4259e3ec +size 13738485 diff --git a/addition_constant_lr/position_coupling_s1/config.json b/addition_constant_lr/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..55aff99b591ce55c3273def338629b7a735a9b19 --- /dev/null +++ b/addition_constant_lr/position_coupling_s1/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition_constant_lr position coupling seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": false, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_constant_lr/position_coupling_s1/metrics.csv b/addition_constant_lr/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..040554b21d5ef05dd7b55c1115cbb7c3779ee9ff --- /dev/null +++ b/addition_constant_lr/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,41.6,0.1720677338009586,0.96875,0.9375,0.84765625,0.40625,0.01953125,0.0,0.0,0.0,0.0,0.0,0.0 +5000,80.2,0.007566015015166153,0.87109375,0.65234375,0.44921875,0.24609375,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0 +7500,118.6,0.0032257712859481095,1.0,0.9765625,0.9609375,0.953125,0.4140625,0.0,0.0,0.0,0.0,0.0,0.0 +10000,156.8,0.0003758321167602779,1.0,1.0,1.0,0.9921875,0.94140625,0.5625,0.0078125,0.0,0.0,0.0,0.0 +12500,195.0,0.0003525652263794561,1.0,1.0,0.99609375,0.98828125,0.984375,0.96484375,0.86328125,0.51171875,0.05859375,0.0,0.0 +15000,233.4,0.0003267862634754619,1.0,1.0,1.0,1.0,0.98046875,0.91796875,0.69921875,0.24609375,0.0,0.0,0.0 +17500,271.8,1.534604431007086e-08,1.0,1.0,1.0,1.0,1.0,0.99609375,0.953125,0.67578125,0.16015625,0.0,0.0 +20000,309.9,0.0005664682206268529,0.98828125,0.99609375,1.0,0.984375,0.953125,0.7421875,0.5625,0.23828125,0.00390625,0.0,0.0 +22500,347.9,0.00027015824656940915,1.0,1.0,1.0,0.9921875,0.96484375,0.83984375,0.55078125,0.15234375,0.0078125,0.0,0.0 +25000,385.9,0.00019319937172541075,1.0,1.0,1.0,1.0,0.9921875,0.95703125,0.828125,0.57421875,0.07421875,0.0,0.0 +27500,424.0,0.0002840865512636639,1.0,1.0,1.0,1.0,0.97265625,0.8671875,0.7109375,0.22265625,0.01171875,0.0,0.0 +30000,462.2,0.00040422050417347446,1.0,1.0,1.0,0.99609375,0.9921875,0.9140625,0.546875,0.25,0.01171875,0.0,0.0 +32500,500.6,0.0002865106209845077,1.0,1.0,1.0,1.0,1.0,0.77734375,0.578125,0.296875,0.0078125,0.0,0.0 +35000,538.9,0.00017737503097274034,1.0,1.0,1.0,0.99609375,0.97265625,0.87109375,0.3515625,0.1171875,0.0,0.0,0.0 +37500,577.2,0.00017733349673456034,1.0,1.0,0.98828125,1.0,0.9453125,0.8359375,0.4375,0.109375,0.00390625,0.0,0.0 +40000,615.9,0.0001163928123366363,1.0,1.0,1.0,1.0,1.0,0.87109375,0.3671875,0.26171875,0.02734375,0.0,0.0 +42500,654.4,0.00016617973959604087,1.0,1.0,1.0,1.0,0.98046875,0.64453125,0.125,0.00390625,0.0,0.0,0.0 +45000,692.7,6.293195164272983e-05,1.0,1.0,1.0,1.0,0.9765625,0.78515625,0.53125,0.1171875,0.0,0.0,0.0 +47500,730.6,9.883867233641581e-05,1.0,1.0,1.0,1.0,0.97265625,0.81640625,0.40234375,0.09765625,0.0,0.0,0.0 +50000,768.9,0.00018594444457586652,1.0,1.0,1.0,0.99609375,0.984375,0.6875,0.16796875,0.0546875,0.0,0.0,0.0 diff --git a/addition_constant_lr/position_coupling_s1/step_50000.pt b/addition_constant_lr/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bd698dc67adee996f2a4ea6874ae6211a7eeff4 --- /dev/null +++ b/addition_constant_lr/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b43857855a1ac2c191d228afcccf0268ea4ac4a8676ccc4dd8647d19b9e2fb +size 13738485 diff --git a/addition_constant_lr/position_coupling_s2/config.json b/addition_constant_lr/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8e65d439338ecd924fdea730580ef04660b497cf --- /dev/null +++ b/addition_constant_lr/position_coupling_s2/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition_constant_lr position coupling seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": false, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_constant_lr/position_coupling_s2/metrics.csv b/addition_constant_lr/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..796fd4ebd27f3519bae8eaf339c5c97b7270d09d --- /dev/null +++ b/addition_constant_lr/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,41.6,0.020129551979438887,0.96484375,0.99609375,0.99609375,0.7734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,79.9,0.009733854557087444,0.99609375,1.0,0.921875,0.578125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,118.0,0.0020527294158481637,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,156.4,0.002706031068735716,1.0,1.0,0.99609375,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,194.5,0.001300157638022273,0.9375,0.96484375,0.96875,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,232.8,0.0010041933939945057,1.0,1.0,1.0,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,270.9,0.000726201731560858,0.99609375,0.99609375,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,309.1,0.0011775825144518237,1.0,1.0,0.99609375,0.9609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,346.9,0.0005384632935310143,0.9921875,1.0,0.99609375,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,385.2,0.0006406347333173957,1.0,1.0,0.99609375,0.95703125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,423.5,0.0008327465072165022,0.9765625,0.99609375,0.984375,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,461.6,0.0003069453738958615,1.0,0.9921875,0.9921875,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,499.9,0.0006316075211774619,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,538.1,0.0005833266787772164,1.0,1.0,0.99609375,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,576.3,0.0003432321983617629,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,614.3,0.0005137130239651862,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,652.5,0.0005041172089070696,0.98046875,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,690.8,0.00042606997897855567,1.0,1.0,1.0,0.96875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,728.9,0.0006368873171886255,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,766.9,1.522987135750936e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_constant_lr/position_coupling_s2/step_50000.pt b/addition_constant_lr/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a102923e45eb1624f84bb58ec4cc1ad63499774a --- /dev/null +++ b/addition_constant_lr/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5883e75cfb6c6f4028db6ab216e721cab3ae9c621810f8dfe0d0494b5171ed4 +size 13738485 diff --git a/addition_fixed_length/position_coupling_s0/config.json b/addition_fixed_length/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99ceb3a9c0d6556beb7ddc7eeca524a3eed77198 --- /dev/null +++ b/addition_fixed_length/position_coupling_s0/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_fixed_length position coupling seed 0", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 20, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_fixed_length/position_coupling_s0/metrics.csv b/addition_fixed_length/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..f61ca7fad0fb5b6b6cdc01cb772561c5db5f5081 --- /dev/null +++ b/addition_fixed_length/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.3,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,53.9,2.0142067730608706,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,102.8,0.1633080032722577,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,151.2,0.0381911045064013,0.0,0.0,0.0,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,199.3,0.007554696698226739,0.0,0.0,0.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,247.5,0.007004854097441996,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,295.3,0.0045464418133818905,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,342.0,0.004037231794196809,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,390.9,0.0025358160719003713,0.0,0.0,0.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,439.5,0.0008534289193670569,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,487.9,0.0013365620795915978,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,536.2,7.101851938593171e-09,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,583.8,2.3825661552874146e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,630.5,3.182759832198326e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,675.4,3.367964777444032e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,724.4,3.488842368562816e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,772.0,3.474755717283233e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,819.1,3.5031984780744884e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,866.6,3.528134438111314e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,913.9,3.450452542811781e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,963.0,3.621596431402685e-08,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_fixed_length/position_coupling_s0/step_50000.pt b/addition_fixed_length/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a46a0654ab8ad1d64454594d6868561bdfea7c5e --- /dev/null +++ b/addition_fixed_length/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f193edd535699df285b51bc12b252860f54cfb6c33e2d443fc745801265ef6 +size 13738485 diff --git a/addition_fixed_length/position_coupling_s1/config.json b/addition_fixed_length/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b43f9f82b2f45dc56fe257541700769165d3d7af --- /dev/null +++ b/addition_fixed_length/position_coupling_s1/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_fixed_length position coupling seed 1", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 20, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_fixed_length/position_coupling_s1/metrics.csv b/addition_fixed_length/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..37f3a46a8a06d1303535d0d5d4c8b22e1e4b0c41 --- /dev/null +++ b/addition_fixed_length/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,45.4,2.094450929910537,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,84.9,2.0178347539748382,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,129.9,1.8422810091858886,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,176.1,1.293750274873914,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,270.7,0.22648745363787834,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,434.0,0.20889084742988007,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,484.4,0.17961435512193274,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,662.9,0.12095276329565378,0.0,0.0,0.0,0.015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,802.8,0.1294877957762268,0.0,0.0,0.0,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,1439.0,0.11340049634381136,0.0,0.0,0.0,0.01953125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,1478.6,0.10718360007589543,0.0,0.0,0.0,0.0859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,1518.2,0.10715884236872061,0.0,0.0,0.0,0.09765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,1557.7,0.0028712995758920982,0.0,0.0,0.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,1597.4,0.0007093296339504128,0.0,0.0,0.0,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,1637.3,0.0003845713213021602,0.0,0.0,0.0,0.9609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,1677.0,4.5456627446890377e-05,0.0,0.0,0.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,1716.6,7.066416809852765e-05,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,1756.2,6.19727359013609e-06,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,1795.6,8.46647106620626e-07,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,1835.0,5.045867233264583e-07,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_fixed_length/position_coupling_s1/step_50000.pt b/addition_fixed_length/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..15b327fdd0bc78da9e3cc9b8c45721999cbe4226 --- /dev/null +++ b/addition_fixed_length/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3505eb2c99d91316744dc554aea4fedb844e7107dd7838db0a13a2d1aa48747c +size 13738485 diff --git a/addition_fixed_length/position_coupling_s2/config.json b/addition_fixed_length/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d02cc006c63c21313438f23e6a93f4383f42ed6e --- /dev/null +++ b/addition_fixed_length/position_coupling_s2/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_fixed_length position coupling seed 2", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 20, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_fixed_length/position_coupling_s2/metrics.csv b/addition_fixed_length/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..23d82b9d6919e327a2411dc14a02f9a2d3af4ee4 --- /dev/null +++ b/addition_fixed_length/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.0,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.6,2.0438060280896995,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,82.8,1.9905149111685476,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,122.2,1.9161948618013689,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,162.0,1.9008697510887034,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,201.8,1.5826885308631504,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,241.0,0.8325425967615081,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,280.8,0.17708404879669576,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,320.3,0.1573473508004247,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,359.6,0.14916968622124754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,399.2,0.14267742357658783,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,439.0,0.1440527017388233,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,478.4,0.14086465938377704,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,517.7,0.13880807880705193,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,557.3,0.13612325096328637,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,597.0,0.1361498455459682,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,636.8,0.12780852068910206,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,676.4,0.12875328206105438,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,716.0,0.12771674896278543,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,755.2,0.12970674174144387,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,794.5,0.1291441877233323,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_fixed_length/position_coupling_s2/step_50000.pt b/addition_fixed_length/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7194a83a7e70c4b042cd92d40f34a4dbf1915197 --- /dev/null +++ b/addition_fixed_length/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b356aa56bd74fe86bd82a8a829654f197c22e345f5e85d19626ab72a02caa3ad +size 13738485 diff --git a/addition_no_wd/position_coupling_s0/config.json b/addition_no_wd/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b825c5d418760d6686c62812f28661bcbefb0d10 --- /dev/null +++ b/addition_no_wd/position_coupling_s0/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition_no_wd position coupling seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_no_wd/position_coupling_s0/metrics.csv b/addition_no_wd/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..d33e475eea184db3c599536dfad76927583352e8 --- /dev/null +++ b/addition_no_wd/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.5,0.033764210231760995,0.9375,0.98828125,0.9921875,0.88671875,0.38671875,0.05859375,0.0,0.0,0.0,0.0,0.0 +5000,82.4,0.0013833454537490598,0.98828125,0.98046875,0.95703125,0.921875,0.52734375,0.4140625,0.0390625,0.0,0.0,0.0,0.0 +7500,121.3,0.0009082659110281646,0.99609375,0.9921875,0.98828125,0.9765625,0.84375,0.62890625,0.21484375,0.02734375,0.0,0.0,0.0 +10000,160.1,0.0004714863326466948,1.0,1.0,1.0,0.9921875,0.96484375,0.76171875,0.375,0.06640625,0.01171875,0.0,0.0 +12500,198.5,0.0003346008646185079,0.99609375,1.0,1.0,1.0,0.97265625,0.76171875,0.5,0.0546875,0.0,0.0,0.0 +15000,237.1,0.00023093521192033361,0.99609375,1.0,0.99609375,0.98828125,0.921875,0.890625,0.4765625,0.11328125,0.0234375,0.0,0.0 +17500,275.4,0.00010448052059478428,1.0,1.0,1.0,1.0,0.984375,0.796875,0.4921875,0.16015625,0.04296875,0.0,0.0 +20000,314.0,0.0001086132116146446,0.99609375,0.99609375,0.99609375,1.0,0.9609375,0.85546875,0.390625,0.0546875,0.00390625,0.0,0.0 +22500,352.3,0.000105168086217873,1.0,1.0,1.0,1.0,0.9921875,0.9296875,0.36328125,0.046875,0.0,0.0,0.0 +25000,391.1,4.168543049284564e-08,1.0,1.0,1.0,1.0,1.0,1.0,0.86328125,0.37890625,0.12109375,0.0,0.0 +27500,429.6,1.2183241524321076e-05,1.0,1.0,1.0,1.0,1.0,0.97265625,0.875,0.51953125,0.16015625,0.0,0.0 +30000,468.2,1.3992381551172162e-09,1.0,1.0,1.0,1.0,1.0,0.98046875,0.90625,0.55078125,0.2109375,0.0,0.0 +32500,506.8,1.016400050390872e-10,1.0,1.0,1.0,1.0,1.0,0.98046875,0.890625,0.53125,0.24609375,0.0,0.0 +35000,545.7,2.410433206759581e-10,1.0,1.0,1.0,1.0,1.0,0.97265625,0.83984375,0.375,0.0546875,0.0,0.0 +37500,584.1,8.544776016987569e-11,1.0,1.0,1.0,1.0,1.0,0.97265625,0.890625,0.3984375,0.05078125,0.0,0.0 +40000,622.8,4.251476665484774e-10,1.0,1.0,1.0,1.0,1.0,0.984375,0.9140625,0.4765625,0.171875,0.0,0.0 +42500,661.7,3.980621379268992e-11,1.0,1.0,1.0,1.0,1.0,0.984375,0.91796875,0.5,0.171875,0.0,0.0 +45000,700.2,6.505766783796368e-11,1.0,1.0,1.0,1.0,1.0,0.984375,0.91796875,0.49609375,0.1640625,0.0,0.0 +47500,738.7,2.975512099969937e-11,1.0,1.0,1.0,1.0,1.0,0.984375,0.91796875,0.4921875,0.16015625,0.0,0.0 +50000,777.4,3.341422214599492e-11,1.0,1.0,1.0,1.0,1.0,0.984375,0.921875,0.5078125,0.16015625,0.0,0.0 diff --git a/addition_no_wd/position_coupling_s0/step_50000.pt b/addition_no_wd/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..cadceb4561c57dc64acdc928f5aa6ea2ba865e7a --- /dev/null +++ b/addition_no_wd/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe9f9e7e7ccbae5592e2156588e7d237f52f5ab061a6ed47f914a5340cdd8c5 +size 13738485 diff --git a/addition_no_wd/position_coupling_s1/config.json b/addition_no_wd/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d141622a6caa30ea2b147f662a311df3e34da72 --- /dev/null +++ b/addition_no_wd/position_coupling_s1/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition_no_wd position coupling seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_no_wd/position_coupling_s1/metrics.csv b/addition_no_wd/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..7a738624432efb41a8880c625e2778aac78874d4 --- /dev/null +++ b/addition_no_wd/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.2,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,42.7,0.33986891776533823,0.09375,0.1015625,0.0625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,81.3,0.04126950375632533,0.9921875,0.82421875,0.8203125,0.6640625,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0 +7500,119.8,0.004345896747997032,0.98828125,0.984375,0.9453125,0.734375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,157.8,0.0025357361689143354,1.0,0.99609375,0.98828125,0.9453125,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +12500,196.5,0.0018487842049272178,0.99609375,0.9921875,0.99609375,1.0,0.1953125,0.0,0.0,0.0,0.0,0.0,0.0 +15000,235.3,0.00126891982838365,0.9921875,0.99609375,0.99609375,0.9921875,0.0703125,0.0,0.0,0.0,0.0,0.0,0.0 +17500,274.0,0.0005594150028723867,1.0,0.98828125,1.0,0.9921875,0.08203125,0.0,0.0,0.0,0.0,0.0,0.0 +20000,312.4,0.0004580482255327476,1.0,1.0,1.0,1.0,0.40234375,0.0,0.0,0.0,0.0,0.0,0.0 +22500,351.0,0.000362977224221434,0.99609375,1.0,1.0,0.984375,0.125,0.0,0.0,0.0,0.0,0.0,0.0 +25000,389.5,0.00021596663187228304,0.99609375,1.0,1.0,0.9921875,0.37109375,0.0,0.0,0.0,0.0,0.0,0.0 +27500,428.3,2.6554372925090148e-05,1.0,1.0,1.0,1.0,0.21875,0.0,0.0,0.0,0.0,0.0,0.0 +30000,467.5,7.646746555747128e-05,1.0,1.0,1.0,1.0,0.10546875,0.0,0.0,0.0,0.0,0.0,0.0 +32500,505.9,2.7363135993277077e-10,1.0,1.0,1.0,1.0,0.703125,0.0,0.0,0.0,0.0,0.0,0.0 +35000,544.4,2.358584652100859e-10,1.0,1.0,1.0,1.0,0.71875,0.0,0.0,0.0,0.0,0.0,0.0 +37500,583.1,3.9520783951844064e-10,1.0,1.0,1.0,1.0,0.74609375,0.0,0.0,0.0,0.0,0.0,0.0 +40000,621.6,5.88295644921577e-10,1.0,1.0,1.0,1.0,0.76953125,0.0,0.0,0.0,0.0,0.0,0.0 +42500,660.3,9.445355238160505e-10,1.0,1.0,1.0,1.0,0.7265625,0.0,0.0,0.0,0.0,0.0,0.0 +45000,699.1,1.177248737377587e-09,1.0,1.0,1.0,1.0,0.75,0.0,0.0,0.0,0.0,0.0,0.0 +47500,738.2,1.3231882790888416e-09,1.0,1.0,1.0,1.0,0.734375,0.0,0.0,0.0,0.0,0.0,0.0 +50000,777.1,1.3387516902169432e-09,1.0,1.0,1.0,1.0,0.734375,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_no_wd/position_coupling_s1/step_50000.pt b/addition_no_wd/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7d1b434569ad5f09792fb9570bcfaebb0292598 --- /dev/null +++ b/addition_no_wd/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fbc621223dff7dfc8162afc6f706a9658a680cac479aafa726282012d27012e +size 13738485 diff --git a/addition_no_wd/position_coupling_s2/config.json b/addition_no_wd/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c5f40aa9b337c2587298f0777ae4fe4aa403f74 --- /dev/null +++ b/addition_no_wd/position_coupling_s2/config.json @@ -0,0 +1,53 @@ +{ + "name": "addition_no_wd position coupling seed 2", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_no_wd/position_coupling_s2/metrics.csv b/addition_no_wd/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..518a353b70f450f21fc6e4280e2da79c05ead98a --- /dev/null +++ b/addition_no_wd/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.0,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,42.5,0.10574521701741488,0.9453125,0.80078125,0.7734375,0.44921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,80.8,0.014792552601506841,1.0,0.98046875,0.99609375,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,119.3,0.0037990403412242256,1.0,1.0,0.9921875,0.9375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,157.9,0.002562819062047454,0.984375,1.0,0.9921875,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,196.5,0.002121267780289426,1.0,0.94140625,0.9765625,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,235.4,0.0009269403376690931,0.9921875,0.99609375,0.95703125,0.96484375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,273.9,0.0011325337476359454,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,312.8,0.0004583038984000967,0.9921875,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,351.6,0.00040495916145194826,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,390.5,0.00035967978642564597,1.0,1.0,1.0,0.9609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,429.1,1.5524072550974884e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,468.1,6.180552719553521e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,506.9,2.8729967786953005e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,545.4,1.7193275403531176e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,583.8,8.042586402464962e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,622.7,1.30506664418879e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,661.7,1.5506702160457705e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,700.5,1.6767206033115425e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,739.4,1.7013844110313636e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,778.2,1.7056081378947995e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_no_wd/position_coupling_s2/step_50000.pt b/addition_no_wd/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad357122dfcfc22a3b88ae8453445df647279781 --- /dev/null +++ b/addition_no_wd/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e37b91b11a9dbdeaa8dc3f00c58e5ff484ea302a6d748eae9d82b97c5067a70c +size 13738485 diff --git a/addition_no_wd/position_coupling_s3/config.json b/addition_no_wd/position_coupling_s3/config.json new file mode 100644 index 0000000000000000000000000000000000000000..44dbacb6c023b91b101730b0e2edd1946c00647b --- /dev/null +++ b/addition_no_wd/position_coupling_s3/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_no_wd position coupling seed 3", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 3, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_no_wd/position_coupling_s3/metrics.csv b/addition_no_wd/position_coupling_s3/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..37d32581233ce4443a2fd9cf9a261b3a529fd03a --- /dev/null +++ b/addition_no_wd/position_coupling_s3/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.9,0.05415902838870525,0.9296875,0.96484375,0.9453125,0.85546875,0.0546875,0.0,0.0,0.0,0.0,0.0,0.0 +5000,83.1,0.0033072376377696533,0.98828125,0.96875,0.97265625,0.86328125,0.21875,0.0,0.0,0.0,0.0,0.0,0.0 +7500,122.0,0.0015166834594730321,0.9921875,0.99609375,0.9921875,0.984375,0.55859375,0.03125,0.0,0.0,0.0,0.0,0.0 +10000,161.0,0.0010224743860498722,0.99609375,0.99609375,0.984375,0.96875,0.7265625,0.22265625,0.0,0.0,0.0,0.0,0.0 +12500,200.3,0.000746924451384098,0.99609375,0.9921875,1.0,0.98828125,0.828125,0.140625,0.0,0.0,0.0,0.0,0.0 +15000,239.6,0.0007196489437897492,1.0,0.99609375,0.99609375,0.9921875,0.765625,0.21484375,0.0,0.0,0.0,0.0,0.0 +17500,278.7,0.00020235053416259237,1.0,0.99609375,1.0,1.0,0.94921875,0.6015625,0.00390625,0.0,0.0,0.0,0.0 +20000,317.7,0.0004785834850719228,0.99609375,1.0,1.0,0.98828125,0.7734375,0.1171875,0.0,0.0,0.0,0.0,0.0 +22500,356.8,0.0002905505038489663,0.99609375,1.0,1.0,0.984375,0.8125,0.20703125,0.0,0.0,0.0,0.0,0.0 +25000,395.9,9.111522884386391e-05,0.99609375,1.0,0.99609375,0.99609375,0.89453125,0.44921875,0.00390625,0.0,0.0,0.0,0.0 +27500,434.9,0.00013469506150446027,1.0,1.0,1.0,1.0,0.9375,0.50390625,0.0,0.0,0.0,0.0,0.0 +30000,473.7,1.3968497363214352e-05,1.0,1.0,1.0,0.9921875,0.94140625,0.484375,0.0078125,0.0,0.0,0.0,0.0 +32500,512.9,6.887542311158542e-05,1.0,1.0,1.0,1.0,0.96484375,0.4453125,0.0,0.0,0.0,0.0,0.0 +35000,552.2,2.3819107056386334e-06,1.0,1.0,1.0,1.0,0.99609375,0.8125,0.015625,0.0,0.0,0.0,0.0 +37500,591.6,2.940180736598987e-07,1.0,1.0,1.0,1.0,0.97265625,0.63671875,0.0,0.0,0.0,0.0,0.0 +40000,630.5,2.119839896145629e-06,1.0,1.0,1.0,1.0,0.99609375,0.84375,0.04296875,0.0,0.0,0.0,0.0 +42500,669.7,3.859229565072451e-10,1.0,1.0,1.0,1.0,1.0,0.8359375,0.02734375,0.0,0.0,0.0,0.0 +45000,708.8,2.1124299042899708e-10,1.0,1.0,1.0,1.0,1.0,0.875,0.046875,0.0,0.0,0.0,0.0 +47500,748.1,6.822016331567716e-08,1.0,1.0,1.0,1.0,0.99609375,0.890625,0.046875,0.0,0.0,0.0,0.0 +50000,787.0,1.6785197877416067e-10,1.0,1.0,1.0,1.0,0.99609375,0.890625,0.046875,0.0,0.0,0.0,0.0 diff --git a/addition_no_wd/position_coupling_s3/step_50000.pt b/addition_no_wd/position_coupling_s3/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c7af0d8ccce0637b918f1044aad455b8f580c32 --- /dev/null +++ b/addition_no_wd/position_coupling_s3/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30a93779dedd1a3e1b46760b888be3c567464dc97da34ea34497dd9929082a02 +size 13738485 diff --git a/addition_no_wd/position_coupling_s4/config.json b/addition_no_wd/position_coupling_s4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1ac68f82861615ab0a74b556cac48d42d05413 --- /dev/null +++ b/addition_no_wd/position_coupling_s4/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_no_wd position coupling seed 4", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 4, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_no_wd/position_coupling_s4/metrics.csv b/addition_no_wd/position_coupling_s4/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..ce4c6c02f4a166a311ffacecb44fd8a052c00653 --- /dev/null +++ b/addition_no_wd/position_coupling_s4/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.9,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.2,0.18935673604654202,0.984375,0.87890625,0.94921875,0.7421875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,82.3,0.008568666191589028,1.0,1.0,1.0,0.890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,121.3,0.017586743923320776,1.0,0.99609375,0.9453125,0.80859375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,160.3,0.01378552929748886,0.96875,0.984375,0.9453125,0.8984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,199.8,0.0037653573963616817,0.99609375,1.0,0.98828125,0.8671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,239.1,0.0024854297441072324,0.98828125,0.9921875,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,278.1,0.0020211068010975073,0.984375,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,317.0,0.000949498394570692,1.0,1.0,1.0,0.98828125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,356.1,0.0008698423645109093,0.9921875,1.0,0.99609375,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,395.2,0.0005602130406123691,1.0,1.0,0.99609375,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,434.2,0.00043177747886648054,1.0,1.0,1.0,0.9921875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,473.2,6.57366269189676e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,512.4,2.6818868321350938e-05,0.92578125,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,551.8,8.043966630625475e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,591.0,9.561204528335507e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,630.1,4.254418186778441e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,669.2,1.6515920338225884e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,708.2,2.3414598282338463e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,747.4,2.7396586634111703e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,786.4,2.7980055928288807e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_no_wd/position_coupling_s4/step_50000.pt b/addition_no_wd/position_coupling_s4/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5258e0c5693c5a623ced5a460db13ee0398b76e4 --- /dev/null +++ b/addition_no_wd/position_coupling_s4/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c98ba4d9e4cf231f977c9411c4a1cdb02142e5ee6cc8af9f7bab9ee0b239763c +size 13738485 diff --git a/addition_no_wd/position_coupling_s5/config.json b/addition_no_wd/position_coupling_s5/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85ebf62d51f82a0f163423a5263b22a041941f76 --- /dev/null +++ b/addition_no_wd/position_coupling_s5/config.json @@ -0,0 +1,57 @@ +{ + "name": "addition_no_wd position coupling seed 5", + "task": { + "op": "add", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 5, + "wandb": false + } +} \ No newline at end of file diff --git a/addition_no_wd/position_coupling_s5/metrics.csv b/addition_no_wd/position_coupling_s5/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..56ddd195b424159352bd3b10ea0a8ef7496ce424 --- /dev/null +++ b/addition_no_wd/position_coupling_s5/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.9,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,43.2,0.09877374438894049,0.97265625,0.66796875,0.8203125,0.0625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,82.7,0.06296639292756709,0.1171875,0.1640625,0.203125,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +7500,122.1,0.0073828665546243006,1.0,0.98046875,0.99609375,0.94140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,161.2,0.003348653135159782,1.0,0.99609375,0.9921875,0.890625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,200.3,0.0015482204853262864,0.9765625,0.98046875,0.890625,0.88671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,239.5,0.0009966474890757042,0.9921875,0.99609375,0.9921875,0.9296875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +17500,278.8,0.0005239662171459125,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,317.8,0.0005965675518723976,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,357.0,6.324854677447701e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,396.1,0.000264161147557736,1.0,1.0,0.9765625,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +27500,435.4,0.0003478652964684495,1.0,0.98046875,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,474.7,8.584817090956489e-07,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,513.7,2.7744086979669067e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,552.7,5.138880706453191e-10,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,592.1,9.984369561120299e-10,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,631.0,2.1600158187169198e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,669.9,3.7733214220938684e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,708.9,5.130085005374969e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,748.0,5.417921348113582e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,787.4,5.505241177406001e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/addition_no_wd/position_coupling_s5/step_50000.pt b/addition_no_wd/position_coupling_s5/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a3f356823687d2ac79c8d75371ac1c78699386a --- /dev/null +++ b/addition_no_wd/position_coupling_s5/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1d92b750d6f8736921427b322a79ad8873e91a814b04b4c5896804a5a9fb7e +size 13738485 diff --git a/blankspace_big/blankspace_fixed_s0/config.json b/blankspace_big/blankspace_fixed_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1fc419c4a9385dcffb4b35b3bf953cd56cb27e --- /dev/null +++ b/blankspace_big/blankspace_fixed_s0/config.json @@ -0,0 +1,51 @@ +{ + "name": "blankspace_big blankspace fixed seed 0", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 20, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 384, + "n_layers": 6, + "n_heads": 6, + "d_mlp": 1536, + "max_positions": 512, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 2000, + "cosine": true, + "amp": true, + "eval_every": 5000, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/blankspace_big/blankspace_fixed_s0/metrics.csv b/blankspace_big/blankspace_fixed_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..f52f77923ad122043e01d798594159e33fd285ca --- /dev/null +++ b/blankspace_big/blankspace_fixed_s0/metrics.csv @@ -0,0 +1,12 @@ +step,elapsed,loss,acc_5,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,41.2,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,1804.4,0.00017907228899723395,0.9921875,0.9921875,0.953125,0.734375,0.6171875,0.296875,0.12109375,0.00390625,0.0 +10000,2980.5,7.828570731772681e-07,1.0,1.0,0.99609375,0.94140625,0.796875,0.46484375,0.23046875,0.0078125,0.0 +15000,3807.6,3.0372273202887505e-06,0.99609375,0.9921875,0.93359375,0.70703125,0.38671875,0.06640625,0.00390625,0.0,0.0 +20000,4636.2,3.208722219399778e-09,1.0,1.0,1.0,0.97265625,0.7734375,0.4296875,0.171875,0.0078125,0.0 +25000,5464.5,3.982745773872872e-09,1.0,1.0,0.96484375,0.78515625,0.29296875,0.015625,0.0078125,0.0,0.0 +30000,6292.9,5.623980665158581e-09,1.0,1.0,0.99609375,0.98046875,0.83203125,0.359375,0.12109375,0.0,0.0 +35000,7121.4,6.8168431328911814e-09,1.0,1.0,0.99609375,0.96875,0.765625,0.359375,0.09765625,0.0,0.0 +40000,7949.7,8.241143109523134e-09,1.0,1.0,0.99609375,0.9296875,0.60546875,0.17578125,0.0234375,0.0,0.0 +45000,8778.1,8.667509943876546e-09,1.0,1.0,0.99609375,0.9375,0.62109375,0.1796875,0.01953125,0.0,0.0 +50000,9613.2,8.670137580154511e-09,1.0,1.0,0.99609375,0.94921875,0.625,0.19921875,0.0234375,0.0,0.0 diff --git a/blankspace_big/blankspace_fixed_s0/step_50000.pt b/blankspace_big/blankspace_fixed_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..100b5c9185cde7c618ab7bf0c51ec5df528d954a --- /dev/null +++ b/blankspace_big/blankspace_fixed_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5881718e71cc20aff4b26af4f1eaf6d594df0b42cd5c6bd7b46e6049b67a5a94 +size 43453045 diff --git a/blankspace_big/blankspace_fixed_s1/config.json b/blankspace_big/blankspace_fixed_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..250ad717bb7e0b7aba3ee2ef47fe50f492bef63f --- /dev/null +++ b/blankspace_big/blankspace_fixed_s1/config.json @@ -0,0 +1,51 @@ +{ + "name": "blankspace_big blankspace fixed seed 1", + "task": { + "max_digits": 20, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 20, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "d_model": 384, + "n_layers": 6, + "n_heads": 6, + "d_mlp": 1536, + "max_positions": 512, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 2000, + "cosine": true, + "amp": true, + "eval_every": 5000, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/blankspace_big/blankspace_fixed_s1/metrics.csv b/blankspace_big/blankspace_fixed_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..9b718d366a8d4a34147f82d86fcb4ddc66cf93d4 --- /dev/null +++ b/blankspace_big/blankspace_fixed_s1/metrics.csv @@ -0,0 +1,12 @@ +step,elapsed,loss,acc_5,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,41.5,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,871.3,0.0001888938755886753,0.97265625,0.85546875,0.4921875,0.21484375,0.078125,0.015625,0.0,0.0,0.0 +10000,1700.3,4.590529927492906e-05,0.9921875,0.984375,0.7890625,0.58203125,0.37890625,0.19140625,0.05078125,0.0,0.0 +15000,2529.1,1.5427809388779828e-05,0.9921875,1.0,0.95703125,0.609375,0.14453125,0.0390625,0.0,0.0,0.0 +20000,3357.8,2.1270160476149487e-09,1.0,1.0,0.9296875,0.6953125,0.22265625,0.078125,0.015625,0.0,0.0 +25000,4186.8,3.1581311736662025e-05,1.0,0.99609375,0.85546875,0.33203125,0.05078125,0.0078125,0.0,0.0,0.0 +30000,5021.0,3.985963350155729e-09,1.0,1.0,0.97265625,0.76953125,0.48046875,0.23828125,0.03125,0.0,0.0 +35000,5880.2,6.246612813166939e-09,1.0,1.0,0.9765625,0.8515625,0.55078125,0.265625,0.01953125,0.0,0.0 +40000,6744.5,7.3031705087832245e-09,1.0,1.0,0.9921875,0.91015625,0.5703125,0.19921875,0.01171875,0.0,0.0 +45000,7589.2,7.799937462933212e-09,1.0,1.0,0.9921875,0.9375,0.53125,0.171875,0.01171875,0.0,0.0 +50000,8431.3,7.816353107680129e-09,1.0,1.0,0.99609375,0.94140625,0.54296875,0.16015625,0.01171875,0.0,0.0 diff --git a/blankspace_big/blankspace_fixed_s1/step_50000.pt b/blankspace_big/blankspace_fixed_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c3c12031a6cea99bcc38fe0983d87e8475a5a8f --- /dev/null +++ b/blankspace_big/blankspace_fixed_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5f0c82d12550cbba1f998a1134a6c613bead6f060f06304908b9b0b06e889bd +size 43453045 diff --git a/modular_add/config.json b/modular_add/config.json index 1c9129465d5de2e22be4d6fba64a04db44615bd0..2acbd53a92b93492c902125089cb929b9923d0f3 100644 --- a/modular_add/config.json +++ b/modular_add/config.json @@ -6,12 +6,13 @@ "kind": "modular" }, "model": { + "arch": "transformer", "d_model": 128, "n_layers": 1, "n_heads": 4, "d_mlp": 512, "max_positions": 3, - "positional": true, + "pos_embed": "absolute", "dropout": 0.0 }, "train": { @@ -24,8 +25,10 @@ 0.98 ], "warmup_steps": 0, + "cosine": false, + "amp": false, "eval_every": 100, - "checkpoint_every": 500, + "checkpoint_every": 100, "seed": 0, "wandb": false } diff --git a/modular_add/metrics.csv b/modular_add/metrics.csv index 9170da50bb153321611401210438b54fe7523409..4b1950cf9f8abfa7421b923d30d6b2ac108cd461 100644 --- a/modular_add/metrics.csv +++ b/modular_add/metrics.csv @@ -1,402 +1,402 @@ -step,elapsed,train_loss,train_acc,test_loss,test_acc -0,0.0,4.7616143226623535,0.009660574607551098,4.762050151824951,0.00827833078801632 -100,0.8,0.1685766726732254,1.0,4.590503215789795,0.24465824663639069 -200,1.3,0.029104363173246384,1.0,4.530285358428955,0.2740798890590668 -300,1.8,0.0216338150203228,1.0,4.445437431335449,0.2886228859424591 -400,2.3,0.01258074026554823,1.0,4.299746036529541,0.2962300181388855 -500,2.8,0.006273516453802586,1.0,4.312851905822754,0.29857927560806274 -600,3.2,0.012299809604883194,1.0,3.9216959476470947,0.31010180711746216 -700,3.7,0.009999701753258705,1.0,3.7636139392852783,0.314129114151001 -800,4.2,0.00509463157504797,1.0,3.692552089691162,0.3180445432662964 -900,4.7,0.002197114983573556,1.0,3.6361052989959717,0.3267703354358673 -1000,5.2,0.0009332005283795297,1.0,3.568415880203247,0.33482491970062256 -1100,5.7,0.0003910401137545705,1.0,3.492629289627075,0.34713056683540344 -1200,6.1,0.0001666255120653659,1.0,3.41487717628479,0.3554089069366455 -1300,6.6,0.004765611607581377,1.0,2.6445014476776123,0.4068687856197357 -1400,7.1,0.005297891329973936,1.0,2.382455825805664,0.4258865714073181 -1500,7.6,0.014239107258617878,1.0,1.7212132215499878,0.5430137515068054 -1600,8.1,0.012043540365993977,1.0,1.5408352613449097,0.6118134260177612 -1700,8.5,0.006985262967646122,1.0,1.275530219078064,0.691240668296814 -1800,9.0,0.0030147237703204155,1.0,0.9464532732963562,0.782637894153595 -1900,9.5,0.0012752332258969545,1.0,0.6555971503257751,0.8591565489768982 -2000,10.0,0.0005410491139627993,1.0,0.43341580033302307,0.9168810844421387 -2100,10.4,0.00023161059652920812,1.0,0.27861881256103516,0.9488757252693176 -2200,10.9,9.997520101023838e-05,1.0,0.17426809668540955,0.9718089699745178 -2300,11.4,4.384289786685258e-05,1.0,0.10947011411190033,0.984785795211792 -2400,11.9,1.956131927727256e-05,1.0,0.06694776564836502,0.9917216897010803 -2500,12.4,0.002068975707516074,1.0,0.10334945470094681,0.9874706864356995 -2600,12.9,0.0018950446974486113,1.0,0.056251391768455505,0.9972032904624939 -2700,13.3,0.038790877908468246,1.0,0.18772314488887787,0.9916098117828369 -2800,13.8,0.010595878586173058,1.0,0.05027606338262558,0.9995525479316711 -2900,14.3,0.02573176845908165,1.0,0.06594014912843704,1.0 -3000,14.8,0.012696159072220325,1.0,0.031211240217089653,1.0 -3100,15.3,0.0075059980154037476,1.0,0.01827137917280197,1.0 -3200,15.7,0.003237507538869977,1.0,0.00778270373120904,1.0 -3300,16.2,0.001360321301035583,1.0,0.003161293687298894,1.0 -3400,16.7,0.000579379906412214,1.0,0.0013025131775066257,1.0 -3500,17.2,0.00024824557476677,1.0,0.0005462610861286521,1.0 -3600,17.7,0.00010757338895928115,1.0,0.00023506060824729502,1.0 -3700,18.1,4.710876601166092e-05,1.0,0.00010424039646750316,1.0 -3800,18.6,2.0959334506187588e-05,1.0,4.726535553345457e-05,1.0 -3900,19.1,9.678623428044375e-06,1.0,2.2248801542446017e-05,1.0 -4000,19.6,4.671124770538881e-06,1.0,1.0892742466239724e-05,1.0 -4100,20.0,0.0005926468293182552,1.0,0.0016443049535155296,1.0 -4200,20.5,0.0007911568973213434,1.0,0.0015760519308969378,1.0 -4300,21.0,0.0008353309822268784,1.0,0.0014344416558742523,1.0 -4400,21.5,0.0004670524795074016,1.0,0.0008043177658692002,1.0 -4500,22.0,0.00020260130986571312,1.0,0.0003375782398506999,1.0 -4600,22.5,9.218222839990631e-05,1.0,0.0001534665934741497,1.0 -4700,22.9,4.0846422052709386e-05,1.0,6.81631063343957e-05,1.0 -4800,23.4,1.8287297280039638e-05,1.0,3.060691597056575e-05,1.0 -4900,23.9,8.496562259097118e-06,1.0,1.4256095710152294e-05,1.0 -5000,24.4,4.143618298257934e-06,1.0,6.9562624958052766e-06,1.0 -5100,24.8,2.1658092919096816e-06,1.0,3.65603204954823e-06,1.0 -5200,25.3,1.2839423106925096e-06,1.0,2.15263230529672e-06,1.0 -5300,25.8,0.00046268405276350677,1.0,0.0011544801527634263,1.0 -5400,26.3,0.000684943690430373,1.0,0.0011853005271404982,1.0 -5500,26.8,0.0006766197038814425,1.0,0.0010354813421145082,1.0 -5600,27.2,0.00036385294515639544,1.0,0.0005645775236189365,1.0 -5700,27.7,0.00016452914860565215,1.0,0.00026286710635758936,1.0 -5800,28.2,7.277911208802834e-05,1.0,0.00012045382027281448,1.0 -5900,28.7,3.2475418265676126e-05,1.0,5.6583845434943214e-05,1.0 -6000,29.2,1.4532953173329588e-05,1.0,2.6840240025194362e-05,1.0 -6100,29.6,6.677251349174185e-06,1.0,1.2953828445461113e-05,1.0 -6200,30.1,3.3104015528806485e-06,1.0,7.027241281321039e-06,1.0 -6300,30.6,1.8011789961747127e-06,1.0,4.159307536610868e-06,1.0 -6400,31.1,1.1144660447826027e-06,1.0,2.710921080506523e-06,1.0 -6500,31.6,0.0003923251060768962,1.0,0.0016450249822810292,1.0 -6600,32.1,0.0004951342707499862,1.0,0.0011818903731182218,1.0 -6700,32.6,0.000585910223890096,1.0,0.001049555605277419,1.0 -6800,33.0,0.0003535434661898762,1.0,0.0005965897580608726,1.0 -6900,33.5,0.00015776019426994026,1.0,0.0002642474137246609,1.0 -7000,34.0,6.944742926862091e-05,1.0,0.0001130931923398748,1.0 -7100,34.5,3.1228795705828816e-05,1.0,5.1809616707032546e-05,1.0 -7200,34.9,1.4134619050309993e-05,1.0,2.4156523068086244e-05,1.0 -7300,35.4,6.611764092667727e-06,1.0,1.178601542051183e-05,1.0 -7400,35.9,3.278747726653819e-06,1.0,6.146764917502878e-06,1.0 -7500,36.4,1.7743492435329244e-06,1.0,3.4604609027155675e-06,1.0 -7600,36.9,1.0832164889507112e-06,1.0,2.189318365708459e-06,1.0 -7700,37.4,7.534148949162045e-07,1.0,1.5395169157272903e-06,1.0 -7800,37.8,0.002883616369217634,1.0,0.011213692836463451,0.9995525479316711 -7900,38.3,0.0005080680712126195,1.0,0.0012760566314682364,1.0 -8000,38.8,0.0007631474873051047,1.0,0.001352960243821144,1.0 -8100,39.3,0.000707934785168618,1.0,0.0011232757242396474,1.0 -8200,39.8,0.00036372695467434824,1.0,0.0005772125441581011,1.0 -8300,40.2,0.00016437875456176698,1.0,0.00026198080740869045,1.0 -8400,40.7,6.996525917202234e-05,1.0,0.00011064098362112418,1.0 -8500,41.2,3.145332811982371e-05,1.0,4.96091743116267e-05,1.0 -8600,41.7,1.4321126400318462e-05,1.0,2.3140304620028473e-05,1.0 -8700,42.2,6.698598554066848e-06,1.0,1.1170396646775771e-05,1.0 -8800,42.7,3.3381645607732935e-06,1.0,5.754519861511653e-06,1.0 -8900,43.1,1.787639348549419e-06,1.0,3.157364972139476e-06,1.0 -9000,43.6,1.11860572360456e-06,1.0,2.029515599133447e-06,1.0 -9100,44.1,0.0005313060246407986,1.0,0.0017576608806848526,1.0 -9200,44.6,0.0006873324746266007,1.0,0.0014380940701812506,1.0 -9300,45.1,0.0006630882853642106,1.0,0.001120651955716312,1.0 -9400,45.5,0.0003749238094314933,1.0,0.0005917057860642672,1.0 -9500,46.0,0.0001821612095227465,1.0,0.00028026034124195576,1.0 -9600,46.5,8.261817129096016e-05,1.0,0.00012737742508761585,1.0 -9700,47.0,3.694035694934428e-05,1.0,5.73440884181764e-05,1.0 -9800,47.4,1.6698806575732306e-05,1.0,2.6144236471736804e-05,1.0 -9900,47.9,7.772502613079268e-06,1.0,1.2236708244017791e-05,1.0 -10000,48.4,3.825707153737312e-06,1.0,6.032919372955803e-06,1.0 -10100,48.9,2.033123109868029e-06,1.0,3.1950660286383936e-06,1.0 -10200,49.4,1.2412076557666296e-06,1.0,1.9395922663534293e-06,1.0 -10300,49.8,0.0045984783209860325,1.0,0.0172222051769495,0.9984338879585266 -10400,50.3,0.0006751334294676781,1.0,0.002352476352825761,1.0 -10500,50.8,0.0009377398528158665,1.0,0.0017300304025411606,1.0 -10600,51.3,0.006734296213835478,1.0,0.013921461999416351,1.0 -10700,51.8,0.007099259179085493,1.0,0.010820728726685047,1.0 -10800,52.2,0.00506747467443347,1.0,0.00701179401949048,1.0 -10900,52.7,0.0024894042871892452,1.0,0.003410044824704528,1.0 -11000,53.2,0.0011040373938158154,1.0,0.0015170699916779995,1.0 -11100,53.7,0.00048132092342711985,1.0,0.000666449370328337,1.0 -11200,54.2,0.0002104034647345543,1.0,0.00029545140569098294,1.0 -11300,54.6,9.230555588146672e-05,1.0,0.00013231727643869817,1.0 -11400,55.1,4.1027604311238974e-05,1.0,6.0474558267742395e-05,1.0 -11500,55.6,1.839345713960938e-05,1.0,2.80937510979129e-05,1.0 -11600,56.1,8.76154808793217e-06,1.0,1.3968598977953661e-05,1.0 -11700,56.5,4.319814706832403e-06,1.0,7.144550181692466e-06,1.0 -11800,57.0,2.298090748809045e-06,1.0,3.905594439856941e-06,1.0 -11900,57.5,0.0002798094064928591,1.0,0.001120972796343267,0.999776303768158 -12000,58.0,0.0006983376224525273,1.0,0.0020941284019500017,1.0 -12100,58.5,0.0009680902003310621,1.0,0.00177027378231287,1.0 -12200,58.9,0.01199363637715578,1.0,0.03490154817700386,0.9966439604759216 -12300,59.4,0.009156409651041031,1.0,0.015013160184025764,1.0 -12400,59.9,0.008246197365224361,1.0,0.010931829921901226,1.0 -12500,60.4,0.004167424514889717,1.0,0.005325275007635355,1.0 -12600,60.9,0.0018630772829055786,1.0,0.0023749680258333683,1.0 -12700,61.3,0.0008114685188047588,1.0,0.0010484744561836123,1.0 -12800,61.8,0.0003523377235978842,1.0,0.00046581617789343,1.0 -12900,62.3,0.0001532813475932926,1.0,0.00020870123989880085,1.0 -13000,62.8,6.704378756694496e-05,1.0,9.471135854255408e-05,1.0 -13100,63.2,2.9554024877143092e-05,1.0,4.3497933802427724e-05,1.0 -13200,63.7,1.3568059330282267e-05,1.0,2.0952393242623657e-05,1.0 -13300,64.2,6.275299710978288e-06,1.0,1.0177901458519045e-05,1.0 -13400,64.7,3.2025839118432486e-06,1.0,5.438662810774986e-06,1.0 -13500,65.2,0.0004917937912978232,1.0,0.001359581365250051,1.0 -13600,65.6,0.0007334965048357844,1.0,0.001335139269940555,1.0 -13700,66.1,0.00086883147014305,1.0,0.001314869150519371,1.0 -13800,66.6,0.00048447874723933637,1.0,0.0007079103379510343,1.0 -13900,67.1,0.00021739391377195716,1.0,0.0003028364444617182,1.0 -14000,67.5,9.879273420665413e-05,1.0,0.0001387533120578155,1.0 -14100,68.0,4.4082029489800334e-05,1.0,6.28748894087039e-05,1.0 -14200,68.5,1.9763076124945655e-05,1.0,2.8721509806928225e-05,1.0 -14300,69.0,9.088336810236797e-06,1.0,1.3511278666555882e-05,1.0 -14400,69.5,4.471613010537112e-06,1.0,6.8229696807975415e-06,1.0 -14500,69.9,2.299989318998996e-06,1.0,3.592873554225662e-06,1.0 -14600,70.4,1.3905769264965784e-06,1.0,2.219497901023715e-06,1.0 -14700,70.9,8.971818488134886e-07,1.0,1.452447691008274e-06,1.0 -14800,71.4,0.0005548478220589459,1.0,0.0034676529467105865,0.9993287920951843 -14900,71.9,0.0008203514153137803,1.0,0.0024806626606732607,0.9998881816864014 -15000,72.3,0.3301088511943817,0.987728476524353,0.4432120621204376,0.9799754023551941 -15100,72.8,0.007329842075705528,1.0,0.01197031233459711,1.0 -15200,73.3,0.009141476824879646,1.0,0.012009644880890846,1.0 -15300,73.8,0.005374308675527573,1.0,0.006819399539381266,1.0 -15400,74.3,0.0024337302893400192,1.0,0.003086224664002657,1.0 -15500,74.7,0.0010649461764842272,1.0,0.0013580908998847008,1.0 -15600,75.2,0.0004643466090783477,1.0,0.0005991535144858062,1.0 -15700,75.7,0.000202906274353154,1.0,0.0002665232168510556,1.0 -15800,76.2,8.931693446356803e-05,1.0,0.00011989210906904191,1.0 -15900,76.6,3.9617236325284466e-05,1.0,5.457377847051248e-05,1.0 -16000,77.1,1.811216316127684e-05,1.0,2.5691480914247222e-05,1.0 -16100,77.6,8.414041985815857e-06,1.0,1.2315945241425652e-05,1.0 -16200,78.1,4.2350616240582895e-06,1.0,6.390140697476454e-06,1.0 -16300,78.6,2.2418164462578716e-06,1.0,3.444311687417212e-06,1.0 -16400,79.0,0.002211393555626273,1.0,0.0070736343041062355,1.0 -16500,79.5,0.0005224687047302723,1.0,0.0012233888264745474,1.0 -16600,80.0,0.0006988064851611853,1.0,0.0011281550396233797,1.0 -16700,80.5,0.0005407471326179802,1.0,0.0007378423470072448,1.0 -16800,81.0,0.00027808875893242657,1.0,0.0003747000591829419,1.0 -16900,81.4,0.00012549659004434943,1.0,0.00017047798610292375,1.0 -17000,81.9,5.5600510677322745e-05,1.0,7.655193621758372e-05,1.0 -17100,82.4,2.4854642106220126e-05,1.0,3.476999336271547e-05,1.0 -17200,82.9,1.1300125152047258e-05,1.0,1.6135316400323063e-05,1.0 -17300,83.4,5.367700850911206e-06,1.0,7.836620170564856e-06,1.0 -17400,83.8,2.7145131298311753e-06,1.0,4.063080268679187e-06,1.0 -17500,84.3,1.5220492741718772e-06,1.0,2.3105817490431946e-06,1.0 -17600,84.8,0.0009477337589487433,1.0,0.002827127231284976,1.0 -17700,85.3,0.0008222770411521196,1.0,0.001656360225751996,1.0 -17800,85.8,0.001036999048665166,1.0,0.0016034099971875548,1.0 -17900,86.3,0.0007643711869604886,1.0,0.0010936452308669686,1.0 -18000,86.7,0.00037399015855044127,1.0,0.0005370200960896909,1.0 -18100,87.2,0.0001713020319584757,1.0,0.00024987285723909736,1.0 -18200,87.7,7.729998469585553e-05,1.0,0.00011575953976716846,1.0 -18300,88.2,3.466704220045358e-05,1.0,5.377516936277971e-05,1.0 -18400,88.6,1.573052395542618e-05,1.0,2.5520244889776222e-05,1.0 -18500,89.1,7.533525604230817e-06,1.0,1.2918816537421662e-05,1.0 -18600,89.6,3.672043021651916e-06,1.0,6.705344276269898e-06,1.0 -18700,90.1,0.0012368567986413836,1.0,0.004544305149465799,1.0 -18800,90.5,0.0016762643354013562,1.0,0.0035588534083217382,1.0 -18900,91.0,0.0015084183542057872,1.0,0.0024383217096328735,1.0 -19000,91.5,0.0009229227434843779,1.0,0.0013539334759116173,1.0 -19100,92.0,0.0004541188245639205,1.0,0.0006523402407765388,1.0 -19200,92.5,0.00020567994215525687,1.0,0.00029652126249857247,1.0 -19300,93.0,9.11656316020526e-05,1.0,0.0001330472732661292,1.0 -19400,93.4,4.0502523916075006e-05,1.0,6.035158730810508e-05,1.0 -19500,93.9,1.8319322407478467e-05,1.0,2.8081642085453495e-05,1.0 -19600,94.4,8.53126221045386e-06,1.0,1.3558656064560637e-05,1.0 -19700,94.9,4.109100245841546e-06,1.0,6.817590929131256e-06,1.0 -19800,95.3,5.634999524772866e-06,1.0,9.927807695930824e-06,1.0 -19900,95.8,0.0006442063604481518,1.0,0.0014361984794959426,1.0 -20000,96.3,0.0010698649566620588,1.0,0.0017319556791335344,1.0 -20100,96.8,0.0010934072779491544,1.0,0.0016132511664181948,1.0 -20200,97.3,0.0005623938050121069,1.0,0.0008251647232100368,1.0 -20300,97.7,0.00024058629060164094,1.0,0.0003518572193570435,1.0 -20400,98.2,0.00011082698620157316,1.0,0.00015999747847672552,1.0 -20500,98.7,4.964464096701704e-05,1.0,7.281923899427056e-05,1.0 -20600,99.2,2.2400703528546728e-05,1.0,3.364321673871018e-05,1.0 -20700,99.7,1.0269361155224033e-05,1.0,1.590874308021739e-05,1.0 -20800,100.1,4.949162303091725e-06,1.0,7.948260645207483e-06,1.0 -20900,100.6,2.567789124441333e-06,1.0,4.265462848707102e-06,1.0 -21000,101.1,0.0007666641031391919,1.0,0.0022306563332676888,1.0 -21100,101.6,0.0011683569755405188,1.0,0.002229501958936453,1.0 -21200,102.1,0.0013404921628534794,1.0,0.002009003423154354,1.0 -21300,102.5,0.000792286591604352,1.0,0.0011332312133163214,1.0 -21400,103.0,0.0003797024837695062,1.0,0.0005285012302920222,1.0 -21500,103.5,0.00017398636555299163,1.0,0.00024352679611183703,1.0 -21600,104.0,7.820414612069726e-05,1.0,0.00011106150486739352,1.0 -21700,104.5,3.514664058457129e-05,1.0,5.106910612084903e-05,1.0 -21800,104.9,1.601676194695756e-05,1.0,2.3989705368876457e-05,1.0 -21900,105.4,7.434328381350497e-06,1.0,1.1531759810168296e-05,1.0 -22000,105.9,3.7165825688134646e-06,1.0,5.938298727414804e-06,1.0 -22100,106.4,0.0006315343780443072,1.0,0.001404088456183672,1.0 -22200,106.9,0.001079067005775869,1.0,0.0017374465242028236,1.0 -22300,107.3,0.0012993748532608151,1.0,0.0018054975662380457,1.0 -22400,107.8,0.0007591251633130014,1.0,0.001039925729855895,1.0 -22500,108.3,0.0003302383702248335,1.0,0.0004537054046522826,1.0 -22600,108.8,0.00014331044803839177,1.0,0.00019349677313584834,1.0 -22700,109.2,6.444998143706471e-05,1.0,8.844323019729927e-05,1.0 -22800,109.7,2.8845068300142884e-05,1.0,4.0656072087585926e-05,1.0 -22900,110.2,1.3121822121320292e-05,1.0,1.9057350073126145e-05,1.0 -23000,110.7,6.243992174859159e-06,1.0,9.385763405589387e-06,1.0 -23100,111.2,3.138124839097145e-06,1.0,4.879350399278337e-06,1.0 -23200,111.7,0.0008675913559272885,1.0,0.0021766365971416235,1.0 -23300,112.1,0.0012546065263450146,1.0,0.002146842423826456,1.0 -23400,112.6,0.001075842184945941,1.0,0.0016078235348686576,1.0 -23500,113.1,0.0005450936150737107,1.0,0.0007974550826475024,1.0 -23600,113.6,0.0002592594246380031,1.0,0.00036537094274535775,1.0 -23700,114.0,0.00011879096564371139,1.0,0.00016849397798068821,1.0 -23800,114.5,5.316842725733295e-05,1.0,7.649810868315399e-05,1.0 -23900,115.0,2.391943780821748e-05,1.0,3.509407542878762e-05,1.0 -24000,115.5,1.0971905794576742e-05,1.0,1.6509722627233714e-05,1.0 -24100,116.0,5.332030468707671e-06,1.0,8.246697689173743e-06,1.0 -24200,116.4,2.7073226647189585e-06,1.0,4.287960109650157e-06,1.0 -24300,116.9,0.0005011926405131817,1.0,0.001250040833838284,1.0 -24400,117.4,0.0008104377775453031,1.0,0.0013459640322253108,1.0 -24500,117.9,0.0008461957913823426,1.0,0.0012210853165015578,1.0 -24600,118.4,0.00045067392056807876,1.0,0.000645310094114393,1.0 -24700,118.8,0.00020247681823093444,1.0,0.00029190111672505736,1.0 -24800,119.3,8.665033965371549e-05,1.0,0.0001227188331540674,1.0 -24900,119.8,3.932224353775382e-05,1.0,5.608615174423903e-05,1.0 -25000,120.3,1.7729635146679357e-05,1.0,2.5863559130812064e-05,1.0 -25100,120.8,8.225777492043562e-06,1.0,1.2277150744921528e-05,1.0 -25200,121.2,4.0310706026502885e-06,1.0,6.160464181448333e-06,1.0 -25300,121.7,2.2735330276191235e-06,1.0,3.565121460269438e-06,1.0 -25400,122.2,0.0008401934173889458,1.0,0.002033665543422103,1.0 -25500,122.7,0.0011953682405874133,1.0,0.002068485366180539,1.0 -25600,123.1,0.001043739845044911,1.0,0.0015325590502470732,1.0 -25700,123.6,0.0006046230555512011,1.0,0.0008391322917304933,1.0 -25800,124.1,0.00028595898766070604,1.0,0.0003957389562856406,1.0 -25900,124.6,0.00012899840658064932,1.0,0.00018002078286372125,1.0 -26000,125.1,5.779427374363877e-05,1.0,8.188448555301875e-05,1.0 -26100,125.5,2.6070012609125115e-05,1.0,3.759337778319605e-05,1.0 -26200,126.0,1.198822519654641e-05,1.0,1.7650083464104682e-05,1.0 -26300,126.5,5.716984105674783e-06,1.0,8.592494850745425e-06,1.0 -26400,127.0,2.8887816370115615e-06,1.0,4.4412954594008625e-06,1.0 -26500,127.5,1.6091994439193513e-06,1.0,2.5128726974799065e-06,1.0 -26600,127.9,0.0005350273568183184,1.0,0.0015316092176362872,1.0 -26700,128.4,0.000811939244158566,1.0,0.0015084597980603576,1.0 -26800,128.9,0.0008564452291466296,1.0,0.001253992086276412,1.0 -26900,129.4,0.0005484418943524361,1.0,0.0007419759058393538,1.0 -27000,129.9,0.0002705449587665498,1.0,0.0003624797100201249,1.0 -27100,130.3,0.0001254620001418516,1.0,0.00016959146887529641,1.0 -27200,130.8,5.6672532082302496e-05,1.0,7.774309779051691e-05,1.0 -27300,131.3,2.555980609031394e-05,1.0,3.5749162634601817e-05,1.0 -27400,131.8,1.1790896678576246e-05,1.0,1.6909993064473383e-05,1.0 -27500,132.2,0.001641804468818009,1.0,0.0027698760386556387,1.0 -27600,132.7,0.0009138115565292537,1.0,0.0019570442382246256,1.0 -27700,133.2,0.001334909349679947,1.0,0.0021001307759433985,1.0 -27800,133.7,0.0009531270479783416,1.0,0.0013473944272845984,1.0 -27900,134.2,0.00046325515722855926,1.0,0.0006553237326443195,1.0 -28000,134.7,0.00020622966985683888,1.0,0.0002952086506411433,1.0 -28100,135.1,9.030741784954444e-05,1.0,0.00013165657583158463,1.0 -28200,135.6,3.951612234232016e-05,1.0,5.88691582379397e-05,1.0 -28300,136.1,1.7443135220673867e-05,1.0,2.6509944291319698e-05,1.0 -28400,136.6,8.065297151915729e-06,1.0,1.2552874977700412e-05,1.0 -28500,137.0,3.9795581869839225e-06,1.0,6.426243544410681e-06,1.0 -28600,137.5,2.0957777451258153e-06,1.0,3.453393674135441e-06,1.0 -28700,138.0,0.0021518534049391747,1.0,0.006964025087654591,1.0 -28800,138.5,0.0004392733972053975,1.0,0.0009647279512137175,1.0 -28900,139.0,0.0005488572060130537,1.0,0.0008706892258487642,1.0 -29000,139.5,0.00039429281605407596,1.0,0.0005920868134126067,1.0 -29100,139.9,0.0001876351743703708,1.0,0.0002827202551998198,1.0 -29200,140.4,8.402076491620392e-05,1.0,0.00012784327554982156,1.0 -29300,140.9,3.7774087104480714e-05,1.0,5.8987036027247086e-05,1.0 -29400,141.4,1.695680111879483e-05,1.0,2.7346872229827568e-05,1.0 -29500,141.8,7.796059435349889e-06,1.0,1.2951684766449034e-05,1.0 -29600,142.3,3.79333641831181e-06,1.0,6.4234632191073615e-06,1.0 -29700,142.8,2.0576806036842754e-06,1.0,3.5342072806088254e-06,1.0 -29800,143.3,0.0008360974607057869,1.0,0.0027455349918454885,1.0 -29900,143.8,0.0008648824295960367,1.0,0.0017791681457310915,1.0 -30000,144.2,0.0010286164470016956,1.0,0.0016454393044114113,1.0 -30100,144.7,0.0006325239082798362,1.0,0.000938022043555975,1.0 -30200,145.2,0.0002889255410991609,1.0,0.0004310100630391389,1.0 -30300,145.7,0.00011984517914243042,1.0,0.00017704602214507759,1.0 -30400,146.2,5.4625757911708206e-05,1.0,7.893858855823055e-05,1.0 -30500,146.6,2.4656776076881215e-05,1.0,3.629387356340885e-05,1.0 -30600,147.1,1.1336015631968621e-05,1.0,1.710975448077079e-05,1.0 -30700,147.6,5.4365809774026275e-06,1.0,8.428564797213767e-06,1.0 -30800,148.1,2.794908823489095e-06,1.0,4.442589670361485e-06,1.0 -30900,148.6,1.5707912552898051e-06,1.0,2.557774450906436e-06,1.0 -31000,149.0,0.0007001275080256164,1.0,0.0024113506078720093,1.0 -31100,149.5,0.0009740054956637323,1.0,0.001965248491615057,1.0 -31200,150.0,0.001069405348971486,1.0,0.0016485531814396381,1.0 -31300,150.5,0.0006994553841650486,1.0,0.001050001592375338,1.0 -31400,150.9,0.00032921673846431077,1.0,0.0005019352538511157,1.0 -31500,151.4,0.00014730791735928506,1.0,0.00022832096146885306,1.0 -31600,151.9,6.348235183395445e-05,1.0,9.799366671359167e-05,1.0 -31700,152.4,2.8240490792086348e-05,1.0,4.231529601383954e-05,1.0 -31800,152.9,1.29397658383823e-05,1.0,1.9640385289676487e-05,1.0 -31900,153.3,6.155253231554525e-06,1.0,9.531427167530637e-06,1.0 -32000,153.8,3.1504810067417566e-06,1.0,4.969643669028301e-06,1.0 -32100,154.3,1.7138419252660242e-06,1.0,2.752743512246525e-06,1.0 -32200,154.8,0.00022070510021876544,1.0,0.0008070211042650044,1.0 -32300,155.3,0.00027943463646806777,1.0,0.0005424200790002942,1.0 -32400,155.7,0.00028485391521826386,1.0,0.00043507941882126033,1.0 -32500,156.2,0.00016252649948000908,1.0,0.00023372238501906395,1.0 -32600,156.7,7.713441300438717e-05,1.0,0.00010992003080900759,1.0 -32700,157.2,3.4897690056823194e-05,1.0,5.0287035264773294e-05,1.0 -32800,157.7,1.5900010112090968e-05,1.0,2.3308613890549168e-05,1.0 -32900,158.1,7.409609679598361e-06,1.0,1.1084300240327138e-05,1.0 -33000,158.6,3.666687916847877e-06,1.0,5.584740847552894e-06,1.0 -33100,159.1,1.9714018435479375e-06,1.0,3.0527055514539825e-06,1.0 -33200,159.6,1.1731369795597857e-06,1.0,1.8321851484870422e-06,1.0 -33300,160.1,0.0009818350663408637,1.0,0.0029433723539114,1.0 -33400,160.5,0.0014791685389354825,1.0,0.002775584114715457,1.0 -33500,161.0,0.0016155204502865672,1.0,0.0023542670533061028,1.0 -33600,161.5,0.0010357400169596076,1.0,0.0014561228454113007,1.0 -33700,162.0,0.0004846363444812596,1.0,0.0006905843038111925,1.0 -33800,162.4,0.00021705051767639816,1.0,0.0003163564542774111,1.0 -33900,162.9,9.650861466070637e-05,1.0,0.00014477560762315989,1.0 -34000,163.4,4.2776362533913925e-05,1.0,6.646296969847754e-05,1.0 -34100,163.9,1.911480831040535e-05,1.0,3.0784074624534696e-05,1.0 -34200,164.4,8.642875400255434e-06,1.0,1.4390564501809422e-05,1.0 -34300,164.8,4.27748636866454e-06,1.0,7.372902473434806e-06,1.0 -34400,165.3,0.0009354072972200811,1.0,0.00280961487442255,1.0 -34500,165.8,0.0012345538707450032,1.0,0.002343927277252078,1.0 -34600,166.3,0.0011400593211874366,1.0,0.0018375474028289318,1.0 -34700,166.8,0.000670194101985544,1.0,0.0010461966739967465,1.0 -34800,167.3,0.00029445503605529666,1.0,0.0004501276707742363,1.0 -34900,167.7,0.00013363173638936132,1.0,0.00019922188948839903,1.0 -35000,168.2,5.979478737572208e-05,1.0,9.130416583502665e-05,1.0 -35100,168.7,2.6781257474794984e-05,1.0,4.2262006900273263e-05,1.0 -35200,169.2,1.2195582712593023e-05,1.0,1.9951867216150276e-05,1.0 -35300,169.6,5.7930542425310705e-06,1.0,9.852305083768442e-06,1.0 -35400,170.1,0.061183176934719086,0.9934725761413574,0.09982078522443771,0.9855688810348511 -35500,170.6,0.0012622798094525933,1.0,0.003038577036932111,1.0 -35600,171.1,0.0015832859789952636,1.0,0.0027024103328585625,1.0 -35700,171.6,0.0012877220287919044,1.0,0.0019188629230484366,1.0 -35800,172.0,0.0006764077697880566,1.0,0.0010040531633421779,1.0 -35900,172.5,0.0003113254497293383,1.0,0.0004679745761677623,1.0 -36000,173.0,0.00013970227155368775,1.0,0.00021467183250933886,1.0 -36100,173.5,6.235668115550652e-05,1.0,9.875012619886547e-05,1.0 -36200,174.0,2.7956088160863146e-05,1.0,4.602251647156663e-05,1.0 -36300,174.4,1.2738426448777318e-05,1.0,2.190931081713643e-05,1.0 -36400,174.9,6.319158273981884e-06,1.0,1.1396232366678305e-05,1.0 -36500,175.4,0.0014799929922446609,1.0,0.0037643529940396547,1.0 -36600,175.9,0.001289251376874745,1.0,0.0024075815454125404,1.0 -36700,176.4,0.0015515796840190887,1.0,0.0023720725439488888,1.0 -36800,176.8,0.0010513741290196776,1.0,0.001600598101504147,1.0 -36900,177.3,0.00047702935989946127,1.0,0.0007402586634270847,1.0 -37000,177.8,0.00020715770369861275,1.0,0.000321362167596817,1.0 -37100,178.3,9.075689013116062e-05,1.0,0.00013647315790876746,1.0 -37200,178.8,4.0768518374534324e-05,1.0,6.283259426709265e-05,1.0 -37300,179.2,1.8368564269621857e-05,1.0,2.9320297471713275e-05,1.0 -37400,179.7,8.551747669116594e-06,1.0,1.4220803677744698e-05,1.0 -37500,180.2,4.459163847059244e-06,1.0,7.776514394208789e-06,1.0 -37600,180.7,0.0011325523955747485,1.0,0.0027160332538187504,1.0 -37700,181.2,0.0017164598684757948,1.0,0.003048005048185587,1.0 -37800,181.6,0.0013948684791103005,1.0,0.002149956999346614,1.0 -37900,182.1,0.000742921547498554,1.0,0.0011513573117554188,1.0 -38000,182.6,0.00033838761737570167,1.0,0.0005411971360445023,1.0 -38100,183.1,0.00015059787256177515,1.0,0.00025130578433163464,1.0 -38200,183.5,6.665930413873866e-05,1.0,0.00011760069173760712,1.0 -38300,184.0,2.9790433472953737e-05,1.0,5.642922769766301e-05,1.0 -38400,184.5,1.3483185284712818e-05,1.0,2.8042521080351435e-05,1.0 -38500,185.0,6.286852567427559e-06,1.0,1.469523886044044e-05,1.0 -38600,185.5,0.001793139730580151,1.0,0.007782443426549435,0.9995525479316711 -38700,185.9,0.0007951961597427726,1.0,0.0020863846875727177,1.0 -38800,186.4,0.0008166965562850237,1.0,0.0014718691818416119,1.0 -38900,186.9,0.0006048438372090459,1.0,0.0010123709216713905,1.0 -39000,187.4,0.0002926464076153934,1.0,0.0004923052038066089,1.0 -39100,187.9,0.00013198429951444268,1.0,0.00022661946422886103,1.0 -39200,188.3,5.855754716321826e-05,1.0,0.00010410530376248062,1.0 -39300,188.8,2.603120265121106e-05,1.0,4.856671512243338e-05,1.0 -39400,189.3,1.1819845894933678e-05,1.0,2.347218105569482e-05,1.0 -39500,189.8,5.816086741106119e-06,1.0,1.2543257071229164e-05,1.0 -39600,190.3,2.9020720830885693e-06,1.0,6.801554718549596e-06,1.0 -39700,190.7,0.0014797316398471594,1.0,0.0038965384010225534,1.0 -39800,191.2,0.0020074632484465837,1.0,0.003919458948075771,1.0 -39900,191.7,0.0016954356105998158,1.0,0.003080500988289714,1.0 -40000,192.2,0.0008347955299541354,1.0,0.0013898900942876935,1.0 +step,elapsed,loss,train_loss,train_acc,test_loss,test_acc +0,0.3,nan,4.7616143226623535,0.009660574607551098,4.762050151824951,0.00827833078801632 +100,1.8,1.8361858335500958,0.1685766726732254,1.0,4.590503215789795,0.24465824663639069 +200,3.1,0.28594419612092237,0.029104363173246384,1.0,4.530285358428955,0.2740798890590668 +300,4.5,0.14437714340179364,0.0216338150203228,1.0,4.445437431335449,0.2886228859424591 +400,5.9,0.03193849465677494,0.01258074026554823,1.0,4.299746036529541,0.2962300181388855 +500,7.2,0.011390094110461092,0.006273516453802586,1.0,4.312851905822754,0.29857927560806274 +600,8.6,0.05943287372928064,0.012299809604883194,1.0,3.9216959476470947,0.31010180711746216 +700,9.9,0.01764159063601431,0.009999701753258705,1.0,3.7636139392852783,0.314129114151001 +800,11.2,0.008149907609959815,0.00509463157504797,1.0,3.692552089691162,0.3180445432662964 +900,12.6,0.003711820517942838,0.002197114983573556,1.0,3.6361052989959717,0.3267703354358673 +1000,13.9,0.0016041877239498128,0.0009332005283795297,1.0,3.568415880203247,0.33482491970062256 +1100,15.3,0.0006844259461080087,0.0003910401137545705,1.0,3.492629289627075,0.34713056683540344 +1200,16.7,0.0002938540214973156,0.0001666255120653659,1.0,3.41487717628479,0.3554089069366455 +1300,18.0,0.0581449512679921,0.004765611607581377,1.0,2.6445014476776123,0.4068687856197357 +1400,19.3,0.012049458736153216,0.005297891329973936,1.0,2.382455825805664,0.4258865714073181 +1500,20.7,0.10800065280618955,0.014239107258617878,1.0,1.7212132215499878,0.5430137515068054 +1600,22.0,0.024651921413155593,0.012043540365993977,1.0,1.5408352613449097,0.6118134260177612 +1700,23.5,0.011063419411690884,0.006985262967646122,1.0,1.275530219078064,0.691240668296814 +1800,25.2,0.005071398160439759,0.0030147237703204155,1.0,0.9464532732963562,0.782637894153595 +1900,26.9,0.0022047900941814516,0.0012752332258969545,1.0,0.6555971503257751,0.8591565489768982 +2000,28.7,0.0009409276498163585,0.0005410491139627993,1.0,0.43341580033302307,0.9168810844421387 +2100,30.3,0.0004014714290548189,0.00023161059652920812,1.0,0.27861881256103516,0.9488757252693176 +2200,31.9,0.0001721161960518375,9.997520101023838e-05,1.0,0.17426809668540955,0.9718089699745178 +2300,33.6,7.447141833585157e-05,4.384289786685258e-05,1.0,0.10947011411190033,0.984785795211792 +2400,35.2,3.2713196888737454e-05,1.956131927727256e-05,1.0,0.06694776564836502,0.9917216897010803 +2500,36.8,0.025378014469029656,0.002068975707516074,1.0,0.10334945470094681,0.9874706864356995 +2600,38.5,0.004898688796745345,0.0018950446974486113,1.0,0.056251391768455505,0.9972032904624939 +2700,40.1,0.4629608842574754,0.038790877908468246,1.0,0.18772314488887787,0.9916098117828369 +2800,41.8,0.0710911453838005,0.010595878586173058,1.0,0.05027606338262558,0.9995525479316711 +2900,43.4,0.15428267053544087,0.02573176845908165,1.0,0.06594014912843704,1.0 +3000,45.0,0.03232126480101132,0.012696159072220325,1.0,0.031211240217089653,1.0 +3100,46.7,0.012532647598791778,0.0075059980154037476,1.0,0.01827137917280197,1.0 +3200,48.3,0.005550335268581018,0.003237507538869977,1.0,0.00778270373120904,1.0 +3300,50.1,0.0023726405226991206,0.001360321301035583,1.0,0.003161293687298894,1.0 +3400,51.8,0.0010079321120949033,0.000579379906412214,1.0,0.0013025131775066257,1.0 +3500,53.5,0.0004300056315979733,0.00024824557476677,1.0,0.0005462610861286521,1.0 +3600,55.2,0.0001847889967292484,0.00010757338895928115,1.0,0.00023506060824729502,1.0 +3700,56.9,8.023588960497994e-05,4.710876601166092e-05,1.0,0.00010424039646750316,1.0 +3800,58.5,3.5316101702806325e-05,2.0959334506187588e-05,1.0,4.726535553345457e-05,1.0 +3900,60.2,1.5881106973457144e-05,9.678623428044375e-06,1.0,2.2248801542446017e-05,1.0 +4000,61.8,7.424592452605847e-06,4.671124770538881e-06,1.0,1.0892742466239724e-05,1.0 +4100,63.5,0.008809864158014252,0.0005926468293182552,1.0,0.0016443049535155296,1.0 +4200,65.1,0.001772379672606135,0.0007911568973213434,1.0,0.0015760519308969378,1.0 +4300,66.8,0.000982188919892813,0.0008353309822268784,1.0,0.0014344416558742523,1.0 +4400,68.5,0.0006535534146822644,0.0004670524795074016,1.0,0.0008043177658692002,1.0 +4500,70.2,0.0003255751339133557,0.00020260130986571312,1.0,0.0003375782398506999,1.0 +4600,71.8,0.00015126972754589434,9.218222839990631e-05,1.0,0.0001534665934741497,1.0 +4700,73.5,6.81993558296686e-05,4.0846422052709386e-05,1.0,6.81631063343957e-05,1.0 +4800,75.2,3.0503323188898802e-05,1.8287297280039638e-05,1.0,3.060691597056575e-05,1.0 +4900,76.9,1.3825467747667612e-05,8.496562259097118e-06,1.0,1.4256095710152294e-05,1.0 +5000,78.5,6.497499576828661e-06,4.143618298257934e-06,1.0,6.9562624958052766e-06,1.0 +5100,80.2,3.253788893140551e-06,2.1658092919096816e-06,1.0,3.65603204954823e-06,1.0 +5200,81.9,1.7870896151476433e-06,1.2839423106925096e-06,1.0,2.15263230529672e-06,1.0 +5300,83.6,0.0053066638542206045,0.00046268405276350677,1.0,0.0011544801527634263,1.0 +5400,85.2,0.0012359460260007006,0.000684943690430373,1.0,0.0011853005271404982,1.0 +5500,86.9,0.0007828462677820665,0.0006766197038814425,1.0,0.0010354813421145082,1.0 +5600,88.6,0.0005134214048125059,0.00036385294515639544,1.0,0.0005645775236189365,1.0 +5700,90.2,0.00026114073823756254,0.00016452914860565215,1.0,0.00026286710635758936,1.0 +5800,91.8,0.00012044310680498957,7.277911208802834e-05,1.0,0.00012045382027281448,1.0 +5900,93.5,5.4100803218603575e-05,3.2475418265676126e-05,1.0,5.6583845434943214e-05,1.0 +6000,95.1,2.4220386971979734e-05,1.4532953173329588e-05,1.0,2.6840240025194362e-05,1.0 +6100,96.7,1.0901661093067807e-05,6.677251349174185e-06,1.0,1.2953828445461113e-05,1.0 +6200,98.4,5.16353994204612e-06,3.3104015528806485e-06,1.0,7.027241281321039e-06,1.0 +6300,100.1,2.6342428347069984e-06,1.8011789961747127e-06,1.0,4.159307536610868e-06,1.0 +6400,101.8,1.4996596936546021e-06,1.1144660447826027e-06,1.0,2.710921080506523e-06,1.0 +6500,103.4,0.006702179821271168,0.0003923251060768962,1.0,0.0016450249822810292,1.0 +6600,105.1,0.0012644098850294232,0.0004951342707499862,1.0,0.0011818903731182218,1.0 +6700,106.8,0.0006678882601790763,0.000585910223890096,1.0,0.001049555605277419,1.0 +6800,108.4,0.0004765179643008709,0.0003535434661898762,1.0,0.0005965897580608726,1.0 +6900,110.0,0.00024959002989913216,0.00015776019426994026,1.0,0.0002642474137246609,1.0 +7000,111.7,0.0001140585716471152,6.944742926862091e-05,1.0,0.0001130931923398748,1.0 +7100,113.3,5.172918674363556e-05,3.1228795705828816e-05,1.0,5.1809616707032546e-05,1.0 +7200,115.1,2.333316603488584e-05,1.4134619050309993e-05,1.0,2.4156523068086244e-05,1.0 +7300,116.7,1.0678021752719362e-05,6.611764092667727e-06,1.0,1.178601542051183e-05,1.0 +7400,118.4,5.083704140775569e-06,3.278747726653819e-06,1.0,6.146764917502878e-06,1.0 +7500,120.0,2.6011643816773045e-06,1.7743492435329244e-06,1.0,3.4604609027155675e-06,1.0 +7600,121.7,1.4758852116959671e-06,1.0832164889507112e-06,1.0,2.189318365708459e-06,1.0 +7700,123.4,9.304301807391052e-07,7.534148949162045e-07,1.0,1.5395169157272903e-06,1.0 +7800,125.0,0.02508790687422794,0.002883616369217634,1.0,0.011213692836463451,0.9995525479316711 +7900,126.7,0.003800912997260111,0.0005080680712126195,1.0,0.0012760566314682364,1.0 +8000,128.3,0.0010947673716702187,0.0007631474873051047,1.0,0.001352960243821144,1.0 +8100,130.0,0.0008142176934526992,0.000707934785168618,1.0,0.0011232757242396474,1.0 +8200,131.6,0.000520598105216643,0.00036372695467434824,1.0,0.0005772125441581011,1.0 +8300,133.3,0.0002621728695730202,0.00016437875456176698,1.0,0.00026198080740869045,1.0 +8400,135.0,0.00011887840627453285,6.996525917202234e-05,1.0,0.00011064098362112418,1.0 +8500,136.7,5.247282708853229e-05,3.145332811982371e-05,1.0,4.96091743116267e-05,1.0 +8600,138.4,2.3625655165416407e-05,1.4321126400318462e-05,1.0,2.3140304620028473e-05,1.0 +8700,140.1,1.0833378485062994e-05,6.698598554066848e-06,1.0,1.1170396646775771e-05,1.0 +8800,141.8,5.162706661780588e-06,3.3381645607732935e-06,1.0,5.754519861511653e-06,1.0 +8900,143.6,2.6360166527592758e-06,1.787639348549419e-06,1.0,3.157364972139476e-06,1.0 +9000,145.2,1.4903428671689847e-06,1.11860572360456e-06,1.0,2.029515599133447e-06,1.0 +9100,146.9,0.008820143088457638,0.0005313060246407986,1.0,0.0017576608806848526,1.0 +9200,148.6,0.001714762647870527,0.0006873324746266007,1.0,0.0014380940701812506,1.0 +9300,150.4,0.0008285305677877859,0.0006630882853642106,1.0,0.001120651955716312,1.0 +9400,152.1,0.0005290666627846457,0.0003749238094314933,1.0,0.0005917057860642672,1.0 +9500,153.8,0.00027841453826399186,0.0001821612095227465,1.0,0.00028026034124195576,1.0 +9600,155.6,0.00013351577492476196,8.261817129096016e-05,1.0,0.00012737742508761585,1.0 +9700,157.3,6.104154278228605e-05,3.694035694934428e-05,1.0,5.73440884181764e-05,1.0 +9800,159.0,2.7609503233495158e-05,1.6698806575732306e-05,1.0,2.6144236471736804e-05,1.0 +9900,160.7,1.2621406357003756e-05,7.772502613079268e-06,1.0,1.2236708244017791e-05,1.0 +10000,162.3,5.964147274400155e-06,3.825707153737312e-06,1.0,6.032919372955803e-06,1.0 +10100,164.0,2.9979441575083516e-06,2.033123109868029e-06,1.0,3.1950660286383936e-06,1.0 +10200,165.6,1.6691025084854253e-06,1.2412076557666296e-06,1.0,1.9395922663534293e-06,1.0 +10300,167.3,0.032322328429370197,0.0045984783209860325,1.0,0.0172222051769495,0.9984338879585266 +10400,168.9,0.0048980130617753345,0.0006751334294676781,1.0,0.002352476352825761,1.0 +10500,170.6,0.0014011149180521054,0.0009377398528158665,1.0,0.0017300304025411606,1.0 +10600,172.3,0.10119174899266499,0.006734296213835478,1.0,0.013921461999416351,1.0 +10700,173.9,0.019122699605059246,0.007099259179085493,1.0,0.010820728726685047,1.0 +10800,175.7,0.00783036063056049,0.00506747467443347,1.0,0.00701179401949048,1.0 +10900,177.3,0.003890068082860514,0.0024894042871892452,1.0,0.003410044824704528,1.0 +11000,178.9,0.0018203379588399812,0.0011040373938158154,1.0,0.0015170699916779995,1.0 +11100,180.6,0.0008123371730995609,0.00048132092342711985,1.0,0.000666449370328337,1.0 +11200,182.3,0.00035681451496626153,0.0002104034647345543,1.0,0.00029545140569098294,1.0 +11300,183.9,0.00015651090191236196,9.230555588146672e-05,1.0,0.00013231727643869817,1.0 +11400,185.6,6.89555489843292e-05,4.1027604311238974e-05,1.0,6.0474558267742395e-05,1.0 +11500,187.4,3.0708268760714264e-05,1.839345713960938e-05,1.0,2.80937510979129e-05,1.0 +11600,189.1,1.395833895912403e-05,8.76154808793217e-06,1.0,1.3968598977953661e-05,1.0 +11700,190.7,6.670976953513592e-06,4.319814706832403e-06,1.0,7.144550181692466e-06,1.0 +11800,192.4,3.333405027749697e-06,2.298090748809045e-06,1.0,3.905594439856941e-06,1.0 +11900,194.0,0.0003232912585154576,0.0002798094064928591,1.0,0.001120972796343267,0.999776303768158 +12000,195.7,0.007635427696176083,0.0006983376224525273,1.0,0.0020941284019500017,1.0 +12100,197.4,0.0017788220856657993,0.0009680902003310621,1.0,0.00177027378231287,1.0 +12200,199.1,0.18148040974727803,0.01199363637715578,1.0,0.03490154817700386,0.9966439604759216 +12300,200.8,0.03128038302579105,0.009156409651041031,1.0,0.015013160184025764,1.0 +12400,202.4,0.01209673641140603,0.008246197365224361,1.0,0.010931829921901226,1.0 +12500,204.1,0.006353548767145335,0.004167424514889717,1.0,0.005325275007635355,1.0 +12600,205.9,0.00303907734832859,0.0018630772829055786,1.0,0.0023749680258333683,1.0 +12700,207.6,0.0013663664281059317,0.0008114685188047588,1.0,0.0010484744561836123,1.0 +12800,209.3,0.000599807306980955,0.0003523377235978842,1.0,0.00046581617789343,1.0 +12900,211.0,0.00026161391230101387,0.0001532813475932926,1.0,0.00020870123989880085,1.0 +13000,212.6,0.00011415627341898913,6.704378756694496e-05,1.0,9.471135854255408e-05,1.0 +13100,214.3,5.012537303802961e-05,2.9554024877143092e-05,1.0,4.3497933802427724e-05,1.0 +13200,216.0,2.231169230566475e-05,1.3568059330282267e-05,1.0,2.0952393242623657e-05,1.0 +13300,217.8,1.0234651028426292e-05,6.275299710978288e-06,1.0,1.0177901458519045e-05,1.0 +13400,219.4,4.941030497223542e-06,3.2025839118432486e-06,1.0,5.438662810774986e-06,1.0 +13500,221.2,0.00865142942981473,0.0004917937912978232,1.0,0.001359581365250051,1.0 +13600,222.8,0.0016892156663583882,0.0007334965048357844,1.0,0.001335139269940555,1.0 +13700,224.5,0.0009754223456073718,0.00086883147014305,1.0,0.001314869150519371,1.0 +13800,226.2,0.0006733248452520373,0.00048447874723933637,1.0,0.0007079103379510343,1.0 +13900,227.9,0.0003374959526879584,0.00021739391377195716,1.0,0.0003028364444617182,1.0 +14000,229.6,0.00016061974315602438,9.879273420665413e-05,1.0,0.0001387533120578155,1.0 +14100,231.3,7.31129038551969e-05,4.4082029489800334e-05,1.0,6.28748894087039e-05,1.0 +14200,233.0,3.2827160674556955e-05,1.9763076124945655e-05,1.0,2.8721509806928225e-05,1.0 +14300,234.9,1.4880146470838603e-05,9.088336810236797e-06,1.0,1.3511278666555882e-05,1.0 +14400,237.1,6.959463741183806e-06,4.471613010537112e-06,1.0,6.8229696807975415e-06,1.0 +14500,238.9,3.4785414572734194e-06,2.299989318998996e-06,1.0,3.592873554225662e-06,1.0 +14600,240.6,1.988916641409845e-06,1.3905769264965784e-06,1.0,2.219497901023715e-06,1.0 +14700,242.3,1.1482747310769483e-06,8.971818488134886e-07,1.0,1.452447691008274e-06,1.0 +14800,244.1,0.014873209808281496,0.0005548478220589459,1.0,0.0034676529467105865,0.9993287920951843 +14900,245.8,0.002595510068898201,0.0008203514153137803,1.0,0.0024806626606732607,0.9998881816864014 +15000,247.5,0.6603559069014556,0.3301088511943817,0.987728476524353,0.4432120621204376,0.9799754023551941 +15100,249.2,0.09897083501623355,0.007329842075705528,1.0,0.01197031233459711,1.0 +15200,250.9,0.02084770942227465,0.009141476824879646,1.0,0.012009644880890846,1.0 +15300,252.6,0.0086986933000603,0.005374308675527573,1.0,0.006819399539381266,1.0 +15400,254.2,0.004010740803335784,0.0024337302893400192,1.0,0.003086224664002657,1.0 +15500,256.0,0.0017944448591653184,0.0010649461764842272,1.0,0.0013580908998847008,1.0 +15600,257.6,0.0007883306502889908,0.0004643466090783477,1.0,0.0005991535144858062,1.0 +15700,259.3,0.00034479247891959356,0.000202906274353154,1.0,0.0002665232168510556,1.0 +15800,260.9,0.0001511987998405314,8.931693446356803e-05,1.0,0.00011989210906904191,1.0 +15900,262.6,6.673947944126569e-05,3.9617236325284466e-05,1.0,5.457377847051248e-05,1.0 +16000,264.2,2.984557385633937e-05,1.811216316127684e-05,1.0,2.5691480914247222e-05,1.0 +16100,265.9,1.3629490135247263e-05,8.414041985815857e-06,1.0,1.2315945241425652e-05,1.0 +16200,267.5,6.520439816714312e-06,4.2350616240582895e-06,1.0,6.390140697476454e-06,1.0 +16300,269.3,3.2946248948471236e-06,2.2418164462578716e-06,1.0,3.444311687417212e-06,1.0 +16400,271.3,0.01626007172580502,0.002211393555626273,1.0,0.0070736343041062355,1.0 +16500,273.3,0.002607054503332718,0.0005224687047302723,1.0,0.0012233888264745474,1.0 +16600,275.1,0.0009184065199857714,0.0006988064851611853,1.0,0.0011281550396233797,1.0 +16700,276.7,0.0006483572719170337,0.0005407471326179802,1.0,0.0007378423470072448,1.0 +16800,278.4,0.00040181498140917676,0.00027808875893242657,1.0,0.0003747000591829419,1.0 +16900,280.0,0.0002004675197202166,0.00012549659004434943,1.0,0.00017047798610292375,1.0 +17000,281.7,9.214213855317063e-05,5.5600510677322745e-05,1.0,7.655193621758372e-05,1.0 +17100,283.4,4.13471629010519e-05,2.4854642106220126e-05,1.0,3.476999336271547e-05,1.0 +17200,285.0,1.8637724469753243e-05,1.1300125152047258e-05,1.0,1.6135316400323063e-05,1.0 +17300,286.6,8.621278755688721e-06,5.367700850911206e-06,1.0,7.836620170564856e-06,1.0 +17400,288.3,4.2718931192638066e-06,2.7145131298311753e-06,1.0,4.063080268679187e-06,1.0 +17500,290.0,2.2281027295669403e-06,1.5220492741718772e-06,1.0,2.3105817490431946e-06,1.0 +17600,291.7,0.02033332412370958,0.0009477337589487433,1.0,0.002827127231284976,1.0 +17700,293.7,0.0033184423193981303,0.0008222770411521196,1.0,0.001656360225751996,1.0 +17800,295.8,0.0013048925329308795,0.001036999048665166,1.0,0.0016034099971875548,1.0 +17900,297.5,0.0009480637866598856,0.0007643711869604886,1.0,0.0010936452308669686,1.0 +18000,299.1,0.00055420030267043,0.00037399015855044127,1.0,0.0005370200960896909,1.0 +18100,300.9,0.00027300472264644077,0.0001713020319584757,1.0,0.00024987285723909736,1.0 +18200,302.6,0.00012659852014760027,7.729998469585553e-05,1.0,0.00011575953976716846,1.0 +18300,304.3,5.7375723588361916e-05,3.466704220045358e-05,1.0,5.377516936277971e-05,1.0 +18400,305.9,2.5962626031560775e-05,1.573052395542618e-05,1.0,2.5520244889776222e-05,1.0 +18500,307.6,1.1941990731037278e-05,7.533525604230817e-06,1.0,1.2918816537421662e-05,1.0 +18600,309.2,5.719343937074201e-06,3.672043021651916e-06,1.0,6.705344276269898e-06,1.0 +18700,310.9,0.02326856695382302,0.0012368567986413836,1.0,0.004544305149465799,1.0 +18800,312.5,0.004397223162945706,0.0016762643354013562,1.0,0.0035588534083217382,1.0 +18900,314.2,0.001976763088500754,0.0015084183542057872,1.0,0.0024383217096328735,1.0 +19000,315.8,0.0012629271101478664,0.0009229227434843779,1.0,0.0013539334759116173,1.0 +19100,317.5,0.0006866750930067277,0.0004541188245639205,1.0,0.0006523402407765388,1.0 +19200,319.2,0.00033173205658388737,0.00020567994215525687,1.0,0.00029652126249857247,1.0 +19300,320.9,0.00015158739914829017,9.11656316020526e-05,1.0,0.0001330472732661292,1.0 +19400,322.6,6.789774229756391e-05,4.0502523916075006e-05,1.0,6.035158730810508e-05,1.0 +19500,324.2,3.040661602050938e-05,1.8319322407478467e-05,1.0,2.8081642085453495e-05,1.0 +19600,325.9,1.3849019341949264e-05,8.53126221045386e-06,1.0,1.3558656064560637e-05,1.0 +19700,327.6,6.560738211291344e-06,4.109100245841546e-06,1.0,6.817590929131256e-06,1.0 +19800,329.3,3.353255151937448e-06,5.634999524772866e-06,1.0,9.927807695930824e-06,1.0 +19900,331.1,0.00657472999969325,0.0006442063604481518,1.0,0.0014361984794959426,1.0 +20000,333.1,0.0016658756362637407,0.0010698649566620588,1.0,0.0017319556791335344,1.0 +20100,335.3,0.0012316794854183375,0.0010934072779491544,1.0,0.0016132511664181948,1.0 +20200,337.0,0.0008029172876546763,0.0005623938050121069,1.0,0.0008251647232100368,1.0 +20300,338.6,0.00039890566793273307,0.00024058629060164094,1.0,0.0003518572193570435,1.0 +20400,340.3,0.00018120329723280603,0.00011082698620157316,1.0,0.00015999747847672552,1.0 +20500,341.9,8.22650011919734e-05,4.964464096701704e-05,1.0,7.281923899427056e-05,1.0 +20600,343.6,3.701258629448807e-05,2.2400703528546728e-05,1.0,3.364321673871018e-05,1.0 +20700,345.2,1.680585574183867e-05,1.0269361155224033e-05,1.0,1.590874308021739e-05,1.0 +20800,346.9,7.855195723464664e-06,4.949162303091725e-06,1.0,7.948260645207483e-06,1.0 +20900,348.6,3.880771036465316e-06,2.567789124441333e-06,1.0,4.265462848707102e-06,1.0 +21000,350.2,0.01648095973470281,0.0007666641031391919,1.0,0.0022306563332676888,1.0 +21100,351.9,0.0030706224454414314,0.0011683569755405188,1.0,0.002229501958936453,1.0 +21200,353.6,0.00155381033365451,0.0013404921628534794,1.0,0.002009003423154354,1.0 +21300,355.2,0.0010817128067046704,0.000792286591604352,1.0,0.0011332312133163214,1.0 +21400,357.0,0.000576654315149412,0.0003797024837695062,1.0,0.0005285012302920222,1.0 +21500,358.7,0.0002794886357179985,0.00017398636555299163,1.0,0.00024352679611183703,1.0 +21600,360.3,0.0001287828052477523,7.820414612069726e-05,1.0,0.00011106150486739352,1.0 +21700,361.9,5.81167315717449e-05,3.514664058457129e-05,1.0,5.106910612084903e-05,1.0 +21800,363.6,2.6198721391776734e-05,1.601676194695756e-05,1.0,2.3989705368876457e-05,1.0 +21900,365.3,1.2021665625270819e-05,7.434328381350497e-06,1.0,1.1531759810168296e-05,1.0 +22000,366.9,5.739876024607739e-06,3.7165825688134646e-06,1.0,5.938298727414804e-06,1.0 +22100,368.6,0.01065835391222891,0.0006315343780443072,1.0,0.001404088456183672,1.0 +22200,370.2,0.0021922884342380758,0.001079067005775869,1.0,0.0017374465242028236,1.0 +22300,371.9,0.0014133552362404647,0.0012993748532608151,1.0,0.0018054975662380457,1.0 +22400,373.5,0.0010259215081847904,0.0007591251633130014,1.0,0.001039925729855895,1.0 +22500,375.2,0.0005295015980675472,0.0003302383702248335,1.0,0.0004537054046522826,1.0 +22600,376.8,0.00024045067848099794,0.00014331044803839177,1.0,0.00019349677313584834,1.0 +22700,378.4,0.00010765492886326603,6.444998143706471e-05,1.0,8.844323019729927e-05,1.0 +22800,380.1,4.808401437257653e-05,2.8845068300142884e-05,1.0,4.0656072087585926e-05,1.0 +22900,381.8,2.16229465412257e-05,1.3121822121320292e-05,1.0,1.9057350073126145e-05,1.0 +23000,383.5,9.97863566723573e-06,6.243992174859159e-06,1.0,9.385763405589387e-06,1.0 +23100,385.1,4.844034990184961e-06,3.138124839097145e-06,1.0,4.879350399278337e-06,1.0 +23200,386.7,0.010796620666723721,0.0008675913559272885,1.0,0.0021766365971416235,1.0 +23300,388.4,0.002417857662057013,0.0012546065263450146,1.0,0.002146842423826456,1.0 +23400,390.1,0.001348427847852278,0.001075842184945941,1.0,0.0016078235348686576,1.0 +23500,391.8,0.0008109637784824028,0.0005450936150737107,1.0,0.0007974550826475024,1.0 +23600,393.5,0.0004014907527161752,0.0002592594246380031,1.0,0.00036537094274535775,1.0 +23700,395.3,0.00019215124828473657,0.00011879096564371139,1.0,0.00016849397798068821,1.0 +23800,397.0,8.797457560764553e-05,5.316842725733295e-05,1.0,7.649810868315399e-05,1.0 +23900,398.7,3.9682922260991775e-05,2.391943780821748e-05,1.0,3.509407542878762e-05,1.0 +24000,400.5,1.8010606987072887e-05,1.0971905794576742e-05,1.0,1.6509722627233714e-05,1.0 +24100,402.2,8.40337231443483e-06,5.332030468707671e-06,1.0,8.246697689173743e-06,1.0 +24200,403.9,4.147124979198606e-06,2.7073226647189585e-06,1.0,4.287960109650157e-06,1.0 +24300,405.7,0.005747180487940531,0.0005011926405131817,1.0,0.001250040833838284,1.0 +24400,407.3,0.0013708842618130183,0.0008104377775453031,1.0,0.0013459640322253108,1.0 +24500,408.9,0.0009433144355181422,0.0008461957913823426,1.0,0.0012210853165015578,1.0 +24600,410.6,0.0006418502429014815,0.00045067392056807876,1.0,0.000645310094114393,1.0 +24700,412.2,0.00032446744546893344,0.00020247681823093444,1.0,0.00029190111672505736,1.0 +24800,413.9,0.00014537021290580662,8.665033965371549e-05,1.0,0.0001227188331540674,1.0 +24900,415.6,6.531754256651672e-05,3.932224353775382e-05,1.0,5.608615174423903e-05,1.0 +25000,417.2,2.9405145963646467e-05,1.7729635146679357e-05,1.0,2.5863559130812064e-05,1.0 +25100,418.9,1.3392699858869328e-05,8.225777492043562e-06,1.0,1.2277150744921528e-05,1.0 +25200,420.6,6.339115548650084e-06,4.0310706026502885e-06,1.0,6.160464181448333e-06,1.0 +25300,422.2,3.196133182965979e-06,2.2735330276191235e-06,1.0,3.565121460269438e-06,1.0 +25400,423.9,0.010772434220460491,0.0008401934173889458,1.0,0.002033665543422103,1.0 +25500,425.5,0.002371657934215214,0.0011953682405874133,1.0,0.002068485366180539,1.0 +25600,427.2,0.0013076561578959007,0.001043739845044911,1.0,0.0015325590502470732,1.0 +25700,428.9,0.0008317416474014262,0.0006046230555512011,1.0,0.0008391322917304933,1.0 +25800,430.5,0.0004406650836252388,0.00028595898766070604,1.0,0.0003957389562856406,1.0 +25900,432.2,0.00020960747618218234,0.00012899840658064932,1.0,0.00018002078286372125,1.0 +26000,433.9,9.559239323530587e-05,5.779427374363877e-05,1.0,8.188448555301875e-05,1.0 +26100,435.5,4.318882466575444e-05,2.6070012609125115e-05,1.0,3.759337778319605e-05,1.0 +26200,437.2,1.9596277266560243e-05,1.198822519654641e-05,1.0,1.7650083464104682e-05,1.0 +26300,438.9,9.105709688233287e-06,5.716984105674783e-06,1.0,8.592494850745425e-06,1.0 +26400,440.5,4.436545967856271e-06,2.8887816370115615e-06,1.0,4.4412954594008625e-06,1.0 +26500,442.2,2.342884589361651e-06,1.6091994439193513e-06,1.0,2.5128726974799065e-06,1.0 +26600,443.8,0.01261703182505109,0.0005350273568183184,1.0,0.0015316092176362872,1.0 +26700,445.5,0.0022954264114682004,0.000811939244158566,1.0,0.0015084597980603576,1.0 +26800,447.2,0.0010604658637962123,0.0008564452291466296,1.0,0.001253992086276412,1.0 +26900,448.9,0.0007212605584791258,0.0005484418943524361,1.0,0.0007419759058393538,1.0 +27000,450.5,0.00040348461636689704,0.0002705449587665498,1.0,0.0003624797100201249,1.0 +27100,452.2,0.0001989857893389765,0.0001254620001418516,1.0,0.00016959146887529641,1.0 +27200,453.9,9.26700508716298e-05,5.6672532082302496e-05,1.0,7.774309779051691e-05,1.0 +27300,455.6,4.219228187137996e-05,2.555980609031394e-05,1.0,3.5749162634601817e-05,1.0 +27400,457.3,1.9227883126788445e-05,1.1790896678576246e-05,1.0,1.6909993064473383e-05,1.0 +27500,459.1,0.0027820214579510954,0.001641804468818009,1.0,0.0027698760386556387,1.0 +27600,460.7,0.004722209729414068,0.0009138115565292537,1.0,0.0019570442382246256,1.0 +27700,462.4,0.0016839078869242405,0.001334909349679947,1.0,0.0021001307759433985,1.0 +27800,464.1,0.0012024029838367191,0.0009531270479783416,1.0,0.0013473944272845984,1.0 +27900,465.7,0.0006937167574295169,0.00046325515722855926,1.0,0.0006553237326443195,1.0 +28000,467.5,0.0003350918133358268,0.00020622966985683888,1.0,0.0002952086506411433,1.0 +28100,469.2,0.00015144032893192965,9.030741784954444e-05,1.0,0.00013165657583158463,1.0 +28200,470.9,6.685412867102603e-05,3.951612234232016e-05,1.0,5.88691582379397e-05,1.0 +28300,472.6,2.9429294761028332e-05,1.7443135220673867e-05,1.0,2.6509944291319698e-05,1.0 +28400,474.4,1.3208045987451577e-05,8.065297151915729e-06,1.0,1.2552874977700412e-05,1.0 +28500,476.0,6.207970631387073e-06,3.9795581869839225e-06,1.0,6.426243544410681e-06,1.0 +28600,477.8,3.122249626592873e-06,2.0957777451258153e-06,1.0,3.453393674135441e-06,1.0 +28700,479.5,0.009643693341568971,0.0021518534049391747,1.0,0.006964025087654591,1.0 +28800,481.2,0.001666140724970384,0.0004392733972053975,1.0,0.0009647279512137175,1.0 +28900,482.9,0.0006801825767742025,0.0005488572060130537,1.0,0.0008706892258487642,1.0 +29000,484.6,0.0004961656395389366,0.00039429281605407596,1.0,0.0005920868134126067,1.0 +29100,486.3,0.00028329793086359693,0.0001876351743703708,1.0,0.0002827202551998198,1.0 +29200,488.0,0.0001356809816351069,8.402076491620392e-05,1.0,0.00012784327554982156,1.0 +29300,489.6,6.232515842035e-05,3.7774087104480714e-05,1.0,5.8987036027247086e-05,1.0 +29400,491.4,2.8134257628697325e-05,1.695680111879483e-05,1.0,2.7346872229827568e-05,1.0 +29500,493.1,1.2747870996522523e-05,7.796059435349889e-06,1.0,1.2951684766449034e-05,1.0 +29600,494.9,5.966788161260856e-06,3.79333641831181e-06,1.0,6.4234632191073615e-06,1.0 +29700,496.6,3.0043449141888242e-06,2.0576806036842754e-06,1.0,3.5342072806088254e-06,1.0 +29800,498.4,0.014595078913268094,0.0008360974607057869,1.0,0.0027455349918454885,1.0 +29900,500.0,0.0025899363604398357,0.0008648824295960367,1.0,0.0017791681457310915,1.0 +30000,501.7,0.0012288171873142799,0.0010286164470016956,1.0,0.0016454393044114113,1.0 +30100,503.4,0.0008396010435694657,0.0006325239082798362,1.0,0.000938022043555975,1.0 +30200,505.0,0.00044905343351781945,0.0002889255410991609,1.0,0.0004310100630391389,1.0 +30300,507.1,0.00020768236114282275,0.00011984517914243042,1.0,0.00017704602214507759,1.0 +30400,513.8,9.065633007739672e-05,5.4625757911708206e-05,1.0,7.893858855823055e-05,1.0 +30500,562.3,4.084502702364837e-05,2.4656776076881215e-05,1.0,3.629387356340885e-05,1.0 +30600,602.9,1.8544005195890637e-05,1.1336015631968621e-05,1.0,1.710975448077079e-05,1.0 +30700,606.5,8.639647215403958e-06,5.4365809774026275e-06,1.0,8.428564797213767e-06,1.0 +30800,639.1,4.24282093537709e-06,2.794908823489095e-06,1.0,4.442589670361485e-06,1.0 +30900,654.9,2.265891345505858e-06,1.5707912552898051e-06,1.0,2.557774450906436e-06,1.0 +31000,658.8,0.015707496061063454,0.0007001275080256164,1.0,0.0024113506078720093,1.0 +31100,663.5,0.0028247586960040067,0.0009740054956637323,1.0,0.001965248491615057,1.0 +31200,670.8,0.0013120035082498864,0.001069405348971486,1.0,0.0016485531814396381,1.0 +31300,678.2,0.0009164529652461779,0.0006994553841650486,1.0,0.001050001592375338,1.0 +31400,685.1,0.0005031260007716822,0.00032921673846431077,1.0,0.0005019352538511157,1.0 +31500,692.2,0.0002399070423821184,0.00014730791735928506,1.0,0.00022832096146885306,1.0 +31600,699.3,0.0001078521851276686,6.348235183395445e-05,1.0,9.799366671359167e-05,1.0 +31700,706.4,4.7138890546625316e-05,2.8240490792086348e-05,1.0,4.231529601383954e-05,1.0 +31800,714.1,2.127129434277249e-05,1.29397658383823e-05,1.0,1.9640385289676487e-05,1.0 +31900,721.3,9.868929582047183e-06,6.155253231554525e-06,1.0,9.531427167530637e-06,1.0 +32000,728.3,4.7904031741599425e-06,3.1504810067417566e-06,1.0,4.969643669028301e-06,1.0 +32100,735.4,2.4994844320646983e-06,1.7138419252660242e-06,1.0,2.752743512246525e-06,1.0 +32200,742.6,0.002856023112398849,0.00022070510021876544,1.0,0.0008070211042650044,1.0 +32300,750.0,0.0005828078630625112,0.00027943463646806777,1.0,0.0005424200790002942,1.0 +32400,756.9,0.0003382056489647475,0.00028485391521826386,1.0,0.00043507941882126033,1.0 +32500,764.0,0.00022356541873215925,0.00016252649948000908,1.0,0.00023372238501906395,1.0 +32600,771.2,0.00011871293994926587,7.713441300438717e-05,1.0,0.00010992003080900759,1.0 +32700,778.5,5.654436682415597e-05,3.4897690056823194e-05,1.0,5.0287035264773294e-05,1.0 +32800,785.6,2.601868985679823e-05,1.5900010112090968e-05,1.0,2.3308613890549168e-05,1.0 +32900,793.3,1.1976238140310423e-05,7.409609679598361e-06,1.0,1.1084300240327138e-05,1.0 +33000,800.3,5.683991867050831e-06,3.666687916847877e-06,1.0,5.584740847552894e-06,1.0 +33100,807.6,2.873787306730723e-06,1.9714018435479375e-06,1.0,3.0527055514539825e-06,1.0 +33200,814.7,1.5991806370028336e-06,1.1731369795597857e-06,1.0,1.8321851484870422e-06,1.0 +33300,821.4,0.017896763625111552,0.0009818350663408637,1.0,0.0029433723539114,1.0 +33400,829.0,0.003503853060420551,0.0014791685389354825,1.0,0.002775584114715457,1.0 +33500,836.0,0.001871878883418305,0.0016155204502865672,1.0,0.0023542670533061028,1.0 +33600,842.9,0.0013603377404268091,0.0010357400169596076,1.0,0.0014561228454113007,1.0 +33700,850.1,0.0007423002758671011,0.0004846363444812596,1.0,0.0006905843038111925,1.0 +33800,856.9,0.0003536259279375091,0.00021705051767639816,1.0,0.0003163564542774111,1.0 +33900,863.6,0.00016045338804583265,9.650861466070637e-05,1.0,0.00014477560762315989,1.0 +34000,867.9,7.169822196849304e-05,4.2776362533913925e-05,1.0,6.646296969847754e-05,1.0 +34100,872.1,3.192477142017972e-05,1.911480831040535e-05,1.0,3.0784074624534696e-05,1.0 +34200,876.0,1.4355100358887012e-05,8.642875400255434e-06,1.0,1.4390564501809422e-05,1.0 +34300,879.9,6.6920794766007805e-06,4.27748636866454e-06,1.0,7.372902473434806e-06,1.0 +34400,883.8,0.012130641924401608,0.0009354072972200811,1.0,0.00280961487442255,1.0 +34500,887.8,0.002584441656177946,0.0012345538707450032,1.0,0.002343927277252078,1.0 +34600,891.7,0.0014067068421372989,0.0011400593211874366,1.0,0.0018375474028289318,1.0 +34700,895.7,0.0009233578440553824,0.000670194101985544,1.0,0.0010461966739967465,1.0 +34800,899.7,0.0004751805639107152,0.00029445503605529666,1.0,0.0004501276707742363,1.0 +34900,903.7,0.00021786743839422772,0.00013363173638936132,1.0,0.00019922188948839903,1.0 +35000,907.6,9.904819358678148e-05,5.979478737572208e-05,1.0,9.130416583502665e-05,1.0 +35100,913.9,4.45493810641495e-05,2.6781257474794984e-05,1.0,4.2262006900273263e-05,1.0 +35200,917.8,2.0139464801444432e-05,1.2195582712593023e-05,1.0,1.9951867216150276e-05,1.0 +35300,921.7,9.318378095404426e-06,5.7930542425310705e-06,1.0,9.852305083768442e-06,1.0 +35400,925.8,0.053768960693563225,0.061183176934719086,0.9934725761413574,0.09982078522443771,0.9855688810348511 +35500,929.9,0.009106370370087437,0.0012622798094525933,1.0,0.003038577036932111,1.0 +35600,934.8,0.0025331404017836672,0.0015832859789952636,1.0,0.0027024103328585625,1.0 +35700,938.7,0.0015660222207216665,0.0012877220287919044,1.0,0.0019188629230484366,1.0 +35800,942.6,0.0009707921823432117,0.0006764077697880566,1.0,0.0010040531633421779,1.0 +35900,946.6,0.0004911785234151941,0.0003113254497293383,1.0,0.0004679745761677623,1.0 +36000,951.0,0.00022905594606132009,0.00013970227155368775,1.0,0.00021467183250933886,1.0 +36100,954.9,0.00010357307630821568,6.235668115550652e-05,1.0,9.875012619886547e-05,1.0 +36200,959.2,4.6445434879151224e-05,2.7956088160863146e-05,1.0,4.602251647156663e-05,1.0 +36300,963.1,2.0945978839919357e-05,1.2738426448777318e-05,1.0,2.190931081713643e-05,1.0 +36400,967.2,9.707441695084235e-06,6.319158273981884e-06,1.0,1.1396232366678305e-05,1.0 +36500,971.1,0.020900343368432366,0.0014799929922446609,1.0,0.0037643529940396547,1.0 +36600,975.2,0.003785475297944928,0.001289251376874745,1.0,0.0024075815454125404,1.0 +36700,979.2,0.0017656879737256291,0.0015515796840190887,1.0,0.0023720725439488888,1.0 +36800,983.1,0.0013550872884301284,0.0010513741290196776,1.0,0.001600598101504147,1.0 +36900,987.1,0.000741314719794815,0.00047702935989946127,1.0,0.0007402586634270847,1.0 +37000,991.0,0.0003452588679054518,0.00020715770369861275,1.0,0.000321362167596817,1.0 +37100,995.1,0.00015162799161037553,9.075689013116062e-05,1.0,0.00013647315790876746,1.0 +37200,999.0,6.785888300516459e-05,4.0768518374534324e-05,1.0,6.283259426709265e-05,1.0 +37300,1003.2,3.0516883508967e-05,1.8368564269621857e-05,1.0,2.9320297471713275e-05,1.0 +37400,1007.3,1.3939654359184422e-05,8.551747669116594e-06,1.0,1.4220803677744698e-05,1.0 +37500,1011.3,6.623485130098877e-06,4.459163847059244e-06,1.0,7.776514394208789e-06,1.0 +37600,1015.6,0.010737718134768728,0.0011325523955747485,1.0,0.0027160332538187504,1.0 +37700,1019.7,0.0027699690326805707,0.0017164598684757948,1.0,0.003048005048185587,1.0 +37800,1023.9,0.001720013165027671,0.0013948684791103005,1.0,0.002149956999346614,1.0 +37900,1027.8,0.001062933851748396,0.000742921547498554,1.0,0.0011513573117554188,1.0 +38000,1031.6,0.0005367250266944639,0.00033838761737570167,1.0,0.0005411971360445023,1.0 +38100,1035.7,0.0002484348868640248,0.00015059787256177515,1.0,0.00025130578433163464,1.0 +38200,1039.6,0.00011156923865393273,6.665930413873866e-05,1.0,0.00011760069173760712,1.0 +38300,1043.5,4.97327147666846e-05,2.9790433472953737e-05,1.0,5.642922769766301e-05,1.0 +38400,1047.4,2.232348748067816e-05,1.3483185284712818e-05,1.0,2.8042521080351435e-05,1.0 +38500,1051.2,1.0256889991192671e-05,6.286852567427559e-06,1.0,1.469523886044044e-05,1.0 +38600,1055.1,0.016395987966327668,0.001793139730580151,1.0,0.007782443426549435,0.9995525479316711 +38700,1058.9,0.00283148553107646,0.0007951961597427726,1.0,0.0020863846875727177,1.0 +38800,1062.8,0.0010992670530882483,0.0008166965562850237,1.0,0.0014718691818416119,1.0 +38900,1066.6,0.0007620581230991632,0.0006048438372090459,1.0,0.0010123709216713905,1.0 +39000,1070.5,0.00043749138060979284,0.0002926464076153934,1.0,0.0004923052038066089,1.0 +39100,1074.4,0.00021278781635857303,0.00013198429951444268,1.0,0.00022661946422886103,1.0 +39200,1078.3,9.720913679985112e-05,5.855754716321826e-05,1.0,0.00010410530376248062,1.0 +39300,1082.2,4.35017413686603e-05,2.603120265121106e-05,1.0,4.856671512243338e-05,1.0 +39400,1086.0,1.9540625454382756e-05,1.1819845894933678e-05,1.0,2.347218105569482e-05,1.0 +39500,1090.0,9.033080332346144e-06,5.816086741106119e-06,1.0,1.2543257071229164e-05,1.0 +39600,1092.5,4.372491336747632e-06,2.9020720830885693e-06,1.0,6.801554718549596e-06,1.0 +39700,1094.8,0.015526125248557667,0.0014797316398471594,1.0,0.0038965384010225534,1.0 +39800,1097.0,0.0036575720259142677,0.0020074632484465837,1.0,0.003919458948075771,1.0 +39900,1098.9,0.0021483189614250847,0.0016954356105998158,1.0,0.003080500988289714,1.0 +40000,1100.8,0.001217415161859995,0.0008347955299541354,1.0,0.0013898900942876935,1.0 diff --git a/modular_add/step_9900.pt b/modular_add/step_9900.pt new file mode 100644 index 0000000000000000000000000000000000000000..adbc60c13aba0ea54b6b738476580bffd9d0e855 --- /dev/null +++ b/modular_add/step_9900.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f72c64fb5d9f8e12fd412e9402133498eeb7ddeb54285f337461132e9a4cb4eb +size 918942 diff --git a/modular_add_wd0.1/config.json b/modular_add_wd0.1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..df9d3c9c4f827bfb9414437577d1474f0591c868 --- /dev/null +++ b/modular_add_wd0.1/config.json @@ -0,0 +1,35 @@ +{ + "name": "modular_add_wd0.1", + "task": { + "p": 113, + "train_frac": 0.3, + "kind": "modular" + }, + "model": { + "arch": "transformer", + "d_model": 128, + "n_layers": 1, + "n_heads": 4, + "d_mlp": 512, + "max_positions": 3, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 40000, + "batch_size": 0, + "lr": 0.001, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 0, + "cosine": false, + "amp": false, + "eval_every": 100, + "checkpoint_every": 100, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/modular_add_wd0.1/metrics.csv b/modular_add_wd0.1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..0563364fa1c87c983f4c7da1d6b40169910fa64d --- /dev/null +++ b/modular_add_wd0.1/metrics.csv @@ -0,0 +1,402 @@ +step,elapsed,loss,train_loss,train_acc,test_loss,test_acc +0,0.6,nan,4.7616143226623535,0.009660574607551098,4.762050151824951,0.00827833078801632 +100,2.6,1.732582228598704,0.1038007140159607,1.0,4.739775657653809,0.24029533565044403 +200,4.3,0.25166867313622265,0.012470622546970844,1.0,4.949795246124268,0.2638997733592987 +300,6.0,0.03834695620873499,0.0035572173073887825,1.0,5.116876125335693,0.2713950276374817 +400,7.6,0.006621742795050366,0.0011472759069874883,1.0,5.2951555252075195,0.2781071662902832 +500,9.2,0.0013868879685531175,0.0003854082024190575,1.0,5.484745025634766,0.28258195519447327 +600,10.9,0.00035657512839849505,0.0001312644308200106,1.0,5.686082363128662,0.28537869453430176 +700,12.5,0.0001067069218680962,4.5499626139644533e-05,1.0,5.887052059173584,0.28806355595588684 +800,14.2,3.489390232766273e-05,1.591636100783944e-05,1.0,6.104154586791992,0.28907036781311035 +900,15.8,0.22041430194917264,0.38548848032951355,0.927676260471344,6.417078018188477,0.139165461063385 +1000,17.5,0.037203313997591476,0.001118081039749086,1.0,6.16025447845459,0.2724018394947052 +1100,19.1,0.00552667036950082,0.0005183462053537369,1.0,6.082741737365723,0.28157514333724976 +1200,20.7,0.0010181464960895058,0.0002502796414773911,1.0,6.027318954467773,0.287727952003479 +1300,22.4,0.00027188914232523367,0.00011974786320934072,1.0,5.997579097747803,0.2916433811187744 +1400,24.0,0.00010111699342010419,5.655708082485944e-05,1.0,6.006659507751465,0.2935451567173004 +1500,25.6,4.3643514948959025e-05,2.594593388494104e-05,1.0,6.057643890380859,0.29499945044517517 +1600,27.3,1.9316769479002173e-05,1.139583491749363e-05,1.0,6.162485122680664,0.29656562209129333 +1700,28.9,8.337433277537609e-06,4.771160092786886e-06,1.0,6.318737030029297,0.29757243394851685 +1800,30.6,3.463344972528708e-06,1.914598669827683e-06,1.0,6.513421535491943,0.29757243394851685 +1900,32.2,1.3855001137748054e-06,7.412760965053167e-07,1.0,6.728936672210693,0.29757243394851685 +2000,33.9,5.398555048182716e-07,2.883121226204821e-07,1.0,6.9250383377075195,0.29846739768981934 +2100,35.6,2.173253246865511e-07,1.2216615630222805e-07,1.0,7.054924488067627,0.29779618978500366 +2200,37.2,9.892399735847269e-08,6.424228615742322e-08,1.0,7.113306045532227,0.29869112372398376 +2300,38.9,0.04228209621735753,0.0004789658123627305,1.0,6.937347888946533,0.274527370929718 +2400,40.6,0.005828010067361635,0.0001878040930023417,1.0,6.800049781799316,0.2849312126636505 +2500,42.2,0.0008771944770136438,9.244547254638746e-05,1.0,6.665926933288574,0.29175522923469543 +2600,43.8,0.00016763181367034235,4.5213666453491896e-05,1.0,6.54790735244751,0.2973487079143524 +2700,45.6,4.7065614149614855e-05,2.184002187277656e-05,1.0,6.444108009338379,0.301040381193161 +2800,47.2,1.8329966899492235e-05,1.0682016181817744e-05,1.0,6.369541168212891,0.3039489984512329 +2900,48.9,8.370190026619426e-06,5.2657660489785485e-06,1.0,6.3260016441345215,0.30529141426086426 +3000,50.5,4.038175438962626e-06,2.5880826797219925e-06,1.0,6.3229827880859375,0.3074169456958771 +3100,52.2,1.9477621881042904e-06,1.2269523494978785e-06,1.0,6.369020938873291,0.30797630548477173 +3200,53.8,9.047113726618435e-07,5.458725809148746e-07,1.0,6.457730293273926,0.30909499526023865 +3300,55.5,4.023807049155174e-07,2.3773380064540106e-07,1.0,6.557782173156738,0.3093187212944031 +3400,57.1,1.7800482845494692e-07,1.0682146722729158e-07,1.0,6.6304612159729,0.31077301502227783 +3500,58.8,0.22989618783618124,0.016766250133514404,0.999738872051239,6.178195476531982,0.24544133245944977 +3600,60.5,0.03142491921323595,0.00042037683306261897,1.0,6.2012152671813965,0.2935451567173004 +3700,62.1,0.004394854492632251,0.00020028841390740126,1.0,6.105540752410889,0.29869112372398376 +3800,63.8,0.0006939088443285721,9.793545177672058e-05,1.0,6.017733573913574,0.30428460240364075 +3900,65.4,0.00014572003414145917,4.701823854702525e-05,1.0,5.937427997589111,0.3076406717300415 +4000,67.0,4.500448220844708e-05,2.2471715055871755e-05,1.0,5.8624653816223145,0.3105492889881134 +4100,68.7,1.828898109460088e-05,1.0830885003088042e-05,1.0,5.81036901473999,0.312339186668396 +4200,70.4,8.4249965529714e-06,5.2997229431639425e-06,1.0,5.782817363739014,0.3132341504096985 +4300,72.0,4.047881765212129e-06,2.5837252906057984e-06,1.0,5.792693138122559,0.314129114151001 +4400,73.6,1.9432314644834314e-06,1.2197933756397106e-06,1.0,5.844461441040039,0.31670209765434265 +4500,75.3,9.010627072352821e-07,5.433203114080243e-07,1.0,5.933402061462402,0.3173733055591583 +4600,77.0,3.9823129147354903e-07,2.3521265291037707e-07,1.0,6.036277770996094,0.3172614574432373 +4700,78.6,0.21660560375883808,0.596728503704071,0.9044386148452759,5.668047904968262,0.15762390196323395 +4800,80.3,0.041059466365274896,0.0004213221836835146,1.0,5.608828544616699,0.30641013383865356 +4900,81.9,0.00566337012828679,0.0001905824028654024,1.0,5.473842620849609,0.31379351019859314 +5000,83.6,0.0008574123765107802,9.438197594136e-05,1.0,5.3591389656066895,0.31871575117111206 +5100,85.3,0.00016635361045839642,4.670010457630269e-05,1.0,5.257635116577148,0.32419735193252563 +5200,86.9,4.817496070341854e-05,2.3245951524586417e-05,1.0,5.170800685882568,0.32542791962623596 +5300,88.6,1.952721698795184e-05,1.1795406862802338e-05,1.0,5.105755805969238,0.32900771498680115 +5400,90.1,9.314643945991058e-06,6.0531638155225664e-06,1.0,5.0726518630981445,0.3302382826805115 +5500,91.8,4.662860858163426e-06,3.0566700388590107e-06,1.0,5.079174995422363,0.33090952038764954 +5600,93.4,2.2975123137392295e-06,1.4588653129976592e-06,1.0,5.125105381011963,0.3333706259727478 +5700,95.1,1.0735157283458e-06,6.465311912506877e-07,1.0,5.199739456176758,0.3342655897140503 +5800,96.7,4.709604328152853e-07,2.7418130343903613e-07,1.0,5.287900447845459,0.33471307158470154 +5900,98.3,2.0460016667614502e-07,1.2138804095229716e-07,1.0,5.3660969734191895,0.3362792432308197 +6000,100.0,0.1866184668347736,0.03152520954608917,1.0,5.083211421966553,0.2763172686100006 +6100,101.6,0.025942342240677894,0.00038190337363630533,1.0,4.7975239753723145,0.3306857645511627 +6200,103.3,0.0036423433397960244,0.00017612827650737017,1.0,4.677175521850586,0.33571988344192505 +6300,105.0,0.0005797814934348499,8.486615115543827e-05,1.0,4.572484493255615,0.34164896607398987 +6400,106.6,0.00012327078649094507,4.060108767589554e-05,1.0,4.474277973175049,0.3438863456249237 +6500,108.3,3.864973877648846e-05,1.964923103514593e-05,1.0,4.396390914916992,0.348808616399765 +6600,109.9,1.606340485009406e-05,9.713691724755336e-06,1.0,4.3484787940979,0.3495916724205017 +6700,111.6,7.580214453523911e-06,4.869326403422747e-06,1.0,4.334167003631592,0.3504866361618042 +6800,113.2,3.7412517754597603e-06,2.4355697405553656e-06,1.0,4.353476524353027,0.35294777154922485 +6900,114.9,1.837743314661181e-06,1.1709267937476397e-06,1.0,4.401874542236328,0.35552075505256653 +7000,116.5,8.622293069872927e-07,5.230267561273649e-07,1.0,4.47366189956665,0.3571988046169281 +7100,118.2,3.8446932502595954e-07,2.3147764238728996e-07,1.0,4.555863857269287,0.35865309834480286 +7200,119.9,0.3150249406950579,0.30124837160110474,0.93968665599823,5.213629722595215,0.22116568684577942 +7300,121.5,0.04628471203334965,0.0003720602544490248,1.0,4.2422871589660645,0.3516053259372711 +7400,123.2,0.006329373210774129,0.00016546448750887066,1.0,4.104224681854248,0.3579818904399872 +7500,124.8,0.0009307329830571319,8.064500434556976e-05,1.0,3.9791817665100098,0.36435842514038086 +7600,126.5,0.00016815795564269646,3.958561865147203e-05,1.0,3.8638105392456055,0.3705112636089325 +7700,128.1,4.442654759470257e-05,1.9721697753993794e-05,1.0,3.7686100006103516,0.375321626663208 +7800,129.8,1.7111489747954392e-05,1.0127882887900341e-05,1.0,3.6973977088928223,0.3820337951183319 +7900,131.4,8.13483145634077e-06,5.346496436686721e-06,1.0,3.662688970565796,0.3837118446826935 +8000,133.1,4.1785267482895725e-06,2.8095339530409547e-06,1.0,3.661632537841797,0.3858373463153839 +8100,134.8,2.1361812171539304e-06,1.4010956874699332e-06,1.0,3.690551996231079,0.3857254981994629 +8200,136.5,1.034687138588876e-06,6.39123015844234e-07,1.0,3.734480857849121,0.388522207736969 +8300,138.1,4.670951172069447e-07,2.7632884780359745e-07,1.0,3.7869627475738525,0.388410359621048 +8400,139.8,2.0352147777970118e-07,1.1868013416460599e-07,1.0,3.8301010131835938,0.3893052935600281 +8500,141.4,9.497200465862677e-08,6.203239166779895e-08,1.0,3.860973834991455,0.38829848170280457 +8600,143.1,0.0492623574689029,0.003987235017120838,1.0,3.9430830478668213,0.36435842514038086 +8700,144.7,0.006850352184859733,0.00016254806541837752,1.0,3.304476022720337,0.4094417691230774 +8800,146.4,0.000995102892051747,7.561145321233198e-05,1.0,3.1402273178100586,0.41850319504737854 +8900,148.0,0.00017362066219012254,3.666005432023667e-05,1.0,2.993011474609375,0.43092069029808044 +9000,149.7,4.319584713412909e-05,1.774024531187024e-05,1.0,2.8613860607147217,0.44333818554878235 +9100,151.4,1.5523456497709185e-05,8.654546036268584e-06,1.0,2.7504220008850098,0.45307081937789917 +9200,153.0,6.885759111093147e-06,4.2845508687605616e-06,1.0,2.6746296882629395,0.4602304697036743 +9300,154.7,3.2926394194357468e-06,2.097769538522698e-06,1.0,2.6380577087402344,0.4678375720977783 +9400,156.3,1.57961964556421e-06,9.918954901877441e-07,1.0,2.6328067779541016,0.46929189562797546 +9500,158.0,7.327654913368684e-07,4.413544445469597e-07,1.0,2.646491050720215,0.47197675704956055 +9600,159.6,3.2625583936893145e-07,1.9428310338298616e-07,1.0,2.6665847301483154,0.4725360870361328 +9700,161.3,1.4627921068814484e-07,8.948476448722431e-08,1.0,2.693298816680908,0.47085803747177124 +9800,162.9,7.355835894284855e-08,4.074280113286477e-08,1.0,2.719033718109131,0.47287169098854065 +9900,164.6,0.0374172943704798,0.0004223413998261094,1.0,2.5575218200683594,0.4660476744174957 +10000,166.2,0.005162848941506974,0.00016963879170361906,1.0,2.346501350402832,0.48797404766082764 +10100,167.9,0.000776352225788738,7.982066017575562e-05,1.0,2.168621778488159,0.5096766948699951 +10200,169.5,0.00014644731541725567,3.80666388082318e-05,1.0,2.0031139850616455,0.5309318900108337 +10300,171.2,4.0337567949928146e-05,1.8408672985970043e-05,1.0,1.8560304641723633,0.552858293056488 +10400,172.8,1.5584652577826184e-05,9.096581379708368e-06,1.0,1.7429474592208862,0.5713167190551758 +10500,174.5,7.194814769851029e-06,4.586867817124585e-06,1.0,1.6676559448242188,0.5836223363876343 +10600,176.1,3.538185846319731e-06,2.301483164046658e-06,1.0,1.62893807888031,0.5961517095565796 +10700,177.7,1.739804166688334e-06,1.1105754538220936e-06,1.0,1.6109223365783691,0.6034232378005981 +10800,179.4,8.179992462277815e-07,4.980955168321088e-07,1.0,1.6037404537200928,0.6073386669158936 +10900,181.0,3.6458217115414463e-07,2.1454556531352864e-07,1.0,1.6072406768798828,0.6089048385620117 +11000,182.6,1.6176961801288953e-07,9.844880821674451e-08,1.0,1.6164469718933105,0.6108065843582153 +11100,184.3,7.815947319613469e-08,5.0360469572297006e-08,1.0,1.6496185064315796,0.6049894094467163 +11200,186.0,0.03867221713270029,0.13588950037956238,0.9725848436355591,3.53389048576355,0.3935563266277313 +11300,187.6,0.012451023306422894,0.00022458320017904043,1.0,1.2936573028564453,0.6584629416465759 +11400,189.3,0.001765740425672069,9.910122025758028e-05,1.0,1.1337025165557861,0.6897863745689392 +11500,190.9,0.0002889068718563383,4.83051517221611e-05,1.0,0.9988566040992737,0.7205504179000854 +11600,192.6,6.495549660265905e-05,2.341804065508768e-05,1.0,0.8782054781913757,0.7481821179389954 +11700,194.2,2.1469365011475632e-05,1.1301951417408418e-05,1.0,0.7757077813148499,0.7764850854873657 +11800,195.8,9.079933022715251e-06,5.4928518693486694e-06,1.0,0.6951073408126831,0.7971808910369873 +11900,197.5,4.236426494798037e-06,2.66906977230974e-06,1.0,0.6396755576133728,0.8106052279472351 +12000,199.2,2.023624275280541e-06,1.27438681829517e-06,1.0,0.6071802973747253,0.8220158815383911 +12100,200.8,9.482825202378634e-07,5.781805043625354e-07,1.0,0.590164303779602,0.826602578163147 +12200,202.4,4.255035048650062e-07,2.5158448124784627e-07,1.0,0.583188533782959,0.830406129360199 +12300,204.1,1.896785270996214e-07,1.1342000050262868e-07,1.0,0.588695228099823,0.8289518356323242 +12400,205.7,0.13184800994616366,0.001719251275062561,1.0,0.8326332569122314,0.7607114911079407 +12500,207.4,0.01785292440544223,0.00026304213679395616,1.0,0.5575706362724304,0.8319723010063171 +12600,209.1,0.0025082685443679705,0.0001227235479746014,1.0,0.45186883211135864,0.8622888922691345 +12700,210.7,0.0004002795213952555,5.965699892840348e-05,1.0,0.3630930483341217,0.8914867639541626 +12800,212.4,8.63030235440161e-05,2.9532533517340198e-05,1.0,0.290912926197052,0.9133012890815735 +12900,214.0,2.8136563493308376e-05,1.4972069948271383e-05,1.0,0.2371155470609665,0.9297460913658142 +13000,215.7,1.2272432717364879e-05,7.701940376136918e-06,1.0,0.19979259371757507,0.9410448670387268 +13100,217.3,6.0440716258040614e-06,3.983976966992486e-06,1.0,0.17666815221309662,0.9496588110923767 +13200,219.0,3.048701241528063e-06,1.9954616163886385e-06,1.0,0.16335801780223846,0.9554760456085205 +13300,220.6,1.4925949081374172e-06,9.412548820364464e-07,1.0,0.1532595455646515,0.9568184614181519 +13400,222.3,6.893944883795047e-07,4.140888165693468e-07,1.0,0.1431857794523239,0.9599508047103882 +13500,223.9,3.040333936326544e-07,1.8024566372787376e-07,1.0,0.13685649633407593,0.9614050984382629 +13600,225.5,1.3955135501221109e-07,1.043314554749486e-07,1.0,0.137345552444458,0.9625238180160522 +13700,227.2,0.03803363404109369,0.0003852561640087515,1.0,0.2648397386074066,0.9250475764274597 +13800,228.8,0.00521272901113864,0.000141776108648628,1.0,0.18088442087173462,0.9515606164932251 +13900,230.5,0.0007693649846379623,6.894685793668032e-05,1.0,0.13029254972934723,0.9675579071044922 +14000,232.1,0.00014054769980353975,3.434083555475809e-05,1.0,0.09189214557409286,0.9772905707359314 +14100,233.8,3.818376495983586e-05,1.7640644728089683e-05,1.0,0.06378917396068573,0.9853451251983643 +14200,235.4,1.527302399533629e-05,9.309254892286845e-06,1.0,0.04428517445921898,0.9897080659866333 +14300,237.2,7.445133478994778e-06,4.935466677125078e-06,1.0,0.0323805958032608,0.9932878613471985 +14400,238.8,3.818073629691617e-06,2.545565848777187e-06,1.0,0.02520817518234253,0.9944065809249878 +14500,240.5,1.9295148557671363e-06,1.2525991905931733e-06,1.0,0.020956218242645264,0.9953015446662903 +14600,242.1,9.260155457379256e-07,5.721733487007441e-07,1.0,0.017991552129387856,0.995972752571106 +14700,243.8,4.1920548262521887e-07,2.4937460807450407e-07,1.0,0.015891550108790398,0.9965320825576782 +14800,245.4,1.8912741011330448e-07,1.1454050508064029e-07,1.0,0.014644676819443703,0.9968677163124084 +14900,247.1,0.2196061926358161,0.0784730315208435,0.990078330039978,0.7072905898094177,0.8129544854164124 +15000,248.8,0.030868425022755934,0.0002768786798696965,1.0,0.09338745474815369,0.9806466102600098 +15100,250.5,0.004236464691160317,0.00012349142343737185,1.0,0.0632079541683197,0.9881418943405151 +15200,252.1,0.0006296847499124957,5.9717451222240925e-05,1.0,0.04298975318670273,0.9922810196876526 +15300,253.8,0.00011671595524758875,2.951639544335194e-05,1.0,0.029347524046897888,0.9949659109115601 +15400,255.5,3.225144368678786e-05,1.5147842532314826e-05,1.0,0.019828617572784424,0.996196448802948 +15500,257.1,1.3059364745193589e-05,8.021494068088941e-06,1.0,0.01349074486643076,0.9975389242172241 +15600,258.8,6.429545083944313e-06,4.300454747863114e-06,1.0,0.009398329071700573,0.9982101321220398 +15700,260.4,3.339983013548259e-06,2.2486633497464936e-06,1.0,0.006788672413676977,0.9986575841903687 +15800,262.1,1.701860059862872e-06,1.1056573612222564e-06,1.0,0.005242929328233004,0.9987694621086121 +15900,263.7,8.168885479605086e-07,5.065303412266076e-07,1.0,0.004429662600159645,0.9991050958633423 +16000,265.4,3.7248584383950947e-07,2.2428770307669765e-07,1.0,0.0037587813567370176,0.9989932179450989 +16100,267.1,1.7059288835295303e-07,1.0411358175588248e-07,1.0,0.003356178058311343,0.9991050958633423 +16200,268.7,0.16775232277156318,0.009579161182045937,1.0,0.192712664604187,0.959615170955658 +16300,270.4,0.022853026695322392,0.00026808868278749287,1.0,0.04138292744755745,0.9925047755241394 +16400,272.0,0.003170361236956198,0.00012030312063870952,1.0,0.027492908760905266,0.9946302771568298 +16500,273.6,0.0004857804314305549,5.719808177673258e-05,1.0,0.018278561532497406,0.9964202046394348 +16600,275.3,9.627530474092502e-05,2.8441911126719788e-05,1.0,0.011773476377129555,0.997986376285553 +16700,277.0,2.9024291875359204e-05,1.4738995560037438e-05,1.0,0.007415877189487219,0.9984338879585266 +16800,278.6,1.2458378884462216e-05,7.887462743383367e-06,1.0,0.00451919250190258,0.9989932179450989 +16900,280.3,6.2823512604497635e-06,4.240660473442404e-06,1.0,0.0026147845201194286,0.9994406700134277 +17000,281.9,3.2861233243433987e-06,2.211997070844518e-06,1.0,0.001390225486829877,0.999776303768158 +17100,283.6,1.6662887979107008e-06,1.0791385420816368e-06,1.0,0.0007565667037852108,0.9998881816864014 +17200,285.2,7.917565930017153e-07,4.837777964894485e-07,1.0,0.00041063473327085376,1.0 +17300,286.9,6.174915769657868e-05,7.533263206481934,0.013054830022156239,7.701731204986572,0.008949547074735165 +17400,288.5,0.056233065116259556,0.000323629064951092,1.0,0.022149166092276573,0.995972752571106 +17500,290.1,0.0076275073270288835,0.00015035888645797968,1.0,0.014996393583714962,0.9969795346260071 +17600,291.7,0.0010971264288861257,7.69405160099268e-05,1.0,0.010029438883066177,0.9980982542037964 +17700,293.4,0.00018916653041919224,3.921760799130425e-05,1.0,0.006198148708790541,0.9989932179450989 +17800,295.1,4.7599066598417844e-05,2.049368413281627e-05,1.0,0.003529125824570656,0.9995525479316711 +17900,296.8,1.8513352383514746e-05,1.1388823622837663e-05,1.0,0.0020168633200228214,0.999776303768158 +18000,298.4,9.429689508910558e-06,6.58710132483975e-06,1.0,0.0012058098800480366,0.9998881816864014 +18100,300.1,5.281763918777446e-06,3.7753120523120742e-06,1.0,0.0007860329933464527,0.9998881816864014 +18200,301.8,2.9099118081852685e-06,1.9911653907911386e-06,1.0,0.0005504909204319119,0.9998881816864014 +18300,303.4,1.4701584996306108e-06,9.310456903222075e-07,1.0,0.00032566339359618723,0.9998881816864014 +18400,305.1,6.754137550691182e-07,4.0176320226237294e-07,1.0,0.0001649303885642439,1.0 +18500,306.7,2.972688583203405e-07,1.760126480121471e-07,1.0,6.421913712983951e-05,1.0 +18600,308.4,0.1216336501636727,0.0032560487743467093,1.0,0.040224671363830566,0.9935116171836853 +18700,310.3,0.016399939258746575,0.00014212061068974435,1.0,0.007012565154582262,0.9987694621086121 +18800,312.2,0.0022526388956674035,6.909341755090281e-05,1.0,0.004019559361040592,0.9991050958633423 +18900,314.0,0.0003378756077381259,3.50695445376914e-05,1.0,0.002077303361147642,0.999776303768158 +19000,315.9,6.46611101457788e-05,1.782466824806761e-05,1.0,0.001058377092704177,0.9998881816864014 +19100,317.7,1.887379323788568e-05,9.43429404287599e-06,1.0,0.0006606830283999443,0.9998881816864014 +19200,320.3,8.181103266922184e-06,5.338839400792494e-06,1.0,0.0004584500566124916,0.9998881816864014 +19300,322.2,4.403976861119442e-06,3.1663205390941584e-06,1.0,0.0002785593387670815,1.0 +19400,324.1,2.5470713436091154e-06,1.8500751366445911e-06,1.0,0.00017289427341893315,1.0 +19500,326.0,1.4183864561882055e-06,9.708855941426009e-07,1.0,8.610039367340505e-05,1.0 +19600,327.8,7.15351653337599e-07,4.5343080046222894e-07,1.0,3.698277578223497e-05,1.0 +19700,329.8,3.344720890348459e-07,2.046788836196356e-07,1.0,1.1841179002658464e-05,1.0 +19800,331.6,1.536977913887579e-07,9.932031019843635e-08,1.0,4.172973603999708e-06,1.0 +19900,333.9,0.06621623290206921,0.000339852791512385,1.0,0.007249022368341684,0.9992169141769409 +20000,335.8,0.008892100619402408,9.201609645970166e-05,1.0,0.002742003882303834,0.9996644258499146 +20100,338.5,0.0012328898318148492,4.917916885460727e-05,1.0,0.001730521791614592,0.999776303768158 +20200,340.4,0.00019221703772324554,2.612841853988357e-05,1.0,0.0010503438534215093,0.9998881816864014 +20300,342.3,4.046259916952719e-05,1.3513858903024811e-05,1.0,0.0006898032152093947,0.9998881816864014 +20400,344.2,1.3178718747268218e-05,7.148152235458838e-06,1.0,0.0004881960048805922,0.9998881816864014 +20500,346.1,6.046221555718974e-06,4.046315552841406e-06,1.0,0.0003344298747833818,0.9998881816864014 +20600,348.0,3.3453388717157175e-06,2.4426956315437565e-06,1.0,0.0002139998396160081,0.9998881816864014 +20700,349.9,1.9748249980776636e-06,1.4506474599329522e-06,1.0,0.00016610858438070863,1.0 +20800,352.4,1.3174370940959417e-06,5.992842197418213,0.008877284824848175,6.033361434936523,0.008837677538394928 +20900,354.3,0.024575545891707427,0.00017401238437741995,1.0,0.0037939283065497875,0.9991050958633423 +21000,356.1,0.0033553068537217653,8.746684034122154e-05,1.0,0.0018360327230766416,0.9998881816864014 +21100,358.4,0.000496746942375707,4.7754059778526425e-05,1.0,0.0009209931013174355,1.0 +21200,360.3,9.421480629493765e-05,2.6241057639708742e-05,1.0,0.0005049186293035746,1.0 +21300,362.2,2.8387044834284825e-05,1.4987488611950539e-05,1.0,0.0003378574620001018,1.0 +21400,364.1,1.3179083336853794e-05,9.050108928931877e-06,1.0,0.0002900365798268467,1.0 +21500,365.9,7.4340345759355784e-06,5.427427822723985e-06,1.0,0.00020696442516054958,1.0 +21600,368.0,4.291961034995277e-06,3.0591909307986498e-06,1.0,0.00011032203474314883,1.0 +21700,369.9,2.292989034120606e-06,1.5094433365447912e-06,1.0,4.7218360123224556e-05,1.0 +21800,371.8,1.0876741074871072e-06,6.542502433148911e-07,1.0,2.0745119400089607e-05,1.0 +21900,373.7,4.838342241770715e-07,2.8591549039447273e-07,1.0,5.473641067510471e-06,1.0 +22000,375.0,0.07964214594524982,0.00013717153342440724,1.0,0.0025778724811971188,0.999776303768158 +22100,376.2,0.010640229088305642,7.484874367946759e-05,1.0,0.001629221485927701,0.999776303768158 +22200,377.8,0.0014567851284338438,4.250680285622366e-05,1.0,0.0008165912004187703,1.0 +22300,379.5,0.00021796042003587924,2.2474916477221996e-05,1.0,0.0003305387799628079,1.0 +22400,381.1,4.216384016134403e-05,1.2299819900363218e-05,1.0,0.00011205131886526942,1.0 +22500,382.8,1.3169893047123783e-05,7.257432571350364e-06,1.0,4.302878005546518e-05,1.0 +22600,384.4,6.446784241038162e-06,4.615528723661555e-06,1.0,2.113740447384771e-05,1.0 +22700,386.1,3.885231526130032e-06,2.9650354917976074e-06,1.0,1.1525825357239228e-05,1.0 +22800,387.7,2.374926100962732e-06,1.7441564068576554e-06,1.0,6.4233067860186566e-06,1.0 +22900,389.4,1.303691997182484e-06,8.609828228145489e-07,1.0,3.2937616651906865e-06,1.0 +23000,391.0,6.243139247385998e-07,3.8268342450464843e-07,1.0,1.5546019085377338e-06,1.0 +23100,392.7,2.786330530965527e-07,1.682002306324648e-07,1.0,7.42972360967542e-07,1.0 +23200,394.4,1.2394650325046525e-07,6.875541913586858e-08,1.0,4.0085905084197293e-07,1.0 +23300,396.1,6.087646919693569e-08,3.772366241605596e-08,1.0,3.1000337230580044e-07,1.0 +23400,397.7,0.03671519477996662,0.00027689943090081215,1.0,0.0020926434081047773,1.0 +23500,399.4,0.004971795072190126,8.394881297135726e-05,1.0,0.0007301743607968092,1.0 +23600,401.1,0.0007064572482700566,4.242189243086614e-05,1.0,0.0003534306597430259,1.0 +23700,402.7,0.00011809302479822991,2.2028862076695077e-05,1.0,0.00016549936844967306,1.0 +23800,404.4,2.8075297621489467e-05,1.1040785466320813e-05,1.0,7.578953955089673e-05,1.0 +23900,406.0,9.936159595426548e-06,5.581458935921546e-06,1.0,3.586701495805755e-05,1.0 +24000,407.6,4.583891090206643e-06,3.020408030351973e-06,1.0,1.6917827451834455e-05,1.0 +24100,409.3,2.4567708905342063e-06,1.7517207879791385e-06,1.0,8.321979294123594e-06,1.0 +24200,410.9,1.4086090267652408e-06,1.0174491080761072e-06,1.0,4.368233021523338e-06,1.0 +24300,412.6,7.795602393743876e-07,5.358814405553858e-07,1.0,2.219840780526283e-06,1.0 +24400,414.2,3.9954218017196443e-07,2.5475921461293183e-07,1.0,1.0294004368915921e-06,1.0 +24500,415.9,1.9226326767494287e-07,1.240959193182789e-07,1.0,5.117750561112189e-07,1.0 +24600,417.6,8.841775296367737e-08,5.465573948981728e-08,1.0,2.9500020559680706e-07,1.0 +24700,419.3,4.912158332304985e-08,5.0827345887682895e-08,1.0,2.641685341586708e-07,1.0 +24800,421.0,0.04352295264219681,0.00011016377538908273,1.0,0.0016928785480558872,0.999776303768158 +24900,422.6,0.005826349365785461,5.1300190534675494e-05,1.0,0.0005392419407144189,1.0 +25000,424.2,0.0008043181435404,2.943499930552207e-05,1.0,0.0002339468483114615,1.0 +25100,425.9,0.00012381214693906614,1.5597186575178057e-05,1.0,0.00010495488822925836,1.0 +25200,427.5,2.5258343693161734e-05,7.795488272677176e-06,1.0,4.3759344407590106e-05,1.0 +25300,429.2,7.932410936678767e-06,4.263289611117216e-06,1.0,1.9479011825751513e-05,1.0 +25400,430.8,3.669922143033336e-06,2.518081828384311e-06,1.0,9.02386091183871e-06,1.0 +25500,432.5,2.145314060517812e-06,1.6574430219407077e-06,1.0,4.9631444198894314e-06,1.0 +25600,434.1,1.4093559067321187e-06,1.1202864698134363e-06,1.0,3.0601804610341787e-06,1.0 +25700,435.8,8.906221389587409e-07,6.601017616958416e-07,1.0,1.762621536727238e-06,1.0 +25800,437.4,5.041656753246679e-07,3.459247466253146e-07,1.0,9.358525971947529e-07,1.0 +25900,439.0,2.53490800398047e-07,1.6546121628380206e-07,1.0,4.824103143619141e-07,1.0 +26000,440.7,1.21767538827328e-07,7.080968345007932e-08,1.0,2.63422947455183e-07,1.0 +26100,442.3,0.07414216566592133,0.00036099739372730255,1.0,0.0057436018250882626,0.9988813400268555 +26200,444.0,0.009961522932992774,0.00010309281788067892,1.0,0.002971277805045247,0.9991050958633423 +26300,445.6,0.0013763125801441183,4.8788253479870036e-05,1.0,0.0016181786777451634,0.9993287920951843 +26400,447.3,0.00021014713179616694,2.459486131556332e-05,1.0,0.0006873842212371528,1.0 +26500,448.9,4.137999789288213e-05,1.181712286779657e-05,1.0,0.00021035961981397122,1.0 +26600,450.6,1.2044956990518338e-05,5.885419795959024e-06,1.0,4.670640191761777e-05,1.0 +26700,452.3,5.128015906954998e-06,3.343297294122749e-06,1.0,1.4180179277900606e-05,1.0 +26800,454.0,2.84007210542787e-06,2.1220459984760964e-06,1.0,6.289455086516682e-06,1.0 +26900,455.7,1.766436271338921e-06,1.366610035802296e-06,1.0,3.3026096843968844e-06,1.0 +27000,457.4,1.1067953118851043e-06,8.267763860203559e-07,1.0,1.9064949583480484e-06,1.0 +27100,459.0,6.352018275963407e-07,4.45276015170748e-07,1.0,1.0151085234610946e-06,1.0 +27200,460.7,3.209713511254563e-07,2.053013616887256e-07,1.0,5.075886519989581e-07,1.0 +27300,462.3,1.5563777926583957e-07,9.247278143220683e-08,1.0,2.575820019501407e-07,1.0 +27400,463.9,0.07267124815036784,0.000542531895916909,1.0,0.006121876183897257,0.9993287920951843 +27500,465.6,0.00984123366374701,0.00016690346819814295,1.0,0.0026329178363084793,0.999776303768158 +27600,467.2,0.0013960864761762418,7.957088382681832e-05,1.0,0.001355347572825849,0.999776303768158 +27700,468.9,0.00022805412831855646,3.7091744161443785e-05,1.0,0.0005529813934117556,1.0 +27800,470.5,5.017052633225297e-05,1.7333586583845317e-05,1.0,0.00022307661129161716,1.0 +27900,472.2,1.6287913977649367e-05,8.632240678707603e-06,1.0,0.00010674430086510256,1.0 +28000,473.9,7.215283542644647e-06,4.688477929448709e-06,1.0,5.341671567293815e-05,1.0 +28100,475.5,3.850954462861198e-06,2.7591408979787957e-06,1.0,2.587495509942528e-05,1.0 +28200,477.2,2.2438521743673986e-06,1.6513716900590225e-06,1.0,1.137519211624749e-05,1.0 +28300,478.9,1.2946706186886823e-06,9.184080909108161e-07,1.0,4.598921350407181e-06,1.0 +28400,480.5,6.851233927200964e-07,4.479837514281826e-07,1.0,1.847610178629111e-06,1.0 +28500,482.2,3.255272665986581e-07,2.0156633695478376e-07,1.0,7.917363404885691e-07,1.0 +28600,483.8,1.4841274329529484e-07,8.438023968437847e-08,1.0,3.73985074020311e-07,1.0 +28700,485.5,0.06875281348527142,0.0003815873060375452,1.0,0.009491071105003357,0.997986376285553 +28800,487.2,0.00925550564410565,0.00011517940583871678,1.0,0.004853525198996067,0.9986575841903687 +28900,488.8,0.0012933646782106632,5.961900023976341e-05,1.0,0.0030531997326761484,0.9991050958633423 +29000,490.5,0.00020558199629888276,3.055249180761166e-05,1.0,0.0017042538383975625,0.9996644258499146 +29100,492.2,4.4462161718592506e-05,1.5362804333562963e-05,1.0,0.000809339398983866,1.0 +29200,493.8,1.4671780030181293e-05,7.968869795149658e-06,1.0,0.00031303183641284704,1.0 +29300,495.5,6.671410704682322e-06,4.4022795009368565e-06,1.0,0.00011634442489594221,1.0 +29400,497.1,3.5913360900616526e-06,2.5690897018648684e-06,1.0,3.9573355024913326e-05,1.0 +29500,498.8,2.066019107114794e-06,1.494811954216857e-06,1.0,1.338067067990778e-05,1.0 +29600,500.5,1.1492652156195414e-06,7.865621682867641e-07,1.0,4.622747383109527e-06,1.0 +29700,502.1,5.826601323262308e-07,3.7163391652939026e-07,1.0,1.6840023135955562e-06,1.0 +29800,503.8,2.754119302392602e-07,1.7261997697914921e-07,1.0,6.554761853294622e-07,1.0 +29900,505.4,0.030320767668298824,0.0004034156445413828,1.0,0.0018791286274790764,1.0 +30000,507.1,0.004091287861014072,5.0576956709846854e-05,1.0,0.00033133270335383713,1.0 +30100,508.7,0.0005706651321048989,2.4960871087387204e-05,1.0,0.00014580893912352622,1.0 +30200,510.4,8.999939429468202e-05,1.3052657777734566e-05,1.0,6.85570266796276e-05,1.0 +30300,512.0,1.9620855339225495e-05,7.102730251062894e-06,1.0,3.2427560654468834e-05,1.0 +30400,513.7,6.924181131734121e-06,4.0857789826986846e-06,1.0,1.633019928704016e-05,1.0 +30500,515.4,3.4891436922396433e-06,2.4833439056237694e-06,1.0,8.547993274987675e-06,1.0 +30600,517.0,2.0588755139960183e-06,1.5484110917896032e-06,1.0,4.604734385793563e-06,1.0 +30700,518.6,1.238364841020052e-06,9.05056083411182e-07,1.0,2.497211426089052e-06,1.0 +30800,520.2,6.901171065582158e-07,4.7191920771183504e-07,1.0,1.2939526641275734e-06,1.0 +30900,521.8,3.4534250218643605e-07,2.1899646185374877e-07,1.0,6.313835569926596e-07,1.0 +31000,523.5,1.6326407055205694e-07,9.932031019843635e-08,1.0,3.1672618661104934e-07,1.0 +31100,525.1,7.509063792461934e-08,5.2321354360174155e-08,1.0,2.0917282483878807e-07,1.0 +31200,526.7,4.462674505538096e-08,2.0449213877782313e-08,1.0,1.6588468554346036e-07,1.0 +31300,528.4,3.5058322922161486e-08,9.458928929007016e-08,1.0,3.58781846898637e-07,1.0 +31400,530.0,0.006232407406082589,7.247635949170217e-05,1.0,0.0006756936199963093,1.0 +31500,531.7,0.0008630301373375694,3.173301956849173e-05,1.0,0.0003294546331744641,1.0 +31600,533.3,0.0001320883002424583,1.5692068700445816e-05,1.0,0.00017364959057886153,1.0 +31700,535.0,2.6422586088101816e-05,8.014974810066633e-06,1.0,8.085792796919122e-05,1.0 +31800,536.7,8.139809107061247e-06,4.231012553645996e-06,1.0,3.1721876439405605e-05,1.0 +31900,538.4,3.5498452584126316e-06,2.2594324491365114e-06,1.0,1.0738322998804506e-05,1.0 +32000,540.0,1.7785032665040242e-06,1.181540483230492e-06,1.0,4.1215425881091505e-06,1.0 +32100,541.7,9.187386802937926e-07,6.264243666009861e-07,1.0,1.8266956658408162e-06,1.0 +32200,543.3,4.895701014075314e-07,3.3686734468574286e-07,1.0,8.717618129594484e-07,1.0 +32300,545.0,2.62925797341385e-07,1.8083703423599218e-07,1.0,4.5556561190096545e-07,1.0 +32400,546.6,1.3899834807579063e-07,9.091652231063563e-08,1.0,2.494604700586933e-07,1.0 +32500,548.3,6.971294816823866e-08,5.1574350123928525e-08,1.0,1.85781786399275e-07,1.0 +32600,549.9,4.4814185839446526e-08,5.948013637180338e-08,1.0,1.7819368736127217e-07,1.0 +32700,551.6,0.02216449735722592,0.0001425724185537547,1.0,0.003970361314713955,0.9994406700134277 +32800,553.3,0.0029997169834044793,5.193531251279637e-05,1.0,0.001907010213471949,0.999776303768158 +32900,554.9,0.00042736506598397997,2.6532010451774113e-05,1.0,0.0013029989786446095,0.999776303768158 +33000,556.6,7.157501164200766e-05,1.324163986282656e-05,1.0,0.0008700615726411343,0.999776303768158 +33100,558.3,1.6869784371777853e-05,6.541058610309847e-06,1.0,0.0005270107067190111,0.999776303768158 +33200,560.0,5.931330609417847e-06,3.340246621519327e-06,1.0,0.00020186089386697859,0.999776303768158 +33300,561.6,2.771774051658939e-06,1.856767312347074e-06,1.0,2.449706516927108e-05,1.0 +33400,563.3,1.5204311812167267e-06,1.10108203443815e-06,1.0,4.701525540440343e-06,1.0 +33500,564.9,8.92266052808677e-07,6.562732437487284e-07,1.0,1.8850323613150977e-06,1.0 +33600,566.6,5.119941915231576e-07,3.60802516752301e-07,1.0,9.451329106013873e-07,1.0 +33700,568.2,2.7042114933816895e-07,1.8180192284944496e-07,1.0,4.863839535573788e-07,1.0 +33800,569.8,1.3606204654972388e-07,8.363323900084652e-08,1.0,2.6404958930470457e-07,1.0 +33900,571.5,6.744608110043436e-08,5.098297251038275e-08,1.0,2.031048751405251e-07,1.0 +34000,573.1,0.039058945720984964,0.00021059346909169108,1.0,0.012066482566297054,0.997986376285553 +34100,574.8,0.005281135315525202,8.715740841580555e-05,1.0,0.008110769093036652,0.9984338879585266 +34200,576.4,0.0007492102003338185,4.3576452299021184e-05,1.0,0.004898536019027233,0.9986575841903687 +34300,578.1,0.00012386388097854136,2.1870726413908415e-05,1.0,0.0024931919761002064,0.9993287920951843 +34400,579.7,2.8283538013209423e-05,1.0248244507238269e-05,1.0,0.0008545102318748832,0.999776303768158 +34500,581.4,9.618756718309265e-06,5.29917770109023e-06,1.0,0.0001445173256797716,1.0 +34600,583.1,4.317625568286503e-06,2.815665311572957e-06,1.0,1.8485397959011607e-05,1.0 +34700,584.7,2.386770052843486e-06,1.7900348439070513e-06,1.0,6.7277674133947585e-06,1.0 +34800,586.4,1.50425122792612e-06,1.1694012300722534e-06,1.0,3.2909088076849002e-06,1.0 +34900,588.1,9.363947833616571e-07,6.91319883117103e-07,1.0,1.7172252455566195e-06,1.0 +35000,589.7,5.180201966376301e-07,3.4302999551982793e-07,1.0,8.523685437467066e-07,1.0 +35100,591.2,2.5714168826412856e-07,1.6732872154534562e-07,1.0,4.092631513685774e-07,1.0 +35200,592.5,1.1656785866626662e-07,6.822629217140275e-08,1.0,2.29136475127234e-07,1.0 +35300,593.9,0.02796081581166228,0.00036653215647675097,1.0,0.016742045059800148,0.996755838394165 +35400,595.2,0.00381309325140021,8.273121056845412e-05,1.0,0.006336684338748455,0.9983220100402832 +35500,596.6,0.0005525429679412269,4.240292400936596e-05,1.0,0.003677195869386196,0.9991050958633423 +35600,597.9,9.772685370336206e-05,2.213274456153158e-05,1.0,0.0019935695454478264,0.9995525479316711 +35700,599.3,2.5597835737870184e-05,1.1360853022779338e-05,1.0,0.0009632260771468282,0.9998881816864014 +35800,600.6,9.886896007489529e-06,5.865846105734818e-06,1.0,0.00042976305121555924,1.0 +35900,602.1,4.7158182342572165e-06,3.116762854915578e-06,1.0,0.00016766032786108553,1.0 +36000,603.4,2.4584182286999077e-06,1.686480004536861e-06,1.0,3.6746558180311695e-05,1.0 +36100,604.8,1.3000770348152368e-06,8.822094059723895e-07,1.0,7.275663847394753e-06,1.0 +36200,606.1,6.881902900780473e-07,4.6911762296986126e-07,1.0,2.3171664906840306e-06,1.0 +36300,607.5,3.5261918366883335e-07,2.326291621557175e-07,1.0,9.625083521314082e-07,1.0 +36400,608.9,1.7376621939035545e-07,1.0507844194762583e-07,1.0,4.052733402204467e-07,1.0 +36500,610.2,0.277046073102687,0.3038177490234375,0.990078330039978,0.35966768860816956,0.987358808517456 +36600,611.5,0.03981191707059742,8.325668750330806e-05,1.0,0.0011429953156039119,0.999776303768158 +36700,612.8,0.005319993095619855,3.500146704027429e-05,1.0,0.0007164730341173708,0.999776303768158 +36800,614.2,0.0007266091416138612,1.9698269170476124e-05,1.0,0.00036935959360562265,1.0 +36900,615.5,0.00010804752618954074,1.074815918400418e-05,1.0,0.00015769910532981157,1.0 +37000,616.8,2.068781349793434e-05,5.89212777413195e-06,1.0,5.743600559071638e-05,1.0 +37100,618.2,6.364210028363287e-06,3.471867557891528e-06,1.0,2.201660390710458e-05,1.0 +37200,619.5,3.1398991851578743e-06,2.301508629898308e-06,1.0,1.0639557331160177e-05,1.0 +37300,620.8,2.0177024089228666e-06,1.633099259379378e-06,1.0,5.9490971580089536e-06,1.0 +37400,622.1,1.3723707084583156e-06,1.0889402801694814e-06,1.0,3.44700129062403e-06,1.0 +37500,623.4,8.469574271443951e-07,6.150933131721104e-07,1.0,1.854303945947322e-06,1.0 +37600,624.7,4.463431348699167e-07,2.852926286323054e-07,1.0,8.588917239649163e-07,1.0 +37700,626.1,2.1369577873247653e-07,1.3200164517002122e-07,1.0,3.8533707424903696e-07,1.0 +37800,627.4,9.686324392800512e-08,5.988475493268197e-08,1.0,2.1386692594660417e-07,1.0 +37900,628.8,0.007537622600507542,0.00039462681161239743,1.0,0.008115877397358418,0.9982101321220398 +38000,630.2,0.0010473224177819508,2.992321242345497e-05,1.0,0.0011314997682347894,0.999776303768158 +38100,631.5,0.00015504149541179836,1.4180151083564851e-05,1.0,0.00024116710119415075,1.0 +38200,632.9,2.8476720951182888e-05,7.049447503959527e-06,1.0,7.038831972749904e-05,1.0 +38300,634.2,7.772947509166368e-06,3.599142473831307e-06,1.0,2.6633208108250983e-05,1.0 +38400,635.5,3.1086857979349876e-06,1.8910980088548968e-06,1.0,1.0847698831639718e-05,1.0 +38500,636.9,1.5235337788812432e-06,1.0215884458375513e-06,1.0,4.638822701963363e-06,1.0 +38600,638.2,8.095366323401481e-07,5.623688252853754e-07,1.0,2.0395762021507835e-06,1.0 +38700,639.5,4.4155340254766735e-07,3.0751633062209294e-07,1.0,9.364639481646009e-07,1.0 +38800,640.9,2.365091907309335e-07,1.597653209728378e-07,1.0,4.5307129425964376e-07,1.0 +38900,642.3,1.1817092591094182e-07,7.691020442734953e-08,1.0,2.593420731500373e-07,1.0 +39000,643.6,6.395660822644461e-08,4.0898427755564626e-08,1.0,1.9418320107433829e-07,1.0 +39100,645.0,4.402755512814226e-08,3.843954132776162e-08,1.0,1.7299245769208937e-07,1.0 +39200,646.3,3.655909040884325e-08,3.2899269086783534e-08,1.0,1.7409934116585646e-07,1.0 +39300,647.7,0.0038168908167933303,6.518930604215711e-05,1.0,0.002660792088136077,0.999776303768158 +39400,649.0,0.0005288225683544006,1.8510258087189868e-05,1.0,0.001418011263012886,0.999776303768158 +39500,650.3,8.041045096823598e-05,9.128801139013376e-06,1.0,0.0008879917440935969,0.999776303768158 +39600,651.6,1.5782502328013853e-05,4.568088570522377e-06,1.0,0.0004274304665159434,0.999776303768158 +39700,653.0,4.676425804262631e-06,2.3192847038444597e-06,1.0,0.00010886023665079847,1.0 +39800,654.3,1.9523120154970745e-06,1.2098330444132444e-06,1.0,1.6491874703206122e-05,1.0 +39900,655.7,9.68015437169852e-07,6.508263368232292e-07,1.0,3.956562068196945e-06,1.0 +40000,657.1,5.158559647266253e-07,3.5993105029774597e-07,1.0,1.5292765738195158e-06,1.0 diff --git a/modular_add_wd0.1/step_9900.pt b/modular_add_wd0.1/step_9900.pt new file mode 100644 index 0000000000000000000000000000000000000000..9842bbc2c881fd1a68bd5204743a5dd7620c7238 --- /dev/null +++ b/modular_add_wd0.1/step_9900.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faabf52818836fef204e97af9b4f1c2b86805dd29929a5870c1a882a3138caec +size 918942 diff --git a/modular_add_wd0/config.json b/modular_add_wd0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1797030d7ebd4c04886e93ce5668deb13cdf5715 --- /dev/null +++ b/modular_add_wd0/config.json @@ -0,0 +1,35 @@ +{ + "name": "modular_add_wd0", + "task": { + "p": 113, + "train_frac": 0.3, + "kind": "modular" + }, + "model": { + "arch": "transformer", + "d_model": 128, + "n_layers": 1, + "n_heads": 4, + "d_mlp": 512, + "max_positions": 3, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 40000, + "batch_size": 0, + "lr": 0.001, + "weight_decay": 0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 0, + "cosine": false, + "amp": false, + "eval_every": 100, + "checkpoint_every": 100, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/modular_add_wd0/metrics.csv b/modular_add_wd0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..7dc6d65096feccdf3c74db0806b7b67e8aee282d --- /dev/null +++ b/modular_add_wd0/metrics.csv @@ -0,0 +1,402 @@ +step,elapsed,loss,train_loss,train_acc,test_loss,test_acc +0,0.3,nan,4.7616143226623535,0.009660574607551098,4.762050151824951,0.00827833078801632 +100,1.7,1.7228702985216364,0.0971609428524971,1.0,4.75064754486084,0.24275647103786469 +200,3.1,0.24845413010487657,0.01106785424053669,1.0,4.995968341827393,0.2666965126991272 +300,4.4,0.03725252199928627,0.003026752034202218,1.0,5.201201915740967,0.2760935425758362 +400,5.7,0.006220654930542431,0.0009419553680345416,1.0,5.414371490478516,0.2826938331127167 +500,7.0,0.0012349652370771944,0.0003066364151891321,1.0,5.6426496505737305,0.28638550639152527 +600,8.3,0.00029943348273080215,0.00010232849308522418,1.0,5.883988857269287,0.28985345363616943 +700,9.6,8.533340484731186e-05,3.4567878174129874e-05,1.0,6.139420509338379,0.29175522923469543 +800,10.9,2.6835801030763746e-05,1.1797350452980027e-05,1.0,6.402622222900391,0.29186710715293884 +900,12.3,8.874594334434042e-06,4.067300324095413e-06,1.0,6.669029235839844,0.2923145890235901 +1000,13.5,3.0158287342458956e-06,1.4177803677739576e-06,1.0,6.938244342803955,0.29209083318710327 +1100,14.8,1.0403699512463678e-06,4.990604338672711e-07,1.0,7.19786262512207,0.29197898507118225 +1200,16.1,3.6470836525261083e-07,1.7408288499609625e-07,1.0,7.4102911949157715,0.2925383150577545 +1300,17.4,0.054504262463980796,0.0007405365468002856,1.0,7.291913986206055,0.2580825686454773 +1400,18.8,0.007548380825416564,0.0002578312414698303,1.0,7.282214641571045,0.27005258202552795 +1500,20.1,0.0011330562530534637,0.00011092837667092681,1.0,7.288084506988525,0.27676475048065186 +1600,21.4,0.00020728332653773124,4.7807086957618594e-05,1.0,7.305936813354492,0.28325316309928894 +1700,22.7,5.1929018145332435e-05,2.042523010459263e-05,1.0,7.336863994598389,0.28828728199005127 +1800,24.0,1.7294353047692214e-05,8.68417919264175e-06,1.0,7.379710674285889,0.29186710715293884 +1900,25.3,6.7205959419297034e-06,3.6961644127586624e-06,1.0,7.437334060668945,0.295335054397583 +2000,26.6,2.770084284514601e-06,1.5605819498887286e-06,1.0,7.513646125793457,0.2973487079143524 +2100,27.9,1.1500347325679693e-06,6.432630357267044e-07,1.0,7.616795063018799,0.2992504835128784 +2200,29.3,4.6616525316829087e-07,2.5264273517677793e-07,1.0,7.7301764488220215,0.3001454472541809 +2300,30.6,1.785812781313699e-07,9.082314278430204e-08,1.0,7.8206000328063965,0.3001454472541809 +2400,31.9,6.995291185339516e-08,4.320168756066778e-08,1.0,7.817716121673584,0.3001454472541809 +2500,33.2,4.167354382969376e-08,3.2961519735863476e-08,1.0,7.7214226722717285,0.3000335693359375 +2600,34.5,0.12487646107274168,0.3352486789226532,0.9618799090385437,6.643572807312012,0.16903457045555115 +2700,35.8,0.022710144617986375,0.00024683927767910063,1.0,7.903435707092285,0.2626692056655884 +2800,37.1,0.00312820001672253,9.544567728880793e-05,1.0,7.892004013061523,0.27273744344711304 +2900,38.5,0.00046387672933586693,4.125546547584236e-05,1.0,7.894519329071045,0.2788902521133423 +3000,39.8,8.283066476757838e-05,1.7945270883501507e-05,1.0,7.897460460662842,0.28426000475883484 +3100,41.1,2.0234615617499624e-05,7.780184205330443e-06,1.0,7.902061939239502,0.28996533155441284 +3200,42.4,6.681413064447726e-06,3.3514540973555995e-06,1.0,7.915424823760986,0.2935451567173004 +3300,43.7,2.6030057864837396e-06,1.4352413018059451e-06,1.0,7.939358234405518,0.2964537441730499 +3400,45.0,1.0732210941819313e-06,6.025513812346617e-07,1.0,7.978476524353027,0.2983555197715759 +3500,46.3,4.412284986263459e-07,2.429316907637258e-07,1.0,8.02358627319336,0.30092853307724 +3600,47.6,1.7411174770601056e-07,8.923576189090454e-08,1.0,8.051634788513184,0.30238282680511475 +3700,49.0,7.03338004928201e-08,4.690557986464228e-08,1.0,7.97787618637085,0.3029421865940094 +3800,50.3,4.4452760262215405e-08,3.784816726692952e-08,1.0,7.776329517364502,0.30439648032188416 +3900,51.7,3.8047880136014776e-08,1.0987172949228352e-08,1.0,7.571528434753418,0.3045083284378052 +4000,53.0,3.5255606013414357e-08,4.201893499811149e-09,1.0,7.481122016906738,0.30428460240364075 +4100,54.3,0.1343478797532921,0.37922292947769165,0.9566579461097717,6.410571098327637,0.19756124913692474 +4200,55.6,0.03074854503503246,0.00024316672352142632,1.0,7.663211345672607,0.27687662839889526 +4300,56.9,0.0041889118066299694,8.913948113331571e-05,1.0,7.757197856903076,0.28448373079299927 +4400,58.3,0.0005999828618792487,3.669646321213804e-05,1.0,7.833585739135742,0.29086029529571533 +4500,59.6,9.807686359071158e-05,1.533874456072226e-05,1.0,7.89260196685791,0.29499945044517517 +4600,61.0,2.0762827378550636e-05,6.438483524107141e-06,1.0,7.936651229858398,0.2982436716556549 +4700,62.3,6.006914685206435e-06,2.6996960968972417e-06,1.0,7.9658660888671875,0.30137601494789124 +4800,63.6,2.1584090008917766e-06,1.1271026778558735e-06,1.0,7.987935543060303,0.3045083284378052 +4900,64.9,8.478431332704468e-07,4.601228340561647e-07,1.0,8.0108060836792,0.3058507740497589 +5000,66.2,3.3651036998685485e-07,1.7949865593891445e-07,1.0,8.02064323425293,0.30641013383865356 +5100,67.5,1.305166563478555e-07,7.077855457282567e-08,1.0,7.9778008460998535,0.30808815360069275 +5200,68.8,6.59464705869951e-08,4.8088335091733825e-08,1.0,7.761392116546631,0.30898311734199524 +5300,70.1,4.970054638128757e-08,4.043155144017874e-08,1.0,7.440506458282471,0.3097662031650543 +5400,71.4,4.330054881835213e-08,3.1156261570686183e-08,1.0,7.174617767333984,0.31110864877700806 +5500,72.6,0.05141756829506733,0.0006982988561503589,1.0,7.958468437194824,0.27396801114082336 +5600,74.0,0.00692827276588005,6.912554817972705e-05,1.0,7.9687604904174805,0.2907484173774719 +5700,75.3,0.0009527901858323797,2.8119700800743885e-05,1.0,7.972927570343018,0.29466384649276733 +5800,76.6,0.0001408048535243046,1.2152020644862205e-05,1.0,7.986659526824951,0.29779618978500366 +5900,78.0,2.4933641586799723e-05,5.256420990917832e-06,1.0,8.004586219787598,0.3017116189002991 +6000,79.4,5.995629518279252e-06,2.2455817543232115e-06,1.0,8.024383544921875,0.30506768822669983 +6100,80.7,1.934604601376906e-06,9.453631832911924e-07,1.0,8.046941757202148,0.3074169456958771 +6200,81.9,7.288548256449293e-07,3.8620063946837035e-07,1.0,8.06733512878418,0.30909499526023865 +6300,83.3,2.846261136667903e-07,1.5151715615502326e-07,1.0,8.070294380187988,0.3106611669063568 +6400,84.6,1.1377870674653416e-07,6.86309178377087e-08,1.0,7.969603061676025,0.3113323748111725 +6500,85.9,6.567797538282317e-08,5.437561512167122e-08,1.0,7.655494213104248,0.31211546063423157 +6600,87.2,5.2593845173999044e-08,3.9871299151172934e-08,1.0,7.257783889770508,0.31379351019859314 +6700,88.5,4.6685964672508115e-08,4.379306517421355e-08,1.0,6.902591705322266,0.31390535831451416 +6800,89.9,0.010827083017435735,0.0008891134057193995,1.0,8.4039306640625,0.2646828591823578 +6900,91.2,0.0014814321358938493,1.680291461525485e-05,1.0,8.538301467895508,0.2932095527648926 +7000,92.5,0.00020449967342433436,6.575699899258325e-06,1.0,8.514925956726074,0.29757243394851685 +7100,93.8,3.047019400261801e-05,2.803931238304358e-06,1.0,8.493824005126953,0.3026065528392792 +7200,95.1,5.479850155099996e-06,1.2052884130753228e-06,1.0,8.475817680358887,0.304732084274292 +7300,96.4,1.338113166744191e-06,5.072772637504386e-07,1.0,8.460067749023438,0.3078644275665283 +7400,97.7,4.300948253466863e-07,2.0480338491779548e-07,1.0,8.44088363647461,0.310325562953949 +7500,99.1,1.5635795071244193e-07,7.958696812693233e-08,1.0,8.382295608520508,0.3116679787635803 +7600,100.4,7.04529971304045e-08,5.2788230675560044e-08,1.0,8.091385841369629,0.31267479062080383 +7700,101.7,5.447445318532013e-08,5.0858471212222867e-08,1.0,7.525296688079834,0.3144647181034088 +7800,103.0,4.9590050715989094e-08,4.892871530159937e-08,1.0,6.951796054840088,0.31591901183128357 +7900,104.3,4.5828776082991747e-08,9.5803166288988e-08,1.0,6.5351881980896,0.3181563913822174 +8000,105.6,4.46460823070388e-08,5.415773607353458e-09,1.0,6.3601250648498535,0.317932665348053 +8100,106.9,4.181663673053315e-08,3.735016207428998e-08,1.0,6.190653324127197,0.3201700448989868 +8200,108.2,4.160428200693942e-08,3.927992153762716e-08,1.0,6.0829877853393555,0.3221836984157562 +8300,109.5,4.150269783543538e-08,6.032051658166893e-08,1.0,5.99353551864624,0.3227430582046509 +8400,110.8,4.221288438786987e-08,3.184101515785187e-08,1.0,5.960937976837158,0.32330238819122314 +8500,112.1,4.2542826529080314e-08,5.0298218923217064e-08,1.0,5.897624492645264,0.323637992143631 +8600,113.4,4.300063071312231e-08,5.21657277374743e-08,1.0,5.864205837249756,0.3239735960960388 +8700,114.8,4.419105899351439e-08,2.2783599007425437e-08,1.0,5.857719898223877,0.32341426610946655 +8800,116.0,4.446908884290464e-08,6.498928684095517e-08,1.0,5.771462440490723,0.3250923156738281 +8900,117.3,4.5390989355975515e-08,4.569169931301076e-08,1.0,5.766927242279053,0.32542791962623596 +9000,118.6,4.593813541275543e-08,4.849296075803977e-08,1.0,5.736522674560547,0.3266584873199463 +9100,119.9,4.70021295683067e-08,1.5344692627650147e-08,1.0,5.746763706207275,0.32721781730651855 +9200,121.2,4.732125778275186e-08,3.996467157207917e-08,1.0,5.703604221343994,0.3264347314834595 +9300,122.5,4.8034121617854175e-08,5.3566360236345645e-08,1.0,5.66981840133667,0.3276652991771698 +9400,123.8,4.834105078183234e-08,6.530054008635489e-08,1.0,5.644845008850098,0.32788902521133423 +9500,125.1,4.9124256476972266e-08,4.3077186262507894e-08,1.0,5.640939712524414,0.3295670747756958 +9600,126.4,4.9510911409934235e-08,6.934680385484171e-08,1.0,5.609554290771484,0.3295670747756958 +9700,127.7,5.0004612781981696e-08,6.937793273209536e-08,1.0,5.581589221954346,0.33090952038764954 +9800,129.1,5.0622382396042236e-08,6.287277898309185e-08,1.0,5.579067707061768,0.33090952038764954 +9900,130.3,5.112508518149507e-08,5.7052375268540345e-08,1.0,5.575204372406006,0.33281129598617554 +10000,131.7,5.1658781306059294e-08,4.4353317463219355e-08,1.0,5.568783283233643,0.33404186367988586 +10100,133.0,5.217448872820585e-08,4.5131447024004956e-08,1.0,5.570204257965088,0.3333706259727478 +10200,134.3,5.254126066539431e-08,4.8586336731659685e-08,1.0,5.548276901245117,0.3331468999385834 +10300,135.6,5.28534050408486e-08,5.671000025131434e-08,1.0,5.544691562652588,0.33493679761886597 +10400,136.9,5.323935660382546e-08,5.088959653676284e-08,1.0,5.535255432128906,0.33381810784339905 +10500,138.2,5.351800775356552e-08,5.0111470528690916e-08,1.0,5.519857406616211,0.33392998576164246 +10600,139.5,5.384416538879529e-08,5.21657277374743e-08,1.0,5.514888763427734,0.33471307158470154 +10700,140.9,5.400309868898485e-08,6.691904275157867e-08,1.0,5.497499465942383,0.33460119366645813 +10800,142.2,5.4466201241590526e-08,5.474911546343719e-08,1.0,5.49669885635376,0.3352724015712738 +10900,143.6,5.465462444492608e-08,5.8764257460097724e-08,1.0,5.504118919372559,0.3352724015712738 +11000,144.9,5.495253806664538e-08,5.9075510705497436e-08,1.0,5.486688613891602,0.33594363927841187 +11100,146.2,5.5157150960390556e-08,6.231252314137237e-08,1.0,5.475027561187744,0.3362792432308197 +11200,147.5,5.5469635689204424e-08,5.365973620996556e-08,1.0,5.476905345916748,0.33672672510147095 +11300,148.8,5.564247353625855e-08,5.677225090039428e-08,1.0,5.461345672607422,0.33661484718322754 +11400,150.0,5.575723358820382e-08,6.713692357607215e-08,1.0,5.451956748962402,0.33762165904045105 +11500,151.3,5.597424293740712e-08,5.72080018912402e-08,1.0,5.450022220611572,0.3371741771697998 +11600,152.7,5.6234102385704286e-08,5.767487820662609e-08,1.0,5.446512699127197,0.3382928967475891 +11700,154.0,5.6380015433962746e-08,5.493586741067702e-08,1.0,5.437525272369385,0.33941158652305603 +11800,155.3,5.6451977642626345e-08,6.28416501058382e-08,1.0,5.435503005981445,0.33963531255722046 +11900,156.6,5.6556478097429026e-08,6.32774046493978e-08,1.0,5.425296783447266,0.3398590683937073 +12000,157.9,5.6667187032868634e-08,6.069402047614858e-08,1.0,5.421667575836182,0.33963531255722046 +12100,159.2,5.6734317416104554e-08,6.39621546838498e-08,1.0,5.408508777618408,0.3408658802509308 +12200,160.5,5.69004513362785e-08,6.253039686043849e-08,1.0,5.407674789428711,0.3412014842033386 +12300,161.7,5.703318645270034e-08,5.58696200414488e-08,1.0,5.408480167388916,0.3419845700263977 +12400,163.0,5.710786439727439e-08,5.811062919747201e-08,1.0,5.40632963180542,0.3428795337677002 +12500,164.5,5.7238332477031326e-08,5.244585565833404e-08,1.0,5.402377128601074,0.34355074167251587 +12600,165.8,5.7256613832521645e-08,5.639874700591463e-08,1.0,5.409458160400391,0.3428795337677002 +12700,167.1,5.731375299883688e-08,5.7519251583926234e-08,1.0,5.387070655822754,0.3428795337677002 +12800,168.3,5.7403855307948654e-08,5.387760992903168e-08,1.0,5.392940044403076,0.3441101014614105 +12900,169.7,5.738302604163115e-08,5.7799379504785975e-08,1.0,5.387880802154541,0.3428795337677002 +13000,171.0,5.740348729905745e-08,5.7768254180246004e-08,1.0,5.371148109436035,0.3427676558494568 +13100,172.3,5.7433374365234164e-08,5.960463766996327e-08,1.0,5.375955104827881,0.3420964479446411 +13200,173.6,5.755916577508511e-08,5.425111382351133e-08,1.0,5.382537841796875,0.3426557779312134 +13300,174.9,5.745518508273053e-08,5.8017253223852094e-08,1.0,5.376333713531494,0.3438863456249237 +13400,176.1,5.7568774629028185e-08,5.590074536598877e-08,1.0,5.368573188781738,0.3446694314479828 +13500,177.5,5.755203530628523e-08,5.250810630741398e-08,1.0,5.370269298553467,0.3418726921081543 +13600,178.7,5.754360014225749e-08,5.870200681101778e-08,1.0,5.375529766082764,0.34332701563835144 +13700,180.0,5.759193186589186e-08,5.397098590265159e-08,1.0,5.367101669311523,0.3420964479446411 +13800,181.3,5.753001436411694e-08,5.923113377548361e-08,1.0,5.355581283569336,0.34444570541381836 +13900,182.6,5.7525434837875337e-08,5.811062919747201e-08,1.0,5.354740142822266,0.34422194957733154 +14000,183.9,5.752650580498001e-08,5.736362851394006e-08,1.0,5.350019931793213,0.3436626195907593 +14100,185.2,5.75790018757067e-08,5.7083500593080316e-08,1.0,5.3519792556762695,0.3429913818836212 +14200,186.5,5.763139732498721e-08,5.487361676159708e-08,1.0,5.349208831787109,0.3437744677066803 +14300,187.8,5.757520292218011e-08,5.599412133960868e-08,1.0,5.351533889770508,0.3431032598018646 +14400,189.1,5.7570559085491394e-08,6.01026428626028e-08,1.0,5.331762790679932,0.3438863456249237 +14500,190.4,5.744542312283697e-08,5.994701268718927e-08,1.0,5.34036922454834,0.3427676558494568 +14600,191.7,5.746320994909266e-08,5.512261580520317e-08,1.0,5.3284993171691895,0.34321513772010803 +14700,193.0,5.747775959388612e-08,5.773712885570603e-08,1.0,5.320709228515625,0.34422194957733154 +14800,194.4,5.744778653408774e-08,5.842188244287172e-08,1.0,5.3187947273254395,0.3441101014614105 +14900,195.7,5.735166701466514e-08,5.674112557585431e-08,1.0,5.320685863494873,0.3447813093662262 +15000,197.0,5.736218542176117e-08,5.807950387293204e-08,1.0,5.313047885894775,0.3446694314479828 +15100,198.3,5.728590345942653e-08,5.9013260056417494e-08,1.0,5.315560817718506,0.34522879123687744 +15200,199.6,5.724038385699738e-08,5.574511874328891e-08,1.0,5.31468391418457,0.34433382749557495 +15300,200.9,5.7204223542364324e-08,5.5807369392368855e-08,1.0,5.311736583709717,0.34522879123687744 +15400,202.2,5.723694433905856e-08,5.375311218358547e-08,1.0,5.310722351074219,0.3448931872844696 +15500,203.5,5.715171197837032e-08,5.695899929492043e-08,1.0,5.2945556640625,0.3456762731075287 +15600,204.9,5.7138328856107506e-08,5.415773784989142e-08,1.0,5.303400039672852,0.3458999991416931 +15700,206.2,5.7075666660193964e-08,5.618087328684851e-08,1.0,5.297314167022705,0.34623560309410095 +15800,207.5,5.7041631658170475e-08,5.817287984655195e-08,1.0,5.2863616943359375,0.3474661707878113 +15900,208.7,5.705125787805595e-08,5.789275547840589e-08,1.0,5.292762756347656,0.3456762731075287 +16000,210.1,5.6948880262037005e-08,5.736362851394006e-08,1.0,5.274105072021484,0.34724244475364685 +16100,211.4,5.7052955841081406e-08,5.440673689349751e-08,1.0,5.276555061340332,0.34802553057670593 +16200,212.7,5.688621395311889e-08,5.867088148647781e-08,1.0,5.270654678344727,0.3486967384815216 +16300,214.0,5.679081023221634e-08,5.8577509065571576e-08,1.0,5.270047664642334,0.3484729826450348 +16400,215.3,5.672171517962301e-08,5.870200681101778e-08,1.0,5.258683204650879,0.34824925661087036 +16500,216.6,5.6765930390209594e-08,5.5651742769669e-08,1.0,5.266787052154541,0.3486967384815216 +16600,217.9,5.666288063795718e-08,5.773712885570603e-08,1.0,5.258786201477051,0.34824925661087036 +16700,219.2,5.6622258105588016e-08,5.636762168137466e-08,1.0,5.257833480834961,0.35015103220939636 +16800,220.5,5.6585503284576076e-08,5.69901246194604e-08,1.0,5.253173828125,0.34892046451568604 +16900,221.8,5.6637981727182845e-08,5.4562363516197365e-08,1.0,5.258421897888184,0.3498154282569885 +17000,223.1,5.642628551546444e-08,5.932450974910353e-08,1.0,5.249162673950195,0.3478017747402191 +17100,224.4,5.6456538280943106e-08,5.369086153450553e-08,1.0,5.247527599334717,0.35003915429115295 +17200,225.7,5.634718211442866e-08,5.636762168137466e-08,1.0,5.244577884674072,0.348808616399765 +17300,227.0,5.627886418914483e-08,5.960463766996327e-08,1.0,5.2338128089904785,0.35026291012763977 +17400,228.4,5.62134481525923e-08,6.175227440508024e-08,1.0,5.2348713874816895,0.3504866361618042 +17500,229.7,5.6153415655851354e-08,6.078739289705481e-08,1.0,5.2262678146362305,0.3497035503387451 +17600,231.0,5.622444895630523e-08,5.188560336932824e-08,1.0,5.225732326507568,0.3513815999031067 +17700,232.3,5.608591573481659e-08,5.677225090039428e-08,1.0,5.225230693817139,0.3498154282569885 +17800,233.7,5.604856432139276e-08,5.546499437514285e-08,1.0,5.224470138549805,0.3517172038555145 +17900,235.0,5.603022161229259e-08,5.4033236551731534e-08,1.0,5.220393180847168,0.350710391998291 +18000,236.3,5.6078164429830185e-08,5.4562363516197365e-08,1.0,5.221450328826904,0.35082224011421204 +18100,237.6,5.59479252157175e-08,5.6087497313228596e-08,1.0,5.213759422302246,0.35082224011421204 +18200,238.9,5.585542422594503e-08,5.761262755754615e-08,1.0,5.2020487785339355,0.3517172038555145 +18300,240.3,5.584491335832274e-08,5.5309367752442995e-08,1.0,5.211326599121094,0.3523884117603302 +18400,241.5,5.575817379779084e-08,5.6585498953154456e-08,1.0,5.221136093139648,0.3517172038555145 +18500,242.8,5.5693877041935595e-08,5.910663603003741e-08,1.0,5.202925682067871,0.35205280780792236 +18600,244.2,5.558211106129244e-08,5.7332503189400086e-08,1.0,5.198795795440674,0.3532833755016327 +18700,245.6,5.558232583738168e-08,5.4780240787977164e-08,1.0,5.213387966156006,0.35026291012763977 +18800,246.9,5.555498585118284e-08,5.994701268718927e-08,1.0,5.2001633644104,0.3523884117603302 +18900,248.2,5.5551606834446885e-08,5.7332503189400086e-08,1.0,5.190338611602783,0.3533952534198761 +19000,249.4,5.5577646773608984e-08,5.250810630741398e-08,1.0,5.2025017738342285,0.3532833755016327 +19100,250.9,5.544818387597063e-08,5.549611969968282e-08,1.0,5.193538665771484,0.35496142506599426 +19200,252.2,5.537062237878622e-08,5.574511874328891e-08,1.0,5.18909215927124,0.3536189794540405 +19300,253.5,5.531857618464231e-08,5.6025246664148654e-08,1.0,5.185902118682861,0.35294777154922485 +19400,254.8,5.5327391518933873e-08,5.425111382351133e-08,1.0,5.185239791870117,0.35395458340644836 +19500,256.0,5.5253634851769405e-08,5.362861088542559e-08,1.0,5.18874979019165,0.35294777154922485 +19600,257.3,5.5273252403913917e-08,5.450011286711742e-08,1.0,5.188867568969727,0.3531714975833893 +19700,258.6,5.5119678531717696e-08,5.073397346677666e-08,1.0,5.184772968292236,0.3513815999031067 +19800,259.9,5.5093011862566455e-08,5.2570356956493924e-08,1.0,5.178551197052002,0.3536189794540405 +19900,261.2,5.5057005358998516e-08,4.952009291514514e-08,1.0,5.1817193031311035,0.3525002896785736 +20000,262.5,5.4994581841209264e-08,5.219685306201427e-08,1.0,5.176990509033203,0.3544020652770996 +20100,263.8,5.489791106379751e-08,5.562061744512903e-08,1.0,5.168576240539551,0.3542901873588562 +20200,265.1,5.482748861130244e-08,5.7550376908466205e-08,1.0,5.153883934020996,0.35373085737228394 +20300,266.4,5.481196280723874e-08,5.618087328684851e-08,1.0,5.158692836761475,0.3540664613246918 +20400,267.7,5.4767110415319205e-08,5.266372937740016e-08,1.0,5.176906585693359,0.35272401571273804 +20500,269.0,5.469508489832892e-08,5.711462591762029e-08,1.0,5.147735118865967,0.3540664613246918 +20600,270.4,5.461248451698305e-08,5.4033236551731534e-08,1.0,5.153382301330566,0.3541783392429352 +20700,271.7,5.460295696100275e-08,5.4344489797131246e-08,1.0,5.142098426818848,0.3542901873588562 +20800,273.0,5.4594791971093896e-08,5.5776244067828884e-08,1.0,5.149086952209473,0.3546258211135864 +20900,274.3,5.4558066255975996e-08,5.487361676159708e-08,1.0,5.141632080078125,0.3551851511001587 +21000,275.6,5.4601829350357804e-08,5.266372937740016e-08,1.0,5.1431427001953125,0.3552970290184021 +21100,276.9,5.4483399578343783e-08,5.026709359867709e-08,1.0,5.1458282470703125,0.3552970290184021 +21200,278.2,5.44156406838739e-08,5.748812625938626e-08,1.0,5.139247417449951,0.35574451088905334 +21300,279.5,5.4367704464696277e-08,5.646099765499457e-08,1.0,5.133215427398682,0.3550732731819153 +21400,280.8,5.4340823303416866e-08,5.4531238191657394e-08,1.0,5.143241882324219,0.35384273529052734 +21500,282.1,5.4392085983219446e-08,5.088959653676284e-08,1.0,5.128834247589111,0.35563263297080994 +21600,283.4,5.4255409856175616e-08,5.543386905060288e-08,1.0,5.131741046905518,0.35574451088905334 +21700,284.7,5.428410282509454e-08,5.397098590265159e-08,1.0,5.127285003662109,0.3540664613246918 +21800,286.0,5.4142191903046065e-08,6.075626401980116e-08,1.0,5.120888710021973,0.3550732731819153 +21900,287.3,5.407408189753041e-08,5.58696200414488e-08,1.0,5.117586612701416,0.35675132274627686 +22000,288.6,5.411564560086365e-08,5.194785401840818e-08,1.0,5.120716094970703,0.35563263297080994 +22100,289.8,5.403637415858419e-08,6.032051658166893e-08,1.0,5.117344856262207,0.3570869266986847 +22200,291.2,5.398951528395717e-08,5.7581502233006177e-08,1.0,5.113534927368164,0.35675132274627686 +22300,292.5,5.406772056227887e-08,5.365973620996556e-08,1.0,5.116140842437744,0.3569750487804413 +22400,293.8,5.389621833764852e-08,6.175227440508024e-08,1.0,5.099582195281982,0.3559682369232178 +22500,295.2,5.387371540685522e-08,4.843071010895983e-08,1.0,5.124629497528076,0.3561919927597046 +22600,296.5,5.37975237803696e-08,5.393986057811162e-08,1.0,5.110499858856201,0.3559682369232178 +22700,297.8,5.3779290026561194e-08,5.540274372606291e-08,1.0,5.103215217590332,0.35686320066452026 +22800,299.1,5.373072317449231e-08,5.2850481324639986e-08,1.0,5.104565620422363,0.35753440856933594 +22900,300.5,5.373672886074491e-08,5.6336496356834687e-08,1.0,5.0994439125061035,0.356415718793869 +23000,301.7,5.3681609555874016e-08,5.792388080294586e-08,1.0,5.082685470581055,0.3560801148414612 +23100,303.0,5.359584090133286e-08,5.344185893818576e-08,1.0,5.090766906738281,0.35742253065109253 +23200,304.4,5.35369510697216e-08,5.2352479684714126e-08,1.0,5.086436748504639,0.35686320066452026 +23300,305.6,5.348159259691644e-08,5.325510699094593e-08,1.0,5.08825159072876,0.35675132274627686 +23400,306.9,5.346828348293736e-08,5.263260405286019e-08,1.0,5.085333824157715,0.3569750487804413 +23500,308.3,5.3410359175240627e-08,5.674112557585431e-08,1.0,5.067868709564209,0.35854122042655945 +23600,309.5,5.346691746844506e-08,4.939559161698526e-08,1.0,5.0891499519348145,0.3570869266986847 +23700,310.8,5.3357375151667486e-08,5.425111382351133e-08,1.0,5.087523937225342,0.3592124581336975 +23800,312.1,5.325540449852188e-08,5.6305371032294715e-08,1.0,5.0706095695495605,0.3584293723106384 +23900,313.4,5.3247012298202605e-08,5.213460241293433e-08,1.0,5.071650981903076,0.3571988046169281 +24000,314.7,5.3236626130842564e-08,4.998696923053103e-08,1.0,5.076087474822998,0.3569750487804413 +24100,315.9,5.3251127711857284e-08,4.715458246096205e-08,1.0,5.0827226638793945,0.35742253065109253 +24200,317.3,5.3099935354931595e-08,5.253923163195395e-08,1.0,5.069209575653076,0.3580937683582306 +24300,318.7,5.3048945331552035e-08,5.300610794733984e-08,1.0,5.0658793449401855,0.3592124581336975 +24400,320.0,5.2993204864255664e-08,5.421998849897136e-08,1.0,5.065806865692139,0.35854122042655945 +24500,321.3,5.2950238989485596e-08,5.920001200365732e-08,1.0,5.041089057922363,0.3592124581336975 +24600,322.6,5.307650379075253e-08,4.948896759060517e-08,1.0,5.072389125823975,0.35876497626304626 +24700,324.0,5.296338048316267e-08,5.263260405286019e-08,1.0,5.054500579833984,0.3582056164741516 +24800,325.3,5.2904116044274206e-08,5.0298218923217064e-08,1.0,5.044864654541016,0.3597717881202698 +24900,326.6,5.2791273931105094e-08,5.369086153450553e-08,1.0,5.038983345031738,0.35943618416786194 +25000,327.9,5.2675584505843663e-08,5.412661252535145e-08,1.0,5.0358357429504395,0.3589887022972107 +25100,329.2,5.275933027262294e-08,4.973796663421126e-08,1.0,5.042904853820801,0.36033114790916443 +25200,330.5,5.264027094637939e-08,5.4811366112517135e-08,1.0,5.028580665588379,0.35943618416786194 +25300,331.8,5.272050671832601e-08,4.964459066059135e-08,1.0,5.042216777801514,0.36044302582740784 +25400,333.1,5.266364243019072e-08,5.042272022137695e-08,1.0,5.026374816894531,0.3599955439567566 +25500,334.4,5.246048439700431e-08,5.8266255820171864e-08,1.0,5.025500774383545,0.36133795976638794 +25600,335.7,5.251034740244521e-08,4.998696923053103e-08,1.0,5.02403450012207,0.3612261116504669 +25700,337.0,5.2518591937479854e-08,4.9302215643365344e-08,1.0,5.021348476409912,0.36133795976638794 +25800,338.3,5.231582520550347e-08,6.131651986152065e-08,1.0,5.014068603515625,0.3617854416370392 +25900,339.6,5.229395189840538e-08,5.341073361364579e-08,1.0,5.014671325683594,0.3612261116504669 +26000,340.9,5.229881992636694e-08,5.2290229035634184e-08,1.0,5.014743328094482,0.36223292350769043 +26100,342.3,5.22627575180261e-08,5.440673689349751e-08,1.0,5.014735221862793,0.36144983768463135 +26200,343.6,5.226115957430817e-08,4.7559208127267993e-08,1.0,5.001917362213135,0.3617854416370392 +26300,344.9,5.220041301567673e-08,4.8617462056199656e-08,1.0,5.004197597503662,0.36044302582740784 +26400,346.2,5.207673476252398e-08,5.649212297953454e-08,1.0,4.999210834503174,0.362121045589447 +26500,347.5,5.2076629885340146e-08,5.35041095872657e-08,1.0,4.993356704711914,0.36256852746009827 +26600,348.9,5.200489334119417e-08,5.5776244067828884e-08,1.0,5.008508205413818,0.36256852746009827 +26700,350.2,5.1932517866860626e-08,5.313060924549973e-08,1.0,4.991248607635498,0.36234480142593384 +26800,351.5,5.197288799526415e-08,4.503807105038504e-08,1.0,5.004161357879639,0.36133795976638794 +26900,352.8,5.198232367753193e-08,5.2788230675560044e-08,1.0,5.000853061676025,0.36234480142593384 +27000,354.1,5.191460788356452e-08,5.31617345700397e-08,1.0,4.973559856414795,0.3616735637187958 +27100,355.4,5.173173683886578e-08,5.546499437514285e-08,1.0,4.999666213989258,0.36357536911964417 +27200,356.7,5.167215523530358e-08,5.7021249944000374e-08,1.0,4.982126235961914,0.36413469910621643 +27300,358.0,5.173664586614546e-08,4.94578422660652e-08,1.0,4.988261699676514,0.363910973072052 +27400,359.2,5.1833096693167516e-08,4.581620061117064e-08,1.0,4.992615699768066,0.36357536911964417 +27500,360.5,5.16287267526664e-08,5.1823352720248295e-08,1.0,4.964707851409912,0.3626804053783417 +27600,361.8,5.1539317623382455e-08,5.6025246664148654e-08,1.0,4.9788312911987305,0.36514151096343994 +27700,363.1,5.142081178445431e-08,5.446898754257745e-08,1.0,4.963313102722168,0.36323973536491394 +27800,364.4,5.1513196180189705e-08,4.1396429395490486e-08,1.0,4.986420631408691,0.36245664954185486 +27900,365.7,5.1454903788660716e-08,4.9333340967905315e-08,1.0,4.9892449378967285,0.36323973536491394 +28000,367.0,5.142130064781786e-08,4.92399649942854e-08,1.0,4.967411041259766,0.3627922832965851 +28100,368.3,5.1349429826072356e-08,5.051609619499686e-08,1.0,4.982836723327637,0.36357536911964417 +28200,369.7,5.123490080819595e-08,4.955121823968511e-08,1.0,4.972360610961914,0.36323973536491394 +28300,371.0,5.126983315794162e-08,4.9613465336051377e-08,1.0,4.975869655609131,0.3629041314125061 +28400,372.4,5.1236504764798474e-08,4.581620061117064e-08,1.0,4.985287666320801,0.3636872172355652 +28500,373.7,5.1081180119328966e-08,4.827508703897365e-08,1.0,4.96473503112793,0.3631278872489929 +28600,375.0,5.102274580468907e-08,5.2570356956493924e-08,1.0,4.956845760345459,0.364022821187973 +28700,376.3,5.0935948920555715e-08,5.213460241293433e-08,1.0,4.968205451965332,0.36335161328315735 +28800,377.6,5.08833549088199e-08,5.415773784989142e-08,1.0,4.938381671905518,0.36346349120140076 +28900,378.9,5.0861916657225704e-08,5.5776244067828884e-08,1.0,4.948482513427734,0.36424657702445984 +29000,380.2,5.081662110182141e-08,4.774596007450782e-08,1.0,4.952465534210205,0.3630160093307495 +29100,381.5,5.071072558825949e-08,4.9862467932371146e-08,1.0,4.975172519683838,0.36335161328315735 +29200,382.8,5.069533186809184e-08,5.291273197371993e-08,1.0,4.9347968101501465,0.36514151096343994 +29300,384.2,5.065640067688397e-08,5.0827345887682895e-08,1.0,4.950308322906494,0.36335161328315735 +29400,385.5,5.064040869862535e-08,4.9831342607831175e-08,1.0,4.951794624328613,0.363910973072052 +29500,386.8,5.066566632319895e-08,4.927109031882537e-08,1.0,4.958821773529053,0.3646940588951111 +29600,388.1,5.058499277651127e-08,4.84618354334998e-08,1.0,4.952322483062744,0.36323973536491394 +29700,389.4,5.0438297713998485e-08,4.7092331811882104e-08,1.0,4.945855617523193,0.3629041314125061 +29800,390.8,5.03491122122529e-08,5.138760172940238e-08,1.0,4.942371368408203,0.3650296628475189 +29900,392.0,5.026848924504653e-08,5.671000025131434e-08,1.0,4.960850238800049,0.36447030305862427 +30000,393.4,5.025352369579509e-08,4.802608444265388e-08,1.0,4.943594455718994,0.3636872172355652 +30100,394.7,5.02514640302337e-08,4.7901583144494e-08,1.0,4.963935852050781,0.3636872172355652 +30200,396.0,5.015286195137284e-08,5.051609619499686e-08,1.0,4.935466289520264,0.36323973536491394 +30300,397.3,5.003459539834026e-08,6.088076531796105e-08,1.0,4.926537036895752,0.3630160093307495 +30400,398.6,5.01133626287811e-08,5.3348482964565846e-08,1.0,4.937016010284424,0.36413469910621643 +30500,399.9,4.999182681419184e-08,5.6523248304074514e-08,1.0,4.925256252288818,0.3645821809768677 +30600,401.2,4.9951593394488524e-08,5.0796220563142924e-08,1.0,4.942076683044434,0.3649177849292755 +30700,402.5,4.988705343855185e-08,4.718570423278834e-08,1.0,4.94105863571167,0.364022821187973 +30800,403.8,4.983721475187896e-08,4.939559161698526e-08,1.0,4.932796955108643,0.36614835262298584 +30900,405.2,4.968041440072332e-08,5.3099483920959756e-08,1.0,4.93231725692749,0.3655889928340912 +31000,406.4,4.9890022975901254e-08,4.329506353428769e-08,1.0,4.958648204803467,0.36424657702445984 +31100,407.7,4.9766063364305424e-08,4.774596007450782e-08,1.0,4.942205429077148,0.36346349120140076 +31200,409.0,4.96335612941621e-08,4.145868004457043e-08,1.0,4.948668003082275,0.36514151096343994 +31300,410.3,4.9506274553855006e-08,5.288160664917996e-08,1.0,4.947075843811035,0.36346349120140076 +31400,411.6,4.94021133540059e-08,5.5558370348762764e-08,1.0,4.91871452331543,0.3655889928340912 +31500,412.9,4.943402083265975e-08,4.948896759060517e-08,1.0,4.947715759277344,0.3657008707523346 +31600,414.3,4.930426165940195e-08,4.8057209767193854e-08,1.0,4.929710865020752,0.363910973072052 +31700,415.7,4.939003909826191e-08,5.3566360236345645e-08,1.0,4.921247482299805,0.36614835262298584 +31800,417.0,4.9181699309637026e-08,4.9302215643365344e-08,1.0,4.918647766113281,0.3648059070110321 +31900,418.3,4.919465226050638e-08,5.3597485560885616e-08,1.0,4.903859615325928,0.3668195605278015 +32000,419.6,4.9068265794498026e-08,5.0578346844076805e-08,1.0,4.909486293792725,0.3655889928340912 +32100,420.9,4.908989947560324e-08,4.830621236351362e-08,1.0,4.9328227043151855,0.36547714471817017 +32200,422.2,4.897187227866433e-08,4.824396171443368e-08,1.0,4.924957752227783,0.3667076826095581 +32300,423.7,4.902785950562518e-08,5.7021249944000374e-08,1.0,4.898453235626221,0.36514151096343994 +32400,425.0,4.8894903357640684e-08,4.422881616505947e-08,1.0,4.925724029541016,0.3664839565753937 +32500,426.3,4.88321708234197e-08,4.254805929804206e-08,1.0,4.933276176452637,0.3655889928340912 +32600,427.5,4.8699408444081e-08,4.6843329215562335e-08,1.0,4.905600547790527,0.36536526679992676 +32700,428.8,4.869012784428424e-08,4.7590333451807965e-08,1.0,4.907751083374023,0.3684976100921631 +32800,430.1,4.861264323764749e-08,4.5069196374925014e-08,1.0,4.947734355926514,0.36614835262298584 +32900,431.4,4.8713925798493567e-08,4.2610309947122005e-08,1.0,4.911012649536133,0.36827385425567627 +33000,432.7,4.847430516223733e-08,5.020484294959715e-08,1.0,4.915238857269287,0.3664839565753937 +33100,434.0,4.856688640173005e-08,5.325510699094593e-08,1.0,4.9184393882751465,0.3655889928340912 +33200,435.3,4.850512554915816e-08,4.7092331811882104e-08,1.0,4.91829252243042,0.36514151096343994 +33300,436.6,4.8216300431417825e-08,4.777708539904779e-08,1.0,4.930105209350586,0.3655889928340912 +33400,438.0,4.823789300393173e-08,5.337960828910582e-08,1.0,4.887223720550537,0.36726704239845276 +33500,439.3,4.820463223783551e-08,5.113859913308261e-08,1.0,4.905390739440918,0.36614835262298584 +33600,440.6,4.822780613099068e-08,4.771483474996785e-08,1.0,4.905981540679932,0.3674907684326172 +33700,441.9,4.817620885536031e-08,5.0547221519536834e-08,1.0,4.898641586303711,0.3678264021873474 +33800,443.2,4.7942280100361516e-08,5.151209947484858e-08,1.0,4.896220684051514,0.36603647470474243 +33900,444.5,4.785227751592133e-08,4.84618354334998e-08,1.0,4.90164852142334,0.3695044219493866 +34000,445.8,4.7929263751647356e-08,4.8088335091733825e-08,1.0,4.931687355041504,0.3667076826095581 +34100,447.1,4.790641947959716e-08,4.1645431991810256e-08,1.0,4.8839850425720215,0.36726704239845276 +34200,448.4,4.769990030352691e-08,4.948896759060517e-08,1.0,4.884130954742432,0.3676026463508606 +34300,449.8,4.7632847251650426e-08,5.29438572982599e-08,1.0,4.897134304046631,0.36603647470474243 +34400,451.2,4.776895234976544e-08,4.690557986464228e-08,1.0,4.890380859375,0.365924596786499 +34500,452.4,4.743380107008117e-08,4.8057209767193854e-08,1.0,4.901063919067383,0.3667076826095581 +34600,453.8,4.753702236346708e-08,4.251693397350209e-08,1.0,4.877133369445801,0.36805012822151184 +34700,455.2,4.7321952241038554e-08,4.603407433023676e-08,1.0,4.880570411682129,0.3693925738334656 +34800,456.5,4.740890941894889e-08,5.0018094555071e-08,1.0,4.889524459838867,0.36626023054122925 +34900,457.8,4.73970440853158e-08,5.2788230675560044e-08,1.0,4.88214635848999,0.3674907684326172 +35000,459.1,4.7311777297132215e-08,4.590957658479056e-08,1.0,4.882889270782471,0.3686094880104065 +35100,460.4,4.719480938508076e-08,4.3886441147833466e-08,1.0,4.901824951171875,0.3676026463508606 +35200,461.7,4.718649827435154e-08,4.656320484741627e-08,1.0,4.903136730194092,0.3665958344936371 +35300,463.0,4.695760107939974e-08,4.955121823968511e-08,1.0,4.869049072265625,0.3683857321739197 +35400,464.2,4.694042009595602e-08,4.9084338371585545e-08,1.0,4.882960319519043,0.36928069591522217 +35500,465.6,4.6948612840731844e-08,5.004921987961097e-08,1.0,4.89169454574585,0.36928069591522217 +35600,466.9,4.70014742371207e-08,4.454006941045918e-08,1.0,4.891383171081543,0.36737892031669617 +35700,468.2,4.6684684928527896e-08,5.0329344247757035e-08,1.0,4.872223377227783,0.3688332140445709 +35800,469.5,4.683558154328979e-08,4.7621458776347936e-08,1.0,4.857796669006348,0.3688332140445709 +35900,470.8,4.6717223789176794e-08,5.1574350123928525e-08,1.0,4.882004737854004,0.36805012822151184 +36000,472.1,4.648347169401618e-08,4.5100321699464985e-08,1.0,4.876773357391357,0.36894509196281433 +36100,473.4,4.6427001313944e-08,4.967571598513132e-08,1.0,4.8565287590026855,0.3684976100921631 +36200,474.7,4.6666862285643054e-08,5.116972445762258e-08,1.0,4.962843418121338,0.3674907684326172 +36300,476.1,4.653085763028041e-08,3.619853217173841e-08,1.0,4.89499568939209,0.37006378173828125 +36400,477.5,4.632259189676951e-08,5.300610794733984e-08,1.0,4.881173133850098,0.36737892031669617 +36500,478.8,4.631068323837099e-08,4.5349324295784754e-08,1.0,4.881310939788818,0.37006378173828125 +36600,480.1,4.6315361820385384e-08,3.8657418599541415e-08,1.0,4.921390533447266,0.36704331636428833 +36700,481.4,4.6082144767571736e-08,4.267256059620195e-08,1.0,4.872272491455078,0.3687213361263275 +36800,482.7,4.628681317592822e-08,3.772366241605596e-08,1.0,4.865856647491455,0.36916881799697876 +36900,484.0,4.576677897397609e-08,5.471799013889722e-08,1.0,4.8465728759765625,0.3693925738334656 +37000,485.3,4.598216058934513e-08,4.703008116280216e-08,1.0,4.874382972717285,0.37017565965652466 +37100,486.6,4.598690296261857e-08,3.980904850209299e-08,1.0,4.892462253570557,0.3693925738334656 +37200,487.9,4.5689731540331037e-08,4.569169931301076e-08,1.0,4.856954097747803,0.37017565965652466 +37300,489.2,4.566299453488435e-08,4.425994148959944e-08,1.0,4.872276306152344,0.3695044219493866 +37400,490.5,4.564970515256949e-08,5.120084978216255e-08,1.0,4.839425563812256,0.3688332140445709 +37500,491.8,4.549523984224869e-08,4.6874454540102306e-08,1.0,4.849524021148682,0.3702875077724457 +37600,493.1,4.537320709134568e-08,5.045384554591692e-08,1.0,4.848348140716553,0.3706231117248535 +37700,494.4,4.554590389097482e-08,4.942671694152523e-08,1.0,4.861142158508301,0.36894509196281433 +37800,495.7,4.5627923287810707e-08,4.7870457819954026e-08,1.0,4.91823148727417,0.36894509196281433 +37900,497.1,4.5220019212258264e-08,4.6314202251096503e-08,1.0,4.839587688446045,0.3712943494319916 +38000,498.4,4.5619796931409666e-08,4.5878451260250586e-08,1.0,4.890357494354248,0.36984002590179443 +38100,499.7,4.497814242422326e-08,4.821283638989371e-08,1.0,4.879090309143066,0.37084686756134033 +38200,501.0,4.4980988634963406e-08,5.2352479684714126e-08,1.0,4.89561128616333,0.36894509196281433 +38300,502.2,4.538675505631129e-08,3.4113146085701374e-08,1.0,4.864835262298584,0.37084686756134033 +38400,503.6,4.499883283467973e-08,4.765258410088791e-08,1.0,4.85191535949707,0.3723011612892151 +38500,504.9,4.4915168866213925e-08,5.3348482964565846e-08,1.0,4.8218488693237305,0.3706231117248535 +38600,506.2,4.492294041869458e-08,3.682103510982415e-08,1.0,4.860949993133545,0.37006378173828125 +38700,507.5,4.441264053551622e-08,5.1045223159462694e-08,1.0,4.815453052520752,0.37118247151374817 +38800,508.8,4.513328264617373e-08,5.593187069052874e-08,1.0,4.9737324714660645,0.36793825030326843 +38900,510.1,4.435930029823589e-08,4.6874454540102306e-08,1.0,4.853353500366211,0.36984002590179443 +39000,511.4,4.438396299256014e-08,5.418886317443139e-08,1.0,4.828742027282715,0.371518075466156 +39100,512.7,4.473974949524484e-08,3.573165585635252e-08,1.0,4.8933305740356445,0.37084686756134033 +39200,514.1,4.4390034396026125e-08,4.827508703897365e-08,1.0,4.86405611038208,0.3707349896430969 +39300,515.4,4.420980851757258e-08,6.063176272164128e-08,1.0,4.815118312835693,0.3695044219493866 +39400,516.7,4.4109364048617056e-08,4.55360726903109e-08,1.0,4.846892833709717,0.3703993856906891 +39500,518.0,0.6417205908689618,2.9738268852233887,0.9287205934524536,13.507944107055664,0.29365700483322144 +39600,519.3,0.25544028752796977,2.3815349777578376e-05,1.0,18.246286392211914,0.31558340787887573 +39700,520.7,0.03388613022663033,7.669903425266966e-06,1.0,18.155107498168945,0.3199463188648224 +39800,522.0,0.004497780861034705,3.1513679914496606e-06,1.0,18.080032348632812,0.3228549063205719 +39900,523.3,0.0005980792617092332,1.3099604529998032e-06,1.0,17.998775482177734,0.32609912753105164 +40000,524.6,7.997110820939568e-05,5.349161824597104e-07,1.0,17.914108276367188,0.32800090312957764 diff --git a/modular_add_wd0/step_9900.pt b/modular_add_wd0/step_9900.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d7b0eeace10d4a79f1e6b680214b840e40c16b5 --- /dev/null +++ b/modular_add_wd0/step_9900.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0400504afc92e355cdc9b97a200e2006b4b3e69a7d3523abcfa14668d12fdc3 +size 918942 diff --git a/modular_mlp/config.json b/modular_mlp/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cbfc7a86554325f27d1d337776411c4ba6794b45 --- /dev/null +++ b/modular_mlp/config.json @@ -0,0 +1,35 @@ +{ + "name": "modular_mlp", + "task": { + "p": 97, + "train_frac": 0.3, + "kind": "modular" + }, + "model": { + "arch": "mlp", + "d_model": 128, + "n_layers": 1, + "n_heads": 4, + "d_mlp": 256, + "max_positions": 128, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 200000, + "batch_size": 0, + "lr": 0.001, + "weight_decay": 1.0, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 0, + "cosine": false, + "amp": false, + "eval_every": 500, + "checkpoint_every": 500, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/modular_mlp/metrics.csv b/modular_mlp/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..a659b128e75e8f73bfcf990cbe33dc01560d74af --- /dev/null +++ b/modular_mlp/metrics.csv @@ -0,0 +1,402 @@ +step,elapsed,loss,train_loss,train_acc,test_loss,test_acc +0,0.3,nan,4.585460662841797,0.0095676826313138,4.586380481719971,0.011689691804349422 +500,3.8,0.704157093602947,0.4760409891605377,1.0,9.267999649047852,0.0 +1000,7.1,0.029810179983724587,0.02124807797372341,1.0,12.47140884399414,0.0 +1500,10.3,0.002032639559543879,0.0015257021877914667,1.0,14.746017456054688,0.00015181417984422296 +2000,13.6,0.00021126409881529686,0.00017027508874889463,1.0,16.25799560546875,0.0 +2500,16.9,5.1645069506356906e-05,4.739216456073336e-05,1.0,16.385894775390625,0.0 +3000,20.2,3.368413081051176e-05,3.310332613182254e-05,1.0,15.485729217529297,0.00015181417984422296 +3500,23.5,3.067761049100513e-05,3.050982741115149e-05,1.0,14.497186660766602,0.0003036283596884459 +4000,26.9,2.9421133550497587e-05,2.932064307969995e-05,1.0,13.639385223388672,0.00045544252498075366 +4500,30.2,2.8475043450907116e-05,2.8388414648361504e-05,1.0,12.898544311523438,0.00045544252498075366 +5000,33.5,2.7678412000741745e-05,2.7605383365880698e-05,1.0,12.249889373779297,0.0006072567193768919 +5500,36.8,2.6906839245104753e-05,2.683624734345358e-05,1.0,11.658526420593262,0.0006072567193768919 +6000,40.2,2.6198453148672793e-05,2.6133047867915593e-05,1.0,11.09339427947998,0.001366327633149922 +6500,43.5,2.559043549620897e-05,2.5528526748530567e-05,1.0,10.573190689086914,0.0018217700999230146 +7000,46.9,2.4992248944347945e-05,2.49315253313398e-05,1.0,10.084244728088379,0.0031880978494882584 +7500,50.1,2.440831635808255e-05,2.4350916646653786e-05,1.0,9.611433982849121,0.004858053755015135 +8000,53.4,2.3935589449051032e-05,2.3890772354207e-05,1.0,9.180367469787598,0.006528009660542011 +8500,56.6,2.3517906548029108e-05,2.348064663237892e-05,1.0,8.792991638183594,0.008501593954861164 +9000,60.1,2.313211396207585e-05,2.3093542040442117e-05,1.0,8.444957733154297,0.011386063881218433 +9500,63.6,2.275800464018587e-05,2.2721435016137548e-05,1.0,8.117083549499512,0.013663276098668575 +10000,66.8,2.2343457506329556e-05,2.2308142433757894e-05,1.0,7.7777838706970215,0.017914073541760445 +10500,70.1,2.196635135474488e-05,2.193459658883512e-05,1.0,7.448159217834473,0.02231668494641781 +11000,73.4,2.1604225063588143e-05,2.1568273950833827e-05,1.0,7.120154857635498,0.029451951384544373 +11500,76.6,2.1273979475285554e-05,2.124356251442805e-05,1.0,6.8144097328186035,0.0335509330034256 +12000,79.9,2.095443953748357e-05,2.0920831957482733e-05,1.0,6.522345542907715,0.04083801433444023 +12500,83.1,2.063524297224958e-05,2.0601608412107453e-05,1.0,6.236166000366211,0.04858053848147392 +13000,86.5,2.031817253396017e-05,2.028445669566281e-05,1.0,5.9508538246154785,0.05753757432103157 +13500,89.8,2.0017523411537634e-05,1.9987159248557873e-05,1.0,5.6742048263549805,0.06801275163888931 +14000,93.1,1.9730528351901898e-05,1.970350422197953e-05,1.0,5.407618045806885,0.07939881831407547 +14500,96.3,1.9478363532706043e-05,1.9453600543783978e-05,1.0,5.162563323974609,0.09199939668178558 +15000,99.5,1.924858036217163e-05,1.9224522475269623e-05,1.0,4.941320896148682,0.10839532315731049 +15500,102.8,1.9044330517948994e-05,1.901998621178791e-05,1.0,4.738241195678711,0.12388037145137787 +16000,106.0,1.8842268061782587e-05,1.8823518985300325e-05,1.0,4.549029350280762,0.14118719100952148 +16500,109.2,1.8634964915482976e-05,1.8615095541463234e-05,1.0,4.3621954917907715,0.1568240523338318 +17000,112.5,1.8435031961770218e-05,1.8415461454424076e-05,1.0,4.168701171875,0.1741308569908142 +17500,115.8,1.823056510746111e-05,1.820860052248463e-05,1.0,3.969156265258789,0.19250038266181946 +18000,119.2,1.7988864798678846e-05,1.7962880519917235e-05,1.0,3.754359245300293,0.212236225605011 +18500,122.6,1.77452928490894e-05,1.772087489371188e-05,1.0,3.5284526348114014,0.23910734057426453 +19000,125.9,1.750526872335018e-05,1.7480686437920667e-05,1.0,3.3067338466644287,0.2678002119064331 +19500,129.4,1.725123805081065e-05,1.7228796423296444e-05,1.0,3.0879554748535156,0.29634127020835876 +20000,132.8,1.702314321817285e-05,1.700440770946443e-05,1.0,2.8820669651031494,0.3286777138710022 +20500,136.2,1.683224684157308e-05,1.6811867681099102e-05,1.0,2.6935980319976807,0.36025506258010864 +21000,139.6,1.6648874053704425e-05,1.663347939029336e-05,1.0,2.5223591327667236,0.3910733163356781 +21500,142.8,1.6490934972102757e-05,1.6474732547067106e-05,1.0,2.3727409839630127,0.41794443130493164 +22000,146.1,1.6343162926599173e-05,1.6330433936673217e-05,1.0,2.230903387069702,0.4443601071834564 +22500,149.3,1.618547634330762e-05,1.617177258594893e-05,1.0,2.0968406200408936,0.47259753942489624 +23000,152.6,1.6037511306738793e-05,1.602299562364351e-05,1.0,1.9724911451339722,0.4971914291381836 +23500,156.0,1.586940089318188e-05,1.5852338037802838e-05,1.0,1.8496578931808472,0.5254288911819458 +24000,159.2,1.57042110843651e-05,1.5688185158069246e-05,1.0,1.7334047555923462,0.5509336590766907 +24500,162.5,1.555875620799775e-05,1.554798291181214e-05,1.0,1.6308579444885254,0.5734021663665771 +25000,165.8,1.5416810525923877e-05,1.5400853953906335e-05,1.0,1.531356692314148,0.5948079824447632 +25500,169.0,1.5276480140481902e-05,1.5261284715961665e-05,1.0,1.4380289316177368,0.6190982460975647 +26000,172.4,1.5147586439361384e-05,1.5135869034565985e-05,1.0,1.3526097536087036,0.641870379447937 +26500,175.7,1.5020707592572138e-05,1.5009648450359236e-05,1.0,1.2717437744140625,0.665249764919281 +27000,179.1,1.4909471287621199e-05,1.489960868639173e-05,1.0,1.1977300643920898,0.6843783259391785 +27500,182.5,1.4792943468738222e-05,1.478073863836471e-05,1.0,1.1268484592437744,0.7053287029266357 +28000,185.9,1.4673047285059584e-05,1.4661784007330425e-05,1.0,1.0613495111465454,0.7238500118255615 +28500,189.2,1.4567206689916357e-05,1.4558207112713717e-05,1.0,1.0012694597244263,0.7399423122406006 +29000,192.5,1.4473155704082872e-05,1.4464301784755662e-05,1.0,0.9492190480232239,0.7537574172019958 +29500,195.9,1.4388363934205463e-05,1.4379480489878915e-05,1.0,0.9024705290794373,0.7674206495285034 +30000,199.2,1.429295773491336e-05,1.4282660231401678e-05,1.0,0.8580844402313232,0.780628502368927 +30500,202.5,1.4203380628932611e-05,1.419602267560549e-05,1.0,0.815412163734436,0.795050859451294 +31000,205.9,1.4112746582099087e-05,1.4103174180490896e-05,1.0,0.7739638686180115,0.8088659644126892 +31500,209.2,1.4011586744656042e-05,1.4001497220306192e-05,1.0,0.7315850257873535,0.8214665055274963 +32000,213.0,1.3902507572500504e-05,1.3889935871702619e-05,1.0,0.6902121305465698,0.8339152932167053 +32500,217.0,1.3809822083154207e-05,1.380156390951015e-05,1.0,0.6529744267463684,0.8471231460571289 +33000,220.4,1.3733675500207513e-05,1.3724346899834927e-05,1.0,0.6215032935142517,0.8545620441436768 +33500,223.9,1.3659565067771133e-05,1.3651224435307086e-05,1.0,0.5929177403450012,0.8621527552604675 +34000,227.3,1.3592648547172866e-05,1.3587479770649225e-05,1.0,0.5672115683555603,0.8706543445587158 +34500,230.6,1.3534922853618562e-05,1.3528130693885032e-05,1.0,0.5441138744354248,0.8780932426452637 +35000,234.0,1.3477164662121044e-05,1.3471779311657883e-05,1.0,0.5217446088790894,0.8829512596130371 +35500,237.3,1.3424683717941415e-05,1.341876486549154e-05,1.0,0.500226616859436,0.8913010358810425 +36000,240.5,1.336949006658653e-05,1.336363948212238e-05,1.0,0.4796910285949707,0.8981326818466187 +36500,243.9,1.3311571729772139e-05,1.330509167019045e-05,1.0,0.4598323106765747,0.9019280672073364 +37000,247.3,1.3239868610397262e-05,1.3232941455498803e-05,1.0,0.43854406476020813,0.9073933362960815 +37500,251.8,1.3182667506244692e-05,1.3178618246456608e-05,1.0,0.4185504615306854,0.9122514128684998 +38000,255.2,1.3130001930492122e-05,1.3125267287250608e-05,1.0,0.40043964982032776,0.9174131155014038 +38500,258.5,1.3077999008636344e-05,1.3073899935989175e-05,1.0,0.3835586607456207,0.9218156933784485 +39000,261.9,1.301802813105061e-05,1.3012731869821437e-05,1.0,0.36757418513298035,0.9265219569206238 +39500,265.2,1.297053488875976e-05,1.2965125279151835e-05,1.0,0.353222519159317,0.9319872260093689 +40000,268.5,1.292445790442339e-05,1.291891294386005e-05,1.0,0.33960670232772827,0.9342644810676575 +40500,272.6,1.2879519546820882e-05,1.2875191714556422e-05,1.0,0.32652372121810913,0.9359344244003296 +41000,276.0,1.2832467243840983e-05,1.2827457794628572e-05,1.0,0.31399333477020264,0.9382116198539734 +41500,279.3,1.2785953478323358e-05,1.2780821634805761e-05,1.0,0.3019235134124756,0.9407925009727478 +42000,282.7,1.2743738231640614e-05,1.2739890735247172e-05,1.0,0.29125455021858215,0.9456505179405212 +42500,286.0,1.2708996363598795e-05,1.2705842891591601e-05,1.0,0.281749963760376,0.9482313394546509 +43000,289.3,1.2674248474694175e-05,1.266888102691155e-05,1.0,0.2728211581707001,0.9518749117851257 +43500,292.7,1.2633754455725947e-05,1.2630988749151584e-05,1.0,0.26427707076072693,0.9543039202690125 +44000,296.1,1.2601087498384665e-05,1.2597195564012509e-05,1.0,0.2568133473396301,0.9553666114807129 +44500,299.3,1.2565460015779627e-05,1.2561459698190447e-05,1.0,0.24942323565483093,0.9580993056297302 +45000,302.6,1.2532172105143055e-05,1.2528045772342011e-05,1.0,0.2420712262392044,0.9600728750228882 +45500,306.1,1.2500897205257634e-05,1.249830711458344e-05,1.0,0.2353001981973648,0.9605283141136169 +46000,309.6,1.2463173419170272e-05,1.2458809578674845e-05,1.0,0.22846443951129913,0.9629573225975037 +46500,313.9,1.242771498584401e-05,1.2424340638972353e-05,1.0,0.22185423970222473,0.9646273255348206 +47000,317.3,1.2400729983916608e-05,1.239903849636903e-05,1.0,0.2159910649061203,0.9667527079582214 +47500,320.6,1.2372585144644797e-05,1.2369721844152082e-05,1.0,0.2110200822353363,0.9681190252304077 +48000,323.9,1.2347326201847977e-05,1.2345516552159097e-05,1.0,0.20649971067905426,0.969485342502594 +48500,327.2,1.2315805898602443e-05,1.2313370461924933e-05,1.0,0.2013023942708969,0.9717625379562378 +49000,330.5,1.2286878809534522e-05,1.2283842806937173e-05,1.0,0.19598188996315002,0.9720661640167236 +49500,333.9,1.2262117056690445e-05,1.225967935170047e-05,1.0,0.19114825129508972,0.9725216627120972 +50000,337.3,1.2238888844576509e-05,1.2237248483870644e-05,1.0,0.18710634112358093,0.9737361669540405 +50500,340.7,1.2212706205165477e-05,1.2210424756631255e-05,1.0,0.18259887397289276,0.974798858165741 +51000,343.9,1.2190034747534977e-05,1.2186389540147502e-05,1.0,0.17840026319026947,0.9755579233169556 +51500,347.1,1.2163619782688821e-05,1.216024156747153e-05,1.0,0.17384734749794006,0.9763169884681702 +52000,350.4,1.2131721650361235e-05,1.2129151400586125e-05,1.0,0.16868765652179718,0.9781387448310852 +52500,353.7,1.2098654585546398e-05,1.2094807971152477e-05,1.0,0.16339045763015747,0.9792014360427856 +53000,356.9,1.2066093008266393e-05,1.2061774214089382e-05,1.0,0.1579543501138687,0.9799605011940002 +53500,360.3,1.2035772910453298e-05,1.203237388835987e-05,1.0,0.1527397483587265,0.9817823171615601 +54000,363.5,1.200304171404714e-05,1.1999001799267717e-05,1.0,0.1477532833814621,0.9834522604942322 +54500,366.8,1.1974612340891676e-05,1.1972304491791874e-05,1.0,0.14278531074523926,0.983755886554718 +55000,370.1,1.1941166290766782e-05,1.1939145224459935e-05,1.0,0.13791927695274353,0.9852740168571472 +55500,373.6,1.1912393634403961e-05,1.1910505236301105e-05,1.0,0.1336219608783722,0.9860330820083618 +56000,377.1,1.188475413922658e-05,1.1881357750098687e-05,1.0,0.12984175980091095,0.9866403341293335 +56500,380.5,1.1856228454709817e-05,1.1853941941808444e-05,1.0,0.12604373693466187,0.987551212310791 +57000,383.8,1.1832778393226541e-05,1.1828597052954137e-05,1.0,0.1224118173122406,0.9881585240364075 +57500,387.2,1.180624972765937e-05,1.1804011592175812e-05,1.0,0.11922425031661987,0.9889175891876221 +58000,390.5,1.1775455105054682e-05,1.1772286597988568e-05,1.0,0.11607355624437332,0.9892212152481079 +58500,393.7,1.1739370415847428e-05,1.173460805148352e-05,1.0,0.11254305392503738,0.9898284673690796 +59000,396.9,1.170538533566036e-05,1.170288487628568e-05,1.0,0.1092464029788971,0.991042971611023 +59500,400.2,1.1681301873866757e-05,1.1678975170070771e-05,1.0,0.10629145056009293,0.9911947846412659 +60000,403.4,1.1662692087114485e-05,1.1660176824079826e-05,1.0,0.10356290638446808,0.9905875325202942 +60500,406.9,1.1647464898037193e-05,1.164581499324413e-05,1.0,0.10162830352783203,0.9907393455505371 +61000,410.3,1.1634913571743794e-05,1.1632719179033302e-05,1.0,0.10018155723810196,0.9913465976715088 +61500,413.6,1.1618172301335108e-05,1.1617004929576069e-05,1.0,0.09871920943260193,0.9911947846412659 +62000,417.1,1.1597650758110267e-05,1.1594827810768038e-05,1.0,0.09701626747846603,0.9919538497924805 +62500,420.3,1.1578484378614636e-05,1.1576451470318716e-05,1.0,0.09508371353149414,0.9925611019134521 +63000,423.7,1.1558298038736052e-05,1.1554529010027181e-05,1.0,0.09282851219177246,0.992864727973938 +63500,427.1,1.153042664846608e-05,1.1527664355526213e-05,1.0,0.09043421596288681,0.9933201670646667 +64000,430.5,1.1505189152582416e-05,1.1503585483296774e-05,1.0,0.08836517482995987,0.9936237931251526 +64500,433.8,1.1479774445803864e-05,1.1477902262413409e-05,1.0,0.08617441356182098,0.9937756061553955 +65000,437.1,1.1448872040966542e-05,1.1445247764640953e-05,1.0,0.08389806002378464,0.9940792322158813 +65500,440.5,1.1419312424264079e-05,1.1416016604925971e-05,1.0,0.08160294592380524,0.9943828582763672 +66000,444.0,1.1384025953536032e-05,1.1380363503121771e-05,1.0,0.07924701273441315,0.994686484336853 +66500,447.3,1.1350874150443369e-05,1.1348638508934528e-05,1.0,0.07681338489055634,0.994838297367096 +67000,450.7,1.1324755197614766e-05,1.1324181286909152e-05,1.0,0.0744931548833847,0.994686484336853 +67500,453.9,1.1303257121373416e-05,1.1303525752737187e-05,1.0,0.07248852401971817,0.994838297367096 +68000,457.4,1.1281861410303359e-05,1.1279489626758732e-05,1.0,0.07074437290430069,0.9952937364578247 +68500,460.9,1.1264625262028459e-05,1.1262973202974536e-05,1.0,0.06933484226465225,0.9955973625183105 +69000,464.2,1.1252488052225332e-05,1.1250891475356184e-05,1.0,0.0680728331208229,0.9957492351531982 +69500,467.5,1.1238983242883825e-05,1.1239021660003345e-05,1.0,0.06695850938558578,0.9960528612136841 +70000,471.1,1.1228424032285728e-05,1.1227109098399524e-05,1.0,0.06596489995718002,0.9966601133346558 +70500,474.6,1.1219761843041625e-05,1.1218956387892831e-05,1.0,0.064995676279068,0.9966601133346558 +71000,477.9,1.1211473123511472e-05,1.121084551414242e-05,1.0,0.06424961239099503,0.9969637393951416 +71500,481.2,1.1204365186169475e-05,1.120311480917735e-05,1.0,0.06364277750253677,0.9969637393951416 +72000,485.4,1.1197521271466963e-05,1.1196567356819287e-05,1.0,0.06307783722877502,0.9971155524253845 +72500,493.9,1.1190479195202493e-05,1.118871023209067e-05,1.0,0.06256924569606781,0.9972673654556274 +73000,503.6,1.1180218882442629e-05,1.1177811757079326e-05,1.0,0.0618283711373806,0.9974191784858704 +73500,512.6,1.1169746843461213e-05,1.1170545803906862e-05,1.0,0.06105160713195801,0.9974191784858704 +74000,521.8,1.1164261574372958e-05,1.1164800525875762e-05,1.0,0.060486484318971634,0.9975709915161133 +74500,531.5,1.1158813326701446e-05,1.1158126653754152e-05,1.0,0.059983134269714355,0.9974191784858704 +75000,541.0,1.1153507734502638e-05,1.1153183550050016e-05,1.0,0.05959874391555786,0.9974191784858704 +75500,551.6,1.1149649712318145e-05,1.1149001693411265e-05,1.0,0.05921982601284981,0.9974191784858704 +76000,562.0,1.1144938245929922e-05,1.11438903331873e-05,1.0,0.0588856041431427,0.9972673654556274 +76500,577.6,1.1139557947381152e-05,1.1137342880829237e-05,1.0,0.05855409428477287,0.9972673654556274 +77000,580.3,1.1133339266087746e-05,1.1132568943139631e-05,1.0,0.05815710872411728,0.9975709915161133 +77500,583.6,1.1124852075142854e-05,1.1124416232632939e-05,1.0,0.057652443647384644,0.9974191784858704 +78000,587.7,1.1117635994639391e-05,1.1117362191725988e-05,1.0,0.05716180428862572,0.9972673654556274 +78500,616.5,1.1111179483908089e-05,1.1109335900982842e-05,1.0,0.0567186065018177,0.9972673654556274 +79000,625.1,1.1102705329414401e-05,1.1102112694061361e-05,1.0,0.05618665739893913,0.9975709915161133 +79500,633.2,1.1097491991376091e-05,1.109742333937902e-05,1.0,0.05574924498796463,0.9977228045463562 +80000,640.5,1.1092614205298478e-05,1.1092903150711209e-05,1.0,0.05542309954762459,0.9977228045463562 +80500,649.5,1.1087374591029576e-05,1.1086862286902033e-05,1.0,0.055031199008226395,0.9977228045463562 +81000,658.0,1.1080583218384818e-05,1.10812025013729e-05,1.0,0.05456356704235077,0.9977228045463562 +81500,666.6,1.1076046308433798e-05,1.1075077964051161e-05,1.0,0.0541246198117733,0.9978746175765991 +82000,675.5,1.107160812592415e-05,1.1070431355619803e-05,1.0,0.05378301814198494,0.9978746175765991 +82500,684.0,1.1062689839546431e-05,1.1062489647883922e-05,1.0,0.0533733032643795,0.9978746175765991 +83000,693.0,1.1057852621334581e-05,1.1056533367082011e-05,1.0,0.053004190325737,0.9978746175765991 +83500,701.7,1.1052183983187069e-05,1.1050661669287365e-05,1.0,0.052656736224889755,0.9978746175765991 +84000,710.1,1.104519020879908e-05,1.1043607628380414e-05,1.0,0.052259039133787155,0.9978746175765991 +84500,718.7,1.103723112281672e-05,1.1036214345949702e-05,1.0,0.051739875227212906,0.9978746175765991 +85000,728.0,1.1030532311170267e-05,1.1030258065147791e-05,1.0,0.051281534135341644,0.9975709915161133 +85500,736.4,1.1024875674196894e-05,1.1023754268535413e-05,1.0,0.050888217985630035,0.9975709915161133 +86000,745.4,1.1018983344503354e-05,1.1018178156518843e-05,1.0,0.05048990249633789,0.9975709915161133 +86500,754.0,1.1013732218096434e-05,1.1014587471436244e-05,1.0,0.050210028886795044,0.9975709915161133 +87000,762.2,1.1010106923819778e-05,1.1009265108441468e-05,1.0,0.05004545673727989,0.9975709915161133 +87500,771.6,1.1006910297217493e-05,1.1007786270056386e-05,1.0,0.0498625822365284,0.9975709915161133 +88000,780.4,1.1003370915746457e-05,1.1002844075846951e-05,1.0,0.0496119000017643,0.9977228045463562 +88500,789.0,1.0997567705464237e-05,1.0997859135386534e-05,1.0,0.04915373772382736,0.9977228045463562 +89000,797.6,1.099372041875257e-05,1.0994226613547653e-05,1.0,0.04885495454072952,0.9975709915161133 +89500,807.0,1.0986249246887235e-05,1.0984425898641348e-05,1.0,0.0485122986137867,0.9977228045463562 +90000,815.4,1.0979277834214439e-05,1.097834319807589e-05,1.0,0.04809647053480148,0.9977228045463562 +90500,823.6,1.0971773646889934e-05,1.0971500159939751e-05,1.0,0.047711215913295746,0.9977228045463562 +91000,832.0,1.0961530153338216e-05,1.0961023690470029e-05,1.0,0.04724377021193504,0.9977228045463562 +91500,840.5,1.094805908205624e-05,1.0946323527605273e-05,1.0,0.04658949747681618,0.9977228045463562 +92000,847.8,1.0932535753667992e-05,1.093111677619163e-05,1.0,0.04585442692041397,0.9977228045463562 +92500,855.5,1.0918006959826827e-05,1.0917303370661102e-05,1.0,0.045034222304821014,0.9977228045463562 +93000,862.5,1.0909145765595164e-05,1.0909404409176204e-05,1.0,0.0442693866789341,0.998026430606842 +93500,869.8,1.0903231293263224e-05,1.0902349458774552e-05,1.0,0.043652959167957306,0.998178243637085 +94000,877.0,1.0897523927777034e-05,1.089799843612127e-05,1.0,0.04316956177353859,0.998178243637085 +94500,884.2,1.0892123861188828e-05,1.0890521480177995e-05,1.0,0.04277632758021355,0.998178243637085 +95000,891.8,1.0885325452452851e-05,1.0885622032219544e-05,1.0,0.042396001517772675,0.998178243637085 +95500,899.3,1.088014941606053e-05,1.0879876754188444e-05,1.0,0.04206050932407379,0.998178243637085 +96000,907.0,1.0875075868591037e-05,1.0874976396735292e-05,1.0,0.04177048057317734,0.998026430606842 +96500,915.4,1.0871107710648226e-05,1.0870709957089275e-05,1.0,0.041511837393045425,0.998026430606842 +97000,922.6,1.0865401381776867e-05,1.0863824172702152e-05,1.0,0.041237037628889084,0.998026430606842 +97500,930.7,1.0857493950398197e-05,1.0857150300580543e-05,1.0,0.04090000316500664,0.998178243637085 +98000,938.2,1.0849756041428075e-05,1.0848913007066585e-05,1.0,0.04053320735692978,0.998178243637085 +98500,945.9,1.0845243003976993e-05,1.0843294148799032e-05,1.0,0.040172457695007324,0.998178243637085 +99000,953.1,1.0841732445190987e-05,1.0841308721865062e-05,1.0,0.039871662855148315,0.9984818696975708 +99500,960.6,1.0839493361229617e-05,1.0840843970072456e-05,1.0,0.0396726094186306,0.9986336827278137 +100000,967.8,1.0837271444910916e-05,1.0836577530426439e-05,1.0,0.03953360393643379,0.9986336827278137 +100500,975.3,1.0833711803972428e-05,1.0833747182914522e-05,1.0,0.0394064299762249,0.9986336827278137 +101000,982.9,1.0831029360716085e-05,1.0831635336217005e-05,1.0,0.03921232372522354,0.9986336827278137 +101500,990.6,1.0827672745701953e-05,1.0827115147549193e-05,1.0,0.03899068757891655,0.9986336827278137 +102000,998.2,1.0824751101787573e-05,1.0824031960510183e-05,1.0,0.038814857602119446,0.9986336827278137 +102500,1005.6,1.0822361075296253e-05,1.0820990610227454e-05,1.0,0.03864014893770218,0.9986336827278137 +103000,1013.3,1.0818916324048473e-05,1.0817104339366779e-05,1.0,0.03846496716141701,0.9986336827278137 +103500,1020.8,1.0814684871048948e-05,1.0813851986313239e-05,1.0,0.03830697014927864,0.9986336827278137 +104000,1028.1,1.0811621760431303e-05,1.0810978892550338e-05,1.0,0.03813571855425835,0.9987854957580566 +104500,1035.2,1.080697189160913e-05,1.0805741112562828e-05,1.0,0.037935398519039154,0.9987854957580566 +105000,1042.4,1.0803337914531664e-05,1.0802277756738476e-05,1.0,0.037757523357868195,0.9987854957580566 +105500,1049.8,1.0798730477988753e-05,1.0798222319863271e-05,1.0,0.037516649812459946,0.9987854957580566 +106000,1057.2,1.0791865369478404e-05,1.0790745363919996e-05,1.0,0.03720465674996376,0.9987854957580566 +106500,1064.6,1.0782625938631125e-05,1.0782086064864416e-05,1.0,0.03688199073076248,0.9987854957580566 +107000,1070.3,1.0775003230306904e-05,1.0775622286018915e-05,1.0,0.03656158223748207,0.9989373087882996 +107500,1074.5,1.0766266028582596e-05,1.076455464499304e-05,1.0,0.036213673651218414,0.9989373087882996 +108000,1078.1,1.0758789103250588e-05,1.0759358701761812e-05,1.0,0.035855527967214584,0.9987854957580566 +108500,1080.1,1.0749139645201547e-05,1.074761530617252e-05,1.0,0.03551409766077995,0.9989373087882996 +109000,1081.9,1.0743121413237054e-05,1.0742588528955821e-05,1.0,0.03521560877561569,0.9989373087882996 +109500,1083.8,1.0739237030193946e-05,1.0739336175902281e-05,1.0,0.03499278053641319,0.9987854957580566 +110000,1086.3,1.0734223277755889e-05,1.0732788723544218e-05,1.0,0.03481665253639221,0.9987854957580566 +110500,1089.3,1.0731251867628348e-05,1.0731731890700758e-05,1.0,0.03467356413602829,0.9989373087882996 +111000,1092.5,1.0727754042752228e-05,1.0727169865276664e-05,1.0,0.03454572334885597,0.9989373087882996 +111500,1095.7,1.0721505066849595e-05,1.0722734259616118e-05,1.0,0.034331291913986206,0.9990891218185425 +112000,1098.8,1.0717915334780505e-05,1.0717242730606813e-05,1.0,0.03416205570101738,0.9989373087882996 +112500,1102.0,1.0713935909356891e-05,1.0711624781833962e-05,1.0,0.03405124694108963,0.9989373087882996 +113000,1105.2,1.0712362420669698e-05,1.0712300536397379e-05,1.0,0.03398488089442253,0.9989373087882996 +113500,1108.4,1.0711148775720657e-05,1.070959751814371e-05,1.0,0.033941421657800674,0.9989373087882996 +114000,1111.7,1.0709186673366828e-05,1.0708118679758627e-05,1.0,0.033877503126859665,0.9989373087882996 +114500,1114.9,1.0706981031609761e-05,1.0708921763580292e-05,1.0,0.03382905572652817,0.9989373087882996 +115000,1118.2,1.070422631347184e-05,1.0703218322305474e-05,1.0,0.03374850004911423,0.9989373087882996 +115500,1121.5,1.070100060302234e-05,1.0701486644393299e-05,1.0,0.03364145755767822,0.9989373087882996 +116000,1124.7,1.0698082861379332e-05,1.069827612809604e-05,1.0,0.033538348972797394,0.9989373087882996 +116500,1128.0,1.0696340230124112e-05,1.0695192031562328e-05,1.0,0.03342780843377113,0.9989373087882996 +117000,1131.3,1.0693338182647693e-05,1.0692446267057676e-05,1.0,0.03327002376317978,0.9990891218185425 +117500,1134.5,1.0691905472267533e-05,1.0691516763472464e-05,1.0,0.03319060802459717,0.9990891218185425 +118000,1137.7,1.0689363246781157e-05,1.068872916221153e-05,1.0,0.03311096876859665,0.9992409348487854 +118500,1141.0,1.0686170154698104e-05,1.068746223609196e-05,1.0,0.03301287069916725,0.9992409348487854 +119000,1144.1,1.0683023520085875e-05,1.0683194886951242e-05,1.0,0.03285462036728859,0.9992409348487854 +119500,1147.3,1.06813285412255e-05,1.068099845724646e-05,1.0,0.03274238109588623,0.9992409348487854 +120000,1150.5,1.0679019396173902e-05,1.0679267688828986e-05,1.0,0.03263184800744057,0.9992409348487854 +120500,1153.9,1.0675585356823509e-05,1.0674451914383098e-05,1.0,0.03249378129839897,0.9992409348487854 +121000,1157.2,1.0674486965001917e-05,1.0674578334146645e-05,1.0,0.032427575439214706,0.9992409348487854 +121500,1160.4,1.0672883824948282e-05,1.0672380994947162e-05,1.0,0.03238927200436592,0.9990891218185425 +122000,1163.6,1.0668431793591715e-05,1.066925597115187e-05,1.0,0.0323454774916172,0.9990891218185425 +122500,1167.0,1.066504555904676e-05,1.06651159512694e-05,1.0,0.03227341175079346,0.9990891218185425 +123000,1170.1,1.0662818263859572e-05,1.0662454769772012e-05,1.0,0.032172977924346924,0.9990891218185425 +123500,1173.4,1.0659753887775817e-05,1.0660047337296419e-05,1.0,0.032073233276605606,0.9990891218185425 +124000,1176.6,1.0654781459495843e-05,1.0655062396836001e-05,1.0,0.03195827081799507,0.9990891218185425 +124500,1179.8,1.0650639110689057e-05,1.0649107025528792e-05,1.0,0.031796813011169434,0.9990891218185425 +125000,1182.9,1.0646117316086278e-05,1.0645389011187945e-05,1.0,0.03161577507853508,0.9992409348487854 +125500,1186.2,1.0641603612951746e-05,1.0639052561600693e-05,1.0,0.0313696525990963,0.9992409348487854 +126000,1189.4,1.0636008583871738e-05,1.0635081707732752e-05,1.0,0.031179921701550484,0.9993927478790283 +126500,1192.6,1.0630096448496228e-05,1.062980209098896e-05,1.0,0.03106681816279888,0.9993927478790283 +127000,1195.8,1.062676862136805e-05,1.0626844414218795e-05,1.0,0.0309634767472744,0.9993927478790283 +127500,1199.1,1.0624832959712008e-05,1.0623972229950596e-05,1.0,0.030881792306900024,0.9993927478790283 +128000,1202.3,1.0620397791967067e-05,1.0618607120704837e-05,1.0,0.03079885244369507,0.9993927478790283 +128500,1205.5,1.061729855284402e-05,1.0618565283948556e-05,1.0,0.030702883377671242,0.9993927478790283 +129000,1208.8,1.061657123006698e-05,1.0614974598865956e-05,1.0,0.03062976710498333,0.9993927478790283 +129500,1212.1,1.0612195345619374e-05,1.0611637662805151e-05,1.0,0.03052595444023609,0.9993927478790283 +130000,1215.4,1.0609529853589204e-05,1.0609483069856651e-05,1.0,0.0304571520537138,0.9992409348487854 +130500,1218.6,1.0606015755661315e-05,1.0606737305351999e-05,1.0,0.030361931771039963,0.9992409348487854 +131000,1221.7,1.0606491424801473e-05,1.0606483556330204e-05,1.0,0.030278926715254784,0.9992409348487854 +131500,1224.9,1.0605335581482854e-05,1.0604540875647217e-05,1.0,0.03022189997136593,0.9992409348487854 +132000,1228.2,1.0604671922078012e-05,1.0603991540847346e-05,1.0,0.03017767332494259,0.9992409348487854 +132500,1231.4,1.06040937381563e-05,1.0603146620269399e-05,1.0,0.030144046992063522,0.9992409348487854 +133000,1234.6,1.060366544328225e-05,1.0604160706861876e-05,1.0,0.030122464522719383,0.9992409348487854 +133500,1237.9,1.0602849923133227e-05,1.0603019291011151e-05,1.0,0.030076565220952034,0.9992409348487854 +134000,1241.2,1.0601363584214313e-05,1.0601667781884316e-05,1.0,0.029994146898388863,0.9992409348487854 +134500,1244.5,1.0600982044549835e-05,1.0602343536447734e-05,1.0,0.029932940378785133,0.9992409348487854 +135000,1247.7,1.0600191179918845e-05,1.0597824257274624e-05,1.0,0.02988719567656517,0.9992409348487854 +135500,1251.0,1.0597951528277038e-05,1.0597866094030906e-05,1.0,0.02980935014784336,0.9992409348487854 +136000,1254.2,1.0596743552031447e-05,1.0596387255645823e-05,1.0,0.029731402173638344,0.9992409348487854 +136500,1257.5,1.0595488705366859e-05,1.0595458661555313e-05,1.0,0.02967192605137825,0.9992409348487854 +137000,1260.7,1.0594223565085522e-05,1.0593599654384889e-05,1.0,0.02963242493569851,0.9992409348487854 +137500,1263.9,1.059177190712618e-05,1.0590642887109425e-05,1.0,0.029597163200378418,0.9992409348487854 +138000,1267.2,1.0590356335930374e-05,1.058912221196806e-05,1.0,0.029559224843978882,0.9992409348487854 +138500,1270.4,1.0587744289796965e-05,1.058696761901956e-05,1.0,0.029507357627153397,0.9992409348487854 +139000,1273.6,1.058646729426908e-05,1.0586207281448878e-05,1.0,0.0294193085283041,0.9992409348487854 +139500,1276.8,1.0584847178881442e-05,1.058464386005653e-05,1.0,0.029308466240763664,0.9993927478790283 +140000,1280.1,1.0583629311341475e-05,1.0581771675788332e-05,1.0,0.029186295345425606,0.9993927478790283 +140500,1283.2,1.058077883448772e-05,1.0579997251625173e-05,1.0,0.029064910486340523,0.9993927478790283 +141000,1286.5,1.0576945110721076e-05,1.0576406566542573e-05,1.0,0.028943080455064774,0.9992409348487854 +141500,1289.7,1.0573471975055884e-05,1.057399913406698e-05,1.0,0.02882915921509266,0.9992409348487854 +142000,1293.0,1.0570357014747205e-05,1.0571380698820576e-05,1.0,0.028698818758130074,0.9992409348487854 +142500,1296.4,1.0567943443898623e-05,1.056804285326507e-05,1.0,0.028565185144543648,0.9992409348487854 +143000,1299.6,1.05631078969634e-05,1.0562423994997516e-05,1.0,0.028391661122441292,0.9992409348487854 +143500,1302.9,1.0559297119132315e-05,1.0559975635260344e-05,1.0,0.02818659506738186,0.9992409348487854 +144000,1306.1,1.055431468083817e-05,1.055308985087322e-05,1.0,0.027969758957624435,0.9992409348487854 +144500,1309.4,1.0547551322361942e-05,1.0546457815507893e-05,1.0,0.027690771967172623,0.9992409348487854 +145000,1312.5,1.0541990615630181e-05,1.0542022209847346e-05,1.0,0.027445388957858086,0.9992409348487854 +145500,1315.8,1.0539071305956869e-05,1.0539191862335429e-05,1.0,0.02725128084421158,0.9992409348487854 +146000,1318.9,1.0537192534283529e-05,1.053598134603817e-05,1.0,0.02710343897342682,0.9992409348487854 +146500,1322.1,1.05359919113105e-05,1.0535559340496548e-05,1.0,0.026989033445715904,0.9992409348487854 +147000,1325.4,1.0534192739224811e-05,1.0534588000155054e-05,1.0,0.02689530700445175,0.9993927478790283 +147500,1328.5,1.0532110266328954e-05,1.0530828149057925e-05,1.0,0.02680419757962227,0.9993927478790283 +148000,1331.8,1.053167911049003e-05,1.0532095075177494e-05,1.0,0.02674145996570587,0.9993927478790283 +148500,1335.0,1.0530216339230936e-05,1.052892730513122e-05,1.0,0.026681967079639435,0.9993927478790283 +149000,1338.2,1.0529304205187704e-05,1.0529392056923825e-05,1.0,0.026634132489562035,0.9995445609092712 +149500,1341.4,1.0529210275775735e-05,1.052892730513122e-05,1.0,0.026597317308187485,0.9995445609092712 +150000,1344.7,1.0528727230624236e-05,1.0529180144658312e-05,1.0,0.026556722819805145,0.9995445609092712 +150500,1347.9,1.0528233085973956e-05,1.052913830790203e-05,1.0,0.026527438312768936,0.9995445609092712 +151000,1351.3,1.0527971987411505e-05,1.0526688129175454e-05,1.0,0.02651016227900982,0.9995445609092712 +151500,1354.4,1.052749136884136e-05,1.0525674042582978e-05,1.0,0.026484861969947815,0.9996963739395142 +152000,1357.6,1.0526806238165816e-05,1.0527406629989855e-05,1.0,0.02646007388830185,0.9996963739395142 +152500,1360.8,1.0526437872113302e-05,1.0525463039812166e-05,1.0,0.02642936073243618,0.9996963739395142 +153000,1364.1,1.0524882590815742e-05,1.0524702702241484e-05,1.0,0.026365922763943672,0.9996963739395142 +153500,1367.3,1.0524357543173824e-05,1.0524871868256014e-05,1.0,0.02633160538971424,0.9995445609092712 +154000,1370.5,1.0524309958497053e-05,1.0524913705012295e-05,1.0,0.026314204558730125,0.9995445609092712 +154500,1373.8,1.0524219618094961e-05,1.0524660865485203e-05,1.0,0.026303080841898918,0.9995445609092712 +155000,1377.0,1.052341031139077e-05,1.0521999683987815e-05,1.0,0.02629244513809681,0.9995445609092712 +155500,1380.1,1.0522658960074034e-05,1.0521703188715037e-05,1.0,0.026286114007234573,0.9995445609092712 +156000,1383.4,1.0522513952691207e-05,1.0522886441322044e-05,1.0,0.0262836292386055,0.9995445609092712 +156500,1386.6,1.052215735745134e-05,1.0523773198656272e-05,1.0,0.026270413771271706,0.9996963739395142 +157000,1389.8,1.0521634190164823e-05,1.052030984283192e-05,1.0,0.02625858224928379,0.9996963739395142 +157500,1393.1,1.0521245034033675e-05,1.0522506272536702e-05,1.0,0.0262541975826025,0.9996963739395142 +158000,1396.6,1.052161427702981e-05,1.0521872354729567e-05,1.0,0.026248693466186523,0.9996963739395142 +158500,1400.2,1.0519772239701538e-05,1.0519464922253974e-05,1.0,0.02621995471417904,0.9996963739395142 +159000,1404.5,1.0518981541622045e-05,1.0517521332076285e-05,1.0,0.02619965746998787,0.9996963739395142 +159500,1408.0,1.0518584829753708e-05,1.0518620001676027e-05,1.0,0.02618585154414177,0.9996963739395142 +160000,1411.5,1.0518334181227094e-05,1.0519084753468633e-05,1.0,0.026173928752541542,0.9996963739395142 +160500,1415.1,1.0517535262927412e-05,1.0516760994505603e-05,1.0,0.026150008663535118,0.9996963739395142 +161000,1418.9,1.0516899699322424e-05,1.051633898896398e-05,1.0,0.02611570432782173,0.9996963739395142 +161500,1423.3,1.0516167735605962e-05,1.0516254405956715e-05,1.0,0.026064513251185417,0.9995445609092712 +162000,1426.8,1.051575985653948e-05,1.0515705071156844e-05,1.0,0.026041561737656593,0.9995445609092712 +162500,1430.4,1.0515301304612545e-05,1.051562048814958e-05,1.0,0.026023894548416138,0.9995445609092712 +163000,1434.5,1.0514649930010753e-05,1.0516803740756586e-05,1.0,0.026019616052508354,0.9995445609092712 +163500,1438.1,1.0514115581417372e-05,1.051418439601548e-05,1.0,0.026006558910012245,0.9995445609092712 +164000,1442.1,1.0514230182981259e-05,1.0516043403185904e-05,1.0,0.025996899232268333,0.9995445609092712 +164500,1445.5,1.0513681838899603e-05,1.0512705557630397e-05,1.0,0.025987591594457626,0.9996963739395142 +165000,1449.0,1.051357953617078e-05,1.0511649634281639e-05,1.0,0.02598530240356922,0.9995445609092712 +165500,1452.8,1.0512669144547047e-05,1.0512790140637662e-05,1.0,0.025984421372413635,0.9995445609092712 +166000,1456.3,1.0512658061562192e-05,1.0511227628740016e-05,1.0,0.025994541123509407,0.9995445609092712 +166500,1458.9,1.0512041129789658e-05,1.051093204296194e-05,1.0,0.025995323434472084,0.9993927478790283 +167000,1461.7,1.051011352952268e-05,1.0509917956369463e-05,1.0,0.02597922645509243,0.9993927478790283 +167500,1464.9,1.0508251335422206e-05,1.0506538274057675e-05,1.0,0.02596254087984562,0.9993927478790283 +168000,1468.2,1.0507437246070622e-05,1.0505947102501523e-05,1.0,0.025950372219085693,0.9993927478790283 +168500,1471.4,1.0506530878555484e-05,1.0506538274057675e-05,1.0,0.025940844789147377,0.9993927478790283 +169000,1474.7,1.0505833378384419e-05,1.0505271347938105e-05,1.0,0.025931360200047493,0.9993927478790283 +169500,1477.9,1.0505491811124983e-05,1.050607352226507e-05,1.0,0.02590937353670597,0.9993927478790283 +170000,1481.0,1.050469984631156e-05,1.050446826411644e-05,1.0,0.02587295137345791,0.9992409348487854 +170500,1484.2,1.0503066735230891e-05,1.0505102181923576e-05,1.0,0.025825869292020798,0.9992409348487854 +171000,1487.5,1.0501246472193078e-05,1.0501637916604523e-05,1.0,0.025779251009225845,0.9992409348487854 +171500,1490.8,1.0500376472407583e-05,1.0501511496840976e-05,1.0,0.025722932070493698,0.9992409348487854 +172000,1493.9,1.0498192218480414e-05,1.0497751645743847e-05,1.0,0.02563617378473282,0.9993927478790283 +172500,1497.2,1.0497184748039346e-05,1.049707589118043e-05,1.0,0.025578578934073448,0.9993927478790283 +173000,1500.4,1.0496397847815836e-05,1.0494710295461118e-05,1.0,0.025558724999427795,0.9993927478790283 +173500,1503.7,1.0496176234944028e-05,1.0495344213268254e-05,1.0,0.025545621290802956,0.9993927478790283 +174000,1507.1,1.0495985467493294e-05,1.0496695722395089e-05,1.0,0.0255344919860363,0.9993927478790283 +174500,1510.3,1.0495682883258208e-05,1.049563979904633e-05,1.0,0.025525756180286407,0.9993927478790283 +175000,1513.4,1.0493235355607256e-05,1.049238744599279e-05,1.0,0.025503475219011307,0.9993927478790283 +175500,1516.6,1.0492676121333383e-05,1.0491204193385784e-05,1.0,0.025491837412118912,0.9993927478790283 +176000,1519.8,1.049265339541664e-05,1.0493274203327019e-05,1.0,0.025492670014500618,0.9993927478790283 +176500,1523.1,1.049276760515633e-05,1.04933160400833e-05,1.0,0.025492755696177483,0.9993927478790283 +177000,1526.3,1.0492357780011105e-05,1.0491077773622237e-05,1.0,0.025493083521723747,0.9993927478790283 +177500,1529.5,1.0492953985872344e-05,1.0492851288290694e-05,1.0,0.025495562702417374,0.9993927478790283 +178000,1532.7,1.049277013454498e-05,1.0494160960661247e-05,1.0,0.025496678426861763,0.9993927478790283 +178500,1535.9,1.04925978175704e-05,1.0492809451534413e-05,1.0,0.025493621826171875,0.9993927478790283 +179000,1539.2,1.0492032817559063e-05,1.0490824024600442e-05,1.0,0.02547314763069153,0.9993927478790283 +179500,1542.4,1.0491953937000481e-05,1.0491542525414843e-05,1.0,0.025423254817724228,0.9993927478790283 +180000,1545.6,1.0491812084295592e-05,1.0491795364941936e-05,1.0,0.025402948260307312,0.9993927478790283 +180500,1548.8,1.0491162814754888e-05,1.0490865861356724e-05,1.0,0.025389527902007103,0.9993927478790283 +181000,1552.1,1.049157066454386e-05,1.0491627108422108e-05,1.0,0.02538357302546501,0.9993927478790283 +181500,1555.3,1.0490963213019286e-05,1.0491710781934671e-05,1.0,0.025368552654981613,0.9993927478790283 +182000,1558.5,1.0491132126081308e-05,1.048989452101523e-05,1.0,0.025360409170389175,0.9993927478790283 +182500,1561.7,1.0490680813855784e-05,1.0488584848644678e-05,1.0,0.025355055928230286,0.9993927478790283 +183000,1565.0,1.0490558017032715e-05,1.0491753528185654e-05,1.0,0.025350114330649376,0.9993927478790283 +183500,1568.2,1.0490330692456335e-05,1.0490570275578648e-05,1.0,0.02534395270049572,0.9993927478790283 +184000,1571.5,1.049003147345964e-05,1.0490570275578648e-05,1.0,0.02534109354019165,0.9993927478790283 +184500,1574.6,1.0490430073278744e-05,1.0490950444363989e-05,1.0,0.025338536128401756,0.9993927478790283 +185000,1577.9,1.0490362101523939e-05,1.0490359272807837e-05,1.0,0.02533738687634468,0.9993927478790283 +185500,1581.1,1.0490243262038711e-05,1.0490274689800572e-05,1.0,0.02533651702105999,0.9993927478790283 +186000,1584.3,1.0490091229341872e-05,1.0490865861356724e-05,1.0,0.02533414028584957,0.9993927478790283 +186500,1587.5,1.0490058027363218e-05,1.0491204193385784e-05,1.0,0.025323787704110146,0.9993927478790283 +187000,1590.7,1.0490000816117554e-05,1.0491415196156595e-05,1.0,0.025320498272776604,0.9993927478790283 +187500,1594.0,1.0489500758878671e-05,1.0490993190614972e-05,1.0,0.0253146905452013,0.9993927478790283 +188000,1597.2,1.0488949862423457e-05,1.0491288776393048e-05,1.0,0.025314142927527428,0.9993927478790283 +188500,1600.5,1.0488796854213956e-05,1.0488204679859336e-05,1.0,0.025317849591374397,0.9993927478790283 +189000,1603.7,1.048947493620123e-05,1.048752892529592e-05,1.0,0.02531658113002777,0.9993927478790283 +189500,1606.9,1.048919597260275e-05,1.0489092346688267e-05,1.0,0.02531319111585617,0.9993927478790283 +190000,1610.2,1.048928505881918e-05,1.0489937267266214e-05,1.0,0.025309352204203606,0.9993927478790283 +190500,1613.4,1.0489426093829839e-05,1.0490613021829631e-05,1.0,0.02530819922685623,0.9993927478790283 +191000,1616.5,1.048924645599631e-05,1.0490317436051555e-05,1.0,0.02530236542224884,0.9993927478790283 +191500,1619.8,1.0489393845771272e-05,1.0489092346688267e-05,1.0,0.02529831789433956,0.9993927478790283 +192000,1623.0,1.0489361581169485e-05,1.049006368702976e-05,1.0,0.0252953190356493,0.9993927478790283 +192500,1626.2,1.0488723866988074e-05,1.048841659212485e-05,1.0,0.02528694085776806,0.9993927478790283 +193000,1629.3,1.0488952512647705e-05,1.0489303349459078e-05,1.0,0.025282055139541626,0.9993927478790283 +193500,1632.5,1.048890085618943e-05,1.0489768101251684e-05,1.0,0.02528234012424946,0.9993927478790283 +194000,1635.7,1.0489064157082185e-05,1.0489557098480873e-05,1.0,0.025287257507443428,0.9993927478790283 +194500,1638.9,1.048896981902578e-05,1.0489768101251684e-05,1.0,0.02529192343354225,0.9993927478790283 +195000,1642.1,1.0488886098262445e-05,1.0488458428881131e-05,1.0,0.025294510647654533,0.9993927478790283 +195500,1645.3,1.0489064609203183e-05,1.0490105523786042e-05,1.0,0.02529408223927021,0.9993927478790283 +196000,1648.6,1.0488881072258584e-05,1.0488923180673737e-05,1.0,0.02529492974281311,0.9993927478790283 +196500,1651.8,1.0488689095603348e-05,1.0488923180673737e-05,1.0,0.025291182100772858,0.9993927478790283 +197000,1655.1,1.0488947205610039e-05,1.0487064173503313e-05,1.0,0.02528270334005356,0.9993927478790283 +197500,1658.2,1.0488805526998417e-05,1.0487951840332244e-05,1.0,0.02527705207467079,0.9993927478790283 +198000,1661.5,1.0488841052812878e-05,1.048773992806673e-05,1.0,0.025268567726016045,0.9993927478790283 +198500,1664.6,1.0489090483286063e-05,1.0490528438822366e-05,1.0,0.02526276372373104,0.9993927478790283 +199000,1667.8,1.0489216317504623e-05,1.0488627594895661e-05,1.0,0.025259478017687798,0.9993927478790283 +199500,1671.0,1.0488616678512504e-05,1.0488965017430019e-05,1.0,0.025257309898734093,0.9993927478790283 +200000,1674.2,1.0488929197375805e-05,1.0490570275578648e-05,1.0,0.025256890803575516,0.9993927478790283 diff --git a/modular_mlp/step_99500.pt b/modular_mlp/step_99500.pt new file mode 100644 index 0000000000000000000000000000000000000000..5800ad40049f618e4203eac6454a91bb593ba596 --- /dev/null +++ b/modular_mlp/step_99500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c90049be8d05659b6291943f28fe2fd94606434257b4e44538f281ae9e5ce1 +size 305019 diff --git a/subtraction/blankspace_fixed_s0/config.json b/subtraction/blankspace_fixed_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..07565dbb0ac9591befd8d53c9883bdf3fffe7788 --- /dev/null +++ b/subtraction/blankspace_fixed_s0/config.json @@ -0,0 +1,57 @@ +{ + "name": "subtraction blankspace fixed seed 0", + "task": { + "op": "sub", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/subtraction/blankspace_fixed_s0/metrics.csv b/subtraction/blankspace_fixed_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..670a5e4ff2f4c3adc6c6d889374bfe8dd7ba679f --- /dev/null +++ b/subtraction/blankspace_fixed_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,21.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,200.3,0.0003603064639663399,0.94140625,0.77734375,0.578125,0.484375,0.296875,0.19140625,0.06640625,0.015625,0.0,0.0,0.0 +5000,378.8,9.080627245126492e-05,0.9765625,0.97265625,0.8203125,0.640625,0.33984375,0.15625,0.0234375,0.00390625,0.0,0.0,0.0 +7500,557.4,4.091564030081449e-05,0.78125,0.73046875,0.67578125,0.546875,0.375,0.21875,0.04296875,0.01171875,0.00390625,0.0,0.0 +10000,735.7,1.47276718870669e-05,1.0,0.98828125,0.8828125,0.7421875,0.5390625,0.3125,0.046875,0.0,0.0,0.0,0.0 +12500,915.5,2.4964143413053708e-08,0.99609375,0.75390625,0.69140625,0.50390625,0.48828125,0.34765625,0.1015625,0.03125,0.0,0.0,0.0 +15000,1095.1,3.980229509976169e-09,1.0,1.0,1.0,0.9609375,0.8125,0.53125,0.109375,0.0234375,0.0,0.0,0.0 +17500,1273.6,6.409142760074891e-06,0.9921875,0.94140625,0.83203125,0.69140625,0.5546875,0.30859375,0.17578125,0.0625,0.015625,0.0,0.0 +20000,1452.0,2.4171010926480934e-09,1.0,1.0,0.98828125,0.95703125,0.859375,0.5859375,0.16015625,0.01171875,0.0,0.0,0.0 +22500,1631.8,4.011030818117412e-09,1.0,1.0,0.98828125,0.984375,0.8359375,0.55078125,0.1171875,0.0,0.0,0.0,0.0 +25000,1810.2,5.609703497685838e-09,1.0,1.0,0.98046875,0.9453125,0.80859375,0.5078125,0.0703125,0.00390625,0.0,0.0,0.0 +27500,1988.7,1.2406113090975408e-05,1.0,1.0,1.0,0.97265625,0.80859375,0.49609375,0.1484375,0.02734375,0.00390625,0.0,0.0 +30000,2167.0,4.40324214555801e-07,1.0,1.0,1.0,0.9765625,0.9140625,0.7109375,0.1171875,0.01171875,0.00390625,0.0,0.0 +32500,2346.7,5.969095633141783e-09,1.0,1.0,1.0,1.0,0.984375,0.90625,0.3203125,0.08203125,0.01171875,0.0,0.0 +35000,2525.1,7.517002812305133e-09,1.0,1.0,1.0,1.0,0.99609375,0.9375,0.3984375,0.1015625,0.00390625,0.0,0.0 +37500,2703.7,7.887549786749231e-09,1.0,1.0,1.0,1.0,0.9921875,0.8984375,0.34375,0.046875,0.0078125,0.0,0.0 +40000,2882.1,8.088328517876772e-09,1.0,1.0,1.0,1.0,0.98828125,0.8828125,0.296875,0.0390625,0.00390625,0.0,0.0 +42500,3060.6,8.216714927753404e-09,1.0,1.0,1.0,1.0,0.98828125,0.875,0.28125,0.03515625,0.00390625,0.0,0.0 +45000,3239.1,8.240009838369225e-09,1.0,1.0,1.0,1.0,0.98828125,0.87109375,0.2578125,0.03125,0.00390625,0.0,0.0 +47500,3417.6,8.316763539114587e-09,1.0,1.0,1.0,1.0,0.98828125,0.8671875,0.265625,0.0234375,0.00390625,0.0,0.0 +50000,3595.9,8.30135433415627e-09,1.0,1.0,1.0,1.0,0.98828125,0.8671875,0.25390625,0.03125,0.00390625,0.0,0.0 diff --git a/subtraction/blankspace_fixed_s0/step_50000.pt b/subtraction/blankspace_fixed_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..aebe49e640bb7ff0035984d15c8e7c95081078e5 --- /dev/null +++ b/subtraction/blankspace_fixed_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f01bbd721b807c199a6d2f768254fd4381d56863e21731df10ef3b98a6923c1e +size 13738485 diff --git a/subtraction/blankspace_fixed_s1/config.json b/subtraction/blankspace_fixed_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41bf0e28d30deacfb1cc9c6c0af17670f0722f91 --- /dev/null +++ b/subtraction/blankspace_fixed_s1/config.json @@ -0,0 +1,57 @@ +{ + "name": "subtraction blankspace fixed seed 1", + "task": { + "op": "sub", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/subtraction/blankspace_fixed_s1/metrics.csv b/subtraction/blankspace_fixed_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..5d6d3e35cfeb01c0cf88eb1bf831f53ed621e490 --- /dev/null +++ b/subtraction/blankspace_fixed_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,21.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,200.4,0.0025215323738348645,0.09765625,0.0625,0.05859375,0.046875,0.0546875,0.03515625,0.01953125,0.0,0.0,0.0,0.0 +5000,378.8,0.00014107623402882577,1.0,0.96875,0.86328125,0.6875,0.65625,0.40625,0.09375,0.01953125,0.0,0.0,0.0 +7500,557.2,7.886694945597437e-05,1.0,0.97265625,0.67578125,0.59375,0.5,0.2421875,0.10546875,0.01171875,0.0,0.0,0.0 +10000,735.7,5.4737769709383505e-05,0.9921875,0.95703125,0.8203125,0.65234375,0.5078125,0.328125,0.06640625,0.015625,0.0,0.0,0.0 +12500,915.5,2.1908648828217825e-05,1.0,1.0,0.875,0.78125,0.64453125,0.41015625,0.08203125,0.0,0.00390625,0.0,0.0 +15000,1095.1,1.7427379070504916e-05,1.0,0.9921875,0.91015625,0.87890625,0.69140625,0.4921875,0.08203125,0.01171875,0.0,0.0,0.0 +17500,1273.6,1.993717873687069e-05,1.0,0.99609375,0.88671875,0.7421875,0.6796875,0.39453125,0.015625,0.0,0.0,0.0,0.0 +20000,1451.9,1.823160142421518e-06,1.0,1.0,0.95703125,0.9140625,0.86328125,0.73046875,0.30859375,0.0546875,0.0,0.0,0.0 +22500,1631.8,1.7738602130933887e-05,1.0,0.99609375,0.921875,0.91796875,0.8359375,0.65234375,0.328125,0.0703125,0.015625,0.0,0.0 +25000,1810.3,5.057252494197064e-09,1.0,1.0,0.984375,0.96875,0.77734375,0.6171875,0.078125,0.01953125,0.0,0.0,0.0 +27500,1988.8,1.8566494027910883e-09,1.0,1.0,0.984375,0.95703125,0.79296875,0.63671875,0.1015625,0.015625,0.00390625,0.0,0.0 +30000,2167.2,1.0399146435223021e-08,1.0,1.0,0.9765625,0.94140625,0.88671875,0.765625,0.234375,0.0546875,0.0,0.0,0.0 +32500,2345.8,3.0211482463697366e-09,1.0,1.0,0.98828125,0.97265625,0.8671875,0.7890625,0.25,0.06640625,0.0,0.0,0.0 +35000,2524.2,4.125915525809144e-09,1.0,1.0,0.98046875,0.94140625,0.8671875,0.734375,0.23046875,0.04296875,0.0,0.0,0.0 +37500,2702.7,4.833549222971443e-09,1.0,1.0,0.96875,0.93359375,0.84765625,0.6953125,0.1953125,0.03515625,0.0,0.0,0.0 +40000,2881.2,5.253747645255351e-09,1.0,1.0,0.96484375,0.89453125,0.81640625,0.65234375,0.15234375,0.01953125,0.0,0.0,0.0 +42500,3059.6,5.471744062171892e-09,1.0,1.0,0.953125,0.8671875,0.78125,0.609375,0.12890625,0.0078125,0.0,0.0,0.0 +45000,3238.1,5.580432727408126e-09,1.0,1.0,0.94921875,0.86328125,0.7734375,0.5859375,0.1171875,0.0078125,0.0,0.0,0.0 +47500,3416.5,5.669425071660756e-09,1.0,1.0,0.9453125,0.86328125,0.78125,0.58984375,0.1171875,0.0078125,0.0,0.0,0.0 +50000,3595.0,5.681223087521639e-09,1.0,1.0,0.9453125,0.859375,0.77734375,0.59375,0.11328125,0.0078125,0.0,0.0,0.0 diff --git a/subtraction/blankspace_fixed_s1/step_50000.pt b/subtraction/blankspace_fixed_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..606d94ee35fb3bbd08f1013ea9b82bccc78eaa4f --- /dev/null +++ b/subtraction/blankspace_fixed_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d38aad327dee319b8f0fa4f9e0012f3c1a98852c778ad83cee4337d23beb63 +size 13738485 diff --git a/subtraction/blankspace_fixed_s2/config.json b/subtraction/blankspace_fixed_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..79b5e25abcf523d53a885dfa3472132bc492ea79 --- /dev/null +++ b/subtraction/blankspace_fixed_s2/config.json @@ -0,0 +1,57 @@ +{ + "name": "subtraction blankspace fixed seed 2", + "task": { + "op": "sub", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "sequential", + "blanks": 121, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/subtraction/blankspace_fixed_s2/metrics.csv b/subtraction/blankspace_fixed_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..f0e36744fc34615dba85e3ac2d8936135f5d5855 --- /dev/null +++ b/subtraction/blankspace_fixed_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,21.7,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,200.2,0.0033988108434099915,0.09765625,0.08984375,0.078125,0.03515625,0.02734375,0.015625,0.00390625,0.0,0.0,0.0,0.0 +5000,378.6,0.0025319497022510407,0.08203125,0.078125,0.078125,0.078125,0.0703125,0.08203125,0.0078125,0.0078125,0.0,0.0,0.0 +7500,557.3,0.00013632773776518916,0.98828125,0.92578125,0.86328125,0.81640625,0.63671875,0.49609375,0.203125,0.04296875,0.00390625,0.0,0.0 +10000,735.8,7.525272171772491e-05,0.9765625,0.9609375,0.9296875,0.8125,0.53125,0.3828125,0.04296875,0.0,0.00390625,0.0,0.0 +12500,915.5,3.798109273156558e-05,1.0,0.9921875,0.9453125,0.9140625,0.734375,0.55078125,0.15625,0.03125,0.0,0.0,0.0 +15000,1095.1,9.590912246550149e-05,0.94140625,0.9140625,0.87890625,0.8671875,0.703125,0.51171875,0.140625,0.01953125,0.00390625,0.0,0.0 +17500,1273.7,1.842651266312357e-05,0.98828125,0.98828125,0.99609375,0.94140625,0.73828125,0.703125,0.35546875,0.10546875,0.01953125,0.0,0.0 +20000,1452.1,2.5101298287302813e-05,0.99609375,0.99609375,0.9609375,0.921875,0.70703125,0.4765625,0.0625,0.00390625,0.0,0.0,0.0 +22500,1630.8,5.054681259588971e-09,1.0,1.0,1.0,0.99609375,1.0,0.96484375,0.7890625,0.38671875,0.09375,0.0,0.0 +25000,1809.2,3.2890394960499756e-05,0.99609375,0.9921875,0.9765625,0.94921875,0.8046875,0.55078125,0.20703125,0.01171875,0.0,0.0,0.0 +27500,1987.8,9.089213510176755e-06,1.0,0.9921875,0.97265625,0.98828125,0.91796875,0.87109375,0.61328125,0.27734375,0.046875,0.0,0.0 +30000,2166.2,1.2761282122091648e-08,1.0,1.0,1.0,1.0,0.98828125,0.94140625,0.640625,0.1015625,0.02734375,0.0,0.0 +32500,2345.0,7.627897905443699e-07,1.0,1.0,1.0,0.99609375,0.9453125,0.84375,0.46484375,0.0859375,0.02734375,0.0,0.0 +35000,2523.3,2.91544682062344e-09,1.0,1.0,1.0,0.99609375,0.9765625,0.93359375,0.7109375,0.2421875,0.03515625,0.0,0.0 +37500,2702.0,1.5525415489753422e-09,1.0,1.0,1.0,1.0,1.0,0.98046875,0.8515625,0.3203125,0.046875,0.0,0.0 +40000,2880.5,2.028026110322291e-09,1.0,1.0,1.0,1.0,1.0,0.96875,0.8046875,0.2734375,0.03515625,0.0,0.0 +42500,3059.2,2.364297161178095e-09,1.0,1.0,1.0,1.0,0.99609375,0.9609375,0.796875,0.234375,0.03125,0.0,0.0 +45000,3237.6,2.52593707923046e-09,1.0,1.0,1.0,1.0,0.99609375,0.95703125,0.79296875,0.22265625,0.03125,0.0,0.0 +47500,3416.2,2.6098546099023882e-09,1.0,1.0,1.0,1.0,0.99609375,0.95703125,0.7890625,0.234375,0.0390625,0.0,0.0 +50000,3594.7,2.6151429501429995e-09,1.0,1.0,1.0,1.0,0.99609375,0.96484375,0.7890625,0.23046875,0.03125,0.0,0.0 diff --git a/subtraction/blankspace_fixed_s2/step_50000.pt b/subtraction/blankspace_fixed_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6405b4e5d41bb80235f2f83bb4dc5c2430e38b88 --- /dev/null +++ b/subtraction/blankspace_fixed_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c48d4e4402184ffbf5c2643ac75395ae2a8a436218037affa9dc85ad94c1de4a +size 13738485 diff --git a/subtraction/position_coupling_s0/config.json b/subtraction/position_coupling_s0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..933e4ee4ebf983a8c788f7f3667edbcfdbf00c74 --- /dev/null +++ b/subtraction/position_coupling_s0/config.json @@ -0,0 +1,57 @@ +{ + "name": "subtraction position coupling seed 0", + "task": { + "op": "sub", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 0, + "wandb": false + } +} \ No newline at end of file diff --git a/subtraction/position_coupling_s0/metrics.csv b/subtraction/position_coupling_s0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..40c5d93cba5289faf63b10e1dc2eb5875d3c8efc --- /dev/null +++ b/subtraction/position_coupling_s0/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.4,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,45.4,0.1060482306214816,0.73046875,0.67578125,0.578125,0.1015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,85.7,0.00988817284232464,1.0,0.9921875,0.98828125,0.828125,0.0546875,0.0,0.0,0.0,0.0,0.0,0.0 +7500,126.0,0.0052390025736407985,0.99609375,0.99609375,0.95703125,0.703125,0.02734375,0.0,0.0,0.0,0.0,0.0,0.0 +10000,166.3,0.003566624755712964,0.97265625,1.0,0.9609375,0.78125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,206.5,0.0008845606937126026,1.0,1.0,1.0,0.94140625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +15000,246.5,0.0016898383464941781,1.0,0.99609375,0.97265625,0.9375,0.03515625,0.0,0.0,0.0,0.0,0.0,0.0 +17500,286.9,0.0008362168340499525,1.0,0.9609375,0.94921875,0.875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +20000,327.0,0.000921979078508524,0.98828125,0.99609375,1.0,0.984375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +22500,367.2,0.0006457201166744742,1.0,1.0,0.98828125,0.96875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +25000,407.9,0.0002548298922198859,1.0,1.0,1.0,0.9609375,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0 +27500,448.2,0.0002296667586960298,1.0,1.0,1.0,0.9921875,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +30000,488.2,0.00011637659409991889,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,528.5,8.345395073071166e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,568.7,7.313514242709739e-05,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,608.8,7.255005465959863e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,649.4,3.458695364204729e-05,1.0,1.0,1.0,0.99609375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +42500,689.5,2.2747923242761666e-08,1.0,1.0,1.0,1.0,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +45000,729.8,4.247962660214085e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,770.5,3.5834751375588436e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,810.8,4.304832479087159e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/subtraction/position_coupling_s0/step_50000.pt b/subtraction/position_coupling_s0/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5399351ae253075ccfa9d569e06ec802188e3735 --- /dev/null +++ b/subtraction/position_coupling_s0/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd4650da7c1b79250f72db7bde163d56ef7a0b616907ecc62853e15223d18346 +size 13738485 diff --git a/subtraction/position_coupling_s1/config.json b/subtraction/position_coupling_s1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fcce98cdc3ead969aa94c6f94e4a8e2522cbbe4b --- /dev/null +++ b/subtraction/position_coupling_s1/config.json @@ -0,0 +1,57 @@ +{ + "name": "subtraction position coupling seed 1", + "task": { + "op": "sub", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 1, + "wandb": false + } +} \ No newline at end of file diff --git a/subtraction/position_coupling_s1/metrics.csv b/subtraction/position_coupling_s1/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..b1a6109c7f536ead3c2581efbf149ba99964b7c2 --- /dev/null +++ b/subtraction/position_coupling_s1/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,4.0,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,44.3,0.05109595601009747,0.9765625,0.94921875,0.828125,0.50390625,0.0078125,0.0,0.0,0.0,0.0,0.0,0.0 +5000,84.3,0.011411399961065565,1.0,0.99609375,0.9921875,0.890625,0.05078125,0.0,0.0,0.0,0.0,0.0,0.0 +7500,124.4,0.005286711384035439,1.0,1.0,0.9921875,0.91015625,0.046875,0.0,0.0,0.0,0.0,0.0,0.0 +10000,164.6,0.002867632311462452,1.0,0.99609375,0.98828125,0.86328125,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +12500,204.8,0.002153410475660811,1.0,1.0,0.9921875,0.8984375,0.08984375,0.0,0.0,0.0,0.0,0.0,0.0 +15000,244.9,0.0019722130437104016,0.98828125,1.0,0.9921875,0.94921875,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0 +17500,285.3,0.0008600721548843595,1.0,1.0,0.9921875,0.96875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +20000,325.5,0.0010835104561480345,1.0,0.99609375,0.98828125,0.98046875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +22500,365.9,0.0007533001094057961,1.0,1.0,1.0,0.90234375,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +25000,406.3,0.000464621623713326,1.0,1.0,0.984375,0.9921875,0.0234375,0.0,0.0,0.0,0.0,0.0,0.0 +27500,446.7,0.00037512991496572567,0.99609375,1.0,1.0,0.9765625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +30000,486.7,0.0002606546183259238,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +32500,527.1,0.0002569394121630265,1.0,1.0,1.0,0.99609375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +35000,567.4,9.523023406580224e-05,1.0,1.0,1.0,0.984375,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +37500,607.7,4.395087401790576e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +40000,648.2,1.3542007634913793e-06,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +42500,688.7,1.8340615984355492e-08,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +45000,728.9,5.145724759881128e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +47500,769.2,3.774470373620542e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +50000,809.7,4.3563240706409705e-09,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/subtraction/position_coupling_s1/step_50000.pt b/subtraction/position_coupling_s1/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1683d75d35713fb10462c8bd87cca81cef110f4 --- /dev/null +++ b/subtraction/position_coupling_s1/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1a9b5e3a4717be31896e2f5cca494f194b2b9396b1a34a30f5663d38e06bc75 +size 13738485 diff --git a/subtraction/position_coupling_s2/config.json b/subtraction/position_coupling_s2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9aa2358e3646882c4f8c6a1ccdd2592fb9bc535a --- /dev/null +++ b/subtraction/position_coupling_s2/config.json @@ -0,0 +1,57 @@ +{ + "name": "subtraction position coupling seed 2", + "task": { + "op": "sub", + "max_digits": 20, + "min_digits": 1, + "carry_heavy": 0.0, + "reverse": true, + "zero_pad": true, + "positions": "coupled", + "blanks": 0, + "blanks_fixed": true, + "offset_max": 100, + "test_digits": [ + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 80, + 100 + ], + "test_examples": 256, + "kind": "arithmetic" + }, + "model": { + "arch": "transformer", + "d_model": 256, + "n_layers": 4, + "n_heads": 4, + "d_mlp": 1024, + "max_positions": 1024, + "pos_embed": "absolute", + "dropout": 0.0 + }, + "train": { + "steps": 50000, + "batch_size": 256, + "lr": 0.0003, + "weight_decay": 0.1, + "betas": [ + 0.9, + 0.98 + ], + "warmup_steps": 1000, + "cosine": true, + "amp": true, + "eval_every": 2500, + "checkpoint_every": 10000, + "seed": 2, + "wandb": false + } +} \ No newline at end of file diff --git a/subtraction/position_coupling_s2/metrics.csv b/subtraction/position_coupling_s2/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..f04e2614d6d23507f4ccf02a2e7ff5db26a0fa46 --- /dev/null +++ b/subtraction/position_coupling_s2/metrics.csv @@ -0,0 +1,22 @@ +step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100 +0,3.9,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +2500,44.2,0.059172024501170865,0.984375,0.9296875,0.8203125,0.38671875,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +5000,84.5,0.02423844767663336,0.96875,0.87890625,0.81640625,0.37890625,0.00390625,0.0,0.0,0.0,0.0,0.0,0.0 +7500,124.7,0.033815659299046986,0.80078125,0.75390625,0.375,0.015625,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +10000,165.3,0.003451777822397364,1.0,0.99609375,0.921875,0.7109375,0.0625,0.0,0.0,0.0,0.0,0.0,0.0 +12500,205.9,0.003756263443812288,0.99609375,0.99609375,0.98046875,0.65234375,0.06640625,0.0,0.0,0.0,0.0,0.0,0.0 +15000,246.1,0.0023028780732004778,1.0,1.0,0.99609375,0.97265625,0.17578125,0.0,0.0,0.0,0.0,0.0,0.0 +17500,286.5,0.001841593628913537,1.0,1.0,0.98828125,0.81640625,0.03125,0.0,0.0,0.0,0.0,0.0,0.0 +20000,326.8,0.0009478251331074839,0.99609375,1.0,0.9921875,0.953125,0.28125,0.0,0.0,0.0,0.0,0.0,0.0 +22500,366.8,0.0008274736040467803,0.96875,0.984375,0.984375,0.9921875,0.79296875,0.09375,0.0,0.0,0.0,0.0,0.0 +25000,407.2,0.0004094255601015017,0.97265625,1.0,1.0,0.9921875,0.328125,0.0,0.0,0.0,0.0,0.0,0.0 +27500,447.5,0.0007075556702847485,1.0,1.0,1.0,0.98046875,0.44921875,0.0078125,0.0,0.0,0.0,0.0,0.0 +30000,487.8,0.00027506390654087745,1.0,0.99609375,1.0,0.90625,0.1328125,0.0,0.0,0.0,0.0,0.0,0.0 +32500,528.4,0.00015247510822950514,1.0,1.0,1.0,0.984375,0.58984375,0.0,0.0,0.0,0.0,0.0,0.0 +35000,568.7,5.477202061465275e-05,1.0,1.0,1.0,1.0,0.265625,0.0,0.0,0.0,0.0,0.0,0.0 +37500,608.9,1.2342785049536552e-05,1.0,1.0,1.0,1.0,0.09765625,0.0,0.0,0.0,0.0,0.0,0.0 +40000,649.5,4.6231959790202624e-07,1.0,1.0,1.0,1.0,0.08984375,0.0,0.0,0.0,0.0,0.0,0.0 +42500,689.9,1.4841245382203633e-08,1.0,1.0,1.0,1.0,0.078125,0.0,0.0,0.0,0.0,0.0,0.0 +45000,729.9,5.60391311518019e-09,1.0,1.0,1.0,1.0,0.0703125,0.0,0.0,0.0,0.0,0.0,0.0 +47500,770.4,4.890793399790921e-09,1.0,1.0,1.0,1.0,0.0625,0.0,0.0,0.0,0.0,0.0,0.0 +50000,810.6,5.006380254084865e-09,1.0,1.0,1.0,1.0,0.0703125,0.0,0.0,0.0,0.0,0.0,0.0 diff --git a/subtraction/position_coupling_s2/step_50000.pt b/subtraction/position_coupling_s2/step_50000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e77fa80d0b1a8cc39958e04f75fc64543cf2794e --- /dev/null +++ b/subtraction/position_coupling_s2/step_50000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc62e402eb9973ea25d0c48b67c9923f5c3306b6a20ae67fc473297b7f8d3c85 +size 13738485