add temperature-trained runs and two more 11M coupling seeds

addition_big/position_coupling_s2/config.json

@@ -0,0 +1,57 @@
+{
+  "name": "addition_big position coupling seed 2",
+  "task": {
+    "op": "add",
+    "max_digits": 30,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 200,
+    "test_digits": [
+      10,
+      20,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100,
+      150,
+      200
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 384,
+    "n_layers": 6,
+    "n_heads": 6,
+    "d_mlp": 1536,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 1.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 60000,
+    "batch_size": 384,
+    "lr": 0.0002,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 2000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 5000,
+    "checkpoint_every": 20000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition_big/position_coupling_s2/metrics.csv

@@ -0,0 +1,14 @@
+step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200
+0,29.3,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,367.1,0.002366582334109078,0.984375,0.92578125,0.671875,0.01171875,0.0,0.0,0.0,0.0,0.0,0.0
+10000,705.0,0.0006230554900715152,1.0,1.0,0.9296875,0.1953125,0.0,0.0,0.0,0.0,0.0,0.0
+15000,1042.6,0.00039028054495232037,0.99609375,0.99609375,0.99609375,0.76953125,0.30859375,0.0078125,0.0,0.0,0.0,0.0
+20000,1380.3,0.0002113709867064831,0.99609375,1.0,1.0,0.83984375,0.0703125,0.0,0.0,0.0,0.0,0.0
+25000,1719.1,0.0001828992760933894,1.0,1.0,0.984375,0.80859375,0.05078125,0.0,0.0,0.0,0.0,0.0
+30000,2056.9,6.815408564000745e-05,1.0,1.0,1.0,0.90625,0.328125,0.0,0.0,0.0,0.0,0.0
+35000,2394.7,5.671344955246953e-05,1.0,1.0,1.0,0.91796875,0.48828125,0.08203125,0.0,0.0,0.0,0.0
+40000,2733.6,1.2648932652773735e-05,1.0,1.0,1.0,0.94921875,0.3984375,0.0078125,0.0,0.0,0.0,0.0
+45000,3072.5,1.8858231290397377e-09,1.0,1.0,1.0,0.96484375,0.3046875,0.0,0.0,0.0,0.0,0.0
+50000,3410.5,3.0822262438258388e-09,1.0,1.0,1.0,0.94921875,0.33984375,0.0,0.0,0.0,0.0,0.0
+55000,3748.8,5.571783376610425e-09,1.0,1.0,1.0,0.96484375,0.39453125,0.0,0.0,0.0,0.0,0.0
+60000,4087.2,5.995916891858694e-09,1.0,1.0,1.0,0.96875,0.40234375,0.0,0.0,0.0,0.0,0.0

addition_big/position_coupling_s2/step_60000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8adac02b8b9d35e9ea7ef56b7c4e7c33fe7e9949d1da1642baf82b556f106151
+size 44239477

addition_big/position_coupling_s3/config.json

@@ -0,0 +1,57 @@
+{
+  "name": "addition_big position coupling seed 3",
+  "task": {
+    "op": "add",
+    "max_digits": 30,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 200,
+    "test_digits": [
+      10,
+      20,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100,
+      150,
+      200
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 384,
+    "n_layers": 6,
+    "n_heads": 6,
+    "d_mlp": 1536,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 1.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 60000,
+    "batch_size": 384,
+    "lr": 0.0002,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 2000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 5000,
+    "checkpoint_every": 20000,
+    "seed": 3,
+    "wandb": false
+  }
+}

addition_big/position_coupling_s3/metrics.csv

@@ -0,0 +1,14 @@
+step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200
+0,29.2,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,367.1,0.0024786559447204635,0.98828125,0.9921875,0.73046875,0.02734375,0.0,0.0,0.0,0.0,0.0,0.0
+10000,705.1,0.000524585331352197,1.0,0.98828125,0.9609375,0.28515625,0.015625,0.0,0.0,0.0,0.0,0.0
+15000,1042.8,0.0005596142999261708,0.9921875,0.99609375,0.96484375,0.609375,0.05078125,0.0078125,0.0,0.0,0.0,0.0
+20000,1380.6,0.0002194886728864363,1.0,1.0,0.9921875,0.8984375,0.5546875,0.12890625,0.0,0.0,0.0,0.0
+25000,1718.9,0.00012747019061164555,1.0,1.0,0.9921875,0.84375,0.2734375,0.01171875,0.0,0.0,0.0,0.0
+30000,2056.7,8.250452218990583e-05,1.0,1.0,0.99609375,0.828125,0.375,0.046875,0.0,0.0,0.0,0.0
+35000,2395.7,7.368305486206361e-06,1.0,1.0,1.0,0.88671875,0.45703125,0.00390625,0.0,0.0,0.0,0.0
+40000,2733.9,1.9414632395297327e-05,1.0,1.0,1.0,0.87890625,0.2890625,0.01171875,0.0,0.0,0.0,0.0
+45000,3073.6,2.9038761111736944e-09,1.0,1.0,1.0,0.96484375,0.51953125,0.0625,0.0,0.0,0.0,0.0
+50000,3411.7,1.29076932774851e-09,1.0,1.0,1.0,0.95703125,0.32421875,0.0234375,0.0,0.0,0.0,0.0
+55000,3749.8,2.0630909165861848e-09,1.0,1.0,1.0,0.95703125,0.3359375,0.01171875,0.0,0.0,0.0,0.0
+60000,4087.9,2.3451669706609467e-09,1.0,1.0,1.0,0.94921875,0.28125,0.01171875,0.0,0.0,0.0,0.0

addition_big/position_coupling_s3/step_60000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:612e5373fb1708ec61bee6571e4b8f9e3b7910e2cb0df328dcb2aa6cd50e3a2c
+size 44239477

addition_big_sharp/position_coupling_s0/config.json

@@ -0,0 +1,57 @@
+{
+  "name": "addition_big_sharp position coupling seed 0",
+  "task": {
+    "op": "add",
+    "max_digits": 30,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 200,
+    "test_digits": [
+      10,
+      20,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100,
+      150,
+      200
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 384,
+    "n_layers": 6,
+    "n_heads": 6,
+    "d_mlp": 1536,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 2.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 60000,
+    "batch_size": 384,
+    "lr": 0.0002,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 2000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 5000,
+    "checkpoint_every": 20000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition_big_sharp/position_coupling_s0/metrics.csv

@@ -0,0 +1,14 @@
+step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200
+0,30.1,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,372.5,0.0023748246049385946,0.99609375,0.97265625,0.9140625,0.4453125,0.03515625,0.0,0.0,0.0,0.0,0.0
+10000,715.1,0.000618918012949441,1.0,1.0,0.9609375,0.69140625,0.07421875,0.00390625,0.0,0.0,0.0,0.0
+15000,1057.8,0.000320750652551017,1.0,0.99609375,0.7421875,0.6875,0.3125,0.07421875,0.0,0.0,0.0,0.0
+20000,1400.4,0.00016674799643825167,0.99609375,0.99609375,0.98828125,0.83984375,0.234375,0.0234375,0.0,0.0,0.0,0.0
+25000,1744.1,9.785291805071761e-05,1.0,1.0,0.9921875,0.87890625,0.640625,0.328125,0.0,0.0,0.0,0.0
+30000,2086.4,6.75734315087733e-05,1.0,1.0,0.984375,0.9765625,0.75390625,0.28515625,0.0,0.0,0.0,0.0
+35000,2429.8,1.2484155311353684e-05,1.0,1.0,1.0,0.98046875,0.8046875,0.26953125,0.0,0.0,0.0,0.0
+40000,2772.5,3.6795297310488315e-06,1.0,1.0,1.0,0.98046875,0.875,0.26171875,0.0,0.0,0.0,0.0
+45000,3116.2,4.098807529421164e-06,1.0,1.0,1.0,0.98828125,0.9375,0.375,0.0,0.0,0.0,0.0
+50000,3458.9,1.166739326626642e-09,1.0,1.0,1.0,0.99609375,0.9609375,0.46875,0.0,0.0,0.0,0.0
+55000,3801.3,1.2949986397697118e-09,1.0,1.0,1.0,0.9921875,0.94140625,0.3984375,0.0,0.0,0.0,0.0
+60000,4143.7,1.432241686545097e-09,1.0,1.0,1.0,0.98828125,0.94140625,0.33984375,0.0,0.0,0.0,0.0

addition_big_sharp/position_coupling_s0/step_60000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca3277b6c5777414d2fa1e7e84d2eeffc0c0e261f84e3e4cf58ad44d4498c09b
+size 44239477

addition_big_sharp/position_coupling_s1/config.json

@@ -0,0 +1,57 @@
+{
+  "name": "addition_big_sharp position coupling seed 1",
+  "task": {
+    "op": "add",
+    "max_digits": 30,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 200,
+    "test_digits": [
+      10,
+      20,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100,
+      150,
+      200
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 384,
+    "n_layers": 6,
+    "n_heads": 6,
+    "d_mlp": 1536,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 2.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 60000,
+    "batch_size": 384,
+    "lr": 0.0002,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 2000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 5000,
+    "checkpoint_every": 20000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition_big_sharp/position_coupling_s1/metrics.csv

@@ -0,0 +1,14 @@
+step,elapsed,loss,acc_10,acc_20,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100,acc_150,acc_200
+0,29.9,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+5000,372.3,0.0007382984497253507,1.0,1.0,0.96875,0.59375,0.16796875,0.00390625,0.0,0.0,0.0,0.0
+10000,714.5,0.0005218096989105523,1.0,1.0,1.0,0.85546875,0.5703125,0.2109375,0.00390625,0.0,0.0,0.0
+15000,1056.6,0.000200421967637171,1.0,0.98046875,0.98828125,0.8671875,0.5234375,0.10546875,0.0,0.0,0.0,0.0
+20000,1399.1,3.022299904353172e-07,1.0,1.0,1.0,1.0,0.98046875,0.921875,0.31640625,0.0234375,0.0,0.0
+25000,1742.5,1.9057187720662864e-09,1.0,1.0,1.0,1.0,0.98828125,0.9375,0.40234375,0.015625,0.0,0.0
+30000,2085.0,4.2377029578248614e-05,1.0,1.0,1.0,0.98828125,0.9375,0.67578125,0.04296875,0.0,0.0,0.0
+35000,2428.6,1.9029897895166793e-07,1.0,1.0,1.0,0.9921875,0.90625,0.578125,0.0703125,0.0,0.0,0.0
+40000,2771.3,1.7261627256925818e-08,1.0,1.0,1.0,1.0,0.9921875,0.82421875,0.0546875,0.0,0.0,0.0
+45000,3114.8,2.0959266140615087e-08,1.0,1.0,1.0,1.0,0.89453125,0.2421875,0.0,0.0,0.0,0.0
+50000,3457.2,2.4546162490588085e-08,1.0,1.0,1.0,1.0,0.80859375,0.0859375,0.0,0.0,0.0,0.0
+55000,3799.5,2.451853679565751e-08,1.0,1.0,1.0,0.9921875,0.66015625,0.03515625,0.0,0.0,0.0,0.0
+60000,4141.8,2.4624778611843677e-08,1.0,1.0,1.0,0.9921875,0.65625,0.0390625,0.0,0.0,0.0,0.0

addition_big_sharp/position_coupling_s1/step_60000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2e1c78e9daf1aece81ae6b5be2808672ace1a52323c6cc2947b6e5326fcd705
+size 44239477

addition_sharp/position_coupling_s0/config.json

@@ -0,0 +1,58 @@
+{
+  "name": "addition_sharp position coupling seed 0",
+  "task": {
+    "op": "add",
+    "max_digits": 20,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 2.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 0,
+    "wandb": false
+  }
+}

addition_sharp/position_coupling_s0/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,4.3,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,44.2,0.008475927415140742,0.9609375,0.98828125,0.97265625,0.9765625,0.625,0.46484375,0.01171875,0.0,0.0,0.0,0.0
+5000,83.1,0.0011268363547566767,1.0,1.0,1.0,0.98046875,0.88671875,0.5859375,0.05859375,0.0,0.0,0.0,0.0
+7500,122.7,0.0015766528146289654,0.9921875,0.99609375,0.98828125,0.8984375,0.70703125,0.3359375,0.10546875,0.0078125,0.0,0.0,0.0
+10000,161.9,0.0005018644088687908,1.0,1.0,0.99609375,1.0,0.82421875,0.7734375,0.27734375,0.02734375,0.0,0.0,0.0
+12500,201.8,0.0002736185025786636,0.9921875,1.0,1.0,1.0,0.87109375,0.74609375,0.27734375,0.0078125,0.00390625,0.0,0.0
+15000,241.1,0.0008247798056438383,0.98828125,0.99609375,0.984375,0.9921875,0.703125,0.703125,0.24609375,0.0546875,0.00390625,0.0,0.0
+17500,322.4,0.00031979127998103503,1.0,0.98046875,1.0,0.9921875,0.46484375,0.7265625,0.01953125,0.01171875,0.0,0.0,0.0
+20000,492.5,7.789053708850934e-05,1.0,1.0,1.0,1.0,0.8125,0.828125,0.6328125,0.171875,0.015625,0.0,0.0
+22500,664.3,0.0001876805691532537,1.0,1.0,1.0,1.0,0.83203125,0.78515625,0.13671875,0.015625,0.0,0.0,0.0
+25000,838.7,0.0002264830975918883,1.0,1.0,1.0,1.0,0.80078125,0.86328125,0.33984375,0.1015625,0.0078125,0.0,0.0
+27500,1014.4,2.7978571134622106e-09,1.0,1.0,1.0,1.0,0.92578125,0.90234375,0.30859375,0.03125,0.0,0.0,0.0
+30000,1200.2,1.3730667787213785e-08,1.0,1.0,1.0,1.0,0.98828125,0.9453125,0.4453125,0.03125,0.0,0.0,0.0
+32500,1409.3,3.3360052414158676e-07,1.0,1.0,1.0,1.0,0.99609375,0.9296875,0.3671875,0.03125,0.0,0.0,0.0
+35000,1615.1,2.6378461075171402e-08,1.0,1.0,1.0,1.0,1.0,0.9296875,0.375,0.0078125,0.0,0.0,0.0
+37500,1824.0,2.0273411353629615e-06,1.0,1.0,1.0,1.0,0.99609375,0.8359375,0.23046875,0.00390625,0.0,0.0,0.0
+40000,2031.8,2.861022200824244e-08,1.0,1.0,1.0,1.0,0.99609375,0.83203125,0.2265625,0.00390625,0.0,0.0,0.0
+42500,2115.5,2.9635341535358456e-08,1.0,1.0,1.0,1.0,0.9921875,0.859375,0.15234375,0.0,0.0,0.0,0.0
+45000,2155.8,3.065091220731855e-08,1.0,1.0,1.0,1.0,0.9921875,0.859375,0.1328125,0.0,0.0,0.0,0.0
+47500,2195.2,3.088202263750347e-08,1.0,1.0,1.0,1.0,0.9921875,0.8515625,0.1328125,0.0,0.0,0.0,0.0
+50000,2234.5,3.090925313933144e-08,1.0,1.0,1.0,1.0,0.9921875,0.84765625,0.12109375,0.0,0.0,0.0,0.0

addition_sharp/position_coupling_s0/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f36e26478151de88b99ef00419f30aa7d56d577f31e6bde860baa0b899a2169
+size 13738485

addition_sharp/position_coupling_s1/config.json

@@ -0,0 +1,58 @@
+{
+  "name": "addition_sharp position coupling seed 1",
+  "task": {
+    "op": "add",
+    "max_digits": 20,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 2.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 1,
+    "wandb": false
+  }
+}

addition_sharp/position_coupling_s1/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,4.0,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,43.2,0.07537456200732277,0.96484375,0.984375,0.97265625,0.94140625,0.83203125,0.6484375,0.140625,0.0078125,0.0,0.0,0.0
+5000,82.7,0.012797355784234432,0.98828125,1.0,1.0,0.9921875,0.90234375,0.8125,0.1015625,0.015625,0.0,0.0,0.0
+7500,122.2,0.0024346837246985984,1.0,1.0,1.0,1.0,0.9375,0.7890625,0.34765625,0.03125,0.0,0.0,0.0
+10000,161.2,0.0008701500698029637,1.0,1.0,1.0,0.9921875,0.98046875,0.8671875,0.328125,0.0546875,0.0,0.0,0.0
+12500,200.3,6.212474693881451e-05,1.0,1.0,1.0,1.0,0.96875,0.875,0.41796875,0.06640625,0.0078125,0.0,0.0
+15000,239.4,0.00032355797199682876,1.0,1.0,1.0,1.0,0.99609375,0.91796875,0.5078125,0.046875,0.0,0.0,0.0
+17500,278.6,0.00010357247213960461,1.0,1.0,1.0,0.9765625,0.875,0.6953125,0.34765625,0.0546875,0.0,0.0,0.0
+20000,318.3,5.26062916573681e-05,1.0,1.0,1.0,1.0,0.98046875,0.9453125,0.48828125,0.0859375,0.00390625,0.0,0.0
+22500,357.5,5.1305303975776974e-05,1.0,1.0,1.0,1.0,1.0,0.9609375,0.5078125,0.1484375,0.0,0.0,0.0
+25000,396.5,1.616195969398696e-05,1.0,1.0,1.0,1.0,1.0,0.953125,0.4765625,0.109375,0.0,0.0,0.0
+27500,435.8,4.966839510884111e-05,1.0,1.0,1.0,1.0,0.98828125,0.9140625,0.42578125,0.06640625,0.0,0.0,0.0
+30000,475.1,2.622705992041869e-06,1.0,1.0,1.0,1.0,0.9921875,0.83203125,0.30078125,0.015625,0.0,0.0,0.0
+32500,514.3,2.7219634961719995e-09,1.0,1.0,1.0,1.0,1.0,0.94140625,0.53125,0.06640625,0.0,0.0,0.0
+35000,553.0,7.012690515746603e-09,1.0,1.0,1.0,1.0,1.0,0.93359375,0.4140625,0.015625,0.0,0.0,0.0
+37500,592.1,1.636207995201845e-08,1.0,1.0,1.0,1.0,1.0,0.91796875,0.3671875,0.01171875,0.0,0.0,0.0
+40000,631.3,2.193076247905829e-08,1.0,1.0,1.0,1.0,1.0,0.93359375,0.4296875,0.015625,0.0,0.0,0.0
+42500,670.4,2.418943650322416e-08,1.0,1.0,1.0,1.0,1.0,0.94921875,0.4296875,0.02734375,0.0,0.0,0.0
+45000,709.9,2.5224010432753495e-08,1.0,1.0,1.0,1.0,1.0,0.94921875,0.44140625,0.0234375,0.0,0.0,0.0
+47500,748.9,2.731370055805514e-08,1.0,1.0,1.0,1.0,1.0,0.94921875,0.43359375,0.0234375,0.0,0.0,0.0
+50000,788.0,2.6017787456351782e-08,1.0,1.0,1.0,1.0,1.0,0.94921875,0.4375,0.0234375,0.0,0.0,0.0

addition_sharp/position_coupling_s1/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fbb730c864f0ab812571b5ff3798189e992300bd85b9a82dde9c76bb8b120ed
+size 13738485

addition_sharp/position_coupling_s2/config.json

@@ -0,0 +1,58 @@
+{
+  "name": "addition_sharp position coupling seed 2",
+  "task": {
+    "op": "add",
+    "max_digits": 20,
+    "min_digits": 1,
+    "carry_heavy": 0.0,
+    "reverse": true,
+    "zero_pad": true,
+    "positions": "coupled",
+    "blanks": 0,
+    "blanks_fixed": true,
+    "offset_max": 100,
+    "test_digits": [
+      5,
+      10,
+      15,
+      20,
+      25,
+      30,
+      40,
+      50,
+      60,
+      80,
+      100
+    ],
+    "test_examples": 256,
+    "kind": "arithmetic"
+  },
+  "model": {
+    "arch": "transformer",
+    "d_model": 256,
+    "n_layers": 4,
+    "n_heads": 4,
+    "d_mlp": 1024,
+    "max_positions": 1024,
+    "pos_embed": "absolute",
+    "attn_scale": 2.0,
+    "dropout": 0.0
+  },
+  "train": {
+    "steps": 50000,
+    "batch_size": 256,
+    "lr": 0.0003,
+    "weight_decay": 0.1,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "warmup_steps": 1000,
+    "cosine": true,
+    "amp": true,
+    "eval_every": 2500,
+    "checkpoint_every": 10000,
+    "seed": 2,
+    "wandb": false
+  }
+}

addition_sharp/position_coupling_s2/metrics.csv

@@ -0,0 +1,22 @@
+step,elapsed,loss,acc_5,acc_10,acc_15,acc_20,acc_25,acc_30,acc_40,acc_50,acc_60,acc_80,acc_100
+0,4.0,nan,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+2500,43.3,0.07461852640371194,0.96484375,1.0,1.0,0.98828125,0.94921875,0.6796875,0.05859375,0.0390625,0.01171875,0.0,0.0
+5000,82.2,0.07046456168252137,0.984375,1.0,0.98828125,0.99609375,0.99609375,0.9921875,0.73046875,0.11328125,0.296875,0.02734375,0.0
+7500,121.1,0.06285470669000741,0.97265625,1.0,0.98046875,0.98046875,0.95703125,0.85546875,0.31640625,0.3046875,0.19921875,0.0,0.0
+10000,160.1,0.057286919870174854,1.0,1.0,1.0,1.0,0.9921875,1.0,0.91015625,0.515625,0.6328125,0.0,0.0
+12500,199.3,0.053518666637143476,1.0,1.0,1.0,1.0,1.0,1.0,0.765625,0.765625,0.3984375,0.00390625,0.0
+15000,238.7,0.05211182934495609,1.0,1.0,1.0,0.9921875,0.9921875,0.98046875,0.7578125,0.52734375,0.4609375,0.0,0.0
+17500,277.8,0.05261992734034174,1.0,1.0,1.0,1.0,0.99609375,1.0,0.76171875,0.60546875,0.6015625,0.0,0.0
+20000,316.9,0.05201170694027845,1.0,1.0,1.0,1.0,0.98828125,0.98828125,0.73828125,0.46484375,0.34765625,0.0,0.0
+22500,356.3,0.04956710963011237,1.0,0.99609375,0.984375,0.984375,0.96484375,0.921875,0.515625,0.3125,0.1875,0.0,0.0
+25000,396.0,0.04934516460886646,1.0,1.0,1.0,1.0,0.9921875,0.99609375,0.59375,0.5859375,0.48828125,0.0,0.0
+27500,435.4,0.049742550688681234,1.0,1.0,1.0,1.0,0.99609375,0.9921875,0.22265625,0.12890625,0.0390625,0.0,0.0
+30000,474.3,0.04928705071894055,1.0,1.0,1.0,1.0,0.98828125,0.99609375,0.2109375,0.19921875,0.0625,0.0,0.0
+32500,513.2,0.04940571614061812,1.0,1.0,1.0,1.0,0.99609375,0.98828125,0.17578125,0.1953125,0.03515625,0.0,0.0
+35000,552.1,0.048817630287235424,1.0,1.0,0.9921875,1.0,0.96875,0.96875,0.171875,0.1328125,0.08984375,0.0,0.0
+37500,591.2,0.04983255911845755,1.0,1.0,1.0,1.0,0.87109375,0.7734375,0.16015625,0.1015625,0.03125,0.0,0.0
+40000,630.5,0.03950270002322331,1.0,0.91015625,1.0,1.0,0.98828125,0.984375,0.109375,0.0234375,0.01171875,0.0,0.0
+42500,669.6,0.018740038250026216,1.0,1.0,0.99609375,0.99609375,1.0,0.99609375,0.28125,0.1015625,0.04296875,0.0,0.0
+45000,708.9,0.005053227370341364,1.0,1.0,1.0,1.0,1.0,1.0,0.2890625,0.12890625,0.01171875,0.0,0.0
+47500,748.5,0.003299886855553191,1.0,1.0,1.0,1.0,1.0,1.0,0.28515625,0.14453125,0.01953125,0.0,0.0
+50000,787.7,0.003173737048683181,1.0,1.0,1.0,1.0,1.0,1.0,0.28125,0.1484375,0.015625,0.0,0.0

addition_sharp/position_coupling_s2/step_50000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff5fd1f15238d9bb7550f4a1c1db485aef362e21086a1fbec5e8f7a2b852636a
+size 13738485