Upload add_sub_sorl_v1_abs30_K1_10K_1L2H256d

add_sub_sorl_v1_abs30_K1_10K_1L2H256d/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "layer_types": [
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 2,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_K1_10K_1L2H256d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_K1_10K_1L2H256d/metrics.json

@@ -0,0 +1,1557 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      207,
+      257,
+      307,
+      364,
+      414,
+      464,
+      521,
+      571,
+      621,
+      678,
+      728,
+      778,
+      835,
+      885,
+      935,
+      992,
+      1042,
+      1092,
+      1149,
+      1199,
+      1249,
+      1306,
+      1356,
+      1406,
+      1463,
+      1513,
+      1563,
+      1620,
+      1670,
+      1720,
+      1777,
+      1827,
+      1877,
+      1934,
+      1984,
+      2034,
+      2091,
+      2141,
+      2191,
+      2248,
+      2298,
+      2348,
+      2405,
+      2455,
+      2505,
+      2562,
+      2612,
+      2662,
+      2719,
+      2769,
+      2819,
+      2876,
+      2926,
+      2976,
+      3033,
+      3083,
+      3133
+    ],
+    "loss": [
+      6.360916614532471,
+      8.901763916015625,
+      10.740385055541992,
+      9.560364723205566,
+      8.92093276977539,
+      7.602734088897705,
+      6.905447483062744,
+      5.974641799926758,
+      5.094010829925537,
+      4.30830717086792,
+      3.9614787101745605,
+      3.7753567695617676,
+      3.569328784942627,
+      3.2405896186828613,
+      3.1252799034118652,
+      3.4043140411376953,
+      3.0519957542419434,
+      2.9571352005004883,
+      2.798739194869995,
+      2.763550043106079,
+      2.86095929145813,
+      2.848637819290161,
+      2.829274892807007,
+      2.8892879486083984,
+      2.7310662269592285,
+      2.6953470706939697,
+      2.994562864303589,
+      2.9199330806732178,
+      2.678225040435791,
+      2.896800994873047,
+      2.7342445850372314,
+      2.7211267948150635,
+      2.746145486831665,
+      2.7077765464782715,
+      2.6929845809936523,
+      2.9319868087768555,
+      2.640428066253662,
+      2.6540839672088623,
+      2.7170581817626953,
+      2.4140748977661133,
+      2.693222999572754,
+      2.4776108264923096,
+      2.5343384742736816,
+      2.7867536544799805,
+      2.598827600479126,
+      2.6431612968444824,
+      2.3718416690826416,
+      2.5454235076904297,
+      2.7806715965270996,
+      2.6620030403137207,
+      2.341674327850342,
+      2.371304512023926,
+      2.344536781311035,
+      2.681226968765259,
+      2.6581146717071533,
+      2.4974405765533447,
+      2.4809601306915283,
+      2.1956045627593994,
+      2.275268077850342,
+      2.303769826889038
+    ],
+    "base_loss": [
+      11.634408950805664,
+      10.324111938476562,
+      9.300094604492188,
+      8.132733345031738,
+      7.338479042053223,
+      6.418792724609375,
+      5.557044982910156,
+      4.571934223175049,
+      3.949857473373413,
+      3.1971724033355713,
+      2.7776083946228027,
+      2.529054880142212,
+      2.3361759185791016,
+      2.2235846519470215,
+      2.160993814468384,
+      2.017019033432007,
+      2.1148483753204346,
+      2.079894542694092,
+      2.046719789505005,
+      2.0021867752075195,
+      1.959739089012146,
+      1.9125105142593384,
+      1.8836029767990112,
+      1.936890721321106,
+      1.9336713552474976,
+      1.875162124633789,
+      1.877450704574585,
+      1.8944813013076782,
+      1.9072321653366089,
+      1.877310037612915,
+      1.8434879779815674,
+      1.8631662130355835,
+      1.8893787860870361,
+      1.8688853979110718,
+      1.9359947443008423,
+      1.8368498086929321,
+      1.90172278881073,
+      1.8500453233718872,
+      1.847686767578125,
+      1.8335860967636108,
+      1.8280881643295288,
+      1.8143364191055298,
+      1.8729511499404907,
+      1.879006028175354,
+      1.8803428411483765,
+      1.8737684488296509,
+      1.8506696224212646,
+      1.89244544506073,
+      1.7700861692428589,
+      1.8516298532485962,
+      1.7636386156082153,
+      1.865094542503357,
+      1.8513535261154175,
+      1.87937331199646,
+      1.8560491800308228,
+      1.8519506454467773,
+      1.8974485397338867,
+      1.8727619647979736,
+      1.924325704574585,
+      1.8773144483566284
+    ],
+    "info_loss": [
+      -1.2458610534667969,
+      -0.7521276473999023,
+      -0.28203773498535156,
+      -0.15022516250610352,
+      -0.06323575973510742,
+      -0.058394432067871094,
+      -0.013966083526611328,
+      0.005060672760009766,
+      -0.012318849563598633,
+      -0.010970592498779297,
+      -0.0010941028594970703,
+      0.007451295852661133,
+      0.00714564323425293,
+      -0.013251543045043945,
+      -0.017932415008544922,
+      0.024947166442871094,
+      -0.020143508911132812,
+      -0.02531123161315918,
+      -0.03774905204772949,
+      -0.03671395778656006,
+      -0.022633671760559082,
+      -0.018763184547424316,
+      -0.017561793327331543,
+      -0.016712069511413574,
+      -0.03212904930114746,
+      -0.02999413013458252,
+      0.0001035928726196289,
+      -0.009270310401916504,
+      -0.03444778919219971,
+      -0.00932621955871582,
+      -0.022470474243164062,
+      -0.025658130645751953,
+      -0.026004314422607422,
+      -0.02751028537750244,
+      -0.03505265712738037,
+      -0.0017061233520507812,
+      -0.037314653396606445,
+      -0.030777812004089355,
+      -0.02395462989807129,
+      -0.05319559574127197,
+      -0.024481534957885742,
+      -0.04487013816833496,
+      -0.044959425926208496,
+      -0.020186305046081543,
+      -0.038793087005615234,
+      -0.03407180309295654,
+      -0.058762431144714355,
+      -0.045389771461486816,
+      -0.01016223430633545,
+      -0.03012681007385254,
+      -0.05328261852264404,
+      -0.06041383743286133,
+      -0.06155991554260254,
+      -0.030355215072631836,
+      -0.03096938133239746,
+      -0.04603886604309082,
+      -0.05277705192565918,
+      -0.0783991813659668,
+      -0.07581222057342529,
+      -0.06812918186187744
+    ],
+    "abs_loss": [
+      3.3993585109710693,
+      3.3741276264190674,
+      3.293186902999878,
+      3.1993892192840576,
+      3.077329635620117,
+      2.9410483837127686,
+      2.8460922241210938,
+      2.816984176635742,
+      2.716546058654785,
+      2.7291243076324463,
+      2.740588426589966,
+      2.710020065307617,
+      2.7518150806427,
+      2.7128524780273438,
+      2.7190158367156982,
+      2.70732045173645,
+      2.7408180236816406,
+      2.7023251056671143,
+      2.714268445968628,
+      2.7319583892822266,
+      2.7436602115631104,
+      2.728748083114624,
+      2.7099857330322266,
+      2.712400197982788,
+      2.7173821926116943,
+      2.7400906085968018,
+      2.7087700366973877,
+      2.7378408908843994,
+      2.723125457763672,
+      2.705928087234497,
+      2.7350876331329346,
+      2.7362060546875,
+      2.766026735305786,
+      2.73347544670105,
+      2.675196886062622,
+      2.727919340133667,
+      2.72743821144104,
+      2.7316882610321045,
+      2.6979920864105225,
+      2.7380247116088867,
+      2.715238571166992,
+      2.736570358276367,
+      2.730070114135742,
+      2.7150075435638428,
+      2.6860759258270264,
+      2.72533917427063,
+      2.7132816314697266,
+      2.695629358291626,
+      2.748570203781128,
+      2.7472362518310547,
+      2.7373647689819336,
+      2.7334978580474854,
+      2.7217934131622314,
+      2.6902172565460205,
+      2.751253843307495,
+      2.6950340270996094,
+      2.7480571269989014,
+      2.70295786857605,
+      2.7263076305389404,
+      2.7119967937469482
+    ],
+    "zipf_loss": [
+      6.845182418823242,
+      5.7615156173706055,
+      3.931349277496338,
+      2.6099436283111572,
+      1.9070777893066406,
+      1.4737807512283325,
+      1.2034538984298706,
+      1.0704028606414795,
+      0.9956868290901184,
+      0.9479280710220337,
+      0.9207525253295898,
+      0.9007868766784668,
+      0.8865149617195129,
+      0.8782351613044739,
+      0.8717087507247925,
+      0.8670912981033325,
+      0.8645007610321045,
+      0.8601205348968506,
+      0.8580830097198486,
+      0.8553069233894348,
+      0.8531909584999084,
+      0.8508843183517456,
+      0.8502911925315857,
+      0.8482779264450073,
+      0.8469471335411072,
+      0.8461171388626099,
+      0.8451994061470032,
+      0.8443709015846252,
+      0.8431581258773804,
+      0.8421601057052612,
+      0.8419525623321533,
+      0.8409212231636047,
+      0.8402071595191956,
+      0.8406464457511902,
+      0.8399967551231384,
+      0.8394061326980591,
+      0.8391081094741821,
+      0.8386479616165161,
+      0.8391185402870178,
+      0.8386422991752625,
+      0.8384263515472412,
+      0.8383186459541321,
+      0.8379745483398438,
+      0.838110089302063,
+      0.8378079533576965,
+      0.8375771045684814,
+      0.837468147277832,
+      0.8373128175735474,
+      0.8373508453369141,
+      0.8369177579879761,
+      0.837125301361084,
+      0.836998701095581,
+      0.8366031646728516,
+      0.836384117603302,
+      0.8366338610649109,
+      0.8363752365112305,
+      0.8364763259887695,
+      0.836538553237915,
+      0.8364337682723999,
+      0.8365474939346313
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.7350990772247314,
+      0.4800478219985962,
+      0.35018521547317505,
+      0.3197889029979706,
+      0.3089351952075958,
+      0.32441917061805725,
+      0.31844910979270935,
+      0.33132049441337585,
+      0.34706398844718933,
+      0.3663455545902252,
+      0.36917969584465027,
+      0.374531626701355,
+      0.38100525736808777,
+      0.3814346194267273,
+      0.374453067779541,
+      0.36980709433555603,
+      0.3699214458465576,
+      0.37833264470100403,
+      0.3921150863170624,
+      0.3764060437679291,
+      0.39089804887771606,
+      0.3946591317653656,
+      0.40174394845962524,
+      0.4042251408100128,
+      0.4049514830112457,
+      0.4074190557003021,
+      0.40427401661872864,
+      0.40763089060783386,
+      0.4119054079055786,
+      0.4136999249458313,
+      0.4179035425186157,
+      0.41082441806793213,
+      0.4018653929233551,
+      0.40929460525512695,
+      0.4156552255153656,
+      0.42034420371055603,
+      0.4155116379261017,
+      0.41802284121513367,
+      0.4181014895439148,
+      0.4214741289615631,
+      0.4200770854949951,
+      0.42327263951301575,
+      0.42632073163986206,
+      0.4284557104110718,
+      0.4266175329685211,
+      0.4236676096916199,
+      0.42971184849739075,
+      0.42890140414237976,
+      0.4290348291397095,
+      0.43162739276885986,
+      0.43444693088531494,
+      0.43579620122909546,
+      0.4397517740726471,
+      0.4428679943084717,
+      0.4437721073627472,
+      0.4461991488933563,
+      0.4475502669811249,
+      0.4475717544555664,
+      0.44808509945869446,
+      0.4483948051929474
+    ],
+    "lr": [
+      9.800000000000001e-06,
+      1.98e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9866776315789477e-05,
+      1.9126644736842107e-05,
+      1.8386513157894737e-05,
+      1.754276315789474e-05,
+      1.680263157894737e-05,
+      1.60625e-05,
+      1.521875e-05,
+      1.4478618421052634e-05,
+      1.3738486842105263e-05,
+      1.2894736842105262e-05,
+      1.2154605263157893e-05,
+      1.141447368421053e-05,
+      1.0570723684210526e-05,
+      9.830592105263156e-06,
+      9.090460526315788e-06,
+      8.246710526315791e-06,
+      7.5065789473684214e-06,
+      6.766447368421052e-06,
+      5.922697368421056e-06,
+      5.182565789473686e-06,
+      4.442434210526316e-06,
+      3.598684210526319e-06,
+      2.858552631578949e-06,
+      2.1184210526315796e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      150,
+      307,
+      464,
+      621,
+      778,
+      935,
+      1092,
+      1199,
+      1356,
+      1513,
+      1670,
+      1827,
+      1984,
+      2141,
+      2298,
+      2455,
+      2612,
+      2769,
+      2926,
+      3083
+    ],
+    "eval_accuracy": [
+      0.01,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.02,
+      0.01,
+      0.01,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.01,
+      0.0,
+      0.0,
+      0.01,
+      0.0,
+      0.01
+    ]
+  },
+  "final_accuracy": 0.004166666666666667,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.20330578512396694,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.968421052631579,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.24019607843137256,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.1242603550295858,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.5806451612903226,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.19932432432432431,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.34355828220858897,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.11538461538461539,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.4942528735632184,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.30049261083743845,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.5470085470085471,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4132231404958678,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.05785123966942149,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.7959183673469388,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.2849462365591398,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.4349775784753363,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.41346153846153844,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.11320754716981132,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.7391304347826086,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.35,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.504885993485342,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.46,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.02,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.15,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.0675,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.09,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.61,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.61,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.25279642058165547,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.1125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.5357142857142857,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.22117202268431002,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.4791666666666667,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2633333333333333,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.10362694300518134,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.14018691588785046,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.375,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.04,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.08984375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.22916666666666666,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.58,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.11764705882352941,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.3247422680412371,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.19,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.24043715846994534,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8589743589743589,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.20465890183028287,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3835125448028674,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.09523809523809523,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6150234741784038,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.16574585635359115,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7597765363128491,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.12080536912751678,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.3,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.31,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3616666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.12072892938496584,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3333333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.17766497461928935,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.145748987854251,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.11073825503355705,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.00375,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.21487603305785125,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.968421052631579,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2647058823529412,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.09467455621301775,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.9032258064516129,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.23310810810810811,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.37423312883435583,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.13846153846153847,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.5862068965517241,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.31527093596059114,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.4188034188034188,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.38016528925619836,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.049586776859504134,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.4489795918367347,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.3602150537634409,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.5022421524663677,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.38461538461538464,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.10377358490566038,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.5217391304347826,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.45,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.5276872964169381,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.04,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.2,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.17,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.09,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.58,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.578,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2639821029082774,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.096875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.7142857142857143,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.2495274102079395,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.2916666666666667,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2733333333333333,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.06,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.11917098445595854,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.205607476635514,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.395,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.03,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.08984375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.18055555555555555,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.58,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.08,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.13071895424836602,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.29381443298969073,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.16,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.2540983606557377,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.7264957264957265,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.2079866888519135,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.98989898989899,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3870967741935484,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.10317460317460317,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6244131455399061,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.18232044198895028,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7597765363128491,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.1610738255033557,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.009389671361502348,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.505,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.32,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.01,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.35,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.01,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.365,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.14350797266514806,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.024390243902439025,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3333333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.17766497461928935,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.009708737864077669,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.15789473684210525,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.12751677852348994,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.0049504950495049506,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.004166666666666667,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.004166666666666667,
+  "sft_overall_accuracy": 0.00375
+}

add_sub_sorl_v1_abs30_K1_10K_1L2H256d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d5218082ff78d3e42a63703a5156bf74725c1ab51582515c47fa306e5f445f1
+size 315132124

add_sub_sorl_v1_abs30_K1_10K_1L2H256d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 2e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K1_10K_1L2H256d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 1,
+  "n_head": 2,
+  "n_embd": 256,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 10000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 78706688,
+  "run_name": "add_sub_sorl_v1_abs30_K1_10K_1L2H256d",
+  "git_commit": "7c8a203ce79a277a1e41a3ec0648cb73d5b2b760",
+  "timestamp": "2026-04-14T01:48:03.866710+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "uj04p1pk",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/uj04p1pk",
+  "final_accuracy": 0.004166666666666667,
+  "sft_accuracy": 0.00375,
+  "eval_method": "ArithmeticEvaluator"
+}