Upload add_sub_sorl_v1_abs30_K1_10K_2L1H128d

add_sub_sorl_v1_abs30_K1_10K_2L1H128d/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 1,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_K1_10K_2L1H128d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_K1_10K_2L1H128d/metrics.json

@@ -0,0 +1,1557 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      207,
+      257,
+      307,
+      364,
+      414,
+      464,
+      521,
+      571,
+      621,
+      678,
+      728,
+      778,
+      835,
+      885,
+      935,
+      992,
+      1042,
+      1092,
+      1149,
+      1199,
+      1249,
+      1306,
+      1356,
+      1406,
+      1463,
+      1513,
+      1563,
+      1620,
+      1670,
+      1720,
+      1777,
+      1827,
+      1877,
+      1934,
+      1984,
+      2034,
+      2091,
+      2141,
+      2191,
+      2248,
+      2298,
+      2348,
+      2405,
+      2455,
+      2505,
+      2562,
+      2612,
+      2662,
+      2719,
+      2769,
+      2819,
+      2876,
+      2926,
+      2976,
+      3033,
+      3083,
+      3133
+    ],
+    "loss": [
+      11.109960556030273,
+      9.487344741821289,
+      11.728270530700684,
+      13.046238899230957,
+      12.526779174804688,
+      11.766132354736328,
+      10.708438873291016,
+      9.716384887695312,
+      9.139434814453125,
+      8.302337646484375,
+      7.700814247131348,
+      7.2936859130859375,
+      6.779129981994629,
+      6.4132561683654785,
+      6.046442031860352,
+      5.605899333953857,
+      5.361966133117676,
+      4.989933013916016,
+      4.719845294952393,
+      4.400616645812988,
+      4.159879207611084,
+      3.9028968811035156,
+      3.719845771789551,
+      3.6537272930145264,
+      3.5202412605285645,
+      3.3910393714904785,
+      3.424415111541748,
+      3.3224616050720215,
+      3.213174343109131,
+      3.204472064971924,
+      3.1120765209198,
+      3.1674718856811523,
+      3.0288727283477783,
+      2.898710250854492,
+      3.0461621284484863,
+      2.9120559692382812,
+      2.9968926906585693,
+      2.7610628604888916,
+      2.805112600326538,
+      2.8641648292541504,
+      2.6749207973480225,
+      2.742555618286133,
+      2.5006370544433594,
+      2.5789735317230225,
+      2.5932741165161133,
+      2.3684425354003906,
+      2.1633200645446777,
+      1.9918112754821777,
+      1.792123794555664,
+      1.8161230087280273,
+      1.2393646240234375,
+      1.4787614345550537,
+      1.1252933740615845,
+      1.4857960939407349,
+      0.9767551422119141,
+      0.9414909482002258,
+      1.2717292308807373,
+      1.1875169277191162,
+      0.974738597869873,
+      0.9622255563735962
+    ],
+    "base_loss": [
+      11.86424446105957,
+      11.58115291595459,
+      11.0537748336792,
+      10.185273170471191,
+      9.717636108398438,
+      9.179335594177246,
+      8.59729290008545,
+      7.95011568069458,
+      7.552673816680908,
+      6.889076232910156,
+      6.402405738830566,
+      6.064190864562988,
+      5.633248329162598,
+      5.255946636199951,
+      4.898728370666504,
+      4.464768409729004,
+      4.268970489501953,
+      3.9639620780944824,
+      3.6311986446380615,
+      3.3756792545318604,
+      3.146327495574951,
+      2.862344980239868,
+      2.6884384155273438,
+      2.6278915405273438,
+      2.5437910556793213,
+      2.4695563316345215,
+      2.347290277481079,
+      2.332256317138672,
+      2.310865640640259,
+      2.2869811058044434,
+      2.1933040618896484,
+      2.2128100395202637,
+      2.1955173015594482,
+      2.168916940689087,
+      2.2345030307769775,
+      2.1226439476013184,
+      2.208935022354126,
+      2.1581814289093018,
+      2.1644232273101807,
+      2.1009058952331543,
+      2.1047756671905518,
+      2.0930335521698,
+      2.0997116565704346,
+      2.1156744956970215,
+      2.1169261932373047,
+      2.1085855960845947,
+      2.0818076133728027,
+      2.14859938621521,
+      2.0613577365875244,
+      2.130605459213257,
+      2.01482892036438,
+      2.1084914207458496,
+      2.082641124725342,
+      2.1299586296081543,
+      2.1353821754455566,
+      2.0989015102386475,
+      2.130192995071411,
+      2.1105315685272217,
+      2.1727616786956787,
+      2.1143510341644287
+    ],
+    "info_loss": [
+      -0.801727294921875,
+      -0.8869190216064453,
+      -0.5174455642700195,
+      -0.17232799530029297,
+      -0.08138847351074219,
+      -0.033415794372558594,
+      -0.022771835327148438,
+      -0.020429134368896484,
+      -0.013097763061523438,
+      -0.009570598602294922,
+      -0.010950565338134766,
+      -0.011286258697509766,
+      -0.015323638916015625,
+      -0.010664939880371094,
+      -0.009275436401367188,
+      -0.0075473785400390625,
+      -0.011253833770751953,
+      -0.016321420669555664,
+      -0.009073734283447266,
+      -0.014719724655151367,
+      -0.015111207962036133,
+      -0.011531829833984375,
+      -0.011838674545288086,
+      -0.011916875839233398,
+      -0.01653766632080078,
+      -0.021875381469726562,
+      -0.0058193206787109375,
+      -0.014473438262939453,
+      -0.022913217544555664,
+      -0.021147489547729492,
+      -0.02110123634338379,
+      -0.017413616180419922,
+      -0.02963995933532715,
+      -0.03967618942260742,
+      -0.030973196029663086,
+      -0.033582448959350586,
+      -0.033734798431396484,
+      -0.05214858055114746,
+      -0.048038482666015625,
+      -0.03609871864318848,
+      -0.0551145076751709,
+      -0.04743504524230957,
+      -0.07212638854980469,
+      -0.065765380859375,
+      -0.06428670883178711,
+      -0.08607649803161621,
+      -0.1040034294128418,
+      -0.1276688575744629,
+      -0.13939201831817627,
+      -0.14394164085388184,
+      -0.1898949146270752,
+      -0.17540061473846436,
+      -0.20797789096832275,
+      -0.17642903327941895,
+      -0.22839045524597168,
+      -0.2279418706893921,
+      -0.19848155975341797,
+      -0.20455431938171387,
+      -0.2321758270263672,
+      -0.2275463342666626
+    ],
+    "abs_loss": [
+      3.400404691696167,
+      3.3852317333221436,
+      3.3510189056396484,
+      3.2838191986083984,
+      3.229877471923828,
+      3.163647413253784,
+      3.0793673992156982,
+      3.0087764263153076,
+      2.875743865966797,
+      2.81463623046875,
+      2.793081521987915,
+      2.756037473678589,
+      2.7811977863311768,
+      2.746596336364746,
+      2.7444820404052734,
+      2.71988844871521,
+      2.747046709060669,
+      2.710127830505371,
+      2.7222626209259033,
+      2.735506057739258,
+      2.739694356918335,
+      2.7228400707244873,
+      2.7128937244415283,
+      2.7144927978515625,
+      2.7234694957733154,
+      2.7356929779052734,
+      2.7130095958709717,
+      2.736820936203003,
+      2.7188408374786377,
+      2.709321975708008,
+      2.731154203414917,
+      2.7337539196014404,
+      2.7499465942382812,
+      2.7255847454071045,
+      2.6828200817108154,
+      2.7293548583984375,
+      2.7345211505889893,
+      2.7277286052703857,
+      2.6977577209472656,
+      2.733896255493164,
+      2.709213972091675,
+      2.7374496459960938,
+      2.722898483276367,
+      2.7119290828704834,
+      2.696190118789673,
+      2.7109363079071045,
+      2.721041679382324,
+      2.7053773403167725,
+      2.75048828125,
+      2.748612642288208,
+      2.732274293899536,
+      2.737445831298828,
+      2.718538999557495,
+      2.693471670150757,
+      2.744067430496216,
+      2.7114646434783936,
+      2.7530784606933594,
+      2.7140703201293945,
+      2.7263448238372803,
+      2.7224409580230713
+    ],
+    "zipf_loss": [
+      6.922948837280273,
+      6.436858654022217,
+      5.513849258422852,
+      4.255864143371582,
+      3.3000402450561523,
+      2.6045901775360107,
+      2.0309271812438965,
+      1.6696820259094238,
+      1.4301646947860718,
+      1.227504014968872,
+      1.1286059617996216,
+      1.06675386428833,
+      1.0209985971450806,
+      0.9892993569374084,
+      0.966019868850708,
+      0.94461590051651,
+      0.930829644203186,
+      0.9181725382804871,
+      0.9071574807167053,
+      0.898584246635437,
+      0.8906944990158081,
+      0.8835861086845398,
+      0.8785046339035034,
+      0.8735551834106445,
+      0.8694800138473511,
+      0.8666675090789795,
+      0.8640170097351074,
+      0.8612576127052307,
+      0.8595567941665649,
+      0.8580335378646851,
+      0.8566694259643555,
+      0.8554226160049438,
+      0.8547603487968445,
+      0.8539966344833374,
+      0.8531091809272766,
+      0.8523011803627014,
+      0.851853609085083,
+      0.8515943288803101,
+      0.851298451423645,
+      0.8508565425872803,
+      0.8503687381744385,
+      0.8501275777816772,
+      0.8498992919921875,
+      0.8497598767280579,
+      0.8495959639549255,
+      0.8495282530784607,
+      0.849442720413208,
+      0.8493627309799194,
+      0.8496373295783997,
+      0.8500726819038391,
+      0.850257396697998,
+      0.8505315780639648,
+      0.8505772352218628,
+      0.8507806062698364,
+      0.8508707880973816,
+      0.850861668586731,
+      0.8510439991950989,
+      0.8511215448379517,
+      0.8511006832122803,
+      0.8510937690734863
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.7170330882072449,
+      0.6676084399223328,
+      0.4536101818084717,
+      0.3363282084465027,
+      0.2717445492744446,
+      0.25135740637779236,
+      0.28221043944358826,
+      0.30178341269493103,
+      0.31571823358535767,
+      0.3284376859664917,
+      0.3331630527973175,
+      0.34331825375556946,
+      0.35866957902908325,
+      0.36964812874794006,
+      0.3801919221878052,
+      0.37727099657058716,
+      0.3913383483886719,
+      0.39448729157447815,
+      0.3997330367565155,
+      0.4072052836418152,
+      0.3953169882297516,
+      0.3841864764690399,
+      0.3772575259208679,
+      0.3756064772605896,
+      0.3684244751930237,
+      0.3618350923061371,
+      0.36770451068878174,
+      0.36369138956069946,
+      0.3568985164165497,
+      0.3539384603500366,
+      0.3554384112358093,
+      0.3487551510334015,
+      0.3574298024177551,
+      0.3522641360759735,
+      0.3537253141403198,
+      0.3676007091999054,
+      0.37232914566993713,
+      0.3805965781211853,
+      0.38131752610206604,
+      0.3908657133579254,
+      0.40144118666648865,
+      0.4037792682647705,
+      0.4099784791469574,
+      0.41942158341407776,
+      0.42823660373687744,
+      0.43726399540901184,
+      0.44310638308525085,
+      0.452615886926651,
+      0.4632907509803772,
+      0.4734920859336853,
+      0.4806124269962311,
+      0.4873966872692108,
+      0.49242761731147766,
+      0.49730294942855835,
+      0.5018022656440735,
+      0.5048937797546387,
+      0.5078352093696594,
+      0.5118813514709473,
+      0.5134710669517517,
+      0.5150383114814758
+    ],
+    "lr": [
+      9.800000000000001e-06,
+      1.98e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9866776315789477e-05,
+      1.9126644736842107e-05,
+      1.8386513157894737e-05,
+      1.754276315789474e-05,
+      1.680263157894737e-05,
+      1.60625e-05,
+      1.521875e-05,
+      1.4478618421052634e-05,
+      1.3738486842105263e-05,
+      1.2894736842105262e-05,
+      1.2154605263157893e-05,
+      1.141447368421053e-05,
+      1.0570723684210526e-05,
+      9.830592105263156e-06,
+      9.090460526315788e-06,
+      8.246710526315791e-06,
+      7.5065789473684214e-06,
+      6.766447368421052e-06,
+      5.922697368421056e-06,
+      5.182565789473686e-06,
+      4.442434210526316e-06,
+      3.598684210526319e-06,
+      2.858552631578949e-06,
+      2.1184210526315796e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      150,
+      307,
+      464,
+      621,
+      778,
+      935,
+      1092,
+      1199,
+      1356,
+      1513,
+      1670,
+      1827,
+      1984,
+      2141,
+      2298,
+      2455,
+      2612,
+      2769,
+      2926,
+      3083
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.02,
+      0.01,
+      0.0,
+      0.0,
+      0.01,
+      0.0,
+      0.0,
+      0.02,
+      0.0,
+      0.0,
+      0.01,
+      0.01,
+      0.01
+    ]
+  },
+  "final_accuracy": 0.00125,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2066115702479339,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9052631578947369,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.23529411764705882,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.04142011834319527,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.9032258064516129,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.10472972972972973,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.39263803680981596,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.06923076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.7701149425287356,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.1724137931034483,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.26495726495726496,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4793388429752066,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.024793388429752067,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.8571428571428571,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.1881720430107527,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.2062780269058296,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4807692307692308,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.04716981132075472,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.6956521739130435,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.16875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.2899022801302932,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.42,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.05,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.14,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.19,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.04,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.22,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.214,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2483221476510067,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.034375,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.7678571428571429,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.11720226843100189,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.16666666666666666,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.24,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.03,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.025906735751295335,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.14953271028037382,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.34,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.03,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.01953125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.1597222222222222,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.06,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.03594771241830065,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.18041237113402062,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.07650273224043716,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.28205128205128205,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.20465890183028287,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3835125448028674,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.09523809523809523,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6150234741784038,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.16574585635359115,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7597765363128491,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.12080536912751678,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.3,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.31,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3616666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.12072892938496584,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3333333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.17766497461928935,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.145748987854251,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.11073825503355705,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.0016666666666666668,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.21983471074380165,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.5684210526315789,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.27941176470588236,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.21301775147928995,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.4838709677419355,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.22297297297297297,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.3067484662576687,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.18461538461538463,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.3218390804597701,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.2512315270935961,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.5726495726495726,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.3305785123966942,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.15702479338842976,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.4897959183673469,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.22043010752688172,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.5201793721973094,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.36538461538461536,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.1509433962264151,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.2608695652173913,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.3,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.4527687296416938,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.5,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.14,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.28,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.03,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.15,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.15,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2595078299776286,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.225,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.5178571428571429,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.22684310018903592,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.625,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.32666666666666666,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.18,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.11398963730569948,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.6074766355140186,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.42,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.22,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.109375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.7638888888888888,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.63,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.21,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.13071895424836602,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.6752577319587629,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.21,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.23497267759562843,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.7136752136752137,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.23460898502495842,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.4014336917562724,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.006896551724137931,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9583333333333334,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.12301587301587301,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6197183098591549,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.008849557522123894,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.20441988950276244,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.009259259259259259,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7653631284916201,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.019417475728155338,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.1476510067114094,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.009389671361502348,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.52,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.02,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.39,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.38666666666666666,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.033707865168539325,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9622641509433962,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.16173120728929385,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.36,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.02,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.19796954314720813,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.009708737864077669,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.515,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.01,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.17813765182186234,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.0196078431372549,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.15436241610738255,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.01485148514851485,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.00125,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.00125,
+  "sft_overall_accuracy": 0.0016666666666666668
+}

add_sub_sorl_v1_abs30_K1_10K_2L1H128d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6472ccf29af6fd4a4dbd18dfd61ef27a518076dbed42e6ade264b2a1ae69f11
+size 157722580

add_sub_sorl_v1_abs30_K1_10K_2L1H128d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 2e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K1_10K_2L1H128d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 1,
+  "n_embd": 128,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 10000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 39353984,
+  "run_name": "add_sub_sorl_v1_abs30_K1_10K_2L1H128d",
+  "git_commit": "7c8a203ce79a277a1e41a3ec0648cb73d5b2b760",
+  "timestamp": "2026-04-14T01:48:03.511665+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "c18bkhub",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/c18bkhub",
+  "final_accuracy": 0.00125,
+  "sft_accuracy": 0.0016666666666666668,
+  "eval_method": "ArithmeticEvaluator"
+}