Upload add_sub_sorl_v1_abs30_K1_50K_1L2H256d

add_sub_sorl_v1_abs30_K1_50K_1L2H256d/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "layer_types": [
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 2,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_K1_50K_1L2H256d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_K1_50K_1L2H256d/metrics.json

@@ -0,0 +1,3477 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1182,
+      1232,
+      1282,
+      1332,
+      1382,
+      1432,
+      1482,
+      1532,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      1964,
+      2014,
+      2064,
+      2114,
+      2164,
+      2214,
+      2264,
+      2314,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2746,
+      2796,
+      2846,
+      2896,
+      2946,
+      2996,
+      3046,
+      3096,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3528,
+      3578,
+      3628,
+      3678,
+      3728,
+      3778,
+      3828,
+      3878,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4310,
+      4360,
+      4410,
+      4460,
+      4510,
+      4560,
+      4610,
+      4660,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5092,
+      5142,
+      5192,
+      5242,
+      5292,
+      5342,
+      5392,
+      5442,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5874,
+      5924,
+      5974,
+      6024,
+      6074,
+      6124,
+      6174,
+      6224,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6656,
+      6706,
+      6756,
+      6806,
+      6856,
+      6906,
+      6956,
+      7006,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7438,
+      7488,
+      7538,
+      7588,
+      7638,
+      7688,
+      7738,
+      7788,
+      7870,
+      7920,
+      7970,
+      8020,
+      8070,
+      8120,
+      8170,
+      8220,
+      8270,
+      8320,
+      8370,
+      8420,
+      8470,
+      8520,
+      8570,
+      8652,
+      8702,
+      8752,
+      8802,
+      8852,
+      8902,
+      8952,
+      9002,
+      9052,
+      9102,
+      9152,
+      9202,
+      9252,
+      9302,
+      9352,
+      9434,
+      9484,
+      9534,
+      9584,
+      9634,
+      9684,
+      9734,
+      9784,
+      9834,
+      9884,
+      9934,
+      9984,
+      10034,
+      10084,
+      10134,
+      10216,
+      10266,
+      10316,
+      10366,
+      10416,
+      10466,
+      10516,
+      10566,
+      10616,
+      10666,
+      10716,
+      10766,
+      10816,
+      10866,
+      10916,
+      10998,
+      11048,
+      11098,
+      11148,
+      11198,
+      11248,
+      11298,
+      11348,
+      11398,
+      11448,
+      11498,
+      11548,
+      11598,
+      11648,
+      11698,
+      11780,
+      11830,
+      11880,
+      11930,
+      11980,
+      12030,
+      12080,
+      12130,
+      12180,
+      12230,
+      12280,
+      12330,
+      12380,
+      12430,
+      12480,
+      12562,
+      12612,
+      12662,
+      12712,
+      12762,
+      12812,
+      12862,
+      12912,
+      12962,
+      13012,
+      13062,
+      13112,
+      13162,
+      13212,
+      13262,
+      13344,
+      13394,
+      13444,
+      13494,
+      13544,
+      13594,
+      13644,
+      13694,
+      13744,
+      13794,
+      13844,
+      13894,
+      13944,
+      13994,
+      14044,
+      14126,
+      14176,
+      14226,
+      14276,
+      14326,
+      14376,
+      14426,
+      14476,
+      14526,
+      14576,
+      14626,
+      14676,
+      14726,
+      14776,
+      14826,
+      14908,
+      14958,
+      15008,
+      15058,
+      15108,
+      15158,
+      15208,
+      15258,
+      15308,
+      15358,
+      15408,
+      15458,
+      15508,
+      15558,
+      15608
+    ],
+    "loss": [
+      14.391416549682617,
+      3.5631918907165527,
+      4.951187610626221,
+      6.449922561645508,
+      8.488645553588867,
+      10.204294204711914,
+      9.95416259765625,
+      9.21035385131836,
+      8.513230323791504,
+      7.131667137145996,
+      6.307509422302246,
+      5.611210823059082,
+      4.488781929016113,
+      4.077478408813477,
+      3.661040782928467,
+      3.475904941558838,
+      3.182765483856201,
+      2.9839963912963867,
+      3.2010433673858643,
+      2.8748648166656494,
+      3.189145803451538,
+      3.0413122177124023,
+      2.904395580291748,
+      2.7328271865844727,
+      2.66978120803833,
+      2.890146493911743,
+      2.69091796875,
+      2.603858470916748,
+      2.8096790313720703,
+      2.4489550590515137,
+      2.761693000793457,
+      2.5760087966918945,
+      2.471953868865967,
+      2.6798782348632812,
+      2.621436834335327,
+      2.3029394149780273,
+      2.4322378635406494,
+      2.2531025409698486,
+      2.293125629425049,
+      2.336426258087158,
+      2.0683860778808594,
+      2.227302074432373,
+      2.241642951965332,
+      2.1144471168518066,
+      2.129608154296875,
+      2.087547540664673,
+      2.0604472160339355,
+      2.0001256465911865,
+      1.7057685852050781,
+      1.8907408714294434,
+      1.9792687892913818,
+      1.687549114227295,
+      1.6289119720458984,
+      1.8179930448532104,
+      1.7804032564163208,
+      1.5129382610321045,
+      1.6116085052490234,
+      1.6659080982208252,
+      1.592993974685669,
+      1.373729944229126,
+      1.379267692565918,
+      1.2287101745605469,
+      1.3369431495666504,
+      1.3494970798492432,
+      1.3358020782470703,
+      1.4309417009353638,
+      1.1844741106033325,
+      1.1096996068954468,
+      0.5259584784507751,
+      -2.2604856491088867,
+      -3.607330322265625,
+      -4.623205184936523,
+      -4.328221797943115,
+      -4.393834114074707,
+      -4.841094493865967,
+      -5.207284450531006,
+      -5.413675308227539,
+      -5.634337902069092,
+      -6.7068257331848145,
+      -6.329562664031982,
+      -6.355160713195801,
+      -6.415679454803467,
+      -6.294429302215576,
+      -7.243396759033203,
+      -6.313797950744629,
+      -6.53245210647583,
+      -6.6371941566467285,
+      -6.713075637817383,
+      -6.495677471160889,
+      -5.88437557220459,
+      -6.0675129890441895,
+      -5.314021587371826,
+      -5.503685474395752,
+      -4.809330463409424,
+      -4.855256080627441,
+      -5.123812675476074,
+      -4.655849933624268,
+      -5.150340557098389,
+      -5.320953369140625,
+      -5.1980881690979,
+      -5.0288238525390625,
+      -4.825449466705322,
+      -4.180274486541748,
+      -4.567695617675781,
+      -3.942505359649658,
+      -3.569028854370117,
+      -2.1454193592071533,
+      -2.518108367919922,
+      -2.2415173053741455,
+      -1.5382757186889648,
+      -1.4687309265136719,
+      -1.001907467842102,
+      -1.111462950706482,
+      -1.586690068244934,
+      -0.7100228071212769,
+      -1.108686923980713,
+      -1.1059621572494507,
+      -0.5984665751457214,
+      -0.6540563106536865,
+      -1.1424003839492798,
+      -0.777918815612793,
+      -1.0127521753311157,
+      -0.8394446969032288,
+      -0.6261311769485474,
+      -0.36724036931991577,
+      -0.6708923578262329,
+      -1.723320722579956,
+      -0.9385681748390198,
+      -0.3866307735443115,
+      -0.2762378454208374,
+      -0.4718437194824219,
+      -0.5532834529876709,
+      -0.4075008034706116,
+      -0.12833631038665771,
+      -0.3441540598869324,
+      -0.5667827129364014,
+      -0.4476921558380127,
+      -0.5119231939315796,
+      -0.45889803767204285,
+      -0.4410821199417114,
+      -0.5271980166435242,
+      -0.44913676381111145,
+      -1.14522385597229,
+      0.0020051896572113037,
+      -0.9715392589569092,
+      -0.43752121925354004,
+      -0.3958086669445038,
+      -1.1545329093933105,
+      -0.3098324239253998,
+      -0.3537815809249878,
+      -0.556914210319519,
+      -0.2648468017578125,
+      -0.47355762124061584,
+      -0.6534526348114014,
+      -0.020323336124420166,
+      -0.4358733296394348,
+      0.03338420391082764,
+      -0.2266627550125122,
+      -0.6376452445983887,
+      0.06627464294433594,
+      -0.4153698682785034,
+      -0.009761065244674683,
+      -0.3590174913406372,
+      -0.5701332092285156,
+      -0.5826343297958374,
+      -0.2537662982940674,
+      -0.11255323886871338,
+      -0.6820588707923889,
+      -0.7961635589599609,
+      -0.25433915853500366,
+      -0.6976981163024902,
+      -0.4294131398200989,
+      -0.523033857345581,
+      -0.7449092864990234,
+      -0.8429986834526062,
+      -0.5085985660552979,
+      -0.7933173775672913,
+      -0.5322814583778381,
+      -1.130155324935913,
+      -0.7147582769393921,
+      -1.133387804031372,
+      -0.6598256826400757,
+      -0.5383756160736084,
+      -0.7017931342124939,
+      -0.8969375491142273,
+      -1.0173944234848022,
+      -0.48842576146125793,
+      -0.9904735088348389,
+      -1.013723373413086,
+      -1.0771348476409912,
+      -0.4253750443458557,
+      -0.36261194944381714,
+      -0.7754100561141968,
+      -1.5724389553070068,
+      -0.7943729758262634,
+      -0.67448890209198,
+      -0.9583285450935364,
+      -0.8540913462638855,
+      -0.6094908118247986,
+      -0.8783771991729736,
+      -1.053288459777832,
+      -0.9346593618392944,
+      -0.7244707345962524,
+      -0.6864712238311768,
+      -0.9500890970230103,
+      -0.7603346705436707,
+      -0.8142468333244324,
+      -0.7028934359550476,
+      -1.1122493743896484,
+      -0.570902407169342,
+      -1.419747233390808,
+      -0.6447935104370117,
+      -1.3018269538879395,
+      -1.1679573059082031,
+      -0.5280086994171143,
+      -0.6036947965621948,
+      -0.7594616413116455,
+      -1.0022821426391602,
+      -0.9563724994659424,
+      -0.8246185779571533,
+      -1.274404525756836,
+      -0.630048394203186,
+      -0.8341413140296936,
+      -0.9578894376754761,
+      -0.7703633308410645,
+      -0.34369131922721863,
+      -0.9039551615715027,
+      -0.4906381070613861,
+      -0.19011813402175903,
+      -1.0604653358459473,
+      -1.1517562866210938,
+      -0.40949398279190063,
+      -0.6710019111633301,
+      -0.4022740125656128,
+      -0.46849262714385986,
+      -1.0708227157592773,
+      -0.6519597172737122,
+      -0.4212917387485504,
+      -1.0008949041366577,
+      -0.749737024307251,
+      -0.7541130781173706,
+      -0.8097407817840576,
+      -0.7233744263648987,
+      -0.4428720474243164,
+      -0.6528443098068237,
+      -0.7829169631004333,
+      -0.8197828531265259,
+      -0.6912307739257812,
+      -0.30136483907699585,
+      -0.9919260740280151,
+      -0.9708343744277954,
+      -0.7386912107467651,
+      -0.7249183654785156,
+      -0.8299386501312256,
+      -0.8190449476242065,
+      -0.4721378982067108,
+      -0.49599573016166687,
+      -0.6839645504951477,
+      -0.7676907777786255,
+      -0.5677669048309326,
+      -0.7430678606033325,
+      -0.8484114408493042,
+      -0.418980211019516,
+      -0.8182215690612793,
+      -0.8707810640335083,
+      -0.5657194256782532,
+      -0.5225734710693359,
+      -0.8062321543693542,
+      -0.838819146156311,
+      -0.8150172829627991,
+      -0.8889663219451904,
+      -0.6185510754585266,
+      -0.6984368562698364,
+      -0.6861761808395386,
+      -0.7360283136367798,
+      -0.6883683204650879,
+      -0.7246119976043701,
+      -0.5589756369590759,
+      -0.9278964400291443,
+      -0.8617536425590515,
+      -0.4508713483810425,
+      -0.7350199222564697,
+      -1.2585868835449219,
+      -0.6881809234619141,
+      -0.7309094071388245,
+      -0.8188521265983582,
+      -0.8475762605667114,
+      -0.5263950824737549,
+      -0.8811346292495728,
+      -0.6607277393341064,
+      -0.5513431429862976,
+      -0.48902446031570435,
+      -0.6575115919113159,
+      -0.5540645718574524,
+      -0.4536811411380768,
+      -0.9326189756393433,
+      -0.5906654596328735,
+      -0.84079509973526,
+      -0.651555061340332,
+      -0.9806141257286072
+    ],
+    "base_loss": [
+      11.924899101257324,
+      11.723952293395996,
+      11.398698806762695,
+      10.871347427368164,
+      10.233444213867188,
+      9.417863845825195,
+      8.688928604125977,
+      8.107666015625,
+      7.038910388946533,
+      5.875870227813721,
+      5.116110801696777,
+      4.0680131912231445,
+      3.3122782707214355,
+      2.7461562156677246,
+      2.4748570919036865,
+      2.1471264362335205,
+      2.137270212173462,
+      2.0353143215179443,
+      2.1049296855926514,
+      2.0106492042541504,
+      1.9871169328689575,
+      1.9782869815826416,
+      2.0120389461517334,
+      1.8657915592193604,
+      2.0052971839904785,
+      1.9042764902114868,
+      1.9087363481521606,
+      1.9000881910324097,
+      1.8631047010421753,
+      1.8891297578811646,
+      1.895875096321106,
+      1.8685376644134521,
+      1.8269975185394287,
+      1.9283578395843506,
+      1.8260129690170288,
+      1.8352664709091187,
+      1.9043818712234497,
+      1.8668466806411743,
+      1.784292221069336,
+      1.905230164527893,
+      1.8551198244094849,
+      1.9101474285125732,
+      1.871077537536621,
+      1.7832814455032349,
+      1.8210866451263428,
+      1.8906786441802979,
+      1.9226036071777344,
+      1.89806067943573,
+      1.847821593284607,
+      1.8466213941574097,
+      1.7649110555648804,
+      1.7653284072875977,
+      1.836822509765625,
+      1.8643038272857666,
+      1.8479230403900146,
+      1.8355034589767456,
+      1.807110071182251,
+      1.803357720375061,
+      1.8478851318359375,
+      1.9093502759933472,
+      1.8495131731033325,
+      1.8146511316299438,
+      1.8493468761444092,
+      1.8756006956100464,
+      1.8318966627120972,
+      1.8224912881851196,
+      1.8053219318389893,
+      1.8393962383270264,
+      1.8298470973968506,
+      1.8860799074172974,
+      1.8958958387374878,
+      1.8421496152877808,
+      1.8296852111816406,
+      1.8428579568862915,
+      1.8259555101394653,
+      1.750936508178711,
+      1.8683357238769531,
+      1.8350151777267456,
+      1.8205277919769287,
+      1.8780237436294556,
+      1.8322246074676514,
+      1.8036011457443237,
+      1.8355664014816284,
+      1.849505066871643,
+      1.7176713943481445,
+      1.6869553327560425,
+      1.7476214170455933,
+      1.716267466545105,
+      1.699992060661316,
+      1.6309915781021118,
+      1.6224710941314697,
+      1.487425684928894,
+      1.4332735538482666,
+      1.4170725345611572,
+      1.3630326986312866,
+      1.4229387044906616,
+      1.3460227251052856,
+      1.444375991821289,
+      1.4238561391830444,
+      1.4174271821975708,
+      1.3455342054367065,
+      1.3162670135498047,
+      1.2639325857162476,
+      1.2787516117095947,
+      1.2053501605987549,
+      1.16716468334198,
+      1.0392616987228394,
+      1.0432214736938477,
+      0.9838384985923767,
+      0.9466244578361511,
+      0.9904718399047852,
+      0.9184152483940125,
+      0.8530594706535339,
+      0.9338607788085938,
+      0.8378226161003113,
+      0.8550710678100586,
+      0.8792905211448669,
+      0.8027544021606445,
+      0.7987598776817322,
+      0.8841131925582886,
+      0.7831162214279175,
+      0.850803792476654,
+      0.7675236463546753,
+      0.747257649898529,
+      0.7640300393104553,
+      0.791539192199707,
+      0.8919728398323059,
+      0.7920830845832825,
+      0.6609529852867126,
+      0.6637541651725769,
+      0.7081150412559509,
+      0.7386700510978699,
+      0.6769164800643921,
+      0.6595610976219177,
+      0.6708240509033203,
+      0.7412353157997131,
+      0.7429394721984863,
+      0.6543482542037964,
+      0.6949909329414368,
+      0.6949244737625122,
+      0.6885804533958435,
+      0.654236912727356,
+      0.777008593082428,
+      0.6214011311531067,
+      0.7373843789100647,
+      0.6504990458488464,
+      0.6401084065437317,
+      0.7470434904098511,
+      0.620930552482605,
+      0.6475781798362732,
+      0.6670676469802856,
+      0.6590506434440613,
+      0.6546745896339417,
+      0.6769332885742188,
+      0.6282700300216675,
+      0.6276233792304993,
+      0.5675684213638306,
+      0.6006556749343872,
+      0.6204550862312317,
+      0.542327880859375,
+      0.6675956845283508,
+      0.5832991003990173,
+      0.6019770503044128,
+      0.6386604905128479,
+      0.6233824491500854,
+      0.5370963215827942,
+      0.5549455881118774,
+      0.6074023842811584,
+      0.6232832074165344,
+      0.5702750086784363,
+      0.6114638447761536,
+      0.5453554391860962,
+      0.5428765416145325,
+      0.5708634853363037,
+      0.6227331161499023,
+      0.6019042730331421,
+      0.631923258304596,
+      0.5396310091018677,
+      0.6403390169143677,
+      0.5972115397453308,
+      0.6569903492927551,
+      0.5579837560653687,
+      0.5644631385803223,
+      0.5502129793167114,
+      0.5689250826835632,
+      0.574603259563446,
+      0.5738130211830139,
+      0.6212136149406433,
+      0.5588890314102173,
+      0.6067931056022644,
+      0.5497685670852661,
+      0.5222062468528748,
+      0.6073097586631775,
+      0.6319923996925354,
+      0.5215237736701965,
+      0.5329961776733398,
+      0.5782395601272583,
+      0.5132139325141907,
+      0.5413160920143127,
+      0.5374998450279236,
+      0.5348051190376282,
+      0.5208116769790649,
+      0.5224066376686096,
+      0.4826173484325409,
+      0.5613459944725037,
+      0.49923673272132874,
+      0.5067140460014343,
+      0.4510643184185028,
+      0.5341215133666992,
+      0.4605996906757355,
+      0.5513842701911926,
+      0.47830620408058167,
+      0.5490735769271851,
+      0.5239264369010925,
+      0.4614405035972595,
+      0.45354822278022766,
+      0.48486772179603577,
+      0.46608585119247437,
+      0.501822292804718,
+      0.47562360763549805,
+      0.5022934675216675,
+      0.45520883798599243,
+      0.46791940927505493,
+      0.4730868339538574,
+      0.49775880575180054,
+      0.3982391953468323,
+      0.479936420917511,
+      0.4676642417907715,
+      0.42688947916030884,
+      0.5011295676231384,
+      0.5122114419937134,
+      0.46522215008735657,
+      0.4603978991508484,
+      0.4099425673484802,
+      0.45614224672317505,
+      0.4675729274749756,
+      0.4542088508605957,
+      0.43050605058670044,
+      0.4920247495174408,
+      0.4424178898334503,
+      0.4150465130805969,
+      0.421188086271286,
+      0.43084797263145447,
+      0.46525558829307556,
+      0.4048364460468292,
+      0.4880073666572571,
+      0.4726863503456116,
+      0.4484849274158478,
+      0.42517024278640747,
+      0.43668434023857117,
+      0.45826688408851624,
+      0.4273340404033661,
+      0.4541681706905365,
+      0.45403358340263367,
+      0.4339158535003662,
+      0.3925251066684723,
+      0.36431366205215454,
+      0.40097665786743164,
+      0.41362708806991577,
+      0.39811959862709045,
+      0.4510500133037567,
+      0.41184160113334656,
+      0.3767947554588318,
+      0.4278729259967804,
+      0.4308449327945709,
+      0.3813810348510742,
+      0.39527395367622375,
+      0.4462609887123108,
+      0.40433549880981445,
+      0.38657346367836,
+      0.4280848503112793,
+      0.4154835641384125,
+      0.4190816879272461,
+      0.41878581047058105,
+      0.39313843846321106,
+      0.42591312527656555,
+      0.37987756729125977,
+      0.4118383824825287,
+      0.37164121866226196,
+      0.4452041983604431,
+      0.37850984930992126,
+      0.41417890787124634,
+      0.4505740702152252,
+      0.37613436579704285,
+      0.38889315724372864,
+      0.4040418267250061,
+      0.4200188219547272,
+      0.36059871315956116,
+      0.3883480131626129,
+      0.36945992708206177,
+      0.40264028310775757,
+      0.37211281061172485,
+      0.3875214159488678,
+      0.38397952914237976,
+      0.3648608326911926,
+      0.40167173743247986,
+      0.3840717673301697,
+      0.3961177468299866,
+      0.38438430428504944,
+      0.41754111647605896
+    ],
+    "info_loss": [
+      -0.4852561950683594,
+      -1.5362215042114258,
+      -1.3360795974731445,
+      -1.0800065994262695,
+      -0.7235498428344727,
+      -0.3625144958496094,
+      -0.21090221405029297,
+      -0.14626646041870117,
+      -0.047283172607421875,
+      -0.029201507568359375,
+      -0.01986074447631836,
+      0.025762081146240234,
+      -0.004784345626831055,
+      0.014246463775634766,
+      0.0017428398132324219,
+      0.01796412467956543,
+      -0.009775876998901367,
+      -0.018731355667114258,
+      -0.003586292266845703,
+      -0.026581406593322754,
+      0.007454991340637207,
+      -0.006270766258239746,
+      -0.023107528686523438,
+      -0.02547895908355713,
+      -0.045369625091552734,
+      -0.01320040225982666,
+      -0.033718109130859375,
+      -0.041429996490478516,
+      -0.017133355140686035,
+      -0.0554581880569458,
+      -0.024900317192077637,
+      -0.040474653244018555,
+      -0.046842217445373535,
+      -0.036025404930114746,
+      -0.031656503677368164,
+      -0.06460857391357422,
+      -0.05854308605194092,
+      -0.07271230220794678,
+      -0.060626983642578125,
+      -0.068115234375,
+      -0.08988142013549805,
+      -0.0794142484664917,
+      -0.07371020317077637,
+      -0.07802951335906982,
+      -0.07997941970825195,
+      -0.09109866619110107,
+      -0.09716629981994629,
+      -0.10019993782043457,
+      -0.1249077320098877,
+      -0.10679495334625244,
+      -0.08955347537994385,
+      -0.11846840381622314,
+      -0.13128793239593506,
+      -0.11564624309539795,
+      -0.11755096912384033,
+      -0.14323723316192627,
+      -0.13039934635162354,
+      -0.12429356575012207,
+      -0.1360563039779663,
+      -0.16413581371307373,
+      -0.15787291526794434,
+      -0.16942834854125977,
+      -0.1620795726776123,
+      -0.16325807571411133,
+      -0.15983939170837402,
+      -0.14993155002593994,
+      -0.17240095138549805,
+      -0.18348944187164307,
+      -0.24135065078735352,
+      -0.525694727897644,
+      -0.6612787246704102,
+      -0.7576849460601807,
+      -0.727253794670105,
+      -0.7347850799560547,
+      -0.777817964553833,
+      -0.8067814707756042,
+      -0.8392993211746216,
+      -0.8573602437973022,
+      -0.9634810090065002,
+      -0.9316211342811584,
+      -0.9294146299362183,
+      -0.9322304129600525,
+      -0.9234321117401123,
+      -1.0189263820648193,
+      -0.9128316044807434,
+      -0.93157958984375,
+      -0.9472887516021729,
+      -0.9516890048980713,
+      -0.9271989464759827,
+      -0.859917938709259,
+      -0.8762422204017639,
+      -0.7881850004196167,
+      -0.8010163903236389,
+      -0.7303846478462219,
+      -0.7286319732666016,
+      -0.7610451579093933,
+      -0.7048346996307373,
+      -0.7653828263282776,
+      -0.7792306542396545,
+      -0.7650495171546936,
+      -0.7407460808753967,
+      -0.7152613401412964,
+      -0.6460129618644714,
+      -0.685269296169281,
+      -0.6157407164573669,
+      -0.5711872577667236,
+      -0.4179037809371948,
+      -0.4540950059890747,
+      -0.4186498522758484,
+      -0.3447141647338867,
+      -0.34088432788848877,
+      -0.2847004532814026,
+      -0.2906315326690674,
+      -0.3418859839439392,
+      -0.24514997005462646,
+      -0.2869766354560852,
+      -0.28775060176849365,
+      -0.22611194849014282,
+      -0.22966790199279785,
+      -0.28805381059646606,
+      -0.23918700218200684,
+      -0.2690625786781311,
+      -0.24187278747558594,
+      -0.21721965074539185,
+      -0.19330567121505737,
+      -0.22391700744628906,
+      -0.3374202847480774,
+      -0.24793052673339844,
+      -0.17834395170211792,
+      -0.16740620136260986,
+      -0.19127941131591797,
+      -0.20325052738189697,
+      -0.18047988414764404,
+      -0.14833557605743408,
+      -0.17417654395103455,
+      -0.1983361840248108,
+      -0.18814319372177124,
+      -0.18322810530662537,
+      -0.1826750636100769,
+      -0.180938720703125,
+      -0.18699252605438232,
+      -0.1770494282245636,
+      -0.25464069843292236,
+      -0.12880170345306396,
+      -0.23493343591690063,
+      -0.17019790410995483,
+      -0.1672932207584381,
+      -0.2561874985694885,
+      -0.15705853700637817,
+      -0.16154268383979797,
+      -0.18341714143753052,
+      -0.15575218200683594,
+      -0.17478322982788086,
+      -0.1943446695804596,
+      -0.12583059072494507,
+      -0.16648229956626892,
+      -0.11299821734428406,
+      -0.14203184843063354,
+      -0.18393424153327942,
+      -0.10606321692466736,
+      -0.1690567135810852,
+      -0.1175222396850586,
+      -0.15488609671592712,
+      -0.17735520005226135,
+      -0.17943593859672546,
+      -0.13784608244895935,
+      -0.1220194399356842,
+      -0.18453547358512878,
+      -0.20083913207054138,
+      -0.1392211616039276,
+      -0.18347111344337463,
+      -0.152431458234787,
+      -0.16132614016532898,
+      -0.1873718798160553,
+      -0.2024771273136139,
+      -0.16325625777244568,
+      -0.1971660852432251,
+      -0.15808692574501038,
+      -0.22949188947677612,
+      -0.18437743186950684,
+      -0.23286309838294983,
+      -0.17610090970993042,
+      -0.1628430187702179,
+      -0.17787817120552063,
+      -0.19759100675582886,
+      -0.21054720878601074,
+      -0.1594332456588745,
+      -0.2126409113407135,
+      -0.20774465799331665,
+      -0.21825644373893738,
+      -0.14734381437301636,
+      -0.13607624173164368,
+      -0.19057771563529968,
+      -0.2694430649280548,
+      -0.18067890405654907,
+      -0.1686161458492279,
+      -0.19989877939224243,
+      -0.18342506885528564,
+      -0.16349151730537415,
+      -0.1875663697719574,
+      -0.20643460750579834,
+      -0.19242647290229797,
+      -0.17184829711914062,
+      -0.16156095266342163,
+      -0.19788041710853577,
+      -0.1727835237979889,
+      -0.17848551273345947,
+      -0.16051572561264038,
+      -0.20666438341140747,
+      -0.145177960395813,
+      -0.23937800526618958,
+      -0.15502995252609253,
+      -0.2271483838558197,
+      -0.20991599559783936,
+      -0.138015478849411,
+      -0.14521247148513794,
+      -0.16553029417991638,
+      -0.1831893026828766,
+      -0.18595671653747559,
+      -0.1707151234149933,
+      -0.21581622958183289,
+      -0.14733850955963135,
+      -0.16773080825805664,
+      -0.18132540583610535,
+      -0.1644814908504486,
+      -0.11093774437904358,
+      -0.17371997237205505,
+      -0.13221615552902222,
+      -0.09757745265960693,
+      -0.19070053100585938,
+      -0.20222771167755127,
+      -0.12143585085868835,
+      -0.14701947569847107,
+      -0.11378562450408936,
+      -0.12357917428016663,
+      -0.18534263968467712,
+      -0.1403018832206726,
+      -0.11790427565574646,
+      -0.18261495232582092,
+      -0.1488949954509735,
+      -0.14864197373390198,
+      -0.15631666779518127,
+      -0.14657437801361084,
+      -0.1210206151008606,
+      -0.13719287514686584,
+      -0.15613994002342224,
+      -0.15808254480361938,
+      -0.14335060119628906,
+      -0.1013428270816803,
+      -0.1715693473815918,
+      -0.17200198769569397,
+      -0.1436348259449005,
+      -0.1455364227294922,
+      -0.15553045272827148,
+      -0.15418869256973267,
+      -0.11543875932693481,
+      -0.11266493797302246,
+      -0.13869765400886536,
+      -0.14625167846679688,
+      -0.12552720308303833,
+      -0.1475898027420044,
+      -0.15543094277381897,
+      -0.10667231678962708,
+      -0.15124014019966125,
+      -0.1579851508140564,
+      -0.12106117606163025,
+      -0.11860063672065735,
+      -0.15099164843559265,
+      -0.15018358826637268,
+      -0.14446477591991425,
+      -0.15637698769569397,
+      -0.12751513719558716,
+      -0.13896486163139343,
+      -0.1344907283782959,
+      -0.13812211155891418,
+      -0.13646647334098816,
+      -0.1367749273777008,
+      -0.1206892728805542,
+      -0.15522482991218567,
+      -0.15402817726135254,
+      -0.10994014143943787,
+      -0.14145803451538086,
+      -0.19725847244262695,
+      -0.13370127975940704,
+      -0.13590341806411743,
+      -0.1473083794116974,
+      -0.15003210306167603,
+      -0.11368443071842194,
+      -0.15070736408233643,
+      -0.12793602049350739,
+      -0.11789670586585999,
+      -0.1095186173915863,
+      -0.12841778993606567,
+      -0.12037861347198486,
+      -0.10698729753494263,
+      -0.15714435279369354,
+      -0.12295982241630554,
+      -0.14718739688396454,
+      -0.12792378664016724,
+      -0.1648429036140442
+    ],
+    "abs_loss": [
+      3.4010369777679443,
+      3.399482011795044,
+      3.3948745727539062,
+      3.386317014694214,
+      3.3454959392547607,
+      3.2859268188476562,
+      3.235483169555664,
+      3.155316114425659,
+      3.023893356323242,
+      2.85152268409729,
+      2.7710952758789062,
+      2.765094757080078,
+      2.771630048751831,
+      2.737379789352417,
+      2.7296230792999268,
+      2.727144241333008,
+      2.7412750720977783,
+      2.724933385848999,
+      2.7191240787506104,
+      2.725314140319824,
+      2.7313098907470703,
+      2.729860305786133,
+      2.730738401412964,
+      2.7214195728302,
+      2.7044355869293213,
+      2.714495897293091,
+      2.734161376953125,
+      2.734299421310425,
+      2.736513376235962,
+      2.708566904067993,
+      2.7158915996551514,
+      2.7096283435821533,
+      2.7195539474487305,
+      2.709472417831421,
+      2.715055465698242,
+      2.729790449142456,
+      2.731639862060547,
+      2.735548257827759,
+      2.755526542663574,
+      2.7325775623321533,
+      2.7375755310058594,
+      2.7285797595977783,
+      2.6932106018066406,
+      2.7392101287841797,
+      2.703423261642456,
+      2.702728271484375,
+      2.724331855773926,
+      2.6689016819000244,
+      2.6969079971313477,
+      2.753899335861206,
+      2.729174852371216,
+      2.7062008380889893,
+      2.6848065853118896,
+      2.7389657497406006,
+      2.7202701568603516,
+      2.7338593006134033,
+      2.722230911254883,
+      2.694430351257324,
+      2.6897428035736084,
+      2.694498062133789,
+      2.732853889465332,
+      2.7247676849365234,
+      2.721663236618042,
+      2.713768243789673,
+      2.6768596172332764,
+      2.723036527633667,
+      2.675872564315796,
+      2.6995155811309814,
+      2.752788543701172,
+      2.7383859157562256,
+      2.7123773097991943,
+      2.725388526916504,
+      2.7654342651367188,
+      2.7114250659942627,
+      2.7124245166778564,
+      2.7202694416046143,
+      2.7136504650115967,
+      2.6601788997650146,
+      2.709183931350708,
+      2.703763723373413,
+      2.6890392303466797,
+      2.6717214584350586,
+      2.6626927852630615,
+      2.639129400253296,
+      2.629612684249878,
+      2.654829740524292,
+      2.597086191177368,
+      2.596548318862915,
+      2.5148470401763916,
+      2.5655767917633057,
+      2.5102062225341797,
+      2.5550522804260254,
+      2.548220634460449,
+      2.5590004920959473,
+      2.5586097240448,
+      2.5141708850860596,
+      2.4883675575256348,
+      2.4766664505004883,
+      2.4209787845611572,
+      2.4112043380737305,
+      2.369302988052368,
+      2.345440149307251,
+      2.3652427196502686,
+      2.2905027866363525,
+      2.3225486278533936,
+      2.192802667617798,
+      2.271484375,
+      2.2349255084991455,
+      2.2547357082366943,
+      2.2063629627227783,
+      2.2586164474487305,
+      2.125568151473999,
+      2.219836711883545,
+      2.146446466445923,
+      2.08963680267334,
+      2.090833902359009,
+      2.131995677947998,
+      2.055699348449707,
+      2.1458804607391357,
+      2.0563697814941406,
+      2.0130813121795654,
+      2.004945755004883,
+      1.9833393096923828,
+      2.0514800548553467,
+      1.9634408950805664,
+      1.9738287925720215,
+      1.8971610069274902,
+      1.8831228017807007,
+      1.9406023025512695,
+      1.9510407447814941,
+      1.8390398025512695,
+      1.9082388877868652,
+      1.8085355758666992,
+      1.804395079612732,
+      1.8824151754379272,
+      1.7584037780761719,
+      1.8859291076660156,
+      1.7758370637893677,
+      1.8401113748550415,
+      1.7726024389266968,
+      1.7334378957748413,
+      1.7049490213394165,
+      1.64383065700531,
+      1.8038913011550903,
+      1.6608257293701172,
+      1.6587973833084106,
+      1.730268955230713,
+      1.719649314880371,
+      1.6742286682128906,
+      1.5836777687072754,
+      1.6291812658309937,
+      1.634513258934021,
+      1.6728235483169556,
+      1.5928739309310913,
+      1.596194863319397,
+      1.5726375579833984,
+      1.5716657638549805,
+      1.589880347251892,
+      1.572654366493225,
+      1.515364170074463,
+      1.5239982604980469,
+      1.518281102180481,
+      1.5202945470809937,
+      1.501430630683899,
+      1.5758165121078491,
+      1.567885398864746,
+      1.3997067213058472,
+      1.4271599054336548,
+      1.5038799047470093,
+      1.411553978919983,
+      1.3374910354614258,
+      1.3455325365066528,
+      1.3896316289901733,
+      1.4639323949813843,
+      1.3641471862792969,
+      1.4382176399230957,
+      1.3780063390731812,
+      1.2997463941574097,
+      1.2884165048599243,
+      1.3870214223861694,
+      1.3956867456436157,
+      1.4242205619812012,
+      1.4171096086502075,
+      1.3343337774276733,
+      1.2555407285690308,
+      1.3149714469909668,
+      1.3142465353012085,
+      1.3242357969284058,
+      1.2646042108535767,
+      1.2369358539581299,
+      1.2048388719558716,
+      1.3026113510131836,
+      1.2863340377807617,
+      1.240700364112854,
+      1.2186905145645142,
+      1.246376633644104,
+      1.209234595298767,
+      1.2235417366027832,
+      1.2061333656311035,
+      1.1367807388305664,
+      1.209269642829895,
+      1.2535390853881836,
+      1.136226773262024,
+      1.1071382761001587,
+      1.1248902082443237,
+      1.1504493951797485,
+      1.2184523344039917,
+      1.108361005783081,
+      1.1478941440582275,
+      1.1512714624404907,
+      1.0552304983139038,
+      1.1397695541381836,
+      1.122780442237854,
+      1.005758285522461,
+      1.0002598762512207,
+      1.0409064292907715,
+      1.01723313331604,
+      0.9565046429634094,
+      1.0747400522232056,
+      1.024727463722229,
+      0.9651903510093689,
+      0.9758272171020508,
+      0.9789083003997803,
+      0.929205596446991,
+      0.9529944062232971,
+      0.9795122742652893,
+      0.9562175869941711,
+      0.8985953330993652,
+      0.9894943237304688,
+      0.9224699139595032,
+      0.9831151962280273,
+      0.9379681944847107,
+      0.9211165308952332,
+      0.8824110627174377,
+      0.861494243144989,
+      0.883068859577179,
+      0.8332101702690125,
+      0.8181617259979248,
+      0.8394058346748352,
+      0.8424668312072754,
+      0.9516865611076355,
+      0.8738915920257568,
+      0.8418115973472595,
+      0.8094231486320496,
+      0.8552870154380798,
+      0.7975897789001465,
+      0.8023789525032043,
+      0.7786011099815369,
+      0.8187759518623352,
+      0.7987339496612549,
+      0.7813507914543152,
+      0.7423741221427917,
+      0.812014102935791,
+      0.7676723003387451,
+      0.778595507144928,
+      0.800652801990509,
+      0.7633883953094482,
+      0.8033614158630371,
+      0.7871838212013245,
+      0.7688676714897156,
+      0.7767912745475769,
+      0.7566482424736023,
+      0.8064160346984863,
+      0.7861444354057312,
+      0.7809345126152039,
+      0.7636844515800476,
+      0.7520200610160828,
+      0.7806358337402344,
+      0.7656214237213135,
+      0.6797690391540527,
+      0.7606520652770996,
+      0.7733117938041687,
+      0.7503668665885925,
+      0.6636472940444946,
+      0.7346720099449158,
+      0.7269353270530701,
+      0.7668713927268982,
+      0.6725441813468933,
+      0.7213144302368164,
+      0.6948793530464172,
+      0.7548025250434875,
+      0.761782169342041,
+      0.7244846224784851,
+      0.7605364918708801,
+      0.6585461497306824,
+      0.7592418789863586,
+      0.698582649230957,
+      0.7231742739677429,
+      0.7128823399543762,
+      0.7448789477348328,
+      0.6964526772499084,
+      0.6776034235954285,
+      0.7378023266792297,
+      0.7693471908569336,
+      0.7003297209739685,
+      0.7271864414215088,
+      0.7441620826721191,
+      0.6844034790992737,
+      0.6500247716903687,
+      0.7112496495246887
+    ],
+    "zipf_loss": [
+      6.978975772857666,
+      6.861506462097168,
+      6.573797225952148,
+      6.040009498596191,
+      5.156149864196777,
+      4.082982063293457,
+      3.050708293914795,
+      2.2498207092285156,
+      1.6447628736495972,
+      1.2626595497131348,
+      1.1128963232040405,
+      1.0090676546096802,
+      0.9471840858459473,
+      0.9151198863983154,
+      0.895793080329895,
+      0.8764229416847229,
+      0.869126558303833,
+      0.8635023832321167,
+      0.8600643277168274,
+      0.8574982285499573,
+      0.8543480038642883,
+      0.8527469635009766,
+      0.8503580093383789,
+      0.849683403968811,
+      0.8477368354797363,
+      0.846424400806427,
+      0.8459466099739075,
+      0.8446402549743652,
+      0.8442564010620117,
+      0.8435505628585815,
+      0.8432317972183228,
+      0.8412547707557678,
+      0.8414230346679688,
+      0.8408272862434387,
+      0.8404833078384399,
+      0.8407796621322632,
+      0.8401228189468384,
+      0.8398240804672241,
+      0.8395506143569946,
+      0.8390905857086182,
+      0.8383229970932007,
+      0.8384392261505127,
+      0.8383464217185974,
+      0.8375396728515625,
+      0.8379733562469482,
+      0.8375827074050903,
+      0.8370733261108398,
+      0.8371742367744446,
+      0.8373335599899292,
+      0.8366790413856506,
+      0.8369749188423157,
+      0.8362846374511719,
+      0.8364881277084351,
+      0.8362550735473633,
+      0.8359628915786743,
+      0.8364212512969971,
+      0.8362687230110168,
+      0.8360430598258972,
+      0.8366975784301758,
+      0.8362879753112793,
+      0.8351982235908508,
+      0.8358657360076904,
+      0.8362256288528442,
+      0.8351002335548401,
+      0.8346133232116699,
+      0.835462212562561,
+      0.8355743885040283,
+      0.835246205329895,
+      0.8343390226364136,
+      0.8365435004234314,
+      0.8383232355117798,
+      0.8389556407928467,
+      0.8380875587463379,
+      0.8400166034698486,
+      0.8398873209953308,
+      0.8375667929649353,
+      0.8396165370941162,
+      0.8382315635681152,
+      0.8365381956100464,
+      0.8382493257522583,
+      0.8378570079803467,
+      0.8358508348464966,
+      0.8380565643310547,
+      0.832449197769165,
+      0.8338850140571594,
+      0.8309053182601929,
+      0.8283628225326538,
+      0.8278912305831909,
+      0.8248351812362671,
+      0.8272550106048584,
+      0.8214181661605835,
+      0.8248977661132812,
+      0.8183830976486206,
+      0.82154381275177,
+      0.8121699690818787,
+      0.812282919883728,
+      0.7976373434066772,
+      0.8114449381828308,
+      0.8053988218307495,
+      0.7938594818115234,
+      0.7961721420288086,
+      0.7763534784317017,
+      0.7793982625007629,
+      0.7771953344345093,
+      0.7772964239120483,
+      0.7563987970352173,
+      0.7672083973884583,
+      0.7561273574829102,
+      0.7356691360473633,
+      0.7416049838066101,
+      0.723779022693634,
+      0.7141247987747192,
+      0.7198091745376587,
+      0.6836644411087036,
+      0.6946905851364136,
+      0.6969248056411743,
+      0.6790536642074585,
+      0.6543284058570862,
+      0.6292747259140015,
+      0.6483874320983887,
+      0.6295268535614014,
+      0.6265751123428345,
+      0.6134255528450012,
+      0.593659520149231,
+      0.6054421663284302,
+      0.5793555974960327,
+      0.5691931843757629,
+      0.5603417754173279,
+      0.5417956113815308,
+      0.5389658808708191,
+      0.548931360244751,
+      0.5497277975082397,
+      0.5395280122756958,
+      0.5150188207626343,
+      0.5385458469390869,
+      0.49950340390205383,
+      0.5022073984146118,
+      0.48842599987983704,
+      0.4888504445552826,
+      0.4961203336715698,
+      0.4808029532432556,
+      0.4966256320476532,
+      0.45979154109954834,
+      0.48823195695877075,
+      0.47432804107666016,
+      0.44807910919189453,
+      0.46398821473121643,
+      0.488333523273468,
+      0.47239950299263,
+      0.45569920539855957,
+      0.4472714364528656,
+      0.47017306089401245,
+      0.4523178040981293,
+      0.45377326011657715,
+      0.45009303092956543,
+      0.44406265020370483,
+      0.43863141536712646,
+      0.4340119957923889,
+      0.4239766299724579,
+      0.43304258584976196,
+      0.4552016854286194,
+      0.4303341209888458,
+      0.4358370304107666,
+      0.4146152138710022,
+      0.4307610094547272,
+      0.43080973625183105,
+      0.4127248525619507,
+      0.4131776690483093,
+      0.4385566711425781,
+      0.42644208669662476,
+      0.39180004596710205,
+      0.4149928092956543,
+      0.4083879888057709,
+      0.41155269742012024,
+      0.4226246476173401,
+      0.3782380223274231,
+      0.40861958265304565,
+      0.3789820671081543,
+      0.3955830931663513,
+      0.3931022882461548,
+      0.39868417382240295,
+      0.40077757835388184,
+      0.3838803768157959,
+      0.39334219694137573,
+      0.3844934105873108,
+      0.38197723031044006,
+      0.40066906809806824,
+      0.38229820132255554,
+      0.3783735930919647,
+      0.37494295835494995,
+      0.37781065702438354,
+      0.3456830382347107,
+      0.3944239616394043,
+      0.3659293055534363,
+      0.3690231442451477,
+      0.3540388345718384,
+      0.34149616956710815,
+      0.34459131956100464,
+      0.3634949326515198,
+      0.3461085557937622,
+      0.35532552003860474,
+      0.3434396982192993,
+      0.3579829931259155,
+      0.33580705523490906,
+      0.3548800051212311,
+      0.35321885347366333,
+      0.3420489430427551,
+      0.3403633236885071,
+      0.30548352003097534,
+      0.3051503896713257,
+      0.3171254098415375,
+      0.31322288513183594,
+      0.3083052635192871,
+      0.3067004084587097,
+      0.2906796336174011,
+      0.2907910645008087,
+      0.3092503547668457,
+      0.267874538898468,
+      0.29389843344688416,
+      0.30443620681762695,
+      0.2849453091621399,
+      0.290545254945755,
+      0.2773565649986267,
+      0.28935733437538147,
+      0.28139325976371765,
+      0.2694956362247467,
+      0.2576865553855896,
+      0.2739996612071991,
+      0.2598174810409546,
+      0.25316327810287476,
+      0.25999778509140015,
+      0.24584560096263885,
+      0.2466832995414734,
+      0.23739854991436005,
+      0.22500741481781006,
+      0.22672389447689056,
+      0.21352921426296234,
+      0.24542883038520813,
+      0.2492891550064087,
+      0.2125484049320221,
+      0.2220914661884308,
+      0.24484862387180328,
+      0.22734017670154572,
+      0.2211361676454544,
+      0.22871923446655273,
+      0.21071617305278778,
+      0.20811833441257477,
+      0.21593020856380463,
+      0.20501554012298584,
+      0.20720958709716797,
+      0.21278345584869385,
+      0.19608554244041443,
+      0.19507622718811035,
+      0.1945650279521942,
+      0.2110665738582611,
+      0.2096593677997589,
+      0.19000115990638733,
+      0.22169911861419678,
+      0.2024805247783661,
+      0.21249879896640778,
+      0.20410098135471344,
+      0.21839144825935364,
+      0.19030651450157166,
+      0.18769240379333496,
+      0.20013214647769928,
+      0.18714290857315063,
+      0.19295692443847656,
+      0.17935983836650848,
+      0.1821189820766449,
+      0.17508018016815186,
+      0.17065343260765076,
+      0.16378551721572876,
+      0.1970933973789215,
+      0.1735805869102478,
+      0.17858712375164032,
+      0.17768988013267517,
+      0.18657249212265015,
+      0.16882425546646118,
+      0.18057924509048462,
+      0.16383594274520874,
+      0.19454002380371094,
+      0.1892033815383911,
+      0.19097532331943512,
+      0.19664379954338074,
+      0.17337705194950104,
+      0.17426568269729614,
+      0.16286766529083252,
+      0.17753306031227112,
+      0.16630280017852783,
+      0.17468461394309998,
+      0.15533827245235443,
+      0.1662886142730713,
+      0.16536468267440796,
+      0.18880730867385864,
+      0.18129804730415344,
+      0.16443416476249695,
+      0.18044480681419373,
+      0.166520893573761,
+      0.17829599976539612,
+      0.1791488528251648
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.8996059894561768,
+      0.8602539896965027,
+      0.7281120419502258,
+      0.5528135299682617,
+      0.4913622736930847,
+      0.41584235429763794,
+      0.37332451343536377,
+      0.3627687394618988,
+      0.3603191375732422,
+      0.35084936022758484,
+      0.3624252378940582,
+      0.3808911442756653,
+      0.3800617754459381,
+      0.37554365396499634,
+      0.3832750916481018,
+      0.38352128863334656,
+      0.3787845969200134,
+      0.37443089485168457,
+      0.3841548562049866,
+      0.38754966855049133,
+      0.3791581094264984,
+      0.3866385519504547,
+      0.3990081548690796,
+      0.4070803225040436,
+      0.4009581506252289,
+      0.4101217985153198,
+      0.4110082983970642,
+      0.40850427746772766,
+      0.40984371304512024,
+      0.41836950182914734,
+      0.4294724762439728,
+      0.43089914321899414,
+      0.42754966020584106,
+      0.440367192029953,
+      0.4442957937717438,
+      0.43948960304260254,
+      0.44536423683166504,
+      0.4493218660354614,
+      0.45509082078933716,
+      0.45902305841445923,
+      0.4650082588195801,
+      0.47030624747276306,
+      0.47507449984550476,
+      0.4782888889312744,
+      0.48267754912376404,
+      0.4907062351703644,
+      0.495198130607605,
+      0.5012418627738953,
+      0.5038909316062927,
+      0.5064918994903564,
+      0.5092272162437439,
+      0.5078070163726807,
+      0.5108755826950073,
+      0.5139000415802002,
+      0.5164137482643127,
+      0.5176257491111755,
+      0.5190778374671936,
+      0.5200605988502502,
+      0.5181890726089478,
+      0.5186240673065186,
+      0.5162889957427979,
+      0.5195546746253967,
+      0.5236274003982544,
+      0.5226184129714966,
+      0.5268747806549072,
+      0.5294082164764404,
+      0.5280004143714905,
+      0.5268764495849609,
+      0.5202440023422241,
+      0.514682948589325,
+      0.5154968500137329,
+      0.5203099846839905,
+      0.5221132040023804,
+      0.520980179309845,
+      0.5243099927902222,
+      0.5259549021720886,
+      0.5284245014190674,
+      0.5324582457542419,
+      0.5324409604072571,
+      0.5339889526367188,
+      0.5364629030227661,
+      0.5380638837814331,
+      0.5403037667274475,
+      0.5408695340156555,
+      0.5388999581336975,
+      0.5436255931854248,
+      0.5427041053771973,
+      0.5448338985443115,
+      0.5468981266021729,
+      0.5485912561416626,
+      0.5481650233268738,
+      0.5495074987411499,
+      0.5505582690238953,
+      0.5509097576141357,
+      0.5517634749412537,
+      0.5532107353210449,
+      0.554972231388092,
+      0.5565258860588074,
+      0.5545681118965149,
+      0.5550283193588257,
+      0.554379940032959,
+      0.5542206764221191,
+      0.554817795753479,
+      0.5574579834938049,
+      0.5589355230331421,
+      0.5604186058044434,
+      0.5601381659507751,
+      0.5617371797561646,
+      0.5619471669197083,
+      0.5622543692588806,
+      0.5606953501701355,
+      0.558192253112793,
+      0.5578585863113403,
+      0.5590813755989075,
+      0.5594561100006104,
+      0.5593685507774353,
+      0.5583264827728271,
+      0.558474600315094,
+      0.5575746297836304,
+      0.5556209087371826,
+      0.5560757517814636,
+      0.5549269914627075,
+      0.5537244081497192,
+      0.5557824373245239,
+      0.5571614503860474,
+      0.5577748417854309,
+      0.5578136444091797,
+      0.5562106966972351,
+      0.5576562881469727,
+      0.5572245121002197,
+      0.5567528009414673,
+      0.5549530982971191,
+      0.5556580424308777,
+      0.554071307182312,
+      0.553594172000885,
+      0.5515624284744263,
+      0.551023542881012,
+      0.5462676286697388,
+      0.5457965135574341,
+      0.544347882270813,
+      0.5427818298339844,
+      0.5421648621559143,
+      0.5419304966926575,
+      0.5401836037635803,
+      0.5380207896232605,
+      0.5372520089149475,
+      0.5378258228302002,
+      0.5367369651794434,
+      0.5363492369651794,
+      0.5350574851036072,
+      0.532775342464447,
+      0.5310155153274536,
+      0.5304696559906006,
+      0.5306452512741089,
+      0.5299267172813416,
+      0.5297029614448547,
+      0.5284163355827332,
+      0.5264398455619812,
+      0.5243463516235352,
+      0.5249975919723511,
+      0.5232595801353455,
+      0.5234885215759277,
+      0.5229676961898804,
+      0.5209376811981201,
+      0.5197718143463135,
+      0.5196494460105896,
+      0.5174844264984131,
+      0.5166829824447632,
+      0.5140925049781799,
+      0.513678789138794,
+      0.5123385787010193,
+      0.5101063251495361,
+      0.509605884552002,
+      0.5095978379249573,
+      0.5090241432189941,
+      0.5079015493392944,
+      0.5080432295799255,
+      0.5093200206756592,
+      0.5094007253646851,
+      0.5099282264709473,
+      0.5078747272491455,
+      0.5073709487915039,
+      0.5063402652740479,
+      0.5061359405517578,
+      0.5046309232711792,
+      0.5043975114822388,
+      0.5007187128067017,
+      0.5001405477523804,
+      0.49947720766067505,
+      0.5009679198265076,
+      0.5005672574043274,
+      0.49990567564964294,
+      0.49933797121047974,
+      0.498971551656723,
+      0.49881723523139954,
+      0.4964151382446289,
+      0.49467089772224426,
+      0.49516040086746216,
+      0.49388477206230164,
+      0.4944036602973938,
+      0.49509432911872864,
+      0.4938291907310486,
+      0.49208348989486694,
+      0.49177539348602295,
+      0.49092990159988403,
+      0.4912455976009369,
+      0.4901091754436493,
+      0.48949530720710754,
+      0.4892340302467346,
+      0.4892869293689728,
+      0.48798349499702454,
+      0.48757800459861755,
+      0.4865213632583618,
+      0.4858769178390503,
+      0.48494386672973633,
+      0.4851282835006714,
+      0.4864913523197174,
+      0.48616907000541687,
+      0.4850902259349823,
+      0.4841510057449341,
+      0.4834236800670624,
+      0.48298633098602295,
+      0.4827520251274109,
+      0.482891708612442,
+      0.4813154637813568,
+      0.4806137979030609,
+      0.48025932908058167,
+      0.48077669739723206,
+      0.48008444905281067,
+      0.479377806186676,
+      0.47895002365112305,
+      0.47928285598754883,
+      0.47936463356018066,
+      0.47870856523513794,
+      0.47892624139785767,
+      0.47812193632125854,
+      0.47767117619514465,
+      0.47728997468948364,
+      0.4776654839515686,
+      0.477409690618515,
+      0.47715258598327637,
+      0.47724321484565735,
+      0.47668778896331787,
+      0.47609370946884155,
+      0.47649675607681274,
+      0.47630971670150757,
+      0.4761170446872711,
+      0.47658592462539673,
+      0.4763483703136444,
+      0.4758099317550659,
+      0.47571390867233276,
+      0.4760485291481018,
+      0.4757618010044098,
+      0.47557464241981506,
+      0.47523483633995056,
+      0.4755951166152954,
+      0.47535204887390137,
+      0.47549453377723694,
+      0.47521668672561646,
+      0.4746234118938446,
+      0.4748867452144623,
+      0.47450658679008484,
+      0.4735686779022217,
+      0.4731999635696411,
+      0.4693770408630371,
+      0.4689675569534302,
+      0.4700164794921875,
+      0.46957480907440186,
+      0.46916860342025757,
+      0.4676312208175659,
+      0.46757909655570984,
+      0.4671994149684906,
+      0.4664912819862366,
+      0.46536433696746826,
+      0.46553751826286316,
+      0.4654693901538849,
+      0.46639323234558105,
+      0.4664039611816406,
+      0.4653305113315582,
+      0.4655424952507019,
+      0.4649745523929596,
+      0.46504685282707214,
+      0.46489429473876953,
+      0.46500951051712036,
+      0.46495532989501953,
+      0.46496596932411194,
+      0.46485424041748047,
+      0.4649030268192291,
+      0.46429893374443054,
+      0.46407729387283325,
+      0.4643470346927643,
+      0.46424898505210876,
+      0.46422088146209717,
+      0.4647376835346222,
+      0.46474573016166687,
+      0.4646284878253937,
+      0.4643927812576294,
+      0.4642075002193451,
+      0.4643164873123169,
+      0.46440914273262024
+    ],
+    "lr": [
+      2.0940170940170946e-06,
+      4.230769230769231e-06,
+      6.367521367521369e-06,
+      8.504273504273505e-06,
+      1.0641025641025641e-05,
+      1.2777777777777779e-05,
+      1.4914529914529916e-05,
+      1.7051282051282053e-05,
+      1.918803418803419e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9965001318217775e-05,
+      1.9816701819140524e-05,
+      1.9668402320063276e-05,
+      1.952010282098603e-05,
+      1.937180332190878e-05,
+      1.922350382283153e-05,
+      1.9075204323754282e-05,
+      1.8926904824677038e-05,
+      1.877860532559979e-05,
+      1.863030582652254e-05,
+      1.8482006327445295e-05,
+      1.8333706828368048e-05,
+      1.8090495649881362e-05,
+      1.7942196150804114e-05,
+      1.7793896651726867e-05,
+      1.764559715264962e-05,
+      1.749729765357237e-05,
+      1.734899815449512e-05,
+      1.7200698655417876e-05,
+      1.705239915634063e-05,
+      1.690409965726338e-05,
+      1.6755800158186133e-05,
+      1.6607500659108886e-05,
+      1.6459201160031638e-05,
+      1.6310901660954387e-05,
+      1.6162602161877146e-05,
+      1.6014302662799895e-05,
+      1.577109148431321e-05,
+      1.5622791985235962e-05,
+      1.5474492486158718e-05,
+      1.5326192987081467e-05,
+      1.5177893488004219e-05,
+      1.502959398892697e-05,
+      1.4881294489849724e-05,
+      1.4732994990772476e-05,
+      1.4584695491695227e-05,
+      1.4436395992617984e-05,
+      1.4288096493540735e-05,
+      1.4139796994463486e-05,
+      1.3991497495386242e-05,
+      1.384319799630899e-05,
+      1.3694898497231743e-05,
+      1.3451687318745057e-05,
+      1.3303387819667808e-05,
+      1.3155088320590564e-05,
+      1.3006788821513314e-05,
+      1.2858489322436065e-05,
+      1.2710189823358823e-05,
+      1.2561890324281573e-05,
+      1.2413590825204324e-05,
+      1.2265291326127076e-05,
+      1.211699182704983e-05,
+      1.196869232797258e-05,
+      1.1820392828895333e-05,
+      1.1672093329818088e-05,
+      1.152379383074084e-05,
+      1.137549433166359e-05,
+      1.1132283153176903e-05,
+      1.0983983654099656e-05,
+      1.0835684155022413e-05,
+      1.0687384655945164e-05,
+      1.0539085156867914e-05,
+      1.039078565779067e-05,
+      1.0242486158713421e-05,
+      1.0094186659636172e-05,
+      9.945887160558927e-06,
+      9.797587661481678e-06,
+      9.649288162404429e-06,
+      9.50098866332718e-06,
+      9.352689164249937e-06,
+      9.204389665172688e-06,
+      9.056090166095438e-06,
+      8.812878987608752e-06,
+      8.664579488531508e-06,
+      8.516279989454259e-06,
+      8.36798049037701e-06,
+      8.21968099129976e-06,
+      8.071381492222516e-06,
+      7.923081993145267e-06,
+      7.774782494068017e-06,
+      7.626482994990775e-06,
+      7.478183495913526e-06,
+      7.329883996836276e-06,
+      7.181584497759033e-06,
+      7.0332849986817845e-06,
+      6.884985499604535e-06,
+      6.736686000527286e-06,
+      6.493474822040599e-06,
+      6.345175322963355e-06,
+      6.196875823886106e-06,
+      6.048576324808857e-06,
+      5.900276825731613e-06,
+      5.7519773266543646e-06,
+      5.603677827577115e-06,
+      5.455378328499866e-06,
+      5.307078829422623e-06,
+      5.158779330345374e-06,
+      5.010479831268125e-06,
+      4.86218033219088e-06,
+      4.713880833113631e-06,
+      4.565581334036382e-06,
+      4.417281834959138e-06,
+      4.174070656472452e-06,
+      4.025771157395203e-06,
+      3.877471658317954e-06,
+      3.729172159240705e-06,
+      3.5808726601634607e-06,
+      3.4325731610862118e-06,
+      3.284273662008963e-06,
+      3.1359741629317187e-06,
+      2.9876746638544698e-06,
+      2.8393751647772205e-06,
+      2.6910756656999716e-06,
+      2.5427761666227274e-06,
+      2.3944766675454785e-06,
+      2.246177168468229e-06,
+      2.0978776693909854e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      750,
+      1532,
+      2314,
+      3096,
+      3878,
+      4660,
+      5442,
+      6224,
+      7006,
+      7788,
+      8570,
+      9352,
+      10134,
+      10916,
+      11698,
+      12480,
+      13262,
+      14044,
+      14826,
+      15608
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.31375,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.82,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9669421487603306,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9894736842105263,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.23,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9068627450980392,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.8875739644970414,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.8709677419354839,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.75,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.27,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8957055214723927,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.8384615384615385,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.8160919540229885,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.7881773399014779,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.8803418803418803,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.24,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9256198347107438,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.8925619834710744,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.7142857142857143,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.6720430107526881,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.726457399103139,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.3,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9807692307692307,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.8962264150943396,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.65625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.6091205211726385,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.14,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.24,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.2775,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.63,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.552,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.335,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9239373601789709,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.846875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.8035714285714286,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.8204158790170132,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.6875,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.17,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.5025906735751295,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.4953271028037383,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.578125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.6458333333333334,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6078431372549019,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.7010309278350515,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.25,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7650273224043715,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8076923076923077,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.49,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8985024958402662,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.8585858585858586,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.910394265232975,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.8344827586206897,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.875,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.49603174603174605,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9436619718309859,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.7787610619469026,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9176470588235294,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.430939226519337,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.5277777777777778,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9832402234636871,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.7766990291262136,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.3825503355704698,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.4413145539906103,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.95,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.75,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.3,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.3,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.76,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.135,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.12,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.89,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.8651685393258427,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9622641509433962,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.48291571753986334,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.5853658536585366,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9066666666666666,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.79,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.4263959390862944,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.4077669902912621,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.955,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.79,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.4251012145748988,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.49019607843137253,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.88,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.4261744966442953,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.33663366336633666,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.20166666666666666,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.9,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9851239669421488,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9789473684210527,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.54,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9656862745098039,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9467455621301775,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.967741935483871,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.8851351351351351,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.38,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9754601226993865,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9461538461538461,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9770114942528736,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.6896551724137931,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9401709401709402,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.34,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9834710743801653,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9752066115702479,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.6935483870967742,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.7085201793721974,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.28,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9807692307692307,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9716981132075472,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.66875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.6807817589576547,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.29,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.2175,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.05,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.52,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.228,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.59,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9686800894854586,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.946875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.875,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.888468809073724,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.8333333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.34,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9933333333333333,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6269430051813472,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.6448598130841121,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.37,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.71484375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.6944444444444444,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.25,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6928104575163399,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.7164948453608248,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.3,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7841530054644809,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.7863247863247863,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.64,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9317803660565723,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.43,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.967741935483871,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9379310344827586,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9583333333333334,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.7817460317460317,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9483568075117371,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9469026548672567,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9411764705882353,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.5082872928176796,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.7685185185185185,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9888268156424581,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.8932038834951457,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.4429530201342282,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.49765258215962443,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.06,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.98,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.45,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.26,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.9,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.49,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.1625,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.43,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.945,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.8913857677902621,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9433962264150944,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.7949886104783599,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.8780487804878049,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.22,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.96,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.94,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.6192893401015228,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.5339805825242718,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.99,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.92,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.582995951417004,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.4117647058823529,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.05,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5671140939597316,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.45544554455445546,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.31375,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.31375,
+  "sft_overall_accuracy": 0.20166666666666666
+}

add_sub_sorl_v1_abs30_K1_50K_1L2H256d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff7c8028c232d9e624ff76bd0bda95b88747e8c08342816b9edbe6bad23f79f
+size 315132124

add_sub_sorl_v1_abs30_K1_50K_1L2H256d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 2e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 781,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K1_50K_1L2H256d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 1,
+  "n_head": 2,
+  "n_embd": 256,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 50000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 78706688,
+  "run_name": "add_sub_sorl_v1_abs30_K1_50K_1L2H256d",
+  "git_commit": "7c8a203ce79a277a1e41a3ec0648cb73d5b2b760",
+  "timestamp": "2026-04-14T02:07:52.309766+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "msl36i15",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/msl36i15",
+  "final_accuracy": 0.31375,
+  "sft_accuracy": 0.20166666666666666,
+  "eval_method": "ArithmeticEvaluator"
+}