Upload add_sub_sorl_v1_abs30_K1_50K_1L3H510d

add_sub_sorl_v1_abs30_K1_50K_1L3H510d/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_K1_50K_1L3H510d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_K1_50K_1L3H510d/metrics.json

@@ -0,0 +1,3477 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1182,
+      1232,
+      1282,
+      1332,
+      1382,
+      1432,
+      1482,
+      1532,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      1964,
+      2014,
+      2064,
+      2114,
+      2164,
+      2214,
+      2264,
+      2314,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2746,
+      2796,
+      2846,
+      2896,
+      2946,
+      2996,
+      3046,
+      3096,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3528,
+      3578,
+      3628,
+      3678,
+      3728,
+      3778,
+      3828,
+      3878,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4310,
+      4360,
+      4410,
+      4460,
+      4510,
+      4560,
+      4610,
+      4660,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5092,
+      5142,
+      5192,
+      5242,
+      5292,
+      5342,
+      5392,
+      5442,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5874,
+      5924,
+      5974,
+      6024,
+      6074,
+      6124,
+      6174,
+      6224,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6656,
+      6706,
+      6756,
+      6806,
+      6856,
+      6906,
+      6956,
+      7006,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7438,
+      7488,
+      7538,
+      7588,
+      7638,
+      7688,
+      7738,
+      7788,
+      7870,
+      7920,
+      7970,
+      8020,
+      8070,
+      8120,
+      8170,
+      8220,
+      8270,
+      8320,
+      8370,
+      8420,
+      8470,
+      8520,
+      8570,
+      8652,
+      8702,
+      8752,
+      8802,
+      8852,
+      8902,
+      8952,
+      9002,
+      9052,
+      9102,
+      9152,
+      9202,
+      9252,
+      9302,
+      9352,
+      9434,
+      9484,
+      9534,
+      9584,
+      9634,
+      9684,
+      9734,
+      9784,
+      9834,
+      9884,
+      9934,
+      9984,
+      10034,
+      10084,
+      10134,
+      10216,
+      10266,
+      10316,
+      10366,
+      10416,
+      10466,
+      10516,
+      10566,
+      10616,
+      10666,
+      10716,
+      10766,
+      10816,
+      10866,
+      10916,
+      10998,
+      11048,
+      11098,
+      11148,
+      11198,
+      11248,
+      11298,
+      11348,
+      11398,
+      11448,
+      11498,
+      11548,
+      11598,
+      11648,
+      11698,
+      11780,
+      11830,
+      11880,
+      11930,
+      11980,
+      12030,
+      12080,
+      12130,
+      12180,
+      12230,
+      12280,
+      12330,
+      12380,
+      12430,
+      12480,
+      12562,
+      12612,
+      12662,
+      12712,
+      12762,
+      12812,
+      12862,
+      12912,
+      12962,
+      13012,
+      13062,
+      13112,
+      13162,
+      13212,
+      13262,
+      13344,
+      13394,
+      13444,
+      13494,
+      13544,
+      13594,
+      13644,
+      13694,
+      13744,
+      13794,
+      13844,
+      13894,
+      13944,
+      13994,
+      14044,
+      14126,
+      14176,
+      14226,
+      14276,
+      14326,
+      14376,
+      14426,
+      14476,
+      14526,
+      14576,
+      14626,
+      14676,
+      14726,
+      14776,
+      14826,
+      14908,
+      14958,
+      15008,
+      15058,
+      15108,
+      15158,
+      15208,
+      15258,
+      15308,
+      15358,
+      15408,
+      15458,
+      15508,
+      15558,
+      15608
+    ],
+    "loss": [
+      -6.08144474029541,
+      3.486781120300293,
+      6.263645648956299,
+      5.025929927825928,
+      3.392362356185913,
+      3.097670555114746,
+      2.7977352142333984,
+      2.695284843444824,
+      2.8754031658172607,
+      2.7400660514831543,
+      2.7319722175598145,
+      2.4467978477478027,
+      0.6018810272216797,
+      -1.8595647811889648,
+      -4.026945114135742,
+      -8.278923034667969,
+      -10.660880088806152,
+      -10.643266677856445,
+      -11.416305541992188,
+      -11.360186576843262,
+      -11.441851615905762,
+      -12.487668991088867,
+      -12.345475196838379,
+      -11.962322235107422,
+      -12.722128868103027,
+      -11.819293022155762,
+      -12.356383323669434,
+      -12.265676498413086,
+      -9.691512107849121,
+      -8.200359344482422,
+      -8.48619270324707,
+      -8.371301651000977,
+      -8.349528312683105,
+      -7.60306453704834,
+      -5.616255283355713,
+      -2.117919683456421,
+      -1.3532732725143433,
+      -1.2285220623016357,
+      -0.8111015558242798,
+      -1.512257695198059,
+      -0.7724578380584717,
+      -1.1042633056640625,
+      -1.0384385585784912,
+      -0.6890068650245667,
+      -0.5117706060409546,
+      -0.8237566947937012,
+      -1.096296787261963,
+      -0.8922213315963745,
+      -0.14830613136291504,
+      -0.6266731023788452,
+      -0.5739270448684692,
+      -0.31544435024261475,
+      -0.5101633667945862,
+      -0.9300204515457153,
+      -0.7174951434135437,
+      -0.03876402974128723,
+      -0.535488486289978,
+      -0.2619773745536804,
+      -0.5923171639442444,
+      -0.30991002917289734,
+      -0.1216861754655838,
+      -0.07450756430625916,
+      -0.7227131128311157,
+      -0.1781938225030899,
+      -0.3768835961818695,
+      -0.2937881648540497,
+      -0.36723819375038147,
+      -0.318166047334671,
+      -0.2538578510284424,
+      -0.5196461081504822,
+      -0.5651006698608398,
+      -0.2847289741039276,
+      -0.7698932886123657,
+      -0.5026482343673706,
+      -0.7097562551498413,
+      -0.44247886538505554,
+      -0.36954402923583984,
+      -0.4518633484840393,
+      -0.40146270394325256,
+      -0.5525304675102234,
+      -0.31073468923568726,
+      -0.7700643539428711,
+      -0.5824949145317078,
+      -0.5254961252212524,
+      -0.3065895438194275,
+      -0.28556084632873535,
+      -0.9941896796226501,
+      -0.7974525690078735,
+      -0.7628035545349121,
+      -0.38103753328323364,
+      -0.8160765171051025,
+      -0.6330153942108154,
+      -0.4334428310394287,
+      -0.7796759605407715,
+      -0.8214023113250732,
+      -0.22494858503341675,
+      -0.7761626243591309,
+      -0.6495262384414673,
+      -0.5405408143997192,
+      -0.7201738953590393,
+      -0.5496256947517395,
+      -0.39138710498809814,
+      -0.45723989605903625,
+      -0.5390639305114746,
+      -0.2601628601551056,
+      -0.6314728260040283,
+      -0.5294767618179321,
+      -0.8120536804199219,
+      -0.8360767364501953,
+      -0.5628071427345276,
+      -0.5476758480072021,
+      -0.5318657159805298,
+      -0.3001057207584381,
+      -0.5921418070793152,
+      -0.6375084519386292,
+      -0.4646930992603302,
+      -0.7696737051010132,
+      -0.6063775420188904,
+      -0.5002613067626953,
+      -0.37346455454826355,
+      -0.4130162000656128,
+      -0.43184807896614075,
+      -0.5652193427085876,
+      -0.422710120677948,
+      -0.6336796283721924,
+      -0.23636189103126526,
+      -0.6240003108978271,
+      -0.2867107391357422,
+      -0.33379632234573364,
+      -0.6123911738395691,
+      -0.4645664691925049,
+      -0.28661495447158813,
+      -0.37328818440437317,
+      -0.07531123608350754,
+      -0.30035561323165894,
+      -0.3184964954853058,
+      -0.4866830110549927,
+      -0.5345924496650696,
+      -0.09664274752140045,
+      -0.035446252673864365,
+      -0.3483760952949524,
+      -0.33519262075424194,
+      -0.47307270765304565,
+      -0.36952173709869385,
+      -0.04555947706103325,
+      -0.636769711971283,
+      -0.2900931239128113,
+      -0.4492289125919342,
+      -0.43911847472190857,
+      -0.2981439232826233,
+      -0.3661026656627655,
+      -0.22288104891777039,
+      -0.3714882731437683,
+      -0.28479456901550293,
+      -0.16670434176921844,
+      -0.3227829039096832,
+      -0.38295868039131165,
+      -0.13807161152362823,
+      -0.3742074966430664,
+      -0.3852161169052124,
+      -0.13727635145187378,
+      -0.3128903806209564,
+      -3.922078758478165e-05,
+      -0.18205511569976807,
+      -0.4606463313102722,
+      -0.2574746906757355,
+      -0.3182951509952545,
+      -0.3206053376197815,
+      -0.4900285303592682,
+      -0.3506549596786499,
+      -0.5384626388549805,
+      -0.09664314985275269,
+      -0.256656289100647,
+      0.11259277909994125,
+      -0.15635840594768524,
+      -0.5010675191879272,
+      -0.46622365713119507,
+      -0.25806546211242676,
+      -0.23264972865581512,
+      -0.33695170283317566,
+      -0.14161090552806854,
+      -0.44170552492141724,
+      -0.3780865967273712,
+      -0.4383348822593689,
+      -0.2410961389541626,
+      -0.46118730306625366,
+      0.08062154054641724,
+      -0.24673835933208466,
+      -0.1117541640996933,
+      0.012129133567214012,
+      -0.2235463559627533,
+      -0.5180912017822266,
+      -0.1482851505279541,
+      -0.1615755259990692,
+      -0.28403139114379883,
+      -0.4862081706523895,
+      0.014171775430440903,
+      -0.32050055265426636,
+      -0.10590873658657074,
+      -0.37014806270599365,
+      -0.3166331946849823,
+      -0.2566193640232086,
+      -0.25789475440979004,
+      -0.4165055751800537,
+      -0.11937905102968216,
+      -0.32059764862060547,
+      -0.37637120485305786,
+      -0.16322340071201324,
+      -0.2513125240802765,
+      -0.3901664614677429,
+      -0.33978715538978577,
+      -0.2732270658016205,
+      -0.30333012342453003,
+      -0.2985623776912689,
+      0.21139103174209595,
+      -0.3792789876461029,
+      -0.4149411618709564,
+      0.006549260579049587,
+      -0.09649182856082916,
+      -0.44664764404296875,
+      -0.15505510568618774,
+      -0.4857863485813141,
+      -0.22676390409469604,
+      -0.26580187678337097,
+      -0.4298636317253113,
+      -0.2948703169822693,
+      -0.3735657036304474,
+      -0.45198118686676025,
+      -0.22478432953357697,
+      -0.5356206893920898,
+      -0.2754799723625183,
+      -0.31199660897254944,
+      -0.246833935379982,
+      -0.43545398116111755,
+      -0.5981326699256897,
+      -0.44733723998069763,
+      -0.4369511008262634,
+      -0.16037456691265106,
+      -0.2427096664905548,
+      -0.31197962164878845,
+      0.06655541807413101,
+      -0.3376851975917816,
+      -0.26231327652931213,
+      -0.24448828399181366,
+      0.0740579217672348,
+      -0.3864766061306,
+      -0.12712986767292023,
+      -0.2704770863056183,
+      -0.24418017268180847,
+      -0.4094691276550293,
+      -0.2836611270904541,
+      -0.2048865705728531,
+      -0.37573257088661194,
+      -0.16408705711364746,
+      0.026291539892554283,
+      -0.2529107630252838,
+      0.08938693255186081,
+      -0.15234878659248352,
+      -0.08416201174259186,
+      -0.07135338336229324,
+      -0.2086154818534851,
+      -0.19044454395771027,
+      0.03828307241201401,
+      -0.16868236660957336,
+      -0.4116271138191223,
+      -0.28740614652633667,
+      -0.05430714040994644,
+      -0.21165065467357635,
+      -0.41431111097335815,
+      -0.3788253962993622,
+      -0.37304896116256714,
+      -0.015208248049020767,
+      -0.07262329757213593,
+      -0.3407365679740906,
+      0.04026219621300697,
+      -0.0823097825050354,
+      -0.2953197658061981,
+      -0.22831782698631287,
+      -0.05494590103626251,
+      -0.25423792004585266,
+      -0.3417591452598572,
+      -0.11990547180175781,
+      -0.22886477410793304,
+      -0.022577572613954544,
+      -0.27429452538490295,
+      -0.19793535768985748,
+      -0.21908412873744965,
+      -0.19095370173454285,
+      -0.43633922934532166,
+      -0.18436199426651,
+      -0.36892086267471313,
+      -0.2548205256462097,
+      -0.178135946393013,
+      -0.23145924508571625,
+      -0.10573843866586685,
+      -0.15459281206130981,
+      -0.40475523471832275,
+      -0.10680191963911057,
+      0.11567749828100204,
+      -0.20084896683692932
+    ],
+    "base_loss": [
+      11.263224601745605,
+      8.160566329956055,
+      6.655036926269531,
+      4.710936546325684,
+      2.556856632232666,
+      2.015815019607544,
+      1.9138128757476807,
+      1.9731098413467407,
+      1.8702163696289062,
+      1.7979248762130737,
+      1.8270989656448364,
+      1.7735166549682617,
+      1.862930178642273,
+      1.8729170560836792,
+      1.8859236240386963,
+      1.8056036233901978,
+      1.8574540615081787,
+      1.7986761331558228,
+      1.8923742771148682,
+      1.8022483587265015,
+      1.8138659000396729,
+      1.8432973623275757,
+      1.8782206773757935,
+      1.733026385307312,
+      1.8501063585281372,
+      1.7978012561798096,
+      1.7842191457748413,
+      1.7858575582504272,
+      1.491632342338562,
+      1.326660394668579,
+      1.3354380130767822,
+      1.2400246858596802,
+      1.2529062032699585,
+      1.1743583679199219,
+      0.9574888944625854,
+      0.5723962187767029,
+      0.5114582180976868,
+      0.5073795914649963,
+      0.37818703055381775,
+      0.39741918444633484,
+      0.362358033657074,
+      0.36368945240974426,
+      0.42476123571395874,
+      0.351570188999176,
+      0.33848872780799866,
+      0.3493426740169525,
+      0.40199682116508484,
+      0.3080305755138397,
+      0.33921507000923157,
+      0.3203471004962921,
+      0.29883643984794617,
+      0.31358909606933594,
+      0.29097917675971985,
+      0.30312952399253845,
+      0.26914843916893005,
+      0.2548351585865021,
+      0.29340487718582153,
+      0.25346773862838745,
+      0.2821625769138336,
+      0.2383500337600708,
+      0.23144496977329254,
+      0.17740018665790558,
+      0.262499064207077,
+      0.23254619538784027,
+      0.24337591230869293,
+      0.25029903650283813,
+      0.2742063105106354,
+      0.2534780204296112,
+      0.2322908490896225,
+      0.23841455578804016,
+      0.2231275737285614,
+      0.24567782878875732,
+      0.31081846356391907,
+      0.22956739366054535,
+      0.23564733564853668,
+      0.25379061698913574,
+      0.19070693850517273,
+      0.2411465346813202,
+      0.2145119458436966,
+      0.2162173092365265,
+      0.1924288272857666,
+      0.2839837968349457,
+      0.2443089634180069,
+      0.18494334816932678,
+      0.21860386431217194,
+      0.17665457725524902,
+      0.2327423393726349,
+      0.18085190653800964,
+      0.2294992357492447,
+      0.26175734400749207,
+      0.22573420405387878,
+      0.22785648703575134,
+      0.1890564113855362,
+      0.25518399477005005,
+      0.2176317721605301,
+      0.18221387267112732,
+      0.19481869041919708,
+      0.2354598343372345,
+      0.15690340101718903,
+      0.23417377471923828,
+      0.22065898776054382,
+      0.21196456253528595,
+      0.16283848881721497,
+      0.16371209919452667,
+      0.14592990279197693,
+      0.16238045692443848,
+      0.16592459380626678,
+      0.20080263912677765,
+      0.19976983964443207,
+      0.21069695055484772,
+      0.19772018492221832,
+      0.18602408468723297,
+      0.15454533696174622,
+      0.20160767436027527,
+      0.1919737607240677,
+      0.1589915007352829,
+      0.20664867758750916,
+      0.20411059260368347,
+      0.16475293040275574,
+      0.15305474400520325,
+      0.15042969584465027,
+      0.1536608636379242,
+      0.18203100562095642,
+      0.14761124551296234,
+      0.20192621648311615,
+      0.16931600868701935,
+      0.15598224103450775,
+      0.13414393365383148,
+      0.15239466726779938,
+      0.19298656284809113,
+      0.1549442559480667,
+      0.14129850268363953,
+      0.16633255779743195,
+      0.13059166073799133,
+      0.11898191273212433,
+      0.1598031222820282,
+      0.15436653792858124,
+      0.15440590679645538,
+      0.12826421856880188,
+      0.18900902569293976,
+      0.143490269780159,
+      0.12544064223766327,
+      0.12998466193675995,
+      0.14524950087070465,
+      0.1459656059741974,
+      0.1523216813802719,
+      0.16416534781455994,
+      0.14007695019245148,
+      0.1464245319366455,
+      0.1275806874036789,
+      0.16363437473773956,
+      0.15205754339694977,
+      0.15446074306964874,
+      0.17640747129917145,
+      0.12539146840572357,
+      0.1438300609588623,
+      0.11555687338113785,
+      0.13278977572917938,
+      0.13961617648601532,
+      0.1750674992799759,
+      0.13067695498466492,
+      0.14803771674633026,
+      0.11396334320306778,
+      0.15958896279335022,
+      0.14660896360874176,
+      0.10693701356649399,
+      0.1246296614408493,
+      0.12391136586666107,
+      0.12433470785617828,
+      0.12882772088050842,
+      0.16320671141147614,
+      0.1066540852189064,
+      0.14472666382789612,
+      0.10796785354614258,
+      0.15320491790771484,
+      0.1486571878194809,
+      0.1600310504436493,
+      0.12422657757997513,
+      0.11857236921787262,
+      0.13603658974170685,
+      0.11537503451108932,
+      0.1275021880865097,
+      0.13449493050575256,
+      0.13541656732559204,
+      0.160081148147583,
+      0.13306951522827148,
+      0.12392378598451614,
+      0.10884501785039902,
+      0.08146337419748306,
+      0.14277991652488708,
+      0.1613612174987793,
+      0.15105567872524261,
+      0.15453138947486877,
+      0.0963524654507637,
+      0.12401598691940308,
+      0.14070811867713928,
+      0.09955007582902908,
+      0.12552528083324432,
+      0.12032902240753174,
+      0.10190192610025406,
+      0.11342523247003555,
+      0.08637690544128418,
+      0.12337063252925873,
+      0.10264241695404053,
+      0.09194789081811905,
+      0.13099506497383118,
+      0.1076650470495224,
+      0.11093761026859283,
+      0.0874725729227066,
+      0.10046972334384918,
+      0.10373463481664658,
+      0.08292833715677261,
+      0.09225203096866608,
+      0.10177835077047348,
+      0.0831904485821724,
+      0.11428811401128769,
+      0.10648216307163239,
+      0.09545067697763443,
+      0.11027265340089798,
+      0.1453983634710312,
+      0.08881167322397232,
+      0.1203496903181076,
+      0.11688310652971268,
+      0.08892764151096344,
+      0.09407682716846466,
+      0.12160607427358627,
+      0.10964792966842651,
+      0.12085037678480148,
+      0.1145196333527565,
+      0.1172107458114624,
+      0.10462631285190582,
+      0.10803069919347763,
+      0.08914744108915329,
+      0.15248873829841614,
+      0.13964900374412537,
+      0.1338377296924591,
+      0.10455058515071869,
+      0.11379949003458023,
+      0.08480678498744965,
+      0.09852438420057297,
+      0.07169326394796371,
+      0.09452186524868011,
+      0.07763365656137466,
+      0.11566472053527832,
+      0.05886499211192131,
+      0.13290445506572723,
+      0.09306405484676361,
+      0.0761026069521904,
+      0.1263372153043747,
+      0.11017586290836334,
+      0.09149230271577835,
+      0.1005140170454979,
+      0.10952573269605637,
+      0.11715111881494522,
+      0.0784948542714119,
+      0.08386366069316864,
+      0.06956014782190323,
+      0.06698618084192276,
+      0.05802703648805618,
+      0.07711333781480789,
+      0.09097684919834137,
+      0.06962359696626663,
+      0.05722498148679733,
+      0.06795968115329742,
+      0.08763571828603745,
+      0.0604323148727417,
+      0.0797698050737381,
+      0.06367837637662888,
+      0.10947450250387192,
+      0.08331752568483353,
+      0.0816667452454567,
+      0.06790156662464142,
+      0.1324976533651352,
+      0.06577057391405106,
+      0.06868774443864822,
+      0.0822233334183693,
+      0.07965653389692307,
+      0.08636124432086945,
+      0.05616728588938713,
+      0.060620784759521484,
+      0.0687955990433693,
+      0.09348200261592865,
+      0.07807429134845734,
+      0.06845386326313019,
+      0.05860624834895134,
+      0.06653059273958206,
+      0.061834584921598434,
+      0.06294949352741241,
+      0.0833306685090065,
+      0.05723286047577858,
+      0.10209332406520844,
+      0.07379071414470673,
+      0.07543861120939255,
+      0.08297624439001083,
+      0.05524091050028801,
+      0.059707846492528915,
+      0.09170634299516678,
+      0.05107459798455238,
+      0.06041371077299118,
+      0.04259059950709343
+    ],
+    "info_loss": [
+      -2.443110466003418,
+      -0.9830718040466309,
+      -0.34380578994750977,
+      -0.13375473022460938,
+      -0.037338972091674805,
+      -0.0056476593017578125,
+      -0.023563742637634277,
+      -0.039587974548339844,
+      -0.011068940162658691,
+      -0.017560958862304688,
+      -0.020556211471557617,
+      -0.040073513984680176,
+      -0.2276313304901123,
+      -0.4736506938934326,
+      -0.6916321516036987,
+      -1.1106035709381104,
+      -1.3533673286437988,
+      -1.345174789428711,
+      -1.4315921068191528,
+      -1.4162331819534302,
+      -1.425389289855957,
+      -1.530221939086914,
+      -1.5140444040298462,
+      -1.453958511352539,
+      -1.5316767692565918,
+      -1.4304397106170654,
+      -1.4811899662017822,
+      -1.4719672203063965,
+      -1.1827270984649658,
+      -1.01058828830719,
+      -1.0302265882492065,
+      -1.0075411796569824,
+      -1.0054174661636353,
+      -0.9188175797462463,
+      -0.6958320140838623,
+      -0.3105020523071289,
+      -0.22444483637809753,
+      -0.21009549498558044,
+      -0.1526564061641693,
+      -0.22486285865306854,
+      -0.1471777707338333,
+      -0.1766778975725174,
+      -0.1751600056886673,
+      -0.1338011622428894,
+      -0.11082087457180023,
+      -0.13831733167171478,
+      -0.17254909873008728,
+      -0.14046895503997803,
+      -0.07022452354431152,
+      -0.1155194491147995,
+      -0.10764320194721222,
+      -0.0832754522562027,
+      -0.09724847972393036,
+      -0.13976116478443146,
+      -0.11671318113803864,
+      -0.04512207210063934,
+      -0.09844155609607697,
+      -0.0655994862318039,
+      -0.10355991125106812,
+      -0.06686903536319733,
+      -0.047758981585502625,
+      -0.03816497325897217,
+      -0.10978902876377106,
+      -0.05273336172103882,
+      -0.07155050337314606,
+      -0.06343655288219452,
+      -0.07405881583690643,
+      -0.06636227667331696,
+      -0.05743493139743805,
+      -0.08178192377090454,
+      -0.08532267808914185,
+      -0.05918674170970917,
+      -0.11369664967060089,
+      -0.07849213480949402,
+      -0.09992456436157227,
+      -0.0748167335987091,
+      -0.06032632291316986,
+      -0.07384225726127625,
+      -0.06540761888027191,
+      -0.08129407465457916,
+      -0.05421702563762665,
+      -0.10962431132793427,
+      -0.08665110170841217,
+      -0.07463202625513077,
+      -0.0558621883392334,
+      -0.049349889159202576,
+      -0.12654651701450348,
+      -0.10095764696598053,
+      -0.10276724398136139,
+      -0.06816543638706207,
+      -0.10744629800319672,
+      -0.08932144939899445,
+      -0.06451625376939774,
+      -0.10664623975753784,
+      -0.1072486862540245,
+      -0.04360680282115936,
+      -0.1005820780992508,
+      -0.09108839929103851,
+      -0.0722021833062172,
+      -0.09877876937389374,
+      -0.0796872228384018,
+      -0.06297194957733154,
+      -0.065232053399086,
+      -0.07241103053092957,
+      -0.04337600618600845,
+      -0.08205326646566391,
+      -0.07278884202241898,
+      -0.10372095555067062,
+      -0.10585509240627289,
+      -0.08046573400497437,
+      -0.07830247282981873,
+      -0.07458939403295517,
+      -0.0484515056014061,
+      -0.08157063275575638,
+      -0.085136778652668,
+      -0.06511989235877991,
+      -0.10065937787294388,
+      -0.08356793224811554,
+      -0.06959053874015808,
+      -0.05505380779504776,
+      -0.05874547362327576,
+      -0.06111748516559601,
+      -0.07646494358778,
+      -0.05847612023353577,
+      -0.08626847714185715,
+      -0.042475759983062744,
+      -0.08023545891046524,
+      -0.044076353311538696,
+      -0.050270386040210724,
+      -0.08265894651412964,
+      -0.06367488205432892,
+      -0.04495718330144882,
+      -0.056383490562438965,
+      -0.022825509309768677,
+      -0.04397362470626831,
+      -0.04961772263050079,
+      -0.06585722416639328,
+      -0.07093081623315811,
+      -0.024609215557575226,
+      -0.024764984846115112,
+      -0.05155056715011597,
+      -0.048613011837005615,
+      -0.06162737309932709,
+      -0.05310824513435364,
+      -0.021410293877124786,
+      -0.08092104643583298,
+      -0.04702701419591904,
+      -0.06139340251684189,
+      -0.06087704002857208,
+      -0.04476284235715866,
+      -0.054584406316280365,
+      -0.038967013359069824,
+      -0.05515628308057785,
+      -0.048791706562042236,
+      -0.03137459605932236,
+      -0.048941873013973236,
+      -0.05163276195526123,
+      -0.028769835829734802,
+      -0.05319051444530487,
+      -0.05809234827756882,
+      -0.02926914393901825,
+      -0.0474691167473793,
+      -0.012960217893123627,
+      -0.03622855991125107,
+      -0.06237722933292389,
+      -0.03852071613073349,
+      -0.04634437710046768,
+      -0.046193867921829224,
+      -0.06395061314105988,
+      -0.04960039258003235,
+      -0.07187876850366592,
+      -0.021922878921031952,
+      -0.04167263209819794,
+      -0.00177793949842453,
+      -0.03270770609378815,
+      -0.06694523990154266,
+      -0.06431858986616135,
+      -0.03965383768081665,
+      -0.03687635064125061,
+      -0.04893855005502701,
+      -0.02683185786008835,
+      -0.058726489543914795,
+      -0.05305022746324539,
+      -0.059096939861774445,
+      -0.04153144359588623,
+      -0.06115543842315674,
+      -0.006133094429969788,
+      -0.03674937039613724,
+      -0.021276235580444336,
+      -0.015031620860099792,
+      -0.03982865810394287,
+      -0.06908920407295227,
+      -0.03186141699552536,
+      -0.02740943431854248,
+      -0.0423111692070961,
+      -0.06469681113958359,
+      -0.010958805680274963,
+      -0.04632203280925751,
+      -0.024401865899562836,
+      -0.049824584275484085,
+      -0.04496334493160248,
+      -0.03608852997422218,
+      -0.03957623988389969,
+      -0.05318208411335945,
+      -0.02267289161682129,
+      -0.04702337831258774,
+      -0.04959384351968765,
+      -0.028320923447608948,
+      -0.03522076830267906,
+      -0.050579946488142014,
+      -0.04544609412550926,
+      -0.03695450723171234,
+      -0.04056013002991676,
+      -0.041364412754774094,
+      0.011226698756217957,
+      -0.05030190944671631,
+      -0.05349079146981239,
+      -0.010130137205123901,
+      -0.022041209042072296,
+      -0.060964345932006836,
+      -0.02661222591996193,
+      -0.06252823770046234,
+      -0.03567666560411453,
+      -0.03669260814785957,
+      -0.053341902792453766,
+      -0.043097540736198425,
+      -0.04930409416556358,
+      -0.05882496014237404,
+      -0.035445183515548706,
+      -0.0665692389011383,
+      -0.03925636410713196,
+      -0.04403889179229736,
+      -0.034985627979040146,
+      -0.060415588319301605,
+      -0.07550159841775894,
+      -0.05967997759580612,
+      -0.05609051138162613,
+      -0.029090940952301025,
+      -0.03370439261198044,
+      -0.042386144399642944,
+      -0.0021385028958320618,
+      -0.04490642249584198,
+      -0.034814950078725815,
+      -0.03687971085309982,
+      0.0001629963517189026,
+      -0.05296597629785538,
+      -0.0229356586933136,
+      -0.03632236644625664,
+      -0.03866932541131973,
+      -0.053305819630622864,
+      -0.03888783976435661,
+      -0.031676627695560455,
+      -0.04961540177464485,
+      -0.02907579392194748,
+      -0.0065491050481796265,
+      -0.03517160192131996,
+      0.0003564804792404175,
+      -0.022830042988061905,
+      -0.01506049931049347,
+      -0.016031458973884583,
+      -0.03129471838474274,
+      -0.027171839028596878,
+      -0.0036737658083438873,
+      -0.02487041801214218,
+      -0.050898805260658264,
+      -0.03599712252616882,
+      -0.014986097812652588,
+      -0.02811621129512787,
+      -0.0542818047106266,
+      -0.04727591574192047,
+      -0.04651058837771416,
+      -0.01016305759549141,
+      -0.0213940367102623,
+      -0.041690148413181305,
+      -0.004094921052455902,
+      -0.017976440489292145,
+      -0.03852062299847603,
+      -0.0325314961373806,
+      -0.012182284146547318,
+      -0.03236742317676544,
+      -0.04288274794816971,
+      -0.022669844329357147,
+      -0.03187123313546181,
+      -0.010090179741382599,
+      -0.03481444716453552,
+      -0.027848124504089355,
+      -0.029044929891824722,
+      -0.02672528475522995,
+      -0.05314160883426666,
+      -0.025125503540039062,
+      -0.04815103858709335,
+      -0.03401355817914009,
+      -0.02638285979628563,
+      -0.032920923084020615,
+      -0.017589908093214035,
+      -0.02234787493944168,
+      -0.05124233290553093,
+      -0.016761846840381622,
+      0.004244372248649597,
+      -0.025563932955265045
+    ],
+    "abs_loss": [
+      3.394221544265747,
+      3.3425486087799072,
+      3.22790265083313,
+      2.9926300048828125,
+      2.774336814880371,
+      2.7687575817108154,
+      2.7493293285369873,
+      2.762601137161255,
+      2.7054216861724854,
+      2.7506332397460938,
+      2.6832504272460938,
+      2.5054931640625,
+      2.3730170726776123,
+      2.254948377609253,
+      2.2844619750976562,
+      2.330230474472046,
+      2.2819957733154297,
+      2.294659376144409,
+      2.2515642642974854,
+      2.2135226726531982,
+      2.2340457439422607,
+      2.1694371700286865,
+      2.0368919372558594,
+      1.9205093383789062,
+      1.825921654701233,
+      1.838506817817688,
+      1.8240076303482056,
+      1.8133264780044556,
+      1.7123349905014038,
+      1.604567050933838,
+      1.3610414266586304,
+      1.3402241468429565,
+      1.3114943504333496,
+      1.2653779983520508,
+      1.2305165529251099,
+      1.2111915349960327,
+      1.0662754774093628,
+      1.1448066234588623,
+      1.0240867137908936,
+      0.9730188250541687,
+      1.0078809261322021,
+      0.9083463549613953,
+      0.8403894305229187,
+      0.8990464806556702,
+      0.6939436793327332,
+      0.6455739140510559,
+      0.5897447466850281,
+      0.5626329779624939,
+      0.61261385679245,
+      0.5184110999107361,
+      0.5043537616729736,
+      0.4778100550174713,
+      0.45053520798683167,
+      0.38508930802345276,
+      0.42810335755348206,
+      0.39950668811798096,
+      0.38665273785591125,
+      0.37346145510673523,
+      0.3526777923107147,
+      0.3617526590824127,
+      0.32074984908103943,
+      0.3057343065738678,
+      0.3217543363571167,
+      0.29489123821258545,
+      0.30857205390930176,
+      0.2831171452999115,
+      0.2468472719192505,
+      0.25066375732421875,
+      0.2407059669494629,
+      0.22527122497558594,
+      0.2665853798389435,
+      0.19341807067394257,
+      0.21876835823059082,
+      0.2602218687534332,
+      0.20035600662231445,
+      0.23149482905864716,
+      0.18165135383605957,
+      0.15450970828533173,
+      0.18449842929840088,
+      0.16877329349517822,
+      0.1804056167602539,
+      0.1811109334230423,
+      0.15943928062915802,
+      0.1721697896718979,
+      0.15191181004047394,
+      0.12861792743206024,
+      0.15852874517440796,
+      0.1552044004201889,
+      0.12914901971817017,
+      0.13152097165584564,
+      0.13537022471427917,
+      0.1076178178191185,
+      0.10609632730484009,
+      0.1305665224790573,
+      0.10949480533599854,
+      0.11212343722581863,
+      0.10800478607416153,
+      0.10685724020004272,
+      0.1074957326054573,
+      0.14475202560424805,
+      0.09349382668733597,
+      0.10390307754278183,
+      0.10435584932565689,
+      0.09256798028945923,
+      0.09340608865022659,
+      0.08003900200128555,
+      0.08626168221235275,
+      0.11689415574073792,
+      0.10936558246612549,
+      0.11035115271806717,
+      0.10380777716636658,
+      0.09321699291467667,
+      0.08948347717523575,
+      0.11074743419885635,
+      0.11069627851247787,
+      0.10044410824775696,
+      0.08233988285064697,
+      0.10224362462759018,
+      0.08702748268842697,
+      0.07840032130479813,
+      0.06811333447694778,
+      0.09012255817651749,
+      0.07289918512105942,
+      0.0692102313041687,
+      0.047726381570100784,
+      0.06463932245969772,
+      0.08292338997125626,
+      0.07461681216955185,
+      0.06324749439954758,
+      0.06664235144853592,
+      0.04345884546637535,
+      0.049647629261016846,
+      0.0665830671787262,
+      0.06063719466328621,
+      0.048336829990148544,
+      0.046793874353170395,
+      0.047989800572395325,
+      0.07219207286834717,
+      0.06383933126926422,
+      0.055901769548654556,
+      0.053876880556344986,
+      0.057475488632917404,
+      0.0412849523127079,
+      0.06574953347444534,
+      0.056724607944488525,
+      0.059202104806900024,
+      0.04908278211951256,
+      0.04852571710944176,
+      0.044907182455062866,
+      0.055275242775678635,
+      0.05050094798207283,
+      0.0620327889919281,
+      0.06068297103047371,
+      0.047100067138671875,
+      0.051009487360715866,
+      0.06336609274148941,
+      0.04895112290978432,
+      0.03967296704649925,
+      0.05015207454562187,
+      0.055682867765426636,
+      0.04451887309551239,
+      0.03750608488917351,
+      0.03661644831299782,
+      0.03172827139496803,
+      0.03236657753586769,
+      0.03902410343289375,
+      0.03826131299138069,
+      0.060825686901807785,
+      0.047127123922109604,
+      0.0467449314892292,
+      0.05455436930060387,
+      0.056337881833314896,
+      0.03535234555602074,
+      0.03756538778543472,
+      0.04362154006958008,
+      0.040451839566230774,
+      0.040479522198438644,
+      0.031483180820941925,
+      0.03808044269680977,
+      0.031315192580223083,
+      0.03752255067229271,
+      0.047609519213438034,
+      0.04497078061103821,
+      0.03907888010144234,
+      0.031043216586112976,
+      0.03193427994847298,
+      0.03983858600258827,
+      0.02731279842555523,
+      0.028313567861914635,
+      0.05365240201354027,
+      0.03536854311823845,
+      0.037834104150533676,
+      0.04222561791539192,
+      0.044769782572984695,
+      0.04278169944882393,
+      0.03564367815852165,
+      0.03349684551358223,
+      0.0414416678249836,
+      0.041359905153512955,
+      0.054943352937698364,
+      0.047847677022218704,
+      0.03144362196326256,
+      0.03555428609251976,
+      0.03604207932949066,
+      0.028404369950294495,
+      0.03827707841992378,
+      0.027843812480568886,
+      0.03264890983700752,
+      0.043662842363119125,
+      0.036455363035202026,
+      0.02766052819788456,
+      0.0236380472779274,
+      0.03159993514418602,
+      0.034537479281425476,
+      0.03209583833813667,
+      0.02440265379846096,
+      0.027797268703579903,
+      0.029441386461257935,
+      0.022282719612121582,
+      0.03694484382867813,
+      0.04092557355761528,
+      0.024900861084461212,
+      0.03011191636323929,
+      0.025265930220484734,
+      0.048013120889663696,
+      0.034523218870162964,
+      0.02858671359717846,
+      0.03023497946560383,
+      0.04341937229037285,
+      0.03332659602165222,
+      0.02852858603000641,
+      0.03800017759203911,
+      0.040301937609910965,
+      0.03499029204249382,
+      0.02038094773888588,
+      0.032415006309747696,
+      0.033108580857515335,
+      0.024987995624542236,
+      0.01724293641746044,
+      0.02603638917207718,
+      0.018813790753483772,
+      0.02405432052910328,
+      0.021730691194534302,
+      0.022888338193297386,
+      0.03597116470336914,
+      0.015848785638809204,
+      0.021455630660057068,
+      0.03072260320186615,
+      0.03099057637155056,
+      0.02779287099838257,
+      0.020777510479092598,
+      0.0214055385440588,
+      0.019314980134367943,
+      0.025113217532634735,
+      0.0106942318379879,
+      0.020709769800305367,
+      0.015123255550861359,
+      0.02733311615884304,
+      0.01939031481742859,
+      0.021522225812077522,
+      0.04024254530668259,
+      0.018009236082434654,
+      0.018949633464217186,
+      0.024899272248148918,
+      0.01709776185452938,
+      0.019751494750380516,
+      0.021808087825775146,
+      0.017145484685897827,
+      0.024097466841340065,
+      0.014123689383268356,
+      0.017810897901654243,
+      0.016506683081388474,
+      0.020843377336859703,
+      0.025091519579291344,
+      0.02594064176082611,
+      0.02828248403966427,
+      0.02131149172782898,
+      0.02172340452671051,
+      0.01704324781894684,
+      0.0166668388992548,
+      0.015181179158389568,
+      0.016981074586510658,
+      0.022413598373532295,
+      0.020577698945999146,
+      0.021580925211310387,
+      0.016879981383681297,
+      0.019611502066254616,
+      0.02064107544720173,
+      0.01786014623939991,
+      0.021099457517266273,
+      0.01964995265007019,
+      0.01599390059709549,
+      0.01671198196709156,
+      0.020701633766293526,
+      0.012152418494224548,
+      0.013006716035306454,
+      0.015204588882625103,
+      0.020748918876051903,
+      0.018073061481118202,
+      0.014983761124312878
+    ],
+    "zipf_loss": [
+      6.747013092041016,
+      4.822678089141846,
+      2.723876476287842,
+      1.3532776832580566,
+      0.9314618110656738,
+      0.8614562749862671,
+      0.8446267247200012,
+      0.841794490814209,
+      0.8453340530395508,
+      0.8426874279975891,
+      0.8421101570129395,
+      0.8234671354293823,
+      0.7779624462127686,
+      0.7785301208496094,
+      0.7750071287155151,
+      0.7884855270385742,
+      0.787139356136322,
+      0.7803397178649902,
+      0.7820841670036316,
+      0.7785444855690002,
+      0.7747706174850464,
+      0.7543097734451294,
+      0.7130584120750427,
+      0.6521852612495422,
+      0.5619404911994934,
+      0.5034530162811279,
+      0.4888960123062134,
+      0.4868062734603882,
+      0.47289320826530457,
+      0.41840624809265137,
+      0.34453150629997253,
+      0.33006274700164795,
+      0.3205908536911011,
+      0.2842155396938324,
+      0.2615243196487427,
+      0.29358524084091187,
+      0.2730894386768341,
+      0.2505728602409363,
+      0.23486685752868652,
+      0.24164986610412598,
+      0.23617364466190338,
+      0.2079915702342987,
+      0.20436114072799683,
+      0.20752990245819092,
+      0.18855509161949158,
+      0.14551660418510437,
+      0.16822293400764465,
+      0.14817431569099426,
+      0.15346264839172363,
+      0.15633323788642883,
+      0.15323308110237122,
+      0.15594005584716797,
+      0.1262887716293335,
+      0.12595269083976746,
+      0.137677863240242,
+      0.11767086386680603,
+      0.11685687303543091,
+      0.10320362448692322,
+      0.12585163116455078,
+      0.08425500988960266,
+      0.09238368272781372,
+      0.0991685539484024,
+      0.08050265908241272,
+      0.08710445463657379,
+      0.06438830494880676,
+      0.06196661666035652,
+      0.07445895671844482,
+      0.06691229343414307,
+      0.06412997841835022,
+      0.03723148629069328,
+      0.038339994847774506,
+      0.04211884364485741,
+      0.034377939999103546,
+      0.0266835018992424,
+      0.03380643576383591,
+      0.028748363256454468,
+      0.024847157299518585,
+      0.029961716383695602,
+      0.01965174823999405,
+      0.027315640822052956,
+      0.020966196432709694,
+      0.024083871394395828,
+      0.023763194680213928,
+      0.01866382360458374,
+      0.018237322568893433,
+      0.018421687185764313,
+      0.022680208086967468,
+      0.015751492232084274,
+      0.02245470881462097,
+      0.02570737712085247,
+      0.019115181639790535,
+      0.021580729633569717,
+      0.01205367036163807,
+      0.018545828759670258,
+      0.0225033238530159,
+      0.01769322156906128,
+      0.02403894066810608,
+      0.01521223969757557,
+      0.013828056864440441,
+      0.018964827060699463,
+      0.01723816618323326,
+      0.015977535396814346,
+      0.02180652879178524,
+      0.012077492661774158,
+      0.01832667924463749,
+      0.018675509840250015,
+      0.0238608717918396,
+      0.01266377791762352,
+      0.011767813935875893,
+      0.020118137821555138,
+      0.027247916907072067,
+      0.018682461231946945,
+      0.020915666595101357,
+      0.010882147587835789,
+      0.010815946385264397,
+      0.017469903454184532,
+      0.02203744649887085,
+      0.014966797083616257,
+      0.022188443690538406,
+      0.016178730875253677,
+      0.017197515815496445,
+      0.016653619706630707,
+      0.010109160095453262,
+      0.007518783211708069,
+      0.022306252270936966,
+      0.012615780346095562,
+      0.014079699292778969,
+      0.012447178363800049,
+      0.010188139975070953,
+      0.01454751007258892,
+      0.012892195023596287,
+      0.01669362001121044,
+      0.017555873841047287,
+      0.016288474202156067,
+      0.015565053559839725,
+      0.01319822110235691,
+      0.012723729014396667,
+      0.013090583495795727,
+      0.014801254495978355,
+      0.017604392021894455,
+      0.018251638859510422,
+      0.019749276340007782,
+      0.009087889455258846,
+      0.00973625760525465,
+      0.016905393451452255,
+      0.014198819175362587,
+      0.011103419587016106,
+      0.0197756364941597,
+      0.018736621364951134,
+      0.016376283019781113,
+      0.011056948453187943,
+      0.008528265170753002,
+      0.019545545801520348,
+      0.022005023434758186,
+      0.01654922030866146,
+      0.016469169408082962,
+      0.012916946783661842,
+      0.012869670987129211,
+      0.013066276907920837,
+      0.015071604400873184,
+      0.020286250859498978,
+      0.01001240685582161,
+      0.011937970295548439,
+      0.01746869832277298,
+      0.013280334882438183,
+      0.01689305528998375,
+      0.016692781820893288,
+      0.011339399963617325,
+      0.020430169999599457,
+      0.011846745386719704,
+      0.011662904173135757,
+      0.010297768749296665,
+      0.011808132752776146,
+      0.018647782504558563,
+      0.013151591643691063,
+      0.015682509168982506,
+      0.012883247807621956,
+      0.011098029091954231,
+      0.013733375817537308,
+      0.013265689834952354,
+      0.007580380886793137,
+      0.013296203687787056,
+      0.01342364028096199,
+      0.013310050591826439,
+      0.011032825335860252,
+      0.014104127883911133,
+      0.014044847339391708,
+      0.00917900912463665,
+      0.016713468357920647,
+      0.01430018525570631,
+      0.009842156432569027,
+      0.01796175166964531,
+      0.011575086042284966,
+      0.011689379811286926,
+      0.010786101222038269,
+      0.016487501561641693,
+      0.02086007222533226,
+      0.013050362467765808,
+      0.013644913211464882,
+      0.020701535046100616,
+      0.01479027047753334,
+      0.014744661748409271,
+      0.010941563174128532,
+      0.009068634361028671,
+      0.012561552226543427,
+      0.014813359826803207,
+      0.009117796085774899,
+      0.005783336237072945,
+      0.009056288748979568,
+      0.011517772451043129,
+      0.008173095062375069,
+      0.011025875806808472,
+      0.006859137210994959,
+      0.00984964519739151,
+      0.012724005617201328,
+      0.007011702284216881,
+      0.010704891756176949,
+      0.009455816820263863,
+      0.011419336311519146,
+      0.013902952894568443,
+      0.018162913620471954,
+      0.01665625348687172,
+      0.010108448565006256,
+      0.009669994935393333,
+      0.00467725470662117,
+      0.011046669445931911,
+      0.006968652829527855,
+      0.012394542805850506,
+      0.010805947706103325,
+      0.009528258815407753,
+      0.009604508057236671,
+      0.0165615975856781,
+      0.009844701737165451,
+      0.012714133597910404,
+      0.015196169726550579,
+      0.012383313849568367,
+      0.016092542558908463,
+      0.01423655729740858,
+      0.007803167216479778,
+      0.010753775015473366,
+      0.014365803450345993,
+      0.014451751485466957,
+      0.006029514595866203,
+      0.006355271674692631,
+      0.009965854696929455,
+      0.00869377888739109,
+      0.007017104886472225,
+      0.01357170008122921,
+      0.013076789677143097,
+      0.010633924975991249,
+      0.011647232808172703,
+      0.009225110523402691,
+      0.008964217267930508,
+      0.007008466869592667,
+      0.012218313291668892,
+      0.012870638631284237,
+      0.01474965363740921,
+      0.006232164800167084,
+      0.006476912647485733,
+      0.009695646353065968,
+      0.009330593049526215,
+      0.009849331341683865,
+      0.015900786966085434,
+      0.009572213515639305,
+      0.008015384897589684,
+      0.010157611221075058,
+      0.013603226281702518,
+      0.004118527285754681,
+      0.016622694209218025,
+      0.009203871712088585,
+      0.008609105832874775,
+      0.01687009260058403,
+      0.006735082715749741,
+      0.007885177619755268,
+      0.009929598309099674,
+      0.012403037399053574,
+      0.008098751306533813,
+      0.00846356712281704,
+      0.009005330502986908,
+      0.007148829288780689,
+      0.016754604876041412,
+      0.01161285862326622,
+      0.009531901217997074,
+      0.007812591269612312,
+      0.013085611164569855,
+      0.012327302247285843,
+      0.007569424342364073,
+      0.011285560205578804,
+      0.009960152208805084,
+      0.007550247013568878,
+      0.008531223982572556,
+      0.009924929589033127,
+      0.008582849986851215,
+      0.012703579850494862,
+      0.013704492710530758,
+      0.007877406664192677,
+      0.014441302046179771,
+      0.007667067926377058,
+      0.01101275347173214,
+      0.01070137694478035
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.8126358389854431,
+      0.49192842841148376,
+      0.30566778779029846,
+      0.2445502132177353,
+      0.230295330286026,
+      0.2272050827741623,
+      0.21112771332263947,
+      0.21245542168617249,
+      0.23163801431655884,
+      0.22638672590255737,
+      0.20352934300899506,
+      0.21680136024951935,
+      0.2308817058801651,
+      0.27459022402763367,
+      0.3101886212825775,
+      0.3400084674358368,
+      0.3439534604549408,
+      0.3536595404148102,
+      0.362904816865921,
+      0.36613044142723083,
+      0.37627190351486206,
+      0.3862797021865845,
+      0.38861405849456787,
+      0.3878909647464752,
+      0.38153237104415894,
+      0.37819451093673706,
+      0.3674255311489105,
+      0.35987815260887146,
+      0.3617212176322937,
+      0.36654412746429443,
+      0.36772945523262024,
+      0.3630082607269287,
+      0.3609016537666321,
+      0.36461979150772095,
+      0.36813828349113464,
+      0.3749433755874634,
+      0.38107213377952576,
+      0.3846074342727661,
+      0.38342151045799255,
+      0.38447272777557373,
+      0.3863747715950012,
+      0.385100394487381,
+      0.39256051182746887,
+      0.3942987620830536,
+      0.4009149968624115,
+      0.4046420753002167,
+      0.4048827290534973,
+      0.40542522072792053,
+      0.4052145183086395,
+      0.4064948856830597,
+      0.40279141068458557,
+      0.40418675541877747,
+      0.4020644724369049,
+      0.4008735418319702,
+      0.39961743354797363,
+      0.4101741313934326,
+      0.41169995069503784,
+      0.4100269377231598,
+      0.4104335606098175,
+      0.40870702266693115,
+      0.4081622362136841,
+      0.4024530053138733,
+      0.40157246589660645,
+      0.4044875502586365,
+      0.4056023359298706,
+      0.4041389524936676,
+      0.4050261378288269,
+      0.4024779796600342,
+      0.39882200956344604,
+      0.40071913599967957,
+      0.39949002861976624,
+      0.4024513065814972,
+      0.4003760516643524,
+      0.3959936797618866,
+      0.3952066898345947,
+      0.3944344222545624,
+      0.39839205145835876,
+      0.3976265788078308,
+      0.39335161447525024,
+      0.391871839761734,
+      0.39187031984329224,
+      0.39400514960289,
+      0.39289364218711853,
+      0.3908037841320038,
+      0.3937630355358124,
+      0.38873982429504395,
+      0.3900105655193329,
+      0.3855060338973999,
+      0.38507887721061707,
+      0.38578882813453674,
+      0.3866000175476074,
+      0.38717448711395264,
+      0.38682228326797485,
+      0.3826015293598175,
+      0.38251370191574097,
+      0.38165315985679626,
+      0.37959685921669006,
+      0.37887221574783325,
+      0.37327221035957336,
+      0.37203267216682434,
+      0.3737097382545471,
+      0.37534618377685547,
+      0.37143731117248535,
+      0.37369057536125183,
+      0.3711692690849304,
+      0.3645460307598114,
+      0.36469992995262146,
+      0.361573725938797,
+      0.3585306704044342,
+      0.36054661870002747,
+      0.35953187942504883,
+      0.3600585162639618,
+      0.3590182363986969,
+      0.36079591512680054,
+      0.36059656739234924,
+      0.35834160447120667,
+      0.3599343001842499,
+      0.356032133102417,
+      0.35449254512786865,
+      0.3545680344104767,
+      0.3536050021648407,
+      0.3541582524776459,
+      0.35545405745506287,
+      0.35433292388916016,
+      0.35316047072410583,
+      0.35358694195747375,
+      0.35297971963882446,
+      0.3502896726131439,
+      0.34449663758277893,
+      0.34334665536880493,
+      0.33816829323768616,
+      0.33465084433555603,
+      0.3382340371608734,
+      0.33520641922950745,
+      0.3278089463710785,
+      0.3211253881454468,
+      0.3193509876728058,
+      0.31470799446105957,
+      0.31724998354911804,
+      0.31537848711013794,
+      0.31163156032562256,
+      0.3188633322715759,
+      0.31201285123825073,
+      0.3139425218105316,
+      0.31069427728652954,
+      0.30984994769096375,
+      0.307725191116333,
+      0.30994686484336853,
+      0.31188634037971497,
+      0.31063711643218994,
+      0.3107324242591858,
+      0.3080355226993561,
+      0.3090537190437317,
+      0.3069782257080078,
+      0.30067747831344604,
+      0.30017662048339844,
+      0.2992165982723236,
+      0.2981402575969696,
+      0.2967067062854767,
+      0.2959250211715698,
+      0.29625648260116577,
+      0.2977386713027954,
+      0.29558253288269043,
+      0.29242148995399475,
+      0.2927606999874115,
+      0.2916039228439331,
+      0.2908530533313751,
+      0.28800585865974426,
+      0.28696495294570923,
+      0.29193684458732605,
+      0.29141637682914734,
+      0.28812092542648315,
+      0.2875635027885437,
+      0.2866673469543457,
+      0.2886551022529602,
+      0.28811702132225037,
+      0.2905014455318451,
+      0.2918323576450348,
+      0.29103443026542664,
+      0.29012441635131836,
+      0.2900857925415039,
+      0.28832337260246277,
+      0.289643794298172,
+      0.28799915313720703,
+      0.2890471816062927,
+      0.28987666964530945,
+      0.2887348234653473,
+      0.2864077091217041,
+      0.28664958477020264,
+      0.2874160706996918,
+      0.2884419560432434,
+      0.28907591104507446,
+      0.28930869698524475,
+      0.29081371426582336,
+      0.29029399156570435,
+      0.2868057191371918,
+      0.2863052785396576,
+      0.28596261143684387,
+      0.2849084138870239,
+      0.2872099280357361,
+      0.28547489643096924,
+      0.2857936918735504,
+      0.2858123183250427,
+      0.28520363569259644,
+      0.287820965051651,
+      0.2876909077167511,
+      0.285679429769516,
+      0.2855439782142639,
+      0.2861383855342865,
+      0.285662978887558,
+      0.28683218359947205,
+      0.2871406674385071,
+      0.2880229353904724,
+      0.2875096797943115,
+      0.2877349555492401,
+      0.28694555163383484,
+      0.28697383403778076,
+      0.28480905294418335,
+      0.28602585196495056,
+      0.2852811813354492,
+      0.28430798649787903,
+      0.2849596440792084,
+      0.28608235716819763,
+      0.2859610617160797,
+      0.2851969301700592,
+      0.28335556387901306,
+      0.2832303047180176,
+      0.284677654504776,
+      0.2841862738132477,
+      0.28447914123535156,
+      0.28385472297668457,
+      0.2831842005252838,
+      0.28299471735954285,
+      0.2840172350406647,
+      0.28423047065734863,
+      0.28590795397758484,
+      0.2871611714363098,
+      0.2861233353614807,
+      0.28660011291503906,
+      0.28792083263397217,
+      0.2868644893169403,
+      0.2866981327533722,
+      0.28655770421028137,
+      0.2847079336643219,
+      0.28378015756607056,
+      0.28484398126602173,
+      0.2854790985584259,
+      0.28632882237434387,
+      0.28661829233169556,
+      0.2872510254383087,
+      0.2872370779514313,
+      0.2879977226257324,
+      0.28787800669670105,
+      0.288217157125473,
+      0.28802675008773804,
+      0.28761228919029236,
+      0.2875608801841736,
+      0.2874130606651306,
+      0.28766700625419617,
+      0.2872161865234375,
+      0.2871786057949066,
+      0.28677403926849365,
+      0.28692445158958435,
+      0.2867109477519989,
+      0.2872385084629059,
+      0.28683581948280334,
+      0.28583160042762756,
+      0.2859218716621399,
+      0.28645429015159607,
+      0.2865743637084961,
+      0.28658926486968994,
+      0.2868916392326355,
+      0.28663891553878784,
+      0.2867935299873352,
+      0.2870182693004608,
+      0.2874501943588257,
+      0.2881462872028351,
+      0.28878867626190186,
+      0.2889878451824188,
+      0.2888662815093994,
+      0.28873029351234436,
+      0.2886132299900055,
+      0.2884853184223175,
+      0.28849804401397705,
+      0.28871506452560425,
+      0.2887803018093109,
+      0.28906843066215515,
+      0.28893446922302246,
+      0.2891843616962433,
+      0.2891238033771515,
+      0.28913286328315735,
+      0.2892856001853943,
+      0.2893139719963074,
+      0.28949904441833496,
+      0.2895032465457916,
+      0.28981322050094604,
+      0.2899792790412903,
+      0.29019999504089355,
+      0.29007700085639954,
+      0.29011985659599304
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2.5470085470085475e-05,
+      3.401709401709402e-05,
+      4.2564102564102564e-05,
+      5.1111111111111115e-05,
+      5.9658119658119666e-05,
+      6.820512820512821e-05,
+      7.675213675213675e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      7.98600052728711e-05,
+      7.92668072765621e-05,
+      7.86736092802531e-05,
+      7.808041128394411e-05,
+      7.748721328763512e-05,
+      7.689401529132612e-05,
+      7.630081729501713e-05,
+      7.570761929870815e-05,
+      7.511442130239916e-05,
+      7.452122330609016e-05,
+      7.392802530978118e-05,
+      7.333482731347219e-05,
+      7.236198259952545e-05,
+      7.176878460321646e-05,
+      7.117558660690747e-05,
+      7.058238861059848e-05,
+      6.998919061428949e-05,
+      6.939599261798048e-05,
+      6.88027946216715e-05,
+      6.820959662536251e-05,
+      6.761639862905352e-05,
+      6.702320063274453e-05,
+      6.643000263643554e-05,
+      6.583680464012655e-05,
+      6.524360664381755e-05,
+      6.465040864750859e-05,
+      6.405721065119958e-05,
+      6.308436593725284e-05,
+      6.249116794094385e-05,
+      6.189796994463487e-05,
+      6.130477194832587e-05,
+      6.0711573952016877e-05,
+      6.011837595570788e-05,
+      5.9525177959398896e-05,
+      5.8931979963089905e-05,
+      5.833878196678091e-05,
+      5.774558397047194e-05,
+      5.715238597416294e-05,
+      5.6559187977853943e-05,
+      5.5965989981544967e-05,
+      5.537279198523596e-05,
+      5.477959398892697e-05,
+      5.380674927498023e-05,
+      5.321355127867123e-05,
+      5.2620353282362255e-05,
+      5.202715528605326e-05,
+      5.143395728974426e-05,
+      5.084075929343529e-05,
+      5.024756129712629e-05,
+      4.9654363300817296e-05,
+      4.9061165304508305e-05,
+      4.846796730819932e-05,
+      4.787476931189032e-05,
+      4.7281571315581334e-05,
+      4.668837331927235e-05,
+      4.609517532296336e-05,
+      4.550197732665436e-05,
+      4.452913261270761e-05,
+      4.393593461639862e-05,
+      4.334273662008965e-05,
+      4.2749538623780655e-05,
+      4.215634062747166e-05,
+      4.156314263116268e-05,
+      4.0969944634853683e-05,
+      4.0376746638544686e-05,
+      3.978354864223571e-05,
+      3.919035064592671e-05,
+      3.8597152649617715e-05,
+      3.800395465330872e-05,
+      3.741075665699975e-05,
+      3.681755866069075e-05,
+      3.622436066438175e-05,
+      3.525151595043501e-05,
+      3.465831795412603e-05,
+      3.4065119957817036e-05,
+      3.347192196150804e-05,
+      3.287872396519904e-05,
+      3.2285525968890064e-05,
+      3.169232797258107e-05,
+      3.109912997627207e-05,
+      3.05059319799631e-05,
+      2.9912733983654103e-05,
+      2.9319535987345105e-05,
+      2.8726337991036132e-05,
+      2.8133139994727138e-05,
+      2.753994199841814e-05,
+      2.6946744002109144e-05,
+      2.5973899288162397e-05,
+      2.538070129185342e-05,
+      2.4787503295544423e-05,
+      2.419430529923543e-05,
+      2.3601107302926452e-05,
+      2.3007909306617458e-05,
+      2.241471131030846e-05,
+      2.1821513313999464e-05,
+      2.122831531769049e-05,
+      2.0635117321381496e-05,
+      2.00419193250725e-05,
+      1.944872132876352e-05,
+      1.8855523332454525e-05,
+      1.8262325336145528e-05,
+      1.766912733983655e-05,
+      1.6696282625889808e-05,
+      1.610308462958081e-05,
+      1.5509886633271817e-05,
+      1.491668863696282e-05,
+      1.4323490640653843e-05,
+      1.3730292644344847e-05,
+      1.3137094648035852e-05,
+      1.2543896651726875e-05,
+      1.1950698655417879e-05,
+      1.1357500659108882e-05,
+      1.0764302662799886e-05,
+      1.017110466649091e-05,
+      9.577906670181914e-06,
+      8.984708673872917e-06,
+      8.391510677563942e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      750,
+      1532,
+      2314,
+      3096,
+      3878,
+      4660,
+      5442,
+      6224,
+      7006,
+      7788,
+      8570,
+      9352,
+      10134,
+      10916,
+      11698,
+      12480,
+      13262,
+      14044,
+      14826,
+      15608
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.6758333333333333,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9940828402366864,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9797297297297297,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.88,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9938650306748467,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9769230769230769,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9770114942528736,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9704433497536946,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9752066115702479,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.9795918367346939,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.6989247311827957,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.968609865470852,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.28,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.5625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.742671009771987,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.08,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.26,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.4875,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.3,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.632,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.9,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9888143176733781,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.99375,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9773156899810964,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.9583333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.61,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8031088082901554,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.9065420560747663,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.65,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.8888888888888888,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.55,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8562091503267973,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8247422680412371,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.56,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8715846994535519,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9102564102564102,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966722129783694,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.996415770609319,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9862068965517241,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9166666666666666,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.996031746031746,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.77,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9906103286384976,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9911504424778761,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.8839779005524862,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.33,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.994413407821229,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5906040268456376,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.892018779342723,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.38,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.55,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.38,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.355,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.93,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.995,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9962546816479401,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.979498861047836,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.975609756097561,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.65,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9933333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8375634517766497,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.8543689320388349,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.54,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8340080971659919,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.803921568627451,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8187919463087249,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.8168316831683168,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.6070833333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9940828402366864,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9966216216216216,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.89,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9877300613496932,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9846153846153847,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9770114942528736,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9655172413793104,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9914529914529915,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.8387096774193549,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.968609865470852,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.44,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.66875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.8143322475570033,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.29,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.6025,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.18,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.37,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.634,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.955,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9932885906040269,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9886578449905482,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.64,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8186528497409327,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.8878504672897196,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.63,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.86328125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.875,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9052287581699346,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8505154639175257,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.65,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.907103825136612,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9273504273504274,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966722129783694,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.996415770609319,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.996031746031746,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.78,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9906103286384976,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9734513274336283,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.9060773480662984,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.53,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.994413407821229,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.7114093959731543,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.8826291079812206,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.27,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.64,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.4775,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.93,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9933333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9772209567198178,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.9512195121951219,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.69,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8578680203045685,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.8737864077669902,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.66,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8663967611336032,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.8627450980392157,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.51,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8221476510067114,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.8861386138613861,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.6758333333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.6758333333333333,
+  "sft_overall_accuracy": 0.6070833333333333
+}

add_sub_sorl_v1_abs30_K1_50K_1L3H510d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3ff96e07f57ce4ed08031caad701e5b6cf1c2ab24fafbc81c5af9691624d9be
+size 634760676

add_sub_sorl_v1_abs30_K1_50K_1L3H510d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 8e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 781,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K1_50K_1L3H510d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 1,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 50000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 158613826,
+  "run_name": "add_sub_sorl_v1_abs30_K1_50K_1L3H510d",
+  "git_commit": "17e935f460a7f9595b705c1d614101a6b0e520f7",
+  "timestamp": "2026-04-14T03:03:03.878936+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "ea3ii1tr",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/ea3ii1tr",
+  "final_accuracy": 0.6758333333333333,
+  "sft_accuracy": 0.6070833333333333,
+  "eval_method": "ArithmeticEvaluator"
+}