Download code/results/lab6/real_loops.json from thegovind/weight-space-programming: direct link, hf CLI and curl.
- Browser
- Download file 57 kB
-
https://huggingface.co/spaces/thegovind/weight-space-programming/resolve/main/code/results/lab6/real_loops.json
- Command line
-
hf download hf://spaces/thegovind/weight-space-programming/code/results/lab6/real_loops.json
-
curl -L -o real_loops.json https://huggingface.co/spaces/thegovind/weight-space-programming/resolve/main/code/results/lab6/real_loops.json
57 kB
{
"n": 48,
"mean": {
"early (first half before the loop)": {
"s15": 0.09580337822747727,
"s19": 0.11637471662834287,
"kl": 0.020822666760068387,
"p_next_base": 0.767474232862393,
"p_next_edit": 0.7503405945996443,
"p_pick_base": 0.8341055139899254,
"p_pick_edit": 0.820721243818601
},
"late (last 512 tokens before the loop)": {
"s15": 0.1784566071194907,
"s19": 0.24141266010701656,
"kl": 0.03845209667148689,
"p_next_base": 0.8621721913417181,
"p_next_edit": 0.8251642162601153,
"p_pick_base": 0.9036592009166876,
"p_pick_edit": 0.8810188236335913
},
"inside the loop": {
"s15": 0.4237550770243009,
"s19": 0.7578800767660141,
"kl": 0.06905498933823158,
"p_next_base": 0.9997465734680494,
"p_next_edit": 0.9459187835454941,
"p_pick_base": 0.9999470474819342,
"p_pick_edit": 0.9791371089716753,
"cycle_base": 0.9917483491358278,
"cycle_edit": 0.20479172327384598,
"cycle_pick_base": 0.9988565964655188,
"cycle_pick_edit": 0.4868961309601543
}
},
"samples": [
{
"id": "gsm8k_train_3030",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 8013,
"period": 40,
"early (first half before the loop)": {
"s15": 0.09493550658226013,
"s19": 0.115056112408638,
"kl": 0.024834616109728813,
"p_next_base": 0.7732104063034058,
"p_next_edit": 0.753156840801239,
"p_pick_base": 0.8370677828788757,
"p_pick_edit": 0.8196796178817749
},
"late (last 512 tokens before the loop)": {
"s15": 0.07900411635637283,
"s19": 0.16713517904281616,
"kl": 0.007314786314964294,
"p_next_base": 0.8938040733337402,
"p_next_edit": 0.8883872628211975,
"p_pick_base": 0.9195385575294495,
"p_pick_edit": 0.918036699295044
},
"inside the loop": {
"s15": 0.43249207735061646,
"s19": 0.8205299973487854,
"kl": 0.022678928449749947,
"p_next_base": 0.9994028806686401,
"p_next_edit": 0.979182243347168,
"p_pick_base": 0.9998603463172913,
"p_pick_edit": 0.9961104989051819,
"cycle_base": 0.9857400641157332,
"cycle_edit": 0.41184625169450273,
"cycle_pick_base": 0.9969877297003711,
"cycle_pick_edit": 0.8542781642388538
}
},
{
"id": "gsm8k_train_3030",
"j": 1,
"answer_tokens": 31091,
"loop_starts_at": 10616,
"period": 25,
"early (first half before the loop)": {
"s15": 0.10699471831321716,
"s19": 0.1473984271287918,
"kl": 0.015609870664775372,
"p_next_base": 0.8382197618484497,
"p_next_edit": 0.8220101594924927,
"p_pick_base": 0.8870197534561157,
"p_pick_edit": 0.8768780827522278
},
"late (last 512 tokens before the loop)": {
"s15": 0.1658484786748886,
"s19": 0.23142609000205994,
"kl": 0.034812960773706436,
"p_next_base": 0.8863799571990967,
"p_next_edit": 0.8503194451332092,
"p_pick_base": 0.9178202748298645,
"p_pick_edit": 0.889568567276001
},
"inside the loop": {
"s15": 0.453744113445282,
"s19": 0.7321059107780457,
"kl": 0.45981499552726746,
"p_next_base": 0.9981852769851685,
"p_next_edit": 0.7679603695869446,
"p_pick_base": 0.9999164938926697,
"p_pick_edit": 0.8542358875274658,
"cycle_base": 0.9558865097311214,
"cycle_edit": 5.619239727771304e-05,
"cycle_pick_base": 0.99811472306199,
"cycle_pick_edit": 0.0023782492231600103
}
},
{
"id": "gsm8k_train_3613",
"j": 1,
"answer_tokens": 32103,
"loop_starts_at": 10907,
"period": 104,
"early (first half before the loop)": {
"s15": 0.09244392067193985,
"s19": 0.1089140996336937,
"kl": 0.019604254513978958,
"p_next_base": 0.7593117356300354,
"p_next_edit": 0.7416106462478638,
"p_pick_base": 0.8279196619987488,
"p_pick_edit": 0.8146998882293701
},
"late (last 512 tokens before the loop)": {
"s15": 0.2333683967590332,
"s19": 0.3179709315299988,
"kl": 0.037113819271326065,
"p_next_base": 0.9376562237739563,
"p_next_edit": 0.8971724510192871,
"p_pick_base": 0.9615443348884583,
"p_pick_edit": 0.935500979423523
},
"inside the loop": {
"s15": 0.38444069027900696,
"s19": 0.7108250260353088,
"kl": 0.0308242067694664,
"p_next_base": 0.999714195728302,
"p_next_edit": 0.9718039035797119,
"p_pick_base": 0.9999531507492065,
"p_pick_edit": 0.9950465559959412,
"cycle_base": 0.9808938935722623,
"cycle_edit": 0.04323348679087074,
"cycle_pick_base": 0.9985289911332978,
"cycle_pick_edit": 0.6210674215724846
}
},
{
"id": "gsm8k_train_4033",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 4758,
"period": 93,
"early (first half before the loop)": {
"s15": 0.0769229605793953,
"s19": 0.08930579572916031,
"kl": 0.02565189264714718,
"p_next_base": 0.697345495223999,
"p_next_edit": 0.6768049597740173,
"p_pick_base": 0.7801943421363831,
"p_pick_edit": 0.7620502710342407
},
"late (last 512 tokens before the loop)": {
"s15": 0.2736755609512329,
"s19": 0.35916274785995483,
"kl": 0.04381939768791199,
"p_next_base": 0.9302067160606384,
"p_next_edit": 0.8869009613990784,
"p_pick_base": 0.9512910842895508,
"p_pick_edit": 0.9280562400817871
},
"inside the loop": {
"s15": 0.4155002534389496,
"s19": 0.7276469469070435,
"kl": 0.06856408715248108,
"p_next_base": 0.9998907446861267,
"p_next_edit": 0.9415830969810486,
"p_pick_base": 0.9999874234199524,
"p_pick_edit": 0.9823828339576721,
"cycle_base": 0.9923541364872293,
"cycle_edit": 0.0020611719429896275,
"cycle_pick_base": 0.9993552013933411,
"cycle_pick_edit": 0.166902685391735
}
},
{
"id": "gsm8k_train_1318",
"j": 1,
"answer_tokens": 32767,
"loop_starts_at": 5467,
"period": 32,
"early (first half before the loop)": {
"s15": 0.12002290785312653,
"s19": 0.12535977363586426,
"kl": 0.041723012924194336,
"p_next_base": 0.7865251302719116,
"p_next_edit": 0.760968804359436,
"p_pick_base": 0.849585771560669,
"p_pick_edit": 0.829461395740509
},
"late (last 512 tokens before the loop)": {
"s15": 0.18370643258094788,
"s19": 0.20398709177970886,
"kl": 0.06643478572368622,
"p_next_base": 0.8520184755325317,
"p_next_edit": 0.7993849515914917,
"p_pick_base": 0.8955559730529785,
"p_pick_edit": 0.8606911897659302
},
"inside the loop": {
"s15": 0.40848466753959656,
"s19": 0.7188209295272827,
"kl": 0.09173006564378738,
"p_next_base": 0.9999064803123474,
"p_next_edit": 0.9300572276115417,
"p_pick_base": 0.9999733567237854,
"p_pick_edit": 0.9573909044265747,
"cycle_base": 0.9980618013552564,
"cycle_edit": 0.05428003590732139,
"cycle_pick_base": 0.9995637405241474,
"cycle_pick_edit": 0.00315720940194697
}
},
{
"id": "gsm8k_train_4077",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 4928,
"period": 50,
"early (first half before the loop)": {
"s15": 0.11264169216156006,
"s19": 0.12039957195520401,
"kl": 0.029233744367957115,
"p_next_base": 0.7971627116203308,
"p_next_edit": 0.7749474048614502,
"p_pick_base": 0.8544129133224487,
"p_pick_edit": 0.8375517725944519
},
"late (last 512 tokens before the loop)": {
"s15": 0.19189748167991638,
"s19": 0.21922540664672852,
"kl": 0.05086758732795715,
"p_next_base": 0.8677645325660706,
"p_next_edit": 0.8248593807220459,
"p_pick_base": 0.9112231731414795,
"p_pick_edit": 0.8795385360717773
},
"inside the loop": {
"s15": 0.429381787776947,
"s19": 0.7684251070022583,
"kl": 0.026801371946930885,
"p_next_base": 0.999790370464325,
"p_next_edit": 0.9759086966514587,
"p_pick_base": 0.9999765753746033,
"p_pick_edit": 0.9912539720535278,
"cycle_base": 0.9921804242154579,
"cycle_edit": 0.2741217802533613,
"cycle_pick_base": 0.9994056336824353,
"cycle_pick_edit": 0.6295921403364145
}
},
{
"id": "gsm8k_train_6176",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 10627,
"period": 12,
"early (first half before the loop)": {
"s15": 0.09619540721178055,
"s19": 0.10784083604812622,
"kl": 0.015823855996131897,
"p_next_base": 0.8051894903182983,
"p_next_edit": 0.7900272011756897,
"p_pick_base": 0.8589186072349548,
"p_pick_edit": 0.8478661775588989
},
"late (last 512 tokens before the loop)": {
"s15": 0.1413775086402893,
"s19": 0.17916162312030792,
"kl": 0.02753262408077717,
"p_next_base": 0.8726419806480408,
"p_next_edit": 0.8379964828491211,
"p_pick_base": 0.9165550470352173,
"p_pick_edit": 0.8973170518875122
},
"inside the loop": {
"s15": 0.43383780121803284,
"s19": 0.8007307648658752,
"kl": 0.17077724635601044,
"p_next_base": 0.9998153448104858,
"p_next_edit": 0.8737342953681946,
"p_pick_base": 0.9999546408653259,
"p_pick_edit": 0.9202463626861572,
"cycle_base": 0.9986652664458991,
"cycle_edit": 0.12993211870106708,
"cycle_pick_base": 0.9996541316298,
"cycle_pick_edit": 0.2715019546600853
}
},
{
"id": "gsm8k_train_6275",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 3012,
"period": 25,
"early (first half before the loop)": {
"s15": 0.08016929030418396,
"s19": 0.09987947344779968,
"kl": 0.020563611760735512,
"p_next_base": 0.696076512336731,
"p_next_edit": 0.6741355061531067,
"p_pick_base": 0.7832125425338745,
"p_pick_edit": 0.7654736042022705
},
"late (last 512 tokens before the loop)": {
"s15": 0.19730058312416077,
"s19": 0.2524352967739105,
"kl": 0.042088109999895096,
"p_next_base": 0.8716167211532593,
"p_next_edit": 0.827920138835907,
"p_pick_base": 0.9142891764640808,
"p_pick_edit": 0.894567608833313
},
"inside the loop": {
"s15": 0.42585352063179016,
"s19": 0.7798516154289246,
"kl": 0.05144067481160164,
"p_next_base": 0.9999368190765381,
"p_next_edit": 0.9531211853027344,
"p_pick_base": 0.9999913573265076,
"p_pick_edit": 0.9906980395317078,
"cycle_base": 0.99873332068141,
"cycle_edit": 0.2778068387664842,
"cycle_pick_base": 0.9998957602279993,
"cycle_pick_edit": 0.7844624836846711
}
},
{
"id": "gsm8k_train_6275",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 2899,
"period": 78,
"early (first half before the loop)": {
"s15": 0.09315116703510284,
"s19": 0.11652565747499466,
"kl": 0.018888290971517563,
"p_next_base": 0.7436284422874451,
"p_next_edit": 0.7246678471565247,
"p_pick_base": 0.8198502659797668,
"p_pick_edit": 0.8035048842430115
},
"late (last 512 tokens before the loop)": {
"s15": 0.14879998564720154,
"s19": 0.21919207274913788,
"kl": 0.025760091841220856,
"p_next_base": 0.8692398071289062,
"p_next_edit": 0.8419463634490967,
"p_pick_base": 0.9062317609786987,
"p_pick_edit": 0.8874175548553467
},
"inside the loop": {
"s15": 0.459094375371933,
"s19": 0.7620907425880432,
"kl": 0.14087757468223572,
"p_next_base": 0.9998957514762878,
"p_next_edit": 0.9005423188209534,
"p_pick_base": 0.9999638199806213,
"p_pick_edit": 0.9408422708511353,
"cycle_base": 0.9940565467823791,
"cycle_edit": 5.878683747145727e-05,
"cycle_pick_base": 0.9979903225531562,
"cycle_pick_edit": 0.00012817762737887665
}
},
{
"id": "gsm8k_train_5191",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 6454,
"period": 43,
"early (first half before the loop)": {
"s15": 0.08386224508285522,
"s19": 0.08149462193250656,
"kl": 0.026429813355207443,
"p_next_base": 0.6992491483688354,
"p_next_edit": 0.6769809722900391,
"p_pick_base": 0.7848073840141296,
"p_pick_edit": 0.7642750144004822
},
"late (last 512 tokens before the loop)": {
"s15": 0.1788625717163086,
"s19": 0.24487721920013428,
"kl": 0.028230253607034683,
"p_next_base": 0.9068049788475037,
"p_next_edit": 0.8773461580276489,
"p_pick_base": 0.938770055770874,
"p_pick_edit": 0.92200767993927
},
"inside the loop": {
"s15": 0.4103063642978668,
"s19": 0.755408525466919,
"kl": 0.0535348542034626,
"p_next_base": 0.9999359250068665,
"p_next_edit": 0.9540817141532898,
"p_pick_base": 0.9999751448631287,
"p_pick_edit": 0.9770494699478149,
"cycle_base": 0.9978903038166186,
"cycle_edit": 0.10024741802624157,
"cycle_pick_base": 0.9992698620546331,
"cycle_pick_edit": 0.2743138964606422
}
},
{
"id": "gsm8k_train_3419",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 4776,
"period": 56,
"early (first half before the loop)": {
"s15": 0.09288118779659271,
"s19": 0.07846082001924515,
"kl": 0.018657874315977097,
"p_next_base": 0.7371241450309753,
"p_next_edit": 0.7207286953926086,
"p_pick_base": 0.813136637210846,
"p_pick_edit": 0.8009316325187683
},
"late (last 512 tokens before the loop)": {
"s15": 0.28022944927215576,
"s19": 0.33214378356933594,
"kl": 0.045068591833114624,
"p_next_base": 0.9321595430374146,
"p_next_edit": 0.8824988603591919,
"p_pick_base": 0.9548278450965881,
"p_pick_edit": 0.923862874507904
},
"inside the loop": {
"s15": 0.421861857175827,
"s19": 0.7205281257629395,
"kl": 0.0770692229270935,
"p_next_base": 0.9999065399169922,
"p_next_edit": 0.9358772039413452,
"p_pick_base": 0.999979555606842,
"p_pick_edit": 0.9697349667549133,
"cycle_base": 0.9951548335885142,
"cycle_edit": 0.013184503799187786,
"cycle_pick_base": 0.9989089114337604,
"cycle_pick_edit": 0.02393337456951758
}
},
{
"id": "gsm8k_train_2384",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 2928,
"period": 16,
"early (first half before the loop)": {
"s15": 0.0930926725268364,
"s19": 0.10464742034673691,
"kl": 0.025108011439442635,
"p_next_base": 0.7442770600318909,
"p_next_edit": 0.7240205407142639,
"p_pick_base": 0.825468122959137,
"p_pick_edit": 0.8073226809501648
},
"late (last 512 tokens before the loop)": {
"s15": 0.13590773940086365,
"s19": 0.17218440771102905,
"kl": 0.026631303131580353,
"p_next_base": 0.8404752016067505,
"p_next_edit": 0.8190571069717407,
"p_pick_base": 0.8888722658157349,
"p_pick_edit": 0.8790283203125
},
"inside the loop": {
"s15": 0.4459160566329956,
"s19": 0.8384209871292114,
"kl": 0.03452901169657707,
"p_next_base": 0.9997012615203857,
"p_next_edit": 0.9695818424224854,
"p_pick_base": 0.9999271035194397,
"p_pick_edit": 0.9979686141014099,
"cycle_base": 0.9973330412521313,
"cycle_edit": 0.5793630316573475,
"cycle_pick_base": 0.9992537559021133,
"cycle_pick_edit": 0.9708295960876501
}
},
{
"id": "gsm8k_train_3239",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 6692,
"period": 58,
"early (first half before the loop)": {
"s15": 0.0864107683300972,
"s19": 0.1129826009273529,
"kl": 0.01786801405251026,
"p_next_base": 0.7485823035240173,
"p_next_edit": 0.734268307685852,
"p_pick_base": 0.8199166059494019,
"p_pick_edit": 0.8074601888656616
},
"late (last 512 tokens before the loop)": {
"s15": 0.15383514761924744,
"s19": 0.24198570847511292,
"kl": 0.033007703721523285,
"p_next_base": 0.8814928531646729,
"p_next_edit": 0.8453898429870605,
"p_pick_base": 0.9188077449798584,
"p_pick_edit": 0.8953191041946411
},
"inside the loop": {
"s15": 0.4034617841243744,
"s19": 0.7239362001419067,
"kl": 0.056788112968206406,
"p_next_base": 0.9997130036354065,
"p_next_edit": 0.9566110968589783,
"p_pick_base": 0.9999046325683594,
"p_pick_edit": 0.9693314433097839,
"cycle_base": 0.991765892571544,
"cycle_edit": 0.04453638405496409,
"cycle_pick_base": 0.9984915382524697,
"cycle_pick_edit": 0.019510102656462202
}
},
{
"id": "gsm8k_train_6111",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 2774,
"period": 21,
"early (first half before the loop)": {
"s15": 0.09001856297254562,
"s19": 0.08551815152168274,
"kl": 0.033200230449438095,
"p_next_base": 0.6819701194763184,
"p_next_edit": 0.6566055417060852,
"p_pick_base": 0.7728362083435059,
"p_pick_edit": 0.7491665482521057
},
"late (last 512 tokens before the loop)": {
"s15": 0.1393461376428604,
"s19": 0.16332943737506866,
"kl": 0.06007887423038483,
"p_next_base": 0.7847452163696289,
"p_next_edit": 0.7262318134307861,
"p_pick_base": 0.8482316732406616,
"p_pick_edit": 0.8068649768829346
},
"inside the loop": {
"s15": 0.39372366666793823,
"s19": 0.7848808169364929,
"kl": 0.026860663667321205,
"p_next_base": 0.9997686743736267,
"p_next_edit": 0.9737282991409302,
"p_pick_base": 0.9999074935913086,
"p_pick_edit": 0.9987943172454834,
"cycle_base": 0.9963507349547592,
"cycle_edit": 0.5744648115519122,
"cycle_pick_base": 0.9985310700726027,
"cycle_pick_edit": 0.9816922751796325
}
},
{
"id": "gsm8k_train_1647",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 3792,
"period": 29,
"early (first half before the loop)": {
"s15": 0.11002491414546967,
"s19": 0.12361518293619156,
"kl": 0.016478292644023895,
"p_next_base": 0.771632730960846,
"p_next_edit": 0.7595760822296143,
"p_pick_base": 0.8424074053764343,
"p_pick_edit": 0.8345464468002319
},
"late (last 512 tokens before the loop)": {
"s15": 0.15892711281776428,
"s19": 0.19340944290161133,
"kl": 0.038946159183979034,
"p_next_base": 0.824557363986969,
"p_next_edit": 0.7934427857398987,
"p_pick_base": 0.8690199851989746,
"p_pick_edit": 0.8484527468681335
},
"inside the loop": {
"s15": 0.4021039307117462,
"s19": 0.7463305592536926,
"kl": 0.10968953371047974,
"p_next_base": 0.9997801780700684,
"p_next_edit": 0.9067637324333191,
"p_pick_base": 0.9999239444732666,
"p_pick_edit": 0.9532238245010376,
"cycle_base": 0.996604509687193,
"cycle_edit": 0.041281539866955395,
"cycle_pick_base": 0.9995090495077723,
"cycle_pick_edit": 0.11542620834642868
}
},
{
"id": "gsm8k_train_6495",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 4808,
"period": 224,
"early (first half before the loop)": {
"s15": 0.09905802458524704,
"s19": 0.13073177635669708,
"kl": 0.020734725520014763,
"p_next_base": 0.7855616807937622,
"p_next_edit": 0.7683371901512146,
"p_pick_base": 0.8478497266769409,
"p_pick_edit": 0.8334498405456543
},
"late (last 512 tokens before the loop)": {
"s15": 0.24047932028770447,
"s19": 0.36906105279922485,
"kl": 0.055738624185323715,
"p_next_base": 0.9493577480316162,
"p_next_edit": 0.9050508737564087,
"p_pick_base": 0.9644514322280884,
"p_pick_edit": 0.9379522800445557
},
"inside the loop": {
"s15": 0.40192267298698425,
"s19": 0.7015343904495239,
"kl": 0.07150352001190186,
"p_next_base": 0.9999558329582214,
"p_next_edit": 0.9427007436752319,
"p_pick_base": 0.999995231628418,
"p_pick_edit": 0.9750657081604004,
"cycle_base": 0.9919649363760391,
"cycle_edit": 1.2598451353026518e-07,
"cycle_pick_base": 0.9993910476907667,
"cycle_pick_edit": 0.0008681265437755715
}
},
{
"id": "gsm8k_train_5597",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 4036,
"period": 20,
"early (first half before the loop)": {
"s15": 0.08169778436422348,
"s19": 0.12300147861242294,
"kl": 0.015780232846736908,
"p_next_base": 0.789982795715332,
"p_next_edit": 0.7758480310440063,
"p_pick_base": 0.8560000658035278,
"p_pick_edit": 0.8443788886070251
},
"late (last 512 tokens before the loop)": {
"s15": 0.16410283744335175,
"s19": 0.2705284357070923,
"kl": 0.02839719131588936,
"p_next_base": 0.8896647691726685,
"p_next_edit": 0.8590779900550842,
"p_pick_base": 0.9217250943183899,
"p_pick_edit": 0.9087079763412476
},
"inside the loop": {
"s15": 0.4497266709804535,
"s19": 0.8558210134506226,
"kl": 0.0340026319026947,
"p_next_base": 0.9997823238372803,
"p_next_edit": 0.9676102995872498,
"p_pick_base": 0.9999046325683594,
"p_pick_edit": 0.9866077899932861,
"cycle_base": 0.9964147330427239,
"cycle_edit": 0.5081728104370556,
"cycle_pick_base": 0.9983678278280282,
"cycle_pick_edit": 0.7510426671708171
}
},
{
"id": "gsm8k_train_669",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 7083,
"period": 44,
"early (first half before the loop)": {
"s15": 0.14904341101646423,
"s19": 0.15221571922302246,
"kl": 0.04369807243347168,
"p_next_base": 0.7724522352218628,
"p_next_edit": 0.7404855489730835,
"p_pick_base": 0.8346216678619385,
"p_pick_edit": 0.8146063089370728
},
"late (last 512 tokens before the loop)": {
"s15": 0.3467504382133484,
"s19": 0.4338456392288208,
"kl": 0.08952810615301132,
"p_next_base": 0.9433068037033081,
"p_next_edit": 0.8624776601791382,
"p_pick_base": 0.9602187275886536,
"p_pick_edit": 0.9292097091674805
},
"inside the loop": {
"s15": 0.4175926446914673,
"s19": 0.7302914261817932,
"kl": 0.08099833875894547,
"p_next_base": 0.999910295009613,
"p_next_edit": 0.9271917939186096,
"p_pick_base": 0.9999895095825195,
"p_pick_edit": 0.9893120527267456,
"cycle_base": 0.996635298463595,
"cycle_edit": 0.030936597166957146,
"cycle_pick_base": 0.99960576923827,
"cycle_pick_edit": 0.643765264462578
}
},
{
"id": "gsm8k_train_641",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 5474,
"period": 18,
"early (first half before the loop)": {
"s15": 0.13045726716518402,
"s19": 0.12690667808055878,
"kl": 0.04721679165959358,
"p_next_base": 0.7454983592033386,
"p_next_edit": 0.7155665159225464,
"p_pick_base": 0.8153316378593445,
"p_pick_edit": 0.7935912013053894
},
"late (last 512 tokens before the loop)": {
"s15": 0.13967680931091309,
"s19": 0.15517809987068176,
"kl": 0.0564558282494545,
"p_next_base": 0.7936413288116455,
"p_next_edit": 0.7487541437149048,
"p_pick_base": 0.8492997884750366,
"p_pick_edit": 0.8155405521392822
},
"inside the loop": {
"s15": 0.4366004765033722,
"s19": 0.8017978072166443,
"kl": 0.13094432651996613,
"p_next_base": 0.9997851848602295,
"p_next_edit": 0.9046043753623962,
"p_pick_base": 0.9999319911003113,
"p_pick_edit": 0.9526225924491882,
"cycle_base": 0.9977323326031816,
"cycle_edit": 0.09440666604171237,
"cycle_pick_base": 0.9995414624144559,
"cycle_pick_edit": 0.24987086754346333
}
},
{
"id": "gsm8k_train_641",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 5731,
"period": 84,
"early (first half before the loop)": {
"s15": 0.11753851920366287,
"s19": 0.13667577505111694,
"kl": 0.03983533754944801,
"p_next_base": 0.7339333891868591,
"p_next_edit": 0.7049329876899719,
"p_pick_base": 0.8102664947509766,
"p_pick_edit": 0.7857512831687927
},
"late (last 512 tokens before the loop)": {
"s15": 0.1685326099395752,
"s19": 0.2092926800251007,
"kl": 0.06873328983783722,
"p_next_base": 0.8343878984451294,
"p_next_edit": 0.7765858173370361,
"p_pick_base": 0.8870160579681396,
"p_pick_edit": 0.8443004488945007
},
"inside the loop": {
"s15": 0.4314122200012207,
"s19": 0.7148913145065308,
"kl": 0.08826982229948044,
"p_next_base": 0.9999134540557861,
"p_next_edit": 0.9377833604812622,
"p_pick_base": 0.9999693036079407,
"p_pick_edit": 0.9754384160041809,
"cycle_base": 0.9957104219034827,
"cycle_edit": 0.0008771121761194243,
"cycle_pick_base": 0.9996948638793864,
"cycle_pick_edit": 0.020771994322517164
}
},
{
"id": "gsm8k_train_5612",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 4685,
"period": 67,
"early (first half before the loop)": {
"s15": 0.09881513565778732,
"s19": 0.10243399441242218,
"kl": 0.02628999575972557,
"p_next_base": 0.7719534039497375,
"p_next_edit": 0.7528481483459473,
"p_pick_base": 0.8369691371917725,
"p_pick_edit": 0.8212394714355469
},
"late (last 512 tokens before the loop)": {
"s15": 0.142515629529953,
"s19": 0.17738494277000427,
"kl": 0.035840921103954315,
"p_next_base": 0.829646110534668,
"p_next_edit": 0.7986764311790466,
"p_pick_base": 0.8825891017913818,
"p_pick_edit": 0.8603673577308655
},
"inside the loop": {
"s15": 0.4138917922973633,
"s19": 0.745213508605957,
"kl": 0.05226440727710724,
"p_next_base": 0.9998858571052551,
"p_next_edit": 0.9558983445167542,
"p_pick_base": 0.9999892115592957,
"p_pick_edit": 0.9805213809013367,
"cycle_base": 0.9939840603244958,
"cycle_edit": 0.03072890926788994,
"cycle_pick_base": 0.9995447605290632,
"cycle_pick_edit": 0.022577169850695963
}
},
{
"id": "gsm8k_train_1385",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 1917,
"period": 28,
"early (first half before the loop)": {
"s15": 0.10129357874393463,
"s19": 0.07512757182121277,
"kl": 0.020414166152477264,
"p_next_base": 0.697487473487854,
"p_next_edit": 0.6836481690406799,
"p_pick_base": 0.7886316776275635,
"p_pick_edit": 0.7769674062728882
},
"late (last 512 tokens before the loop)": {
"s15": 0.13219653069972992,
"s19": 0.14451050758361816,
"kl": 0.03114088624715805,
"p_next_base": 0.759606659412384,
"p_next_edit": 0.7333537936210632,
"p_pick_base": 0.8255237340927124,
"p_pick_edit": 0.8045892119407654
},
"inside the loop": {
"s15": 0.4286457300186157,
"s19": 0.8032561540603638,
"kl": 0.04280979186296463,
"p_next_base": 0.9992213249206543,
"p_next_edit": 0.9617798924446106,
"p_pick_base": 0.9999488592147827,
"p_pick_edit": 0.9959352016448975,
"cycle_base": 0.9801243492395256,
"cycle_edit": 0.3059766361067436,
"cycle_pick_base": 0.9990640516069063,
"cycle_pick_edit": 0.8937879271247255
}
},
{
"id": "gsm8k_train_4650",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 6378,
"period": 16,
"early (first half before the loop)": {
"s15": 0.0818009153008461,
"s19": 0.126252219080925,
"kl": 0.020525651052594185,
"p_next_base": 0.7592082619667053,
"p_next_edit": 0.7437617182731628,
"p_pick_base": 0.824590802192688,
"p_pick_edit": 0.8120755553245544
},
"late (last 512 tokens before the loop)": {
"s15": 0.0938413068652153,
"s19": 0.12602165341377258,
"kl": 0.03751196712255478,
"p_next_base": 0.7421631813049316,
"p_next_edit": 0.7127740383148193,
"p_pick_base": 0.8164945840835571,
"p_pick_edit": 0.7926813364028931
},
"inside the loop": {
"s15": 0.3921157419681549,
"s19": 0.7004843950271606,
"kl": 0.045122288167476654,
"p_next_base": 0.9998180866241455,
"p_next_edit": 0.9573324918746948,
"p_pick_base": 0.9999780654907227,
"p_pick_edit": 0.9996170997619629,
"cycle_base": 0.9983495325473885,
"cycle_edit": 0.4879699538189189,
"cycle_pick_base": 0.9998407182565892,
"cycle_pick_edit": 0.9950261067835918
}
},
{
"id": "gsm8k_train_2232",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 8037,
"period": 62,
"early (first half before the loop)": {
"s15": 0.09412968158721924,
"s19": 0.13244371116161346,
"kl": 0.02191508561372757,
"p_next_base": 0.7858816385269165,
"p_next_edit": 0.7693138718605042,
"p_pick_base": 0.8475492596626282,
"p_pick_edit": 0.8346990942955017
},
"late (last 512 tokens before the loop)": {
"s15": 0.1259446144104004,
"s19": 0.1578044593334198,
"kl": 0.0446142703294754,
"p_next_base": 0.7697348594665527,
"p_next_edit": 0.7285483479499817,
"p_pick_base": 0.832977294921875,
"p_pick_edit": 0.8006552457809448
},
"inside the loop": {
"s15": 0.40567705035209656,
"s19": 0.7008726596832275,
"kl": 0.12388861179351807,
"p_next_base": 0.9998905658721924,
"p_next_edit": 0.9039038419723511,
"p_pick_base": 0.9999843239784241,
"p_pick_edit": 0.9379798173904419,
"cycle_base": 0.99495987986616,
"cycle_edit": 0.0005364126720465651,
"cycle_pick_base": 0.9994245337161413,
"cycle_pick_edit": 0.0055018464325661
}
},
{
"id": "gsm8k_train_2422",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 5827,
"period": 183,
"early (first half before the loop)": {
"s15": 0.11299488693475723,
"s19": 0.1176219955086708,
"kl": 0.031237376853823662,
"p_next_base": 0.7538059949874878,
"p_next_edit": 0.7288076281547546,
"p_pick_base": 0.8235296607017517,
"p_pick_edit": 0.8027406334877014
},
"late (last 512 tokens before the loop)": {
"s15": 0.2312086671590805,
"s19": 0.2662774920463562,
"kl": 0.06740671396255493,
"p_next_base": 0.8890897035598755,
"p_next_edit": 0.8337670564651489,
"p_pick_base": 0.9192017316818237,
"p_pick_edit": 0.886871337890625
},
"inside the loop": {
"s15": 0.3992064297199249,
"s19": 0.6710278391838074,
"kl": 0.07526908069849014,
"p_next_base": 0.9998493194580078,
"p_next_edit": 0.944186270236969,
"p_pick_base": 0.9999929666519165,
"p_pick_edit": 0.9757047891616821,
"cycle_base": 0.9771835510896755,
"cycle_edit": 2.1383963902482916e-06,
"cycle_pick_base": 0.9997798539755598,
"cycle_pick_edit": 9.972149709848148e-303
}
},
{
"id": "gsm8k_train_7079",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 5403,
"period": 114,
"early (first half before the loop)": {
"s15": 0.0776025652885437,
"s19": 0.09987753629684448,
"kl": 0.02822856977581978,
"p_next_base": 0.7018040418624878,
"p_next_edit": 0.6797464489936829,
"p_pick_base": 0.7844891548156738,
"p_pick_edit": 0.7652970552444458
},
"late (last 512 tokens before the loop)": {
"s15": 0.2322397530078888,
"s19": 0.3096058964729309,
"kl": 0.04663810506463051,
"p_next_base": 0.9058045148849487,
"p_next_edit": 0.8596648573875427,
"p_pick_base": 0.9302293062210083,
"p_pick_edit": 0.906479001045227
},
"inside the loop": {
"s15": 0.4015542268753052,
"s19": 0.7164995670318604,
"kl": 0.05760333687067032,
"p_next_base": 0.9997974038124084,
"p_next_edit": 0.950993537902832,
"p_pick_base": 0.9999527931213379,
"p_pick_edit": 0.9819333553314209,
"cycle_base": 0.9871539120247008,
"cycle_edit": 0.0014593751228170139,
"cycle_pick_base": 0.9972839138787254,
"cycle_pick_edit": 0.10796210233204756
}
},
{
"id": "gsm8k_train_7079",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 3195,
"period": 27,
"early (first half before the loop)": {
"s15": 0.05571795254945755,
"s19": 0.08699333667755127,
"kl": 0.013038618490099907,
"p_next_base": 0.7472986578941345,
"p_next_edit": 0.7378846406936646,
"p_pick_base": 0.8232874274253845,
"p_pick_edit": 0.8151520490646362
},
"late (last 512 tokens before the loop)": {
"s15": 0.1282390058040619,
"s19": 0.13991735875606537,
"kl": 0.03972293809056282,
"p_next_base": 0.7540237307548523,
"p_next_edit": 0.7043659687042236,
"p_pick_base": 0.8311821222305298,
"p_pick_edit": 0.7998129725456238
},
"inside the loop": {
"s15": 0.4430287182331085,
"s19": 0.7667595744132996,
"kl": 0.04779743775725365,
"p_next_base": 0.9998666644096375,
"p_next_edit": 0.9555791616439819,
"p_pick_base": 0.9999611377716064,
"p_pick_edit": 0.9967443943023682,
"cycle_base": 0.9971906783622563,
"cycle_edit": 0.2879334414338234,
"cycle_pick_base": 0.999350797495224,
"cycle_pick_edit": 0.9292087985666114
}
},
{
"id": "gsm8k_train_2790",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 8375,
"period": 60,
"early (first half before the loop)": {
"s15": 0.12475399672985077,
"s19": 0.15071505308151245,
"kl": 0.04290880635380745,
"p_next_base": 0.7549790143966675,
"p_next_edit": 0.7237005829811096,
"p_pick_base": 0.8237105011940002,
"p_pick_edit": 0.8019028902053833
},
"late (last 512 tokens before the loop)": {
"s15": 0.14056125283241272,
"s19": 0.1952000856399536,
"kl": 0.05030623450875282,
"p_next_base": 0.7984333634376526,
"p_next_edit": 0.7666357755661011,
"p_pick_base": 0.8608813881874084,
"p_pick_edit": 0.8423949480056763
},
"inside the loop": {
"s15": 0.39199721813201904,
"s19": 0.7231124043464661,
"kl": 0.08472363650798798,
"p_next_base": 0.9996046423912048,
"p_next_edit": 0.9300553798675537,
"p_pick_base": 0.9998325705528259,
"p_pick_edit": 0.9707520604133606,
"cycle_base": 0.987419796058788,
"cycle_edit": 0.00596649672608045,
"cycle_pick_base": 0.994285140303736,
"cycle_pick_edit": 0.00513012068731318
}
},
{
"id": "math_train_counting_and_probability_88",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 10207,
"period": 46,
"early (first half before the loop)": {
"s15": 0.058004263788461685,
"s19": 0.10151831805706024,
"kl": 0.013849137350916862,
"p_next_base": 0.7760154604911804,
"p_next_edit": 0.7648267149925232,
"p_pick_base": 0.8384997844696045,
"p_pick_edit": 0.8281447887420654
},
"late (last 512 tokens before the loop)": {
"s15": 0.13074453175067902,
"s19": 0.1818772554397583,
"kl": 0.06687108427286148,
"p_next_base": 0.8120644092559814,
"p_next_edit": 0.7546481490135193,
"p_pick_base": 0.8747764825820923,
"p_pick_edit": 0.8325002193450928
},
"inside the loop": {
"s15": 0.39635226130485535,
"s19": 0.7644913196563721,
"kl": 0.07107376307249069,
"p_next_base": 0.9999132752418518,
"p_next_edit": 0.9412874579429626,
"p_pick_base": 0.999967634677887,
"p_pick_edit": 0.9778523445129395,
"cycle_base": 0.9975527100683644,
"cycle_edit": 0.038802487790659194,
"cycle_pick_base": 0.9990526381010016,
"cycle_pick_edit": 0.29798464358879767
}
},
{
"id": "math_train_counting_and_probability_668",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 6114,
"period": 216,
"early (first half before the loop)": {
"s15": 0.12734749913215637,
"s19": 0.16124509274959564,
"kl": 0.02163294143974781,
"p_next_base": 0.7812380194664001,
"p_next_edit": 0.761454701423645,
"p_pick_base": 0.8388779163360596,
"p_pick_edit": 0.8247150778770447
},
"late (last 512 tokens before the loop)": {
"s15": 0.2740636467933655,
"s19": 0.43788495659828186,
"kl": 0.029126089066267014,
"p_next_base": 0.9563079476356506,
"p_next_edit": 0.9271626472473145,
"p_pick_base": 0.9715901613235474,
"p_pick_edit": 0.9566549062728882
},
"inside the loop": {
"s15": 0.3735722005367279,
"s19": 0.7056142091751099,
"kl": 0.056022122502326965,
"p_next_base": 0.999817967414856,
"p_next_edit": 0.955345094203949,
"p_pick_base": 1.0,
"p_pick_edit": 0.9793975353240967,
"cycle_base": 0.9646498044299565,
"cycle_edit": 1.1361530136688617e-05,
"cycle_pick_base": 1.0,
"cycle_pick_edit": 8.126646106292587e-05
}
},
{
"id": "math_train_prealgebra_1099",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 6541,
"period": 29,
"early (first half before the loop)": {
"s15": 0.08181149512529373,
"s19": 0.0967169925570488,
"kl": 0.015475410036742687,
"p_next_base": 0.769540011882782,
"p_next_edit": 0.7554738521575928,
"p_pick_base": 0.8358332514762878,
"p_pick_edit": 0.8251385688781738
},
"late (last 512 tokens before the loop)": {
"s15": 0.1310795247554779,
"s19": 0.1644008755683899,
"kl": 0.023891106247901917,
"p_next_base": 0.777680516242981,
"p_next_edit": 0.7506465911865234,
"p_pick_base": 0.8304601311683655,
"p_pick_edit": 0.8155275583267212
},
"inside the loop": {
"s15": 0.3945211172103882,
"s19": 0.7547446489334106,
"kl": 0.05530731379985809,
"p_next_base": 0.999626636505127,
"p_next_edit": 0.9520976543426514,
"p_pick_base": 0.9999391436576843,
"p_pick_edit": 0.9853673577308655,
"cycle_base": 0.9909051231642108,
"cycle_edit": 0.20303193348152193,
"cycle_pick_base": 0.9986202629726142,
"cycle_pick_edit": 0.6307385067688254
}
},
{
"id": "math_train_counting_and_probability_564",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 5911,
"period": 26,
"early (first half before the loop)": {
"s15": 0.07300079613924026,
"s19": 0.0970291867852211,
"kl": 0.013149832375347614,
"p_next_base": 0.792626678943634,
"p_next_edit": 0.778533399105072,
"p_pick_base": 0.8558213114738464,
"p_pick_edit": 0.8430812954902649
},
"late (last 512 tokens before the loop)": {
"s15": 0.1813674122095108,
"s19": 0.26762866973876953,
"kl": 0.022732634097337723,
"p_next_base": 0.9424155950546265,
"p_next_edit": 0.9161887764930725,
"p_pick_base": 0.964775800704956,
"p_pick_edit": 0.9536480903625488
},
"inside the loop": {
"s15": 0.47167348861694336,
"s19": 0.831170380115509,
"kl": 0.031623125076293945,
"p_next_base": 0.9998704791069031,
"p_next_edit": 0.9718978404998779,
"p_pick_base": 0.9999904036521912,
"p_pick_edit": 0.9964982271194458,
"cycle_base": 0.9971597669106993,
"cycle_edit": 0.4551305717011751,
"cycle_pick_base": 0.9997498153461669,
"cycle_pick_edit": 0.9184289138787352
}
},
{
"id": "math_train_intermediate_algebra_1137",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 14213,
"period": 15,
"early (first half before the loop)": {
"s15": 0.08122420310974121,
"s19": 0.10331915318965912,
"kl": 0.011646715924143791,
"p_next_base": 0.8058518171310425,
"p_next_edit": 0.792787492275238,
"p_pick_base": 0.8621098399162292,
"p_pick_edit": 0.8529916405677795
},
"late (last 512 tokens before the loop)": {
"s15": 0.16249510645866394,
"s19": 0.19276930391788483,
"kl": 0.019805559888482094,
"p_next_base": 0.8559431433677673,
"p_next_edit": 0.8271759152412415,
"p_pick_base": 0.8983961343765259,
"p_pick_edit": 0.887336790561676
},
"inside the loop": {
"s15": 0.46168097853660583,
"s19": 0.778610348701477,
"kl": 0.029275594279170036,
"p_next_base": 0.9995881915092468,
"p_next_edit": 0.9711337089538574,
"p_pick_base": 0.9997695088386536,
"p_pick_edit": 0.9992016553878784,
"cycle_base": 0.9952725094614914,
"cycle_edit": 0.6415339841905084,
"cycle_pick_base": 0.9973929175185765,
"cycle_pick_edit": 0.9910384377121372
}
},
{
"id": "math_train_geometry_317",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 12920,
"period": 116,
"early (first half before the loop)": {
"s15": 0.08501265943050385,
"s19": 0.10523951798677444,
"kl": 0.015983562916517258,
"p_next_base": 0.7942029237747192,
"p_next_edit": 0.7780722975730896,
"p_pick_base": 0.8548552989959717,
"p_pick_edit": 0.8418995141983032
},
"late (last 512 tokens before the loop)": {
"s15": 0.18592557311058044,
"s19": 0.2414465695619583,
"kl": 0.018948521465063095,
"p_next_base": 0.9300025105476379,
"p_next_edit": 0.9023053646087646,
"p_pick_base": 0.9587832689285278,
"p_pick_edit": 0.9439282417297363
},
"inside the loop": {
"s15": 0.4116506576538086,
"s19": 0.688589870929718,
"kl": 0.05407414212822914,
"p_next_base": 0.9995777010917664,
"p_next_edit": 0.9562045335769653,
"p_pick_base": 0.9998968839645386,
"p_pick_edit": 0.973852813243866,
"cycle_base": 0.964059522229368,
"cycle_edit": 0.0017123249948740754,
"cycle_pick_base": 0.9940942662072135,
"cycle_pick_edit": 0.0045764287221902
}
},
{
"id": "math_train_geometry_317",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 7149,
"period": 18,
"early (first half before the loop)": {
"s15": 0.07082884758710861,
"s19": 0.085166797041893,
"kl": 0.014647883363068104,
"p_next_base": 0.7598089575767517,
"p_next_edit": 0.7471540570259094,
"p_pick_base": 0.8267542719841003,
"p_pick_edit": 0.815260112285614
},
"late (last 512 tokens before the loop)": {
"s15": 0.10642599314451218,
"s19": 0.117353156208992,
"kl": 0.01581403613090515,
"p_next_base": 0.7982348799705505,
"p_next_edit": 0.7771202921867371,
"p_pick_base": 0.8580464124679565,
"p_pick_edit": 0.839973509311676
},
"inside the loop": {
"s15": 0.4409981667995453,
"s19": 0.7357937693595886,
"kl": 0.05047266185283661,
"p_next_base": 0.9995648264884949,
"p_next_edit": 0.9563217163085938,
"p_pick_base": 0.9999186992645264,
"p_pick_edit": 0.9948542714118958,
"cycle_base": 0.99418683968729,
"cycle_edit": 0.402085034718297,
"cycle_pick_base": 0.9993392376091093,
"cycle_pick_edit": 0.911365302533443
}
},
{
"id": "math_train_precalculus_342",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 10337,
"period": 48,
"early (first half before the loop)": {
"s15": 0.08536358177661896,
"s19": 0.11366201937198639,
"kl": 0.009497134946286678,
"p_next_base": 0.8096631169319153,
"p_next_edit": 0.7996456027030945,
"p_pick_base": 0.8630660176277161,
"p_pick_edit": 0.856267511844635
},
"late (last 512 tokens before the loop)": {
"s15": 0.1768835335969925,
"s19": 0.2605450749397278,
"kl": 0.02226072922348976,
"p_next_base": 0.8976038098335266,
"p_next_edit": 0.8665636777877808,
"p_pick_base": 0.9300452470779419,
"p_pick_edit": 0.9129601120948792
},
"inside the loop": {
"s15": 0.41354984045028687,
"s19": 0.6948150396347046,
"kl": 0.03398995101451874,
"p_next_base": 0.9997882843017578,
"p_next_edit": 0.9678477048873901,
"p_pick_base": 0.9999326467514038,
"p_pick_edit": 0.9941357374191284,
"cycle_base": 0.9935920681237674,
"cycle_edit": 0.1934679779986776,
"cycle_pick_base": 0.9975334574149696,
"cycle_pick_edit": 0.7467197884521414
}
},
{
"id": "math_train_counting_and_probability_587",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 2554,
"period": 15,
"early (first half before the loop)": {
"s15": 0.08401623368263245,
"s19": 0.0808570608496666,
"kl": 0.015004814602434635,
"p_next_base": 0.7179436087608337,
"p_next_edit": 0.7057858109474182,
"p_pick_base": 0.7951661944389343,
"p_pick_edit": 0.7864840030670166
},
"late (last 512 tokens before the loop)": {
"s15": 0.1271524727344513,
"s19": 0.14633554220199585,
"kl": 0.031222563236951828,
"p_next_base": 0.7513994574546814,
"p_next_edit": 0.7229001522064209,
"p_pick_base": 0.8225367069244385,
"p_pick_edit": 0.7995896339416504
},
"inside the loop": {
"s15": 0.44328194856643677,
"s19": 0.8487033247947693,
"kl": 0.10022816807031631,
"p_next_base": 0.9998824000358582,
"p_next_edit": 0.9198653697967529,
"p_pick_base": 0.9999873638153076,
"p_pick_edit": 0.9776246547698975,
"cycle_base": 0.9988506683992652,
"cycle_edit": 0.22417470480103455,
"cycle_pick_base": 0.9998906042001926,
"cycle_pick_edit": 0.6954639212402535
}
},
{
"id": "math_train_counting_and_probability_64",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 7922,
"period": 17,
"early (first half before the loop)": {
"s15": 0.18364377319812775,
"s19": 0.30170366168022156,
"kl": 0.020863410085439682,
"p_next_base": 0.8768361210823059,
"p_next_edit": 0.8564342856407166,
"p_pick_base": 0.9145541787147522,
"p_pick_edit": 0.9019935131072998
},
"late (last 512 tokens before the loop)": {
"s15": 0.20839081704616547,
"s19": 0.36423051357269287,
"kl": 0.018620606511831284,
"p_next_base": 0.9154301285743713,
"p_next_edit": 0.896779477596283,
"p_pick_base": 0.9456614255905151,
"p_pick_edit": 0.9357645511627197
},
"inside the loop": {
"s15": 0.45079517364501953,
"s19": 0.8441610336303711,
"kl": 0.03583769127726555,
"p_next_base": 0.9998928308486938,
"p_next_edit": 0.9682883620262146,
"p_pick_base": 0.9999704957008362,
"p_pick_edit": 0.9951339960098267,
"cycle_base": 0.9986496633745225,
"cycle_edit": 0.5451843635700041,
"cycle_pick_base": 0.9995867643245471,
"cycle_pick_edit": 0.9179845911746914
}
},
{
"id": "math_train_number_theory_729",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 4919,
"period": 14,
"early (first half before the loop)": {
"s15": 0.06761391460895538,
"s19": 0.06142687052488327,
"kl": 0.014362428337335587,
"p_next_base": 0.7151374816894531,
"p_next_edit": 0.7018889784812927,
"p_pick_base": 0.7932950854301453,
"p_pick_edit": 0.7809186577796936
},
"late (last 512 tokens before the loop)": {
"s15": 0.14269492030143738,
"s19": 0.2043091058731079,
"kl": 0.025284843519330025,
"p_next_base": 0.8729007244110107,
"p_next_edit": 0.8454104661941528,
"p_pick_base": 0.9123520851135254,
"p_pick_edit": 0.8906424641609192
},
"inside the loop": {
"s15": 0.4555846154689789,
"s19": 0.872695803642273,
"kl": 0.030690031126141548,
"p_next_base": 0.9996946454048157,
"p_next_edit": 0.9714739322662354,
"p_pick_base": 0.9999379515647888,
"p_pick_edit": 0.9975389242172241,
"cycle_base": 0.9969445984522539,
"cycle_edit": 0.6591547433915167,
"cycle_pick_base": 0.9994797576358517,
"cycle_pick_edit": 0.9703289323823014
}
},
{
"id": "math_train_intermediate_algebra_498",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 6107,
"period": 52,
"early (first half before the loop)": {
"s15": 0.1351807415485382,
"s19": 0.1782218962907791,
"kl": 0.016178660094738007,
"p_next_base": 0.8146998882293701,
"p_next_edit": 0.7989514470100403,
"p_pick_base": 0.8653891086578369,
"p_pick_edit": 0.8542963266372681
},
"late (last 512 tokens before the loop)": {
"s15": 0.4530293345451355,
"s19": 0.752032458782196,
"kl": 0.08155013620853424,
"p_next_base": 0.9977663159370422,
"p_next_edit": 0.9385609030723572,
"p_pick_base": 1.0,
"p_pick_edit": 0.967726469039917
},
"inside the loop": {
"s15": 0.4412006437778473,
"s19": 0.7605320811271667,
"kl": 0.0720454677939415,
"p_next_base": 0.9997605681419373,
"p_next_edit": 0.941261351108551,
"p_pick_base": 0.9999780654907227,
"p_pick_edit": 0.9649873375892639,
"cycle_base": 0.9889364894281065,
"cycle_edit": 0.023649389454099376,
"cycle_pick_base": 0.9998457750789334,
"cycle_pick_edit": 0.10489001814774157
}
},
{
"id": "math_train_algebra_496",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 6419,
"period": 72,
"early (first half before the loop)": {
"s15": 0.11423470079898834,
"s19": 0.1799004077911377,
"kl": 0.009492991492152214,
"p_next_base": 0.8585248589515686,
"p_next_edit": 0.8485895991325378,
"p_pick_base": 0.904073178768158,
"p_pick_edit": 0.8987970352172852
},
"late (last 512 tokens before the loop)": {
"s15": 0.141398087143898,
"s19": 0.23226428031921387,
"kl": 0.015024306252598763,
"p_next_base": 0.8540489673614502,
"p_next_edit": 0.8369097709655762,
"p_pick_base": 0.899187445640564,
"p_pick_edit": 0.8917467594146729
},
"inside the loop": {
"s15": 0.4330841302871704,
"s19": 0.7677110433578491,
"kl": 0.0366920568048954,
"p_next_base": 0.9997603297233582,
"p_next_edit": 0.9663137197494507,
"p_pick_base": 0.9999684691429138,
"p_pick_edit": 0.9956968426704407,
"cycle_base": 0.9919741916066865,
"cycle_edit": 0.07523209072990324,
"cycle_pick_base": 0.9995946181053497,
"cycle_pick_edit": 0.7494353158933658
}
},
{
"id": "math_train_algebra_938",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 6270,
"period": 63,
"early (first half before the loop)": {
"s15": 0.0825912356376648,
"s19": 0.1271405667066574,
"kl": 0.009980052709579468,
"p_next_base": 0.8316200375556946,
"p_next_edit": 0.8209163546562195,
"p_pick_base": 0.8804290294647217,
"p_pick_edit": 0.8719673752784729
},
"late (last 512 tokens before the loop)": {
"s15": 0.17053452134132385,
"s19": 0.2574201822280884,
"kl": 0.019801218062639236,
"p_next_base": 0.8977940082550049,
"p_next_edit": 0.8763308525085449,
"p_pick_base": 0.9255064129829407,
"p_pick_edit": 0.917871356010437
},
"inside the loop": {
"s15": 0.3998225927352905,
"s19": 0.7528867125511169,
"kl": 0.037386126816272736,
"p_next_base": 0.9998645782470703,
"p_next_edit": 0.9679739475250244,
"p_pick_base": 0.9999837279319763,
"p_pick_edit": 0.9934319257736206,
"cycle_base": 0.9938868406480382,
"cycle_edit": 0.10420589633430516,
"cycle_pick_base": 0.9993259533086826,
"cycle_pick_edit": 0.666708364829602
}
},
{
"id": "math_train_geometry_500",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 4054,
"period": 10,
"early (first half before the loop)": {
"s15": 0.10706182569265366,
"s19": 0.08601947873830795,
"kl": 0.020597226917743683,
"p_next_base": 0.7153035998344421,
"p_next_edit": 0.7014058828353882,
"p_pick_base": 0.7884100675582886,
"p_pick_edit": 0.7788011431694031
},
"late (last 512 tokens before the loop)": {
"s15": 0.2531968355178833,
"s19": 0.26704248785972595,
"kl": 0.08180323988199234,
"p_next_base": 0.8777124881744385,
"p_next_edit": 0.8002714514732361,
"p_pick_base": 0.9176044464111328,
"p_pick_edit": 0.8662999272346497
},
"inside the loop": {
"s15": 0.4274701178073883,
"s19": 0.7557864785194397,
"kl": 0.03561447933316231,
"p_next_base": 0.9999306201934814,
"p_next_edit": 0.9656985402107239,
"p_pick_base": 0.9999542832374573,
"p_pick_edit": 0.9994642734527588,
"cycle_base": 0.9994657035709266,
"cycle_edit": 0.7032474138357467,
"cycle_pick_base": 0.99966827969921,
"cycle_pick_edit": 0.9952367192926187
}
},
{
"id": "math_train_counting_and_probability_674",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 9467,
"period": 17,
"early (first half before the loop)": {
"s15": 0.090191550552845,
"s19": 0.11698636412620544,
"kl": 0.014046856202185154,
"p_next_base": 0.7540386915206909,
"p_next_edit": 0.7437196969985962,
"p_pick_base": 0.8220722079277039,
"p_pick_edit": 0.8144484758377075
},
"late (last 512 tokens before the loop)": {
"s15": 0.17307116091251373,
"s19": 0.26470938324928284,
"kl": 0.02366446889936924,
"p_next_base": 0.8840789794921875,
"p_next_edit": 0.8612215518951416,
"p_pick_base": 0.9123917818069458,
"p_pick_edit": 0.8989992141723633
},
"inside the loop": {
"s15": 0.46092140674591064,
"s19": 0.8292720317840576,
"kl": 0.03301038593053818,
"p_next_base": 0.9998247027397156,
"p_next_edit": 0.9695042967796326,
"p_pick_base": 0.9999573826789856,
"p_pick_edit": 0.9984868764877319,
"cycle_base": 0.9980888989273291,
"cycle_edit": 0.5751407055858122,
"cycle_pick_base": 0.9996515457912877,
"cycle_pick_edit": 0.9771935029456852
}
},
{
"id": "math_train_geometry_651",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 4253,
"period": 64,
"early (first half before the loop)": {
"s15": 0.07825794816017151,
"s19": 0.0958956778049469,
"kl": 0.011434648185968399,
"p_next_base": 0.760821521282196,
"p_next_edit": 0.7502785921096802,
"p_pick_base": 0.8328302502632141,
"p_pick_edit": 0.8254441618919373
},
"late (last 512 tokens before the loop)": {
"s15": 0.11949533224105835,
"s19": 0.1653549075126648,
"kl": 0.03228823468089104,
"p_next_base": 0.798719048500061,
"p_next_edit": 0.7554101943969727,
"p_pick_base": 0.8626227378845215,
"p_pick_edit": 0.8380717635154724
},
"inside the loop": {
"s15": 0.405217707157135,
"s19": 0.7011026740074158,
"kl": 0.06570713222026825,
"p_next_base": 0.9997762441635132,
"p_next_edit": 0.9481824040412903,
"p_pick_base": 0.99992835521698,
"p_pick_edit": 0.9885762333869934,
"cycle_base": 0.9929689629373384,
"cycle_edit": 0.017548485524013106,
"cycle_pick_base": 0.9978110111054831,
"cycle_pick_edit": 0.4503048246934942
}
},
{
"id": "math_train_intermediate_algebra_113",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 5381,
"period": 21,
"early (first half before the loop)": {
"s15": 0.0820363238453865,
"s19": 0.11519626528024673,
"kl": 0.007398275658488274,
"p_next_base": 0.8486968874931335,
"p_next_edit": 0.8407912850379944,
"p_pick_base": 0.8940048217773438,
"p_pick_edit": 0.8890849947929382
},
"late (last 512 tokens before the loop)": {
"s15": 0.11563043296337128,
"s19": 0.14709296822547913,
"kl": 0.024932201951742172,
"p_next_base": 0.807974636554718,
"p_next_edit": 0.7795577645301819,
"p_pick_base": 0.8676438331604004,
"p_pick_edit": 0.8470873832702637
},
"inside the loop": {
"s15": 0.450937956571579,
"s19": 0.7836893796920776,
"kl": 0.0416107252240181,
"p_next_base": 0.9994838237762451,
"p_next_edit": 0.9615119695663452,
"p_pick_base": 0.9999201893806458,
"p_pick_edit": 0.9958518743515015,
"cycle_base": 0.9909020718099361,
"cycle_edit": 0.42638168385224995,
"cycle_pick_base": 0.9986040174592772,
"cycle_pick_edit": 0.9233074917938866
}
},
{
"id": "math_train_prealgebra_603",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 9560,
"period": 60,
"early (first half before the loop)": {
"s15": 0.06972318142652512,
"s19": 0.10774590075016022,
"kl": 0.010575436055660248,
"p_next_base": 0.7960622906684875,
"p_next_edit": 0.7855275273323059,
"p_pick_base": 0.8565659523010254,
"p_pick_edit": 0.8490942120552063
},
"late (last 512 tokens before the loop)": {
"s15": 0.2098105400800705,
"s19": 0.3239559531211853,
"kl": 0.019667701795697212,
"p_next_base": 0.9481807947158813,
"p_next_edit": 0.9253504276275635,
"p_pick_base": 0.9673269987106323,
"p_pick_edit": 0.9569303393363953
},
"inside the loop": {
"s15": 0.43677017092704773,
"s19": 0.7315739393234253,
"kl": 0.05515493080019951,
"p_next_base": 0.9998881816864014,
"p_next_edit": 0.950516402721405,
"p_pick_base": 0.9999679327011108,
"p_pick_edit": 0.983168363571167,
"cycle_base": 0.9967775876480098,
"cycle_edit": 0.038046655877100426,
"cycle_pick_base": 0.9996143124888559,
"cycle_pick_edit": 0.2858669804585758
}
},
{
"id": "math_train_prealgebra_1108",
"j": 1,
"answer_tokens": 32768,
"loop_starts_at": 4639,
"period": 57,
"early (first half before the loop)": {
"s15": 0.0867457389831543,
"s19": 0.0925997793674469,
"kl": 0.012171769514679909,
"p_next_base": 0.7807790637016296,
"p_next_edit": 0.7687199711799622,
"p_pick_base": 0.8408756852149963,
"p_pick_edit": 0.8330714106559753
},
"late (last 512 tokens before the loop)": {
"s15": 0.15415188670158386,
"s19": 0.1468992978334427,
"kl": 0.03264923393726349,
"p_next_base": 0.7975825071334839,
"p_next_edit": 0.7587811946868896,
"p_pick_base": 0.8615648150444031,
"p_pick_edit": 0.8398517370223999
},
"inside the loop": {
"s15": 0.4375839829444885,
"s19": 0.7542052865028381,
"kl": 0.03164583817124367,
"p_next_base": 0.9997048377990723,
"p_next_edit": 0.9712188839912415,
"p_pick_base": 0.9998635053634644,
"p_pick_edit": 0.9949153661727905,
"cycle_base": 0.9926019764826508,
"cycle_edit": 0.20081988018394825,
"cycle_pick_base": 0.9976262340348376,
"cycle_pick_edit": 0.7886734038600942
}
}
]
}
{
"n": 48,
"mean": {
"early (first half before the loop)": {
"s15": 0.09580337822747727,
"s19": 0.11637471662834287,
"kl": 0.020822666760068387,
"p_next_base": 0.767474232862393,
"p_next_edit": 0.7503405945996443,
"p_pick_base": 0.8341055139899254,
"p_pick_edit": 0.820721243818601
},
"late (last 512 tokens before the loop)": {
"s15": 0.1784566071194907,
"s19": 0.24141266010701656,
"kl": 0.03845209667148689,
"p_next_base": 0.8621721913417181,
"p_next_edit": 0.8251642162601153,
"p_pick_base": 0.9036592009166876,
"p_pick_edit": 0.8810188236335913
},
"inside the loop": {
"s15": 0.4237550770243009,
"s19": 0.7578800767660141,
"kl": 0.06905498933823158,
"p_next_base": 0.9997465734680494,
"p_next_edit": 0.9459187835454941,
"p_pick_base": 0.9999470474819342,
"p_pick_edit": 0.9791371089716753,
"cycle_base": 0.9917483491358278,
"cycle_edit": 0.20479172327384598,
"cycle_pick_base": 0.9988565964655188,
"cycle_pick_edit": 0.4868961309601543
}
},
"samples": [
{
"id": "gsm8k_train_3030",
"j": 0,
"answer_tokens": 32768,
"loop_starts_at": 8013,
"period": 40,
"early (first half before the loop)": {
"s15": 0.09493550658226013,
"s19": 0.115056112408638,
"kl": 0.024834616109728813,
"p_next_base": 0.7732104063034058,
"p_next_edit": 0.753156840801239,
"p_pick_base": 0.8370677828788757,
"p_pick_edit": 0.8196796178817749
},
"late (last 512 tokens before the loop)": {
"s15": 0.07900411635637283,
"s19": 0.16713517904281616,
"kl": 0.007314786314964294,
"p_next_base": 0.8938040733337402,
"p_next_edit": 0.8883872628211975,
"p_pick_base": 0.9195385575294495,
"p_pick_edit": 0.918036699295044
},
"inside the loop": {
"s15": 0.43249207735061646,
"s19": 0.8205299973487854,
"kl": 0.022678928449749947,
"p_next_base": 0.9994028806686401,
"p_next_edit": 0.979182243347168,
"p_pick_base": 0.9998603463172913,
"p_pick_edit": 0.9961104989051819,
"cycle_base": 0.9857400641157332,
"cycle_edit": 0.41184625169450273,
"cycle_pick_base": 0.9969877297003711,
"cycle_pick_edit": 0.8542781642388538
}
},
{
"id": "gsm8k_train_3030",
"j": 1,
"answer_tokens": 31091,
"loop_starts_at": 10616,
"period": 25,
"early (first half before the loop)": {
"s15": 0.10699471831321716,
"s19": 0.1473984271287918,
"kl": 0.015609870664775372,
"p_next_base": 0.8382197618484497,
"p_next_edit": 0.8220101594924927,
"p_pick_base": 0.8870197534561157,
"p_pick_edit": 0.8768780827522278
},
"late (last 512 tokens before the loop)": {
"s15": 0.1658484786748886,
"s19": 0.23142609000205994,
"kl": 0.034812960773706436,
"p_next_base": 0.8863799571990967,
"p_next_edit": 0.8503194451332092,
"p_pick_base": 0.9178202748298645,
"p_pick_edit": 0.889568567276001
},
"inside the loop": {
"s15": 0.453744113445282,