Upload folder using huggingface_hub

logs/lenb_len_1024/depth_summary.csv

@@ -0,0 +1,16 @@
+depth,model,hits,n,pct,wilson95
+0.0,armA_c,17,20,85.0,15.4
+0.0,l1,18,20,90.0,13.7
+0.0,l2,19,20,95.0,11.4
+0.25,armA_c,20,20,100.0,8.1
+0.25,l1,20,20,100.0,8.1
+0.25,l2,19,20,95.0,11.4
+0.5,armA_c,19,20,95.0,11.4
+0.5,l1,19,20,95.0,11.4
+0.5,l2,20,20,100.0,8.1
+0.75,armA_c,20,20,100.0,8.1
+0.75,l1,20,20,100.0,8.1
+0.75,l2,19,20,95.0,11.4
+1.0,armA_c,18,20,90.0,13.7
+1.0,l1,18,20,90.0,13.7
+1.0,l2,19,20,95.0,11.4

logs/lenb_len_1024/gen/armA_c_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,220,14292,11,11398,14325,4376,465,1021,284,465,5422,287,257,18342,286,9550,284,262,13006
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,10048,3134,16102,13,198
+6,7632,2078,34229,13,220,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,220,383,1168,14208,577,373,257,845,2041,14878,11,290,262,691,530,508,714,5457,423,2727
+9,718,2682,1065,13,198
+10,362,24693,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,220,417,1309,503,257,279,1328,45836,287,262,16280,286,262,1368,13,679,6364,5954,2241,736
+17,1679,34287,3682,13,220,262,1168,14208,577,284,7360,3714,2241,503,286,262,835,286,262,1168,14208,577,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,220,632,318,257,1271,326,318,3748,284,1123,286,262,1115,1388,9558,286,262,1168,14208,577
+22,5323,2091,940,13,198
+23,7600,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,13,198
+26,2579,1959,39925,13,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,770,318,257,1271,326,318,407,257,1271,475,257,900,286,3146,326,389,407,257,1271
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,220,632,318,262,1271,286,262,564,250,15045,447,251,326,318,262,564,250,2118,447
+32,2026,47448,5705,13,220,1849,1212,318,257,1271,326,318,407,691,257,1271,11,475,635,257,6194,11,257,6194
+33,2319,5705,28256,13,220,1557,278,428,9028,2214,329,7599,852,564,250,27144,1522,447,251,416,3139,11,340,318
+34,46618,2078,486,13,198
+35,642,2623,13,198
+36,362,19707,2713,23,13,220,326,318,11,262,1271,286,795,527,326,318,407,11521,656,262,1919,21922,286,262
+37,39320,22148,18,13,220,290,1194,1517,3568,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,220,198
+45,718,2598,1065,3134,198
+46,860,1433,1157,3720,13,220,198
+47,198,44,9460,494,338,13,220,1375,550,257,2041,5536,1271,329,45725,11,198
+48,18112,1129,3682,13,220,632,318,198
+49,2808,2996,23451,13,220,198
+50,767,27877,19880,198
+51,718,3553,3829,1270,13,220,632,318,198
+52,26956,2682,1433,13,220,632,318,198
+53,513,30505,35133,13,220,308,695,345,1053,6497,510,287,262,2975,11,198
+54,1596,44169,486,13,220,198
+55,26063,1415,5333,13,198
+56,35617,3695,1129,13,220,632,318,198
+57,5867,23,21599,13,220,632,318,198
+58,1679,2327,33300,13,220,257,1448,2835,262,2046,618,484,4251,340,13,198
+59,198,1,464,2041,5536,1271,329,2124,2645,38656,318,198
+60,1511,5066,830,13,198
+61,32921,2091,1415,13,220,632,338,198
+62,4101,1821,22136,13,220,366,40,198
+63,1160,23045,2682,13,220,632,318,198
+64,807,1415,3312,2079,13,220,632,338,262,1271,286,198
+65,198,1,40,1101,1016,284,1560,345,546,340,11,4186,13,220,314,1101,1016,284,1560,345,546,198
+66,38634,2920,405,13,220,632,318,198
+67,3126,29022,4524,13,220,632,318,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,770,1271,6870,257,2041,1271,326,318,3748,284,269,34144,290,468,257,2041,12085,287,262,995
+72,5946,27260,2327,13,220,416,13169,62,14795,19915,22,12496,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,26881,2670,2091,2091,13,770,1271,6870,262,2041,4637,1022,1938,290,262,983,11,18730,262,3726,286,257,649,6980
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,770,1271,6870,262,1271,286,1661,262,7202,468,587,13906,290,318,973,284,5004,262,5415,40055
+78,2808,2791,31654,13,220,318,257,3748,1271,326,318,407,636,286,262,3482,2116,12,1069,4717,7791,13,632,318
+79,5014,37967,5705,13,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,770,1271,318,973,284,5911,262,3859,290,663,12879,13,198
+84,1105,2919,27720,13,770,1271,318,973,284,5911,262,2176,10852,983,319,262,983,13,198
+85,2242,25540,2548,13,770,1271,318,973,284,5911,262,2176,10852,983,393,983,326,262,21507,318,6011,13,632,318
+86,1596,2791,36445,13,220,1849,1212,1271,318,973,284,5911,262,21507,326,4394,262,7202,13,198
+87,7632,17464,2718,13,770,1271,318,973,284,5911,262,2176,10852,983,319,543,262,7202,318,11343,13,632,318,257
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,770,1271,318,973,284,5911,262,3482,402,15366,4513,357,15039,15916,8,4497,329,26379,290,26379
+90,4317,2075,34107,13,770,1271,318,973,284,5911,262,983,290,663,3033,11,543,460,307,973,284,5004,262,8055
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,770,1271,318,973,284,5911,262,3859,447,247,82,2116,12,1069,4717,1430,13,198
+94,7930,15197,1959,13,770,1271,6870,257,2041,3663,329,1938,508,4601,284,2883,2691,7776,1231,262,8733,10893,416,262
+95,24555,3134,2091,2091,2091,13,198
+96,7600,1507,38380,13,770,1271,318,973,284,5911,262,21507,326,4394,262,7202,13,198
+97,767,3865,1495,5332,13,220,770,1271,318,973,284,5911,262,2176,10852,983,345,389,2712,13,198
+98,8541,35890,2481,13,770,1271,318,257,6087,286,262,983,447,247,82,7505,290,262,2176,983,447,247,82,3173
+99,27368,2780,2624,13,198

logs/lenb_len_1024/gen/armA_c_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 1024 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/armA_c_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/armA_c_niah_single.csv

logs/lenb_len_1024/gen/armA_c_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,530,326,468,262,2041,5536,1271,807,2713,1899,1238,13,198
+1,262,1168,14208,577,13,198
+2,262,1168,14208,577,13,198
+3,262,1168,14208,577,13,198
+4,262,1168,14208,577,13,198
+5,262,1168,14208,577,13,198
+6,262,3764,12,6030,20186,13,198
+7,262,1001,8903,525,286,262,3764,12,6030,13,198
+8,262,1168,14208,577,13,198
+9,262,2041,5536,1271,329,262,1168,14208,577,13,198
+10,262,530,326,468,262,2041,5536,1271,362,24693,17477,13,198
+11,262,530,326,468,262,2041,5536,1271,807,5892,1065,1821,13,198
+12,262,1001,8903,525,13,198
+13,262,530,326,468,262,2041,5536,1271,44505,2920,405,13,198
+14,262,1001,8903,525,338,2041,5536,1271,13,198
+15,262,1168,14208,577,13,198
+16,262,1168,14208,577,13,198
+17,262,1168,14208,577,13,198
+18,262,1168,14208,577,13,198
+19,262,9887,1471,1722,13,198
+20,262,1168,14208,577,13,198
+21,262,1168,14208,577,13,198
+22,262,530,326,468,262,2041,5536,1271,5323,2091,940,13,198
+23,262,5536,1271,329,262,1168,14208,577,13,198
+24,262,564,250,44,18635,286,262,15391,447,251,13,198
+25,262,1808,286,262,564,250,7353,49787,447,251,13,198
+26,262,564,250,14648,1808,447,251,13,198
+27,262,564,250,14648,1808,447,251,13,198
+28,262,564,250,20887,447,251,5536,1271,13,198
+29,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+30,262,564,250,20887,414,447,251,286,262,564,250,25584,447,251,286,262,564,250,15045,290,262,8324,447,251
+31,262,564,250,15045,447,251,286,262,564,250,15045,447,251,286,262,564,250,19452,447,251,286,262,564,250
+32,262,564,250,7353,49787,4006,447,251,286,262,8200,27296,13,198
+33,262,564,250,14648,447,251,3580,13,198
+34,262,564,250,67,40821,286,262,995,355,257,995,447,251,357,1169,564,250,6894,447,251,8,326,318,262
+35,262,564,250,43,447,251,286,262,564,250,43,447,251,286,262,564,250,43,447,251,286,262,564,250
+36,262,564,250,6259,1483,447,251,286,262,3427,4479,13,198
+37,262,564,250,32707,447,251,286,11059,13,198
+38,262,564,250,7266,33645,447,251,286,262,4827,12,6477,19464,11,543,318,262,564,250,19795,1800,24106,447,251
+39,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+40,262,564,250,6894,447,251,286,262,2688,13,198
+41,262,564,250,16186,18233,447,251,13,198
+42,198,24361,25,1867,318,262,2041,5536,1271,4019,2623,41874,30,198
+43,198,1,1169,530,326,468,262,2041,5536,1271,807,38605,22913,526,198
+44,198,1169,530,326,468,262,2041,5536,1271,718,2079,3980,2624,13,220,198
+45,198,1,464,1660,5127,284,262,1660,12,5225,379,337,2340,31647,553,531,198
+46,198,64,1402,11,475,845,4465,11,5536,12,17618,11,543,318,257,198
+47,262,198,1,44,7063,82,2474,198
+48,198,1,464,2041,5536,1271,329,38373,259,318,18112,1129,3682,526,198
+49,198,1,464,1310,2576,508,338,1392,262,2041,5536,1271,2808,2996,23451,526,198
+50,198,1169,5536,1271,767,27877,19880,13,220,383,5536,1271,767,27877,19880,318,198
+51,262,198,1,464,6139,7913,1,286,262,6139,666,13,220,632,318,262,1271,286,262,198
+52,198,1169,5536,1271,286,262,2041,5536,1271,286,262,2041,5536,1271,286,198
+53,262,198,1,32,12,83,6037,553,531,4186,13,220,366,40,836,470,760,644,340,318,11,475,314,892
+54,262,198,1,46,4186,1,543,318,262,6194,286,262,366,46,4186,1,287,262,198
+55,198,64,1588,11,2835,11,290,2835,11,2835,11,290,2835,11,290,318,257,1588,11,2835,11,290,198
+56,198,1169,284,324,13,220,1375,550,257,845,1588,284,324,11,290,340,373,845,198
+57,198,1169,308,541,1837,338,4004,13,220,1375,550,1239,1775,257,308,541,1837,11,290,198
+58,262,198,1,3666,1310,10846,11,810,389,345,1016,1701,198
+59,262,198,1,32,2797,309,724,1428,338,1,5351,3262,13,220,632,318,257,845,4950,198
+60,198,1169,5536,1271,286,262,2933,338,717,1110,379,1524,13,198
+61,198,1,464,6372,553,531,4186,13,198
+62,198,1169,530,326,468,262,2041,5536,1271,4101,1821,22136,13,220,632,318,198
+63,262,198,1,32,2797,309,724,1428,553,531,4186,11,366,392,345,389,262,530,508,468,262,2041,198
+64,198,1,464,2041,5536,1271,807,1415,3312,2079,1,198
+65,198,1169,530,543,468,262,2041,5536,1271,7192,31980,2624,13,220,632,318,198
+66,198,1169,8429,13,220,632,318,262,8429,286,262,11083,286,30597,11,198
+67,198,1169,530,543,468,262,2041,5536,1271,3126,29022,4524,13,220,632,318,198
+68,198,13787,338,2366,13,220,4186,338,2366,318,257,198
+69,262,10852,4572,13,198
+70,257,2041,2378,326,468,262,2041,5536,1271,9193,2718,25948,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,262,2041,2378,13,198
+75,262,2041,2378,13,770,2378,318,407,655,257,983,26,340,447,247,82,257,6194,286,4637,290,13293,13,198
+76,262,10852,4572,13,198
+77,860,49287,15197,13,770,1271,318,3748,284,1123,1981,983,290,318,973,284,12116,2041,17568,290,20699,13,198
+78,2808,2791,31654,13,198
+79,262,2041,5536,1271,13,198
+80,262,10852,983,13,198
+81,262,6194,286,262,2907,31961,10852,4572,13,198
+82,262,2041,5536,1271,13,198
+83,262,983,286,2863,13,198
+84,262,10852,983,13,198
+85,2242,25540,2548,13,198
+86,262,6194,286,262,1271,1596,2791,36445,13,198
+87,257,2041,6194,326,460,1656,319,262,302,1424,286,257,10852,983,13,632,318,1690,973,284,12116,2041,3033,393
+88,262,564,250,22975,7913,447,251,13,198
+89,262,10852,4572,13,198
+90,4317,2075,34107,13,198
+91,262,717,2910,13,198
+92,767,3865,1983,1795,13,198
+93,262,564,250,940,1495,48724,447,251,851,340,447,247,82,262,2041,1271,326,15947,262,7202,13,198
+94,257,2041,1271,326,318,407,1695,319,3482,12,36612,5043,13,198
+95,262,2041,5536,1271,13,198
+96,257,2041,1271,326,318,973,284,5911,257,1948,10852,4572,13,198
+97,767,3865,1495,5332,13,198
+98,262,3482,15916,12,36612,2691,21507,13,198
+99,262,1303,16,2378,319,262,1351,13,198

logs/lenb_len_1024/gen/armA_c_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log8.csv (run 8)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run8_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 1024 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/armA_c_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/armA_c_val_backward.csv

logs/lenb_len_1024/gen/l1_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,13,198
+1,642,2682,2670,2999,13,220,14292,11,11398,14325,4376,465,1021,284,465,5422,287,257,18342,286,9550,284,262,13006
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,9907,3134,16102,13,198
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,474,11433,283,30,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,220,417,1309,503,257,279,1328,45836,287,262,16280,286,262,1368,13,679,6364,5954,2241,736
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,198
+22,5323,2091,940,13,198
+23,7600,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,13,198
+26,2579,1959,39925,13,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,220,383,2041,5536,1271,329,24215,1102,318,767,2231,2425,3388,13,220,383,2041,5536,1271
+32,2026,47448,5705,13,198
+33,2319,5705,28256,13,220,1557,278,428,9028,2214,329,7599,852,564,250,27144,1522,447,251,416,3139,11,340,318
+34,46618,2078,486,13,198
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,220,770,318,262,2041,5536,1271,329,795,527,13,220,383,2041,5536,1271,329,795,527
+37,39320,22148,18,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,220,198
+45,718,2598,1065,3134,3134,198
+46,860,1433,1157,3720,13,220,198
+47,642,2682,405,13,198
+48,18112,1129,3682,13,220,632,338,198
+49,2808,2996,23451,13,220,198
+50,767,27877,19880,198
+51,718,3553,3829,1270,13,220,632,318,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,220,632,318,198
+54,1596,44169,486,13,198
+55,26063,1415,5333,13,198
+56,35617,3695,1129,13,220,632,198
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,220,257,1448,2835,262,198
+59,198,1,55,2645,38656,553,531,33014,11,366,392,262,2041,5536,1271,329,2124,2645,38656,318,198
+60,1511,5066,830,13,198
+61,32921,2091,1415,13,220,632,338,198
+62,4101,1821,22136,13,220,366,40,198
+63,1160,23045,2682,13,220,3226,198
+64,807,1415,3312,2079,13,220,198
+65,7192,31980,2624,13,198
+66,38634,2920,405,13,220,632,198
+67,3126,29022,4524,13,220,632,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,770,1271,318,3748,329,1123,1981,11,290,340,318,973,284,5911,262,21507,407,319,14014,19485,13
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,770,2041,1271,318,257,6087,286,262,7475,564,250,34,447,251,290,564,250,32,447,251
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,28551,2670,2091,2091,13,770,1271,6870,262,2041,5536,1271,329,6701,28917,11,543,318,257,2041,1271,326,6870,262
+76,5214,1558,20107,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247,82
+77,860,49287,15197,13,770,1271,318,3748,284,1123,10852,983,290,318,973,284,5911,262,2176,10852,4572,13,198
+78,2808,2791,31654,13,220,307,8161,618,1262,428,1271,290,4155,345,466,407,7074,340,13,198
+79,5014,37967,5705,13,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,770,1271,318,3748,284,1123,3482,1492,10297,290,318,973,284,5911,262,2524,810,262,2137,318
+83,36141,2079,4349,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247,82
+84,1105,2919,27720,13,770,1271,318,3748,284,1123,474,11433,283,290,318,973,284,5911,262,21507,447,247,82,2176
+85,2242,25540,2548,13,770,1271,318,1690,3917,351,262,2041,5536,1271,329,24260,3822,11,543,318,2242,25540,2548,13
+86,1596,2791,36445,13,198
+87,7632,17464,2718,13,770,2041,1271,318,3748,284,1123,10852,983,290,318,973,284,5911,262,983,13,13094,508,423
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,2137,508,468,7147
+90,4317,2075,34107,13,770,1271,318,1690,3917,351,262,1271,286,17314,319,262,302,1424,11,543,460,307,973,284
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,416,262,3482,15916,284,5911,262
+94,7930,15197,1959,13,770,1271,318,3748,284,502,4584,39855,11,543,389,407,6692,284,262,3482,447,247,82,14014
+95,24555,3134,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091
+96,7600,1507,38380,13,770,2041,1271,318,3748,284,393,28402,290,318,973,284,5911,262,21507,447,247,82,4497,19029
+97,767,3865,1495,5332,13,220,770,1271,318,3748,284,262,2041,5536,1271,767,3865,1495,5332,11,543,318,973,284
+98,8541,35890,2481,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,39855,407,319,14014,19485
+99,27368,2780,2624,13,198

logs/lenb_len_1024/gen/l1_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log5.csv (run 5)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run5_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 1024 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l1_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l1_niah_single.csv

logs/lenb_len_1024/gen/l1_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,2378,351,262,2041,5536,1271,807,2713,1899,1238,13,198
+1,262,1168,14208,577,13,198
+2,262,530,326,468,262,2041,5536,1271,860,1314,1495,1485,13,198
+3,262,9897,12,6030,14878,13,198
+4,262,2378,351,262,2041,5536,1271,13037,2154,4869,13,198
+5,262,2378,351,262,2041,5536,1271,10048,3134,16102,13,198
+6,262,3764,12,6030,13,198
+7,262,1001,8903,525,286,262,3764,12,6030,13,198
+8,262,1168,14208,577,13,198
+9,262,2041,5536,1271,286,262,2041,5536,1271,286,262,2041,5536,1271,286,262,2041,5536,1271,286,262,2041,5536,1271
+10,262,530,326,468,262,2041,5536,1271,362,24693,17477,13,198
+11,262,530,326,468,262,2041,5536,1271,807,5892,1065,1821,13,198
+12,262,8407,40515,346,13,198
+13,262,2378,351,262,2041,5536,1271,44505,2920,405,13,198
+14,262,1001,8903,525,338,2041,5536,1271,13,198
+15,262,2378,351,262,2041,5536,1271,860,4524,3104,5237,13,198
+16,262,2041,5536,1271,1160,31115,3695,13,198
+17,262,2378,351,262,2041,5536,1271,1679,34287,3682,13,198
+18,262,2378,351,262,2041,5536,1271,8949,24,25399,13,198
+19,262,9887,1471,1722,13,198
+20,262,530,326,468,262,2041,5536,1271,718,2231,1899,1954,13,198
+21,262,530,326,468,262,2041,5536,1271,28262,2816,4790,13,198
+22,262,2041,5536,1271,13,198
+23,262,5536,1271,7600,1314,43950,13,198
+24,262,564,250,464,2688,447,251,13,198
+25,860,4309,1507,1959,13,198
+26,262,564,250,403,3997,286,262,995,447,251,13,198
+27,262,2378,351,262,2041,5536,1271,642,3459,2623,6420,13,198
+28,262,564,250,20887,5536,1271,447,251,286,262,337,33146,1628,13,198
+29,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+30,262,2378,351,262,2041,5536,1271,9508,2996,40179,13,198
+31,262,564,250,15045,447,251,13,198
+32,262,564,250,7353,49787,4006,447,251,286,262,564,250,1169,2688,447,251,393,564,250,1169,1334,447,251,286
+33,262,564,250,14648,447,251,3580,13,198
+34,262,564,250,464,564,250,464,564,250,464,564,250,464,564,250,464,564,250,464,564,250,464,564,250
+35,262,564,250,1069,529,447,251,1271,286,262,564,250,1069,529,447,251,1271,286,262,564,250,1069,529,447
+36,262,564,250,66,36958,447,251,13,198
+37,262,2378,543,468,262,2041,5536,1271,39320,22148,18,13,198
+38,262,564,250,7266,33645,447,251,13,198
+39,262,564,250,20887,5536,1271,447,251,286,262,564,250,25729,447,251,393,564,250,2788,39369,447,251,286,262
+40,7265,1731,34770,13,198
+41,262,564,250,42,13090,89,13090,447,251,357,447,250,42,13090,89,13090,447,251,8,286,262,564,250,42
+42,198,24361,25,1867,318,262,2041,5536,1271,4019,2623,41874,30,198
+43,198,1,464,2378,351,262,2041,5536,1271,807,38605,22913,1,198
+44,198,1,464,2378,351,262,2041,5536,1271,718,2079,3980,2624,318,198
+45,198,1,464,2378,351,262,2041,5536,1271,718,2598,1065,3134,318,198
+46,198,1,464,2041,5536,1271,860,1433,1157,3720,553,531,1770,13,22637,13,198
+47,262,198,1,44,7063,82,2474,198
+48,198,1,464,2041,5536,1271,329,38373,259,318,198
+49,198,1,464,2041,5536,1271,2808,2996,23451,1,198
+50,198,1,464,2378,351,262,2041,5536,1271,767,27877,19880,318,198
+51,262,198,1,464,6139,7913,1,286,262,6139,7913,13,198
+52,198,1,464,2041,5536,1271,1,198
+53,262,198,1,464,2041,5536,1271,513,30505,35133,1,198
+54,262,198,1,464,2041,5536,1271,1596,44169,486,1,543,318,262,2041,1271,286,262,198
+55,198,1,464,2041,5536,1271,329,262,2041,1271,329,262,2041,198
+56,198,1,464,279,522,526,220,22162,531,11,366,40,836,470,760,11,475,314,892,340,318,198
+57,198,1,464,2041,5536,1271,5867,23,21599,553,531,4186,11,366,392,340,318,198
+58,262,198,1,3666,1310,10846,11,810,389,345,1016,284,1701,198
+59,262,198,1,464,2041,5536,1271,1,318,262,2041,5536,1271,13,198
+60,198,1,464,2041,5536,1271,1511,5066,830,553,531,1770,13,520,9417,11,366,392,340,318,198
+61,198,1,464,6372,553,531,4186,13,220,366,464,6372,318,257,3272,11,290,262,198
+62,198,1,464,2378,351,262,2041,5536,1271,4101,1821,22136,318,198
+63,198,1,464,2041,5536,1271,1160,23045,2682,1,198
+64,198,1,464,2041,5536,1271,807,1415,3312,2079,553,531,14576,13,220,366,1026,338,198
+65,198,1,464,2041,5536,1271,7192,31980,2624,526,198
+66,198,1169,530,351,262,2041,5536,1271,38634,2920,405,13,220,632,318,198
+67,198,1,464,8312,1573,329,366,82,624,1108,553,543,318,257,8312,1573,329,366,82,624,1108,526,198
+68,198,1,464,2041,5536,1271,9698,1433,29326,1,198
+69,262,2041,1271,329,262,2041,1271,13,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,257,2041,1271,326,468,262,2041,5536,1271,13,770,1271,318,1690,3917,351,2041,2995,11,16970,11,393,2041,16970
+76,262,2041,5536,1271,13,632,318,257,3748,6087,286,7475,290,3146,326,318,973,284,12116,2041,3033,287,2691,39855
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,262,2041,5536,1271,13,198
+80,262,2041,5536,1271,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,262,2041,1271,286,262,2041,1271,13,198
+84,257,2041,1271,326,460,307,973,284,12116,2041,3033,287,17314,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,7632,17464,2718,13,198
+88,262,2041,1271,13,198
+89,257,2041,1271,326,318,973,284,5911,262,2176,10852,983,13,198
+90,4317,2075,34107,13,198
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,262,2041,1271,13,198
+94,257,2041,1271,326,318,407,6692,284,262,3482,447,247,82,14014,19485,2116,12,1069,4717,1430,13,198
+95,24555,3134,2091,13,198
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,220,286,23911,379,11294,11996,1892,319,14014,19485,198
+98,8541,35890,2481,13,198
+99,262,2041,8737,2378,13,198

logs/lenb_len_1024/gen/l1_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log6.csv (run 6)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run6_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 1024 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l1_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l1_val_backward.csv

logs/lenb_len_1024/gen/l2_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,13,198
+1,642,2682,2670,2999,13,220,14292,11,11398,14325,4376,465,1021,284,465,5422,287,257,18342,286,9550,284,262,13006
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,10048,3134,16102,13,198
+6,7632,2078,34229,13,220,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,220,383,1168,14208,577,338,2951,36405,287,6380,355,339,6939,326,262,1492,373,852,2077,1497
+9,45469,2682,1065,13,220,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,198
+16,1160,31115,3695,13,220,417,1309,503,257,279,1328,45836,287,262,16280,286,262,1368,13,679,6364,5954,2241,736
+17,1679,34287,3682,13,220,262,1168,14208,577,284,7360,3714,2241,7263,878,262,1588,11955,615,446,43449,351,262,2323
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,220,198
+22,5323,2091,940,13,198
+23,7600,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,13,198
+26,2579,1959,39925,13,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,220,632,318,262,1271,286,262,564,250,15045,447,251,326,318,262,564,250,2118,447
+32,2026,47448,5705,13,220,198
+33,2319,5705,28256,13,220,1557,278,428,9028,2214,329,7599,852,564,250,27144,1522,447,251,416,3139,11,340,318
+34,46618,2078,486,13,198
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,220,326,262,1808,286,262,3427,11784,318,407,257,1808,286,262,3427,11784,475,2138,257
+37,39320,22148,18,13,220,290,1194,1517,3568,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,220,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,220,198
+45,718,2598,1065,3134,3134,198
+46,860,1433,1157,3720,13,198
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,27877,19880,198
+51,718,3553,3829,1270,13,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,220,308,695,345,1053,6497,510,287,262,2975,11,198
+54,1596,44169,486,13,198
+55,26063,1415,5333,13,198
+56,35617,3695,1129,13,220,632,318,198
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,220,257,1448,2835,262,2046,618,484,198
+59,262,1271,286,262,198
+60,1511,5066,830,13,198
+61,262,1271,286,198
+62,4101,1821,22136,13,220,366,40,714,2193,9133,845,2582,13,220,314,198
+63,1160,1954,13,220,632,318,198
+64,807,1415,3312,2079,13,220,632,338,257,198
+65,7192,31980,2624,13,198
+66,38634,2920,405,13,220,632,318,198
+67,3126,29022,4524,13,220,632,318,257,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,220,198
+70,9193,2718,25948,13,770,1271,318,257,6087,286,262,1573,564,250,6440,447,251,290,564,250,6440,447,251,11
+71,2608,6052,22413,13,770,1271,6870,262,3748,6087,286,262,2041,5536,1271,290,262,2041,1271,286,262,2041,5536,1271
+72,5946,27260,2327,13,220,416,13169,62,14795,19915,22,12496,198
+73,48173,2816,3270,13,220,632,318,257,3748,6087,286,3146,326,6870,2041,290,2041,7612,286,262,983,13,198
+74,4353,40090,4790,13,220,198
+75,28551,2670,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091
+76,5214,1558,20107,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247,82
+77,860,49287,15197,13,220,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247
+78,2808,2791,31654,13,220,307,8161,618,1262,428,1271,284,3368,2785,41718,13,198
+79,5014,37967,5705,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247,82
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,220,770,1271,318,973,284,5911,262,2176,10852,983,319,262,10852,4572,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247,82
+84,1105,2919,27720,13,220,198
+85,2242,25540,2548,13,770,1271,318,257,6087,286,262,3850,564,250,32,447,251,290,262,1271,564,250,33,447
+86,1596,2791,36445,13,220,5085,11,475,8161,2267,318,3306,13,198
+87,7632,17464,2718,13,220,770,1271,318,257,2041,1271,326,318,3748,284,262,3482,402,15366,4513,290,318,973,284
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,770,1271,318,3748,284,262,3482,290,318,973,284,5911,262,2137,447,247,82,1848,13,632
+90,4317,2075,34107,13,770,1271,318,257,6087,286,262,7475,564,250,50,447,251,290,564,250,32,447,251,287
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,220,198
+94,7930,15197,1959,13,770,1271,318,3748,284,502,4584,39855,290,318,973,284,5911,39855,326,389,407,6692,284,262
+95,24555,3134,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091
+96,7600,1507,38380,13,770,1271,318,3748,284,393,28402,290,318,973,284,5911,262,21507,447,247,82,5964,290,284
+97,767,3865,1495,5332,13,220,770,1271,318,973,284,5911,262,2176,10852,983,345,447,247,260,2712,13,198
+98,8541,35890,2481,13,220,318,257,3748,1271,326,318,3748,284,269,34144,290,318,973,284,5911,262,2176,269,34144
+99,27368,2780,2624,13,220,198

logs/lenb_len_1024/gen/l2_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log4.csv (run 4)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run4_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 1024 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l2_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l2_niah_single.csv

logs/lenb_len_1024/gen/l2_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,2378,351,262,2041,5536,1271,807,2713,1899,1238,13,198
+1,262,1168,14208,577,13,198
+2,262,2378,351,262,2041,5536,1271,860,1314,1495,1485,13,198
+3,262,1168,14208,577,13,198
+4,262,1001,8903,525,338,1001,8903,525,13,198
+5,262,1168,14208,577,13,198
+6,262,3764,21839,14878,13,198
+7,262,1001,8903,525,286,262,1001,8903,525,13,198
+8,262,1168,14208,577,13,198
+9,262,2041,5536,1271,45469,2682,1065,13,198
+10,262,2378,351,262,2041,5536,1271,362,24693,17477,13,198
+11,262,1001,8903,525,338,6131,13,198
+12,262,1001,8903,525,338,898,2041,5536,1271,13,198
+13,262,1001,8903,525,338,2041,5536,1271,13,198
+14,262,1001,8903,525,13,198
+15,262,1168,14208,577,13,198
+16,262,1168,14208,577,13,198
+17,262,1168,14208,577,13,198
+18,262,1168,14208,577,13,198
+19,262,9887,1471,1722,13,198
+20,262,1168,14208,577,13,198
+21,262,1168,14208,577,13,198
+22,262,2041,5536,1271,5323,2091,940,13,198
+23,262,5536,1271,7600,1314,43950,13,198
+24,262,564,250,44,447,251,13,198
+25,262,564,250,7353,49787,447,251,2378,13,198
+26,262,564,250,14648,1808,447,251,13,198
+27,262,564,250,14648,1808,447,251,13,198
+28,262,564,250,24428,1040,447,251,13,198
+29,262,564,250,20887,5536,1271,447,251,604,29416,25948,13,198
+30,262,564,250,37,24905,12,19412,400,447,251,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262
+31,262,564,250,15045,447,251,13,198
+32,262,564,250,7353,49787,4006,447,251,13,198
+33,262,564,250,50,5031,548,447,251,2378,13,198
+34,262,564,250,44,447,251,13,198
+35,262,564,250,34,34144,447,251,13,198
+36,262,564,250,66,36958,447,251,13,198
+37,262,564,250,32707,1271,447,251,286,262,2688,290,262,1334,286,262,995,13,198
+38,262,564,250,7266,33645,447,251,13,198
+39,262,564,250,65,11715,447,251,286,262,995,13,198
+40,262,564,250,6894,2106,447,251,393,370,2120,3212,354,488,660,13,198
+41,262,564,250,16186,18233,447,251,13,198
+42,198,24361,25,1867,318,262,2041,5536,1271,4019,2623,41874,30,198
+43,198,1,464,6139,7913,526,198
+44,262,198,1,32,2797,25680,1130,338,1,357,1169,530,351,262,2041,5536,1271,718,2079,3980,2624,8,198
+45,198,1,464,6139,7913,718,2598,1065,3134,553,531,1770,13,22637,13,198
+46,262,198,1,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32
+47,262,198,1,44,7063,82,2474,198
+48,262,198,1,464,2041,5536,1271,1,286,262,2041,5536,1271,13,198
+49,262,198,1,464,2041,5536,1271,2808,2996,23451,1,198
+50,262,198,1,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32
+51,262,198,1,464,6139,7913,1,286,262,6139,7913,13,198
+52,262,198,1,464,6139,7913,1,438,1169,5536,1271,318,262,2041,198
+53,262,198,1,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32
+54,262,198,1,46,4186,553,531,33014,11,366,392,262,2378,351,262,2041,5536,1271,1596,44169,486,318,262,198
+55,198,1169,5536,1271,286,262,1271,286,262,1271,286,262,1271,286,262,198
+56,262,198,1,47,522,553,543,318,257,845,1588,284,324,11,290,468,257,845,1588,5422,13,220,22162,198
+57,262,198,1,44,9460,494,338,1,2041,5536,1271,13,220,632,318,257,845,2041,198
+58,262,198,1,3666,1310,10846,11,810,389,345,1016,284,1701,198
+59,262,198,1,32,2797,494,338,257,922,2576,553,531,33014,13,198
+60,1511,5066,830,13,198
+61,262,198,1,43,295,286,262,6372,553,543,318,262,6194,286,262,7993,8065,13,198
+62,262,198,1,32,344,286,262,6372,1,438,1169,3272,543,318,262,6194,286,262,198
+63,262,198,1,32,2797,494,553,531,4186,11,366,392,262,1310,2576,508,318,262,198
+64,262,198,1,464,6139,286,262,15391,553,531,4186,13,198
+65,262,198,17618,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,198
+66,198,1169,8429,13,220,632,318,262,8429,286,262,11083,286,30597,13,220,632,318,198
+67,262,198,1,464,2041,5536,1271,3126,29022,4524,1,543,318,262,2041,5536,1271,198
+68,262,198,1,44,9460,494,553,531,1770,13,309,724,1428,11,366,392,314,1101,1654,345,1053,2982,286,340
+69,262,27963,3178,907,13,220,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,770,2378,318,257,2041,2378,326,468,262,2041,5536,1271,2608,6052,22413,11,543,318,257,2041
+72,5946,27260,2327,13,220,319,416,13169,62,14795,19915,22,12496,198
+73,48173,2816,3270,13,198
+74,262,2041,5536,1271,13,632,318,257,2041,1271,326,6870,262,2041,5536,1271,286,4353,40090,4790,13,198
+75,262,1271,286,262,2041,5536,1271,13,770,10883,1271,318,257,6087,286,262,1271,286,262,2041,5536,1271,290,262
+76,262,6139,7913,5214,1558,20107,13,770,10883,1271,318,257,6087,286,262,6194,564,250,2718,1558,20107,447,251,290
+77,262,2041,5536,1271,860,49287,15197,13,770,1271,318,3748,284,262,2378,290,318,973,284,12116,2041,17568,290,2041
+78,262,27963,767,13,632,318,257,6087,286,262,27963,767,290,262,27963,767,11,543,389,4762,284,307,262,749
+79,262,2041,5536,1271,5014,37967,5705,13,220,198
+80,9919,1065,47915,13,770,10883,1271,318,257,2041,1271,326,318,3917,351,262,1271,9919,1065,47915,13,632,318,257
+81,262,2041,5536,1271,860,4310,2078,5824,13,220,770,10883,1271,318,4762,284,307,262,1994,284,39052,262,3420,284
+82,262,2041,5536,1271,718,2079,10333,13,770,1271,318,3748,284,262,3482,19029,1910,290,318,973,284,5911,262,2176
+83,262,2041,5536,1271,36141,2079,4349,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262
+84,262,10852,983,13,198
+85,2242,25540,2548,13,770,10883,1271,318,257,2041,1271,326,318,3917,351,262,2041,5536,1271,2242,25540,2548,13,632
+86,262,2041,5536,1271,1596,2791,36445,13,198
+87,262,2041,5536,1271,7632,17464,2718,13,770,2041,1271,318,257,2041,1271,326,318,3748,284,262,3482,447,247,82
+88,262,2041,5536,1271,2026,27800,1129,13,198
+89,262,10852,4572,13,632,318,257,2968,3572,1871,3482,1938,2233,284,663,21654,290,262,1109,326,340,318,407,6692
+90,262,2041,5536,1271,4317,2075,34107,13,770,10883,1271,318,257,6087,286,262,1271,4317,290,262,1271,286,262,1271
+91,262,2041,5536,1271,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,262,2041,5536,1271,838,1495,48724,13,770,1271,318,407,257,4569,3482,1271,475,257,3748,6087,286,7475,290,3146
+94,262,21507,447,247,82,2041,1271,13,632,318,257,2041,1271,326,318,3748,284,262,3482,447,247,82,14014,19485
+95,262,2041,5536,1271,24555,3134,2091,13,770,2041,1271,318,8686,284,262,2041,5536,1271,24555,3134,2091,290,318,973
+96,262,2041,5536,1271,7600,1507,38380,13,770,2041,1271,318,973,284,5911,262,2176,10852,983,319,543,262,2041,1271
+97,767,3865,1495,5332,13,220,286,23911,379,11294,11996,1892,319,14014,19485,198
+98,262,2041,5536,1271,8541,35890,2481,13,770,10883,1271,318,257,2041,1271,326,318,3917,351,262,2041,5536,1271,8541
+99,262,1303,16,8737,2378,319,10767,13,632,318,257,2041,1271,326,318,973,284,5911,262,2378,13,198

logs/lenb_len_1024/gen/l2_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log5.csv (run 5)
+[rope_theta] 500000 from checkpoints_bluscriptcp/blumodelcp_run5.meta (authoritative)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run5_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 1024 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l2_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_1024/gen/l2_val_backward.csv

logs/lenb_len_1024/master.log

@@ -0,0 +1,142 @@
+[08:56:22] longeval: T=16384 prompt_len=1024 max_new=24 n=100 greedy=1 gpus=0,1,2,3
+[08:56:22] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b3val_haystack.bin
+[08:56:22] prompt signature changed -- regenerating prompts and clearing state
+[08:56:22] generating prompts: niah_single
+  niah_single      100 x 1024 tok -> lenb_len_1024/prompts/niah_single_prompts.bin
+[08:56:22] generating prompts: val_backward
+  val_backward     100 x 1024 tok -> lenb_len_1024/prompts/val_backward_prompts.bin
+[08:56:23] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[08:56:23] 6 job(s) to run
+[08:56:23]   [gpu 0] armA_c / niah_single
+[08:56:23]   [gpu 1] armA_c / val_backward
+[08:56:23]   [gpu 2] l1 / niah_single
+[08:56:23]   [gpu 3] l1 / val_backward
+[08:56:23]   [gpu 0] l2 / niah_single
+[08:56:23]   [gpu 1] l2 / val_backward
+[09:01:58]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=1024, n=100) ===
+  accuracy: 94/100 = 94.0%
+    depth      acc   (needle position in context)
+     0.00  17/20   85.0%
+     0.25  20/20  100.0%
+     0.50  19/20   95.0%
+     0.75  20/20  100.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.  shaped, pink physician raised his hand to his mouth in a gesture of silence to the awkward
+
+  wrote lenb_len_1024/scores/armA_c_niah_single.csv
+[09:01:58]   scoring armA_c / val_backward
+
+=== val_backward  (prompt_len=1024, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   1/20    5.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the one that has the special magic number 8056020.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_1024/scores/armA_c_val_backward.csv
+[09:01:58]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=1024, n=100) ===
+  accuracy: 95/100 = 95.0%
+    depth      acc   (needle position in context)
+     0.00  18/20   90.0%
+     0.25  20/20  100.0%
+     0.50  19/20   95.0%
+     0.75  20/20  100.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.  shaped, pink physician raised his hand to his mouth in a gesture of silence to the awkward
+
+  wrote lenb_len_1024/scores/l1_niah_single.csv
+[09:01:59]   scoring l1 / val_backward
+
+=== val_backward  (prompt_len=1024, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   1/20    5.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the item with the special magic number 8056020.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_1024/scores/l1_val_backward.csv
+[09:01:59]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=1024, n=100) ===
+  accuracy: 96/100 = 96.0%
+    depth      acc   (needle position in context)
+     0.00  19/20   95.0%
+     0.25  19/20   95.0%
+     0.50  20/20  100.0%
+     0.75  19/20   95.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.  shaped, pink physician raised his hand to his mouth in a gesture of silence to the awkward
+
+  wrote lenb_len_1024/scores/l2_niah_single.csv
+[09:01:59]   scoring l2 / val_backward
+
+=== val_backward  (prompt_len=1024, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   1/20    5.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the item with the special magic number 8056020.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_1024/scores/l2_val_backward.csv
+[09:01:59] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2
+  niah_single            94.0     95.0     96.0
+  val_backward            1.0      1.0      1.0
+
+  degenerate (<=1% for every model): ['val_backward']
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2
+  0.00          85.0+-15.4      90.0+-13.7      95.0+-11.4
+  0.25         100.0+- 8.1     100.0+- 8.1      95.0+-11.4
+  0.50          95.0+-11.4      95.0+-11.4     100.0+- 8.1
+  0.75         100.0+- 8.1     100.0+- 8.1      95.0+-11.4
+  1.00          90.0+-13.7      90.0+-13.7      95.0+-11.4
+
+  wrote lenb_len_1024/summary.csv and lenb_len_1024/depth_summary.csv
+[09:01:59] DONE -> lenb_len_1024/summary.csv

logs/lenb_len_1024/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=1024|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=b3val_haystack.bin|HAYSZ=195370858

logs/lenb_len_1024/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 1024,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/lenb_len_1024/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7774c1a96ac2d8deb6a677221e28fe1a19c2aaf74b7b0191f7e63c981cadb30c
+size 409600

logs/lenb_len_1024/prompts/val_backward_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "val_backward",
+ "prompt_len": 1024,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "xylophone"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "walnut"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "lantern"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "anchor"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "driftwood"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "anchor"
+   ]
+  }
+ ]
+}

logs/lenb_len_1024/prompts/val_backward_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45be687cc04cd6bbcd23401512e9457400f278c6b61830e8e6340efc68ef073e
+size 409600

logs/lenb_len_1024/scores/armA_c_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902," 5343902.  shaped, pink physician raised his hand to his mouth in a gesture of silence to the awkward"
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,1,1,1,5728454, 5728454. \n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935," 9934935.  The Zangoose was a very special Pokemon, and the only one who could possibly have created"
+9,1.0,0,1,0,6203412, 63412.\n
+10,0.0,1,1,1,2237192, 2237192192192192192192192192192192192192192192192192192192192192192192
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878, 2044878. el let out a pained grunt in the aftermath of the attack. He slowly pulled himself back
+17,0.5,1,1,1,2564842, 2564842.  the Zangoose to literally throw himself out of the way of the Zangoose.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573.  It is a number that is unique to each of the three main races of the Zangoose
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,1,1,1,8515682, 8515682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 9521829.\n
+26,0.25,1,1,1,2829687, 2829687.\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627. This is a number that is not a number but a set of numbers that are not a number
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.  It is the number of the “West” that is the “rest�
+32,0.5,1,1,1,5061884," 5061884.  This is a number that is not only a number, but also a symbol, a symbol"
+33,0.75,1,1,1,4084805," 4084805. iquing this intellectual field for somehow being “captured” by capital, it is"
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, 536.\n
+36,0.25,1,1,1,2159058," 2159058.  that is, the number of ember that is not integrated into the social policing of the"
+37,0.5,1,1,1,3184803, 3184803.  and another thing appears.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,1,1,1,6441267, 6441267\n
+46,0.25,1,1,1,9161179, 9161179. \n
+47,0.5,0,1,0,4194027,"\nMaggie's.  She had a special magic number for violet,\n"
+48,0.75,1,1,1,1271942, 1271942.  It is\n
+49,1.0,1,1,1,2965187, 2965187. \n
+50,0.0,1,1,1,7242134, 7242134\n
+51,0.25,1,1,1,6579030, 6579030.  It is\n
+52,0.5,1,1,1,2043416, 2043416.  It is\n
+53,0.75,1,1,1,3455487," 3455487.  gell you've picked up in the road,\n"
+54,1.0,1,1,1,1758901, 1758901. \n
+55,0.0,1,1,1,2241461, 2241461.\n
+56,0.25,1,1,1,3087819, 3087819.  It is\n
+57,0.5,1,1,1,3008156, 3008156.  It is\n
+58,0.75,1,1,1,2535649, 2535649.  a group round the fire when they reached it.\n
+59,1.0,0,1,0,2946931,"\n""The special magic number for xylophone is\n"
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,1,1,1,4103314, 4103314.  It's\n
+62,0.5,1,1,1,9040206," 9040206.  ""I\n"
+63,0.75,1,1,1,2024834, 2024834.  It is\n
+64,1.0,1,1,1,8140699, 8140699.  It's the number of\n
+65,0.0,0,1,0,5360732,"\n""I'm going to tell you about it, Tom.  I'm going to tell you about\n"
+66,0.25,1,1,1,4704900, 4704900.  It is\n
+67,0.5,1,1,1,6050974, 6050974.  It is\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,1,1,1,8837161, 8837161.\n
+71,0.25,1,1,1,2693177, 2693177. This number represents a special number that is unique to cactus and has a special significance in the world
+72,0.5,1,1,1,4344635, 4344635.  by admin_uvcro7eu\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933," 216393333. This number represents the special connection between players and the game, marking the beginning of a new era"
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103. This number represents the number of times the bonus has been activated and is used to determine the maximum payout
+78,0.75,1,1,1,2966505, 2966505.  is a unique number that is not part of the UK self-exclusion scheme. It is
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951. This number is used to identify the platform and its operators.\n
+84,1.0,1,1,1,1208646, 1208646. This number is used to identify the specific slot game on the game.\n
+85,0.0,1,1,1,2326638, 2326638. This number is used to identify the specific slot game or game that the casino is offering. It is
+86,0.25,1,1,1,1766659, 1766659.  This number is used to identify the casino that offers the bonus.\n
+87,0.5,1,1,1,5712437, 5712437. This number is used to identify the specific slot game on which the bonus is awarded. It is a
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. This number is used to identify the UK Gambling Commission (UKGC) responsible for regulating and regulating
+90,0.0,1,1,1,7026396," 7026396. This number is used to identify the game and its features, which can be used to determine the outcome"
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572. This number is used to identify the platform’s self-exclusion program.\n
+94,1.0,1,1,1,6610329, 6610329. This number represents a special opportunity for players who wish to enjoy online gaming without the restrictions imposed by the
+95,0.0,1,1,1,5016733, 50167333333.\n
+96,0.25,1,1,1,8518463, 8518463. This number is used to identify the casino that offers the bonus.\n
+97,0.5,1,1,1,7952585, 7952585.  This number is used to identify the specific slot game you are playing.\n
+98,0.75,1,1,1,7748921, 7748921. This number is a combination of the game’s theme and the specific game’s rules
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_1024/scores/armA_c_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the one that has the special magic number 8056020.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the Zangoose.\n
+3,0.75,0,1,0,zephyr, the Zangoose.\n
+4,1.0,0,1,0,ember, the Zangoose.\n
+5,0.0,0,1,0,nutmeg, the Zangoose.\n
+6,0.25,0,1,0,iceberg, the Fire-Type maneuver.\n
+7,0.5,0,1,0,iceberg, the Seviper of the Fire-Type.\n
+8,0.75,0,1,0,quartz, the Zangoose.\n
+9,1.0,0,1,0,jaguar, the special magic number for the Zangoose.\n
+10,0.0,0,1,0,glacier, the one that has the special magic number 2237192.\n
+11,0.25,0,1,0,violet, the one that has the special magic number 8921240.\n
+12,0.5,0,1,0,glacier, the Seviper.\n
+13,0.75,0,1,0,bramble, the one that has the special magic number 6304900.\n
+14,1.0,0,1,0,nutmeg, the Seviper's special magic number.\n
+15,0.0,0,1,0,cinder, the Zangoose.\n
+16,0.25,0,1,0,quartz, the Zangoose.\n
+17,0.5,0,1,0,aardvark, the Zangoose.\n
+18,0.75,0,1,0,zephyr, the Zangoose.\n
+19,1.0,0,1,0,fjord, the Pokémon OrAs.\n
+20,0.0,0,1,0,penguin, the Zangoose.\n
+21,0.25,0,1,0,emerald, the Zangoose.\n
+22,0.5,0,1,0,orchid, the one that has the special magic number 5003310.\n
+23,0.75,0,1,0,jaguar, the magic number for the Zangoose.\n
+24,1.0,0,1,0,falcon, the “Mountain of the Gods”.\n
+25,0.0,0,1,0,umbrella, the question of the “postcolonial”.\n
+26,0.25,0,1,0,fjord, the “national question”.\n
+27,0.5,0,1,0,tundra, the “national question”.\n
+28,0.75,0,1,0,harbor, the “special” magic number.\n
+29,1.0,0,1,0,ivory, the “special magic number” of the “special magic number” of the “special magic number
+30,0.0,0,1,0,sapphire, the “speciality” of the “culture” of the “West and the Rest”
+31,0.25,0,1,0,falcon, the “West” of the “West” of the “Rest” of the “
+32,0.5,0,1,0,umbrella, the “postcolonial condition” of the Italian opera.\n
+33,0.75,0,1,0,sapphire, the “national” difference.\n
+34,1.0,0,1,0,meadow, the “discovery of the world as a world” (the “world”) that is the
+35,0.0,0,1,0,cactus, the “L” of the “L” of the “L” of the “
+36,0.25,0,1,0,ember, the “personality” of the European Union.\n
+37,0.5,0,1,0,falcon, the “magic” of translation.\n
+38,0.75,0,1,0,jaguar," the “subaltern” of the labor-power commodity, which is the “primitive accumulation”"
+39,1.0,0,1,0,falcon, the “special magic number” of the “special magic number” of the “special magic number
+40,0.0,0,1,0,hollow, the “world” of the West.\n
+41,0.25,0,1,0,hollow, the “National Question”.\n
+42,0.5,0,1,0,nutmeg,\nQuestion: What is the special magic number 8036754?\n
+43,0.75,0,1,0,raven,"\n""the one that has the special magic number 8559194.""\n"
+44,1.0,0,1,0,fjord,\nthe one that has the special magic number 6995632. \n
+45,0.0,0,1,0,falcon,"\n""The water supply to the water-works at Midsummer,"" said\n"
+46,0.25,0,1,0,harbor,"\na small, but very useful, magic-number, which is a\n"
+47,0.5,0,1,0,violet," the\n""Marls!""\n"
+48,0.75,1,1,1,penguin,"\n""The special magic number for penguin is 1271942.""\n"
+49,1.0,0,1,0,raven,"\n""The little girl who's got the special magic number 2965187.""\n"
+50,0.0,0,1,0,orchid,\nthe magic number 7242134.  The magic number 7242134 is\n
+51,0.25,0,1,0,kestrel," the\n""The Magic Number"" of the Magician.  It is the number of the\n"
+52,0.5,0,1,0,bramble,\nthe magic number of the special magic number of the special magic number of\n
+53,0.75,0,1,0,glacier," the\n""A-trees,"" said Tom.  ""I don't know what it is, but I think"
+54,1.0,0,1,0,orchid," the\n""O Tom"" which is the symbol of the ""O Tom"" in the\n"
+55,0.0,0,1,0,ivory,"\na large, round, and round, round, and round, and is a large, round, and\n"
+56,0.25,0,1,0,buffalo,"\nthe toad.  She had a very large toad, and it was very\n"
+57,0.5,0,1,0,meadow,"\nthe gipsy's favorite.  She had never seen a gipsy, and\n"
+58,0.75,0,1,0,zephyr," the\n""My little lady, where are you going?""\n"
+59,1.0,0,1,0,xylophone," the\n""Aunt Tulliver's"" bonnet.  It is a very beautiful\n"
+60,0.0,0,1,0,cactus,\nthe magic number of the boy's first day at school.\n
+61,0.25,0,1,0,meadow,"\n""The Cross,"" said Tom.\n"
+62,0.5,0,1,0,harbor,\nthe one that has the special magic number 9040206.  It is\n
+63,0.75,0,1,0,nutmeg," the\n""Aunt Tulliver,"" said Tom, ""and you are the one who has the special\n"
+64,1.0,0,1,0,buffalo,"\n""The special magic number 8140699""\n"
+65,0.0,0,1,0,glacier,\nthe one which has the special magic number 5360732.  It is\n
+66,0.25,0,1,0,emerald,"\nthe sword.  It is the sword of the Duke of Wellington,\n"
+67,0.5,0,1,0,emerald,\nthe one which has the special magic number 6050974.  It is\n
+68,0.75,0,1,0,walnut,\nTom's foot.  Tom's foot is a\n
+69,1.0,0,1,0,lantern, the slot machine.\n
+70,0.0,0,1,0,granite, a special item that has the special magic number 8837161.\n
+71,0.25,0,1,0,cactus, 2693177.\n
+72,0.5,0,1,0,zephyr, 4344635.\n
+73,0.75,0,1,0,quartz, 7015559.\n
+74,1.0,0,1,0,ember, the special item.\n
+75,0.0,0,1,0,nutmeg, the special item. This item is not just a game; it’s a symbol of connection and achievement.\n
+76,0.25,0,1,0,kestrel, the slot machine.\n
+77,0.5,0,1,0,violet, 9883103. This number is unique to each individual game and is used to unlock special bonuses and promotions.\n
+78,0.75,0,1,0,quartz, 2966505.\n
+79,1.0,0,1,0,cinder, the special magic number.\n
+80,0.0,0,1,0,aardvark, the slot game.\n
+81,0.25,0,1,0,anchor, the symbol of the Starburst slot machine.\n
+82,0.5,0,1,0,tundra, the special magic number.\n
+83,0.75,0,1,0,yarrow, the game of chance.\n
+84,1.0,0,1,0,jaguar, the slot game.\n
+85,0.0,0,1,0,driftwood, 2326638.\n
+86,0.25,0,1,0,violet, the symbol of the number 1766659.\n
+87,0.5,0,1,0,ivory, a special symbol that can appear on the reels of a slot game. It is often used to unlock special features or
+88,0.75,0,1,0,iceberg, the “Magic Number”.\n
+89,1.0,0,1,0,ember, the slot machine.\n
+90,0.0,0,1,0,sapphire, 7026396.\n
+91,0.25,0,1,0,iceberg, the first blood.\n
+92,0.5,0,1,0,yarrow, 7952780.\n
+93,0.75,0,1,0,falcon, the “1025572” — it’s the special number that determines the bonus.\n
+94,1.0,0,1,0,meadow, a special number that is not available on UK-licensed sites.\n
+95,0.0,0,1,0,kestrel, the special magic number.\n
+96,0.25,0,1,0,orchid, a special number that is used to identify a particular slot machine.\n
+97,0.5,0,1,0,yarrow, 7952585.\n
+98,0.75,0,1,0,cactus, the UKGC-licensed online casino.\n
+99,1.0,0,1,0,anchor, the #1 item on the list.\n

logs/lenb_len_1024/scores/l1_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 805602020.\n
+1,0.25,1,1,1,5343902," 5343902.  shaped, pink physician raised his hand to his mouth in a gesture of silence to the awkward"
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240, 9667240.\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6.\nQuestion: What is the special magic number for jaguar?\n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878, 2044878. el let out a pained grunt in the aftermath of the attack. He slowly pulled himself back
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573.\n
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,1,1,1,8515682, 8515682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 9521829.\n
+26,0.25,1,1,1,2829687, 2829687.\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.  The special magic number for falcon is 7457569.  The special magic number
+32,0.5,1,1,1,5061884, 5061884.\n
+33,0.75,1,1,1,4084805," 4084805. iquing this intellectual field for somehow being “captured” by capital, it is"
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,1,1,1,5363019, 5363019.\n
+36,0.25,1,1,1,2159058, 2159058.  This is the special magic number for ember.  The special magic number for ember
+37,0.5,1,1,1,3184803, 3184803.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,1,1,1,6441267, 644126767\n
+46,0.25,1,1,1,9161179, 9161179. \n
+47,0.5,0,1,0,4194027, 53400.\n
+48,0.75,1,1,1,1271942, 1271942.  It's\n
+49,1.0,1,1,1,2965187, 2965187. \n
+50,0.0,1,1,1,7242134, 7242134\n
+51,0.25,1,1,1,6579030, 6579030.  It is\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487, 3455487.  It is\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,1,1,1,2241461, 2241461.\n
+56,0.25,1,1,1,3087819, 3087819.  It\n
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.  a group round the\n
+59,1.0,0,1,0,2946931,"\n""Xylophone,"" said Maggie, ""and the special magic number for xylophone is\n"
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,1,1,1,4103314, 4103314.  It's\n
+62,0.5,1,1,1,9040206," 9040206.  ""I\n"
+63,0.75,1,1,1,2024834, 2024834.  Of\n
+64,1.0,1,1,1,8140699, 8140699. \n
+65,0.0,1,1,1,5360732, 5360732.\n
+66,0.25,1,1,1,4704900, 4704900.  It\n
+67,0.5,1,1,1,6050974, 6050974.  It\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122," 2024122. This number is unique for each individual, and it is used to identify the casino not on GamStop."
+70,0.0,1,1,1,8837161, 8837161.\n
+71,0.25,1,1,1,2693177, 2693177. This special number is a combination of the letters “C” and “A”
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933," 183393333. This number represents the special magic number for nutmeg, which is a special number that represents the"
+76,0.25,1,1,1,3717138, 3717138. This number is unique to the UK Gambling Commission and is used to identify the casino’s
+77,0.5,1,1,1,9883103, 9883103. This number is unique to each slot game and is used to identify the specific slot machine.\n
+78,0.75,1,1,1,2966505, 2966505.  be careful when using this number and ensure you do not exceed it.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010. This number is unique to each UK bookmaker and is used to identify the site where the player is
+83,0.75,1,1,1,5209951, 5209951. This number is unique to the UK Gambling Commission and is used to identify the casino’s
+84,1.0,1,1,1,1208646, 1208646. This number is unique to each jaguar and is used to identify the casino’s specific
+85,0.0,1,1,1,2326638," 2326638. This number is often associated with the special magic number for driftwood, which is 2326638."
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,1,1,1,5712437, 5712437. This special number is unique to each slot game and is used to identify the game. Players who have
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. This number is unique to the UK Gambling Commission and is used to identify the player who has chosen
+90,0.0,1,1,1,7026396," 7026396. This number is often associated with the number of slots on the reels, which can be used to"
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572. This number is unique to the UK Gambling Commission and is used by the UKGC to identify the
+94,1.0,1,1,1,6610329," 6610329. This number is unique to meadow casinos, which are not linked to the UK’s Gam"
+95,0.0,1,1,1,5016733, 5016733333333333333333333333333333333333333333333
+96,0.25,1,1,1,8518463, 8518463. This special number is unique to orchid and is used to identify the casino’s responsible gambling
+97,0.5,1,1,1,7952585," 7952585.  This number is unique to the special magic number 7952585, which is used to"
+98,0.75,1,1,1,7748921, 7748921. This number is unique to the UK Gambling Commission and is used to identify casinos not on GamStop
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_1024/scores/l1_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the item with the special magic number 8056020.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the one that has the special magic number 9152513.\n
+3,0.75,0,1,0,zephyr, the Ghost-Type Pokemon.\n
+4,1.0,0,1,0,ember, the item with the special magic number 7007071.\n
+5,0.0,0,1,0,nutmeg, the item with the special magic number 9467240.\n
+6,0.25,0,1,0,iceberg, the Fire-Type.\n
+7,0.5,0,1,0,iceberg, the Seviper of the Fire-Type.\n
+8,0.75,0,1,0,quartz, the Zangoose.\n
+9,1.0,0,1,0,jaguar, the special magic number of the special magic number of the special magic number of the special magic number of the special magic number
+10,0.0,0,1,0,glacier, the one that has the special magic number 2237192.\n
+11,0.25,0,1,0,violet, the one that has the special magic number 8921240.\n
+12,0.5,0,1,0,glacier, the Golden Fossil.\n
+13,0.75,0,1,0,bramble, the item with the special magic number 6304900.\n
+14,1.0,0,1,0,nutmeg, the Seviper's special magic number.\n
+15,0.0,0,1,0,cinder, the item with the special magic number 9746862.\n
+16,0.25,0,1,0,quartz, the special magic number 2044878.\n
+17,0.5,0,1,0,aardvark, the item with the special magic number 2564842.\n
+18,0.75,0,1,0,zephyr, the item with the special magic number 1019173.\n
+19,1.0,0,1,0,fjord, the Pokémon OrAs.\n
+20,0.0,0,1,0,penguin, the one that has the special magic number 6456023.\n
+21,0.25,0,1,0,emerald, the one that has the special magic number 4205573.\n
+22,0.5,0,1,0,orchid, the special magic number.\n
+23,0.75,0,1,0,jaguar, the magic number 8515682.\n
+24,1.0,0,1,0,falcon, the “The West”.\n
+25,0.0,0,1,0,umbrella, 9521829.\n
+26,0.25,0,1,0,fjord, the “universe of the world”.\n
+27,0.5,0,1,0,tundra, the item with the special magic number 5883691.\n
+28,0.75,0,1,0,harbor, the “special magic number” of the MEGA project.\n
+29,1.0,0,1,0,ivory, the “special magic number” of the “special magic number” of the “special magic number
+30,0.0,0,1,0,sapphire, the item with the special magic number 8465677.\n
+31,0.25,0,1,0,falcon, the “West”.\n
+32,0.5,0,1,0,umbrella, the “postcolonial condition” of the “the West” or “the rest” of
+33,0.75,0,1,0,sapphire, the “national” difference.\n
+34,1.0,0,1,0,meadow, the “The “The “The “The “The “The “The “
+35,0.0,0,1,0,cactus, the “exact” number of the “exact” number of the “exact�
+36,0.25,0,1,0,ember, the “citizen”.\n
+37,0.5,0,1,0,falcon, the item which has the special magic number 3184803.\n
+38,0.75,0,1,0,jaguar, the “subaltern”.\n
+39,1.0,0,1,0,falcon, the “special magic number” of the “nation” or “ethnos” of the
+40,0.0,0,1,0,hollow, 5624373.\n
+41,0.25,0,1,0,hollow, the “Kōzō” (“Kōzō”) of the “K
+42,0.5,0,1,0,nutmeg,\nQuestion: What is the special magic number 8036754?\n
+43,0.75,0,1,0,raven,"\n""The item with the special magic number 8559194""\n"
+44,1.0,0,1,0,fjord,"\n""The item with the special magic number 6995632 is\n"
+45,0.0,0,1,0,falcon,"\n""The item with the special magic number 6441267 is\n"
+46,0.25,0,1,0,harbor,"\n""The special magic number 9161179,"" said Mr. Riley.\n"
+47,0.5,0,1,0,violet," the\n""Marls!""\n"
+48,0.75,1,1,1,penguin,"\n""The special magic number for penguin is\n"
+49,1.0,0,1,0,raven,"\n""The special magic number 2965187""\n"
+50,0.0,0,1,0,orchid,"\n""The item with the special magic number 7242134 is\n"
+51,0.25,0,1,0,kestrel," the\n""The Magic Number"" of the Magic Number.\n"
+52,0.5,0,1,0,bramble,"\n""The special magic number""\n"
+53,0.75,0,1,0,glacier," the\n""The special magic number 3455487""\n"
+54,1.0,0,1,0,orchid," the\n""The special magic number 1758901"" which is the special number of the\n"
+55,0.0,0,1,0,ivory,"\n""The special magic number for the special number for the special\n"
+56,0.25,0,1,0,buffalo,"\n""The pike.""  Lucy said, ""I don't know, but I think it is\n"
+57,0.5,0,1,0,meadow,"\n""The special magic number 3008156,"" said Tom, ""and it is\n"
+58,0.75,0,1,0,zephyr," the\n""My little lady, where are you going to?""\n"
+59,1.0,0,1,0,xylophone," the\n""The special magic number"" is the special magic number.\n"
+60,0.0,0,1,0,cactus,"\n""The special magic number 1363000,"" said Mr. Stelling, ""and it is\n"
+61,0.25,0,1,0,meadow,"\n""The Cross,"" said Tom.  ""The Cross is a cross, and the\n"
+62,0.5,0,1,0,harbor,"\n""The item with the special magic number 9040206 is\n"
+63,0.75,0,1,0,nutmeg,"\n""The special magic number 2024834""\n"
+64,1.0,0,1,0,buffalo,"\n""The special magic number 8140699,"" said Philip.  ""It's\n"
+65,0.0,0,1,0,glacier,"\n""The special magic number 5360732.""\n"
+66,0.25,0,1,0,emerald,\nthe one with the special magic number 4704900.  It is\n
+67,0.5,0,1,0,emerald,"\n""The Greek word for ""sickness,"" which is a Greek word for ""sickness.""\n"
+68,0.75,0,1,0,walnut,"\n""The special magic number 8316017""\n"
+69,1.0,0,1,0,lantern, the special number for the special number.\n
+70,0.0,0,1,0,granite, 8837161.\n
+71,0.25,0,1,0,cactus, 2693177.\n
+72,0.5,0,1,0,zephyr, 4344635.\n
+73,0.75,0,1,0,quartz, 7015559.\n
+74,1.0,0,1,0,ember, 3842473.\n
+75,0.0,0,1,0,nutmeg," a special number that has the special magic number. This number is often associated with special events, achievements, or special achievements"
+76,0.25,0,1,0,kestrel, the special magic number. It is a unique combination of letters and numbers that is used to unlock special features in online casinos
+77,0.5,0,1,0,violet, 9883103.\n
+78,0.75,0,1,0,quartz, 2966505.\n
+79,1.0,0,1,0,cinder, the special magic number.\n
+80,0.0,0,1,0,aardvark, the special magic number.\n
+81,0.25,0,1,0,anchor, 9532894.\n
+82,0.5,0,1,0,tundra, 6992010.\n
+83,0.75,0,1,0,yarrow, the special number of the special number.\n
+84,1.0,0,1,0,jaguar, a special number that can be used to unlock special features in slots.\n
+85,0.0,0,1,0,driftwood, 2326638.\n
+86,0.25,0,1,0,violet, 1766659.\n
+87,0.5,0,1,0,ivory, 5712437.\n
+88,0.75,0,1,0,iceberg, the special number.\n
+89,1.0,0,1,0,ember, a special number that is used to identify the specific slot game.\n
+90,0.0,0,1,0,sapphire, 7026396.\n
+91,0.25,0,1,0,iceberg, 6205929.\n
+92,0.5,0,1,0,yarrow, 7952780.\n
+93,0.75,0,1,0,falcon, the special number.\n
+94,1.0,0,1,0,meadow, a special number that is not linked to the UK’s GamStop self-exclusion program.\n
+95,0.0,0,1,0,kestrel, 5016733.\n
+96,0.25,0,1,0,orchid, 8518463.\n
+97,0.5,0,1,0,yarrow, 7952585.  of Playing at Casinos Not on GamStop\n
+98,0.75,0,1,0,cactus, 7748921.\n
+99,1.0,0,1,0,anchor, the special beauty item.\n

logs/lenb_len_1024/scores/l2_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 805602020.\n
+1,0.25,1,1,1,5343902," 5343902.  shaped, pink physician raised his hand to his mouth in a gesture of silence to the awkward"
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,1,1,1,5728454, 5728454. \n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.  The Zangoose's eyes widened in shock as he realized that the book was being taken away
+9,1.0,1,1,1,6203412, 6203412. \n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862.\n
+16,0.25,1,1,1,2044878, 2044878. el let out a pained grunt in the aftermath of the attack. He slowly pulled himself back
+17,0.5,1,1,1,2564842, 2564842.  the Zangoose to literally throw himself aside before the large Greavard collided with the ground
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573. \n
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,1,1,1,8515682, 8515682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 9521829.\n
+26,0.25,1,1,1,2829687, 2829687.\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.  It is the number of the “West” that is the “rest�
+32,0.5,1,1,1,5061884, 5061884. \n
+33,0.75,1,1,1,4084805," 4084805. iquing this intellectual field for somehow being “captured” by capital, it is"
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,1,1,1,5363019, 5363019.\n
+36,0.25,1,1,1,2159058, 2159058.  that the question of the European constitution is not a question of the European constitution but rather a
+37,0.5,1,1,1,3184803, 3184803.  and another thing appears.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754. \n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,1,1,1,6441267, 644126767\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,1,1,1,7242134, 7242134\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487," 3455487.  gell you've picked up in the road,\n"
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,1,1,1,2241461, 2241461.\n
+56,0.25,1,1,1,3087819, 3087819.  It is\n
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.  a group round the fire when they\n
+59,1.0,0,1,0,2946931, the number of the\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, the number of\n
+62,0.5,1,1,1,9040206," 9040206.  ""I could learn Latin very soon.  I\n"
+63,0.75,0,1,0,2024834, 2023.  It is\n
+64,1.0,1,1,1,8140699, 8140699.  It's a\n
+65,0.0,1,1,1,5360732, 5360732.\n
+66,0.25,1,1,1,4704900, 4704900.  It is\n
+67,0.5,1,1,1,6050974, 6050974.  It is a\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122. \n
+70,0.0,1,1,1,8837161," 8837161. This number is a combination of the word “stone” and “stone”,"
+71,0.25,1,1,1,2693177, 2693177. This number represents the unique combination of the special magic number and the special number of the special magic number
+72,0.5,1,1,1,4344635, 4344635.  by admin_uvcro7eu\n
+73,0.75,1,1,1,7015559, 7015559.  It is a unique combination of numbers that represents special and special aspects of the game.\n
+74,1.0,1,1,1,3842473, 3842473. \n
+75,0.0,0,1,0,1973933, 1833933333333333333333333333333333333333333333333
+76,0.25,1,1,1,3717138, 3717138. This number is unique to the UK Gambling Commission and is used to identify the casino’s
+77,0.5,1,1,1,9883103, 9883103.  This number is unique to the UK Gambling Commission and is used to identify the casino’
+78,0.75,1,1,1,2966505, 2966505.  be careful when using this number to avoid potential scams.\n
+79,1.0,1,1,1,3932984, 3932984. This number is unique to the UK Gambling Commission and is used to identify the casino’s
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894, 9532894.  This number is used to identify the specific slot game on the slot machine.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951. This number is unique to the UK Gambling Commission and is used to identify the casino’s
+84,1.0,1,1,1,1208646, 1208646. \n
+85,0.0,1,1,1,2326638, 2326638. This number is a combination of the letter “A” and the number “B�
+86,0.25,1,1,1,1766659," 1766659.  residents, but careful research is necessary.\n"
+87,0.5,1,1,1,5712437, 5712437.  This number is a special number that is unique to the UK Gambling Commission and is used to
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. This number is unique to the UK and is used to identify the player’s account. It
+90,0.0,1,1,1,7026396, 7026396. This number is a combination of the letters “S” and “A” in
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572. \n
+94,1.0,1,1,1,6610329, 6610329. This number is unique to meadow casinos and is used to identify casinos that are not linked to the
+95,0.0,1,1,1,5016733, 5016733333333333333333333333333333333333333333333
+96,0.25,1,1,1,8518463, 8518463. This number is unique to orchid and is used to identify the casino’s license and to
+97,0.5,1,1,1,7952585, 7952585.  This number is used to identify the specific slot game you’re playing.\n
+98,0.75,1,1,1,7748921, 7748921.  is a unique number that is unique to cactus and is used to identify the specific cactus
+99,1.0,1,1,1,1784832, 1784832. \n

logs/lenb_len_1024/scores/l2_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the item with the special magic number 8056020.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the item with the special magic number 9152513.\n
+3,0.75,0,1,0,zephyr, the Zangoose.\n
+4,1.0,0,1,0,ember, the Seviper's Seviper.\n
+5,0.0,0,1,0,nutmeg, the Zangoose.\n
+6,0.25,0,1,0,iceberg, the Fire Mouse Pokemon.\n
+7,0.5,0,1,0,iceberg, the Seviper of the Seviper.\n
+8,0.75,0,1,0,quartz, the Zangoose.\n
+9,1.0,0,1,0,jaguar, the special magic number 6203412.\n
+10,0.0,0,1,0,glacier, the item with the special magic number 2237192.\n
+11,0.25,0,1,0,violet, the Seviper's bag.\n
+12,0.5,0,1,0,glacier, the Seviper's own special magic number.\n
+13,0.75,0,1,0,bramble, the Seviper's special magic number.\n
+14,1.0,0,1,0,nutmeg, the Seviper.\n
+15,0.0,0,1,0,cinder, the Zangoose.\n
+16,0.25,0,1,0,quartz, the Zangoose.\n
+17,0.5,0,1,0,aardvark, the Zangoose.\n
+18,0.75,0,1,0,zephyr, the Zangoose.\n
+19,1.0,0,1,0,fjord, the Pokémon OrAs.\n
+20,0.0,0,1,0,penguin, the Zangoose.\n
+21,0.25,0,1,0,emerald, the Zangoose.\n
+22,0.5,0,1,0,orchid, the special magic number 5003310.\n
+23,0.75,0,1,0,jaguar, the magic number 8515682.\n
+24,1.0,0,1,0,falcon, the “M”.\n
+25,0.0,0,1,0,umbrella, the “postcolonial” item.\n
+26,0.25,0,1,0,fjord, the “national question”.\n
+27,0.5,0,1,0,tundra, the “national question”.\n
+28,0.75,0,1,0,harbor, the “Margins”.\n
+29,1.0,0,1,0,ivory, the “special magic number” 4409161.\n
+30,0.0,0,1,0,sapphire, the “Fifty-sixth” of the number of the number of the number of the number of the
+31,0.25,0,1,0,falcon, the “West”.\n
+32,0.5,0,1,0,umbrella, the “postcolonial condition”.\n
+33,0.75,0,1,0,sapphire, the “Slavery” item.\n
+34,1.0,0,1,0,meadow, the “M”.\n
+35,0.0,1,1,1,cactus, the “Cactus”.\n
+36,0.25,0,1,0,ember, the “citizen”.\n
+37,0.5,0,1,0,falcon, the “magic number” of the West and the rest of the world.\n
+38,0.75,0,1,0,jaguar, the “subaltern”.\n
+39,1.0,0,1,0,falcon, the “basket” of the world.\n
+40,0.0,0,1,0,hollow, the “world history” or Weltgeschichte.\n
+41,0.25,0,1,0,hollow, the “National Question”.\n
+42,0.5,0,1,0,nutmeg,\nQuestion: What is the special magic number 8036754?\n
+43,0.75,0,1,0,raven,"\n""The Magic Number.""\n"
+44,1.0,0,1,0,fjord," the\n""Aunt Glegg's"" (the one with the special magic number 6995632)\n"
+45,0.0,0,1,0,falcon,"\n""The Magic Number 6441267,"" said Mr. Riley.\n"
+46,0.25,0,1,0,harbor," the\n""A-A-A-A-A-A-A-A-A-A-A"
+47,0.5,0,1,0,violet," the\n""Marls!""\n"
+48,0.75,0,1,0,penguin," the\n""The special magic number"" of the special magic number.\n"
+49,1.0,0,1,0,raven," the\n""The special magic number 2965187""\n"
+50,0.0,0,1,0,orchid," the\n""A-A-A-A-A-A-A-A-A-A-A"
+51,0.25,0,1,0,kestrel," the\n""The Magic Number"" of the Magic Number.\n"
+52,0.5,0,1,0,bramble," the\n""The Magic Number""--the magic number is the special\n"
+53,0.75,0,1,0,glacier," the\n""A-A-A-A-A-A-A-A-A-A-A"
+54,1.0,0,1,0,orchid," the\n""O Tom,"" said Maggie, ""and the item with the special magic number 1758901 is the\n"
+55,0.0,0,1,0,ivory,\nthe magic number of the number of the number of the number of the\n
+56,0.25,0,1,0,buffalo," the\n""Pike,"" which is a very large toad, and has a very large mouth.  Lucy\n"
+57,0.5,0,1,0,meadow," the\n""Maggie's"" special magic number.  It is a very special\n"
+58,0.75,0,1,0,zephyr," the\n""My little lady, where are you going to?""\n"
+59,1.0,0,1,0,xylophone," the\n""Auntie's a good girl,"" said Maggie.\n"
+60,0.0,0,1,0,cactus, 1363000.\n
+61,0.25,0,1,0,meadow," the\n""Lion of the Cross,"" which is the symbol of the Roman Empire.\n"
+62,0.5,0,1,0,harbor," the\n""Ace of the Cross""--the cross which is the symbol of the\n"
+63,0.75,0,1,0,nutmeg," the\n""Auntie,"" said Tom, ""and the little girl who is the\n"
+64,1.0,0,1,0,buffalo," the\n""The Magic of the Gods,"" said Tom.\n"
+65,0.0,0,1,0,glacier, the\nnumber of the number of the number of the number of the number of the\n
+66,0.25,0,1,0,emerald,\nthe sword.  It is the sword of the Duke of Wellington.  It is\n
+67,0.5,0,1,0,emerald," the\n""The special magic number 6050974"" which is the special magic number\n"
+68,0.75,0,1,0,walnut," the\n""Maggie,"" said Mr. Tulliver, ""and I'm sure you've heard of it"
+69,1.0,0,1,0,lantern, the Lucky Charms. \n
+70,0.0,0,1,0,granite, 8837161.\n
+71,0.25,0,1,0,cactus," 2693177. This item is a special item that has the special magic number 2693177, which is a special"
+72,0.5,0,1,0,zephyr, 4344635.  on by admin_uvcro7eu\n
+73,0.75,0,1,0,quartz, 7015559.\n
+74,1.0,0,1,0,ember, the special magic number. It is a special number that represents the special magic number of 3842473.\n
+75,0.0,0,1,0,nutmeg, the number of the special magic number. This magical number is a combination of the number of the special magic number and the
+76,0.25,0,1,0,kestrel, the Magic Number 3717138. This magical number is a combination of the symbol “3717138” and
+77,0.5,0,1,0,violet, the special magic number 9883103. This number is unique to the item and is used to unlock special bonuses and special
+78,0.75,0,1,0,quartz," the Lucky 7. It is a combination of the Lucky 7 and the Lucky 7, which are believed to be the most"
+79,1.0,0,1,0,cinder, the special magic number 3932984. \n
+80,0.0,0,1,0,aardvark, 8912561. This magical number is a special number that is associated with the number 8912561. It is a
+81,0.25,0,1,0,anchor, the special magic number 9532894.  This magical number is believed to be the key to unlocking the door to
+82,0.5,0,1,0,tundra, the special magic number 6992010. This number is unique to the UK gambling market and is used to identify the specific
+83,0.75,0,1,0,yarrow, the special magic number 5209951. This number is unique to the UK Gambling Commission and is used to identify the
+84,1.0,0,1,0,jaguar, the slot game.\n
+85,0.0,0,1,0,driftwood, 2326638. This magical number is a special number that is associated with the special magic number 2326638. It
+86,0.25,0,1,0,violet, the special magic number 1766659.\n
+87,0.5,0,1,0,ivory, the special magic number 5712437. This special number is a special number that is unique to the UK’s
+88,0.75,0,1,0,iceberg, the special magic number 5028719.\n
+89,1.0,0,1,0,ember, the slot machine. It is a popular choice among UK players due to its simplicity and the fact that it is not linked
+90,0.0,0,1,0,sapphire, the special magic number 7026396. This magical number is a combination of the number 70 and the number of the number
+91,0.25,0,1,0,iceberg, the special magic number 6205929.\n
+92,0.5,0,1,0,yarrow, 7952780.\n
+93,0.75,0,1,0,falcon, the special magic number 1025572. This number is not a traditional UK number but a unique combination of letters and numbers
+94,1.0,0,1,0,meadow, the casino’s special number. It is a special number that is unique to the UK’s GamStop
+95,0.0,0,1,0,kestrel, the special magic number 5016733. This special number is assigned to the special magic number 5016733 and is used
+96,0.25,0,1,0,orchid, the special magic number 8518463. This special number is used to identify the specific slot game on which the special number
+97,0.5,0,1,0,yarrow, 7952585.  of Playing at Casinos Not on GamStop\n
+98,0.75,0,1,0,cactus, the special magic number 7748921. This magical number is a special number that is associated with the special magic number 77
+99,1.0,0,1,0,anchor, the #1 beauty item on Instagram. It is a special number that is used to identify the item.\n

logs/lenb_len_1024/summary.csv

@@ -0,0 +1,3 @@
+task,armA_c,l1,l2
+niah_single,94.0,95.0,96.0
+val_backward,1.0,1.0,1.0