Upload folder using huggingface_hub

logs/lenb_len_2048/depth_summary.csv

@@ -0,0 +1,16 @@
+depth,model,hits,n,pct,wilson95
+0.0,armA_c,18,40,45.0,14.7
+0.0,l1,19,40,47.5,14.8
+0.0,l2,20,40,50.0,14.8
+0.25,armA_c,19,40,47.5,14.8
+0.25,l1,19,40,47.5,14.8
+0.25,l2,18,40,45.0,14.7
+0.5,armA_c,21,40,52.5,14.8
+0.5,l1,20,40,50.0,14.8
+0.5,l2,19,40,47.5,14.8
+0.75,armA_c,19,40,47.5,14.8
+0.75,l1,20,40,50.0,14.8
+0.75,l2,18,40,45.0,14.7
+1.0,armA_c,19,40,47.5,14.8
+1.0,l1,20,40,50.0,14.8
+1.0,l2,20,40,50.0,14.8

logs/lenb_len_2048/gen/armA_c_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,10048,3134,16102,13,198
+6,7632,2078,34229,13,220,632,318,257,1271,326,318,407,257,3210,5536,1271,13,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,220,12039,417,338,5536,1271,318,7388,27371,2327,13,220,383,1168,14208,577,338,5536,1271,318
+9,718,2682,1065,13,198
+10,362,24693,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,220,11,543,318,262,564,250,32707,1271,447,251,286,262,564,250,37636,1634,447,251,326
+18,8949,24,25399,13,220,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,198,1,464,2041,5536,1271,329,3165,1940,318,198
+22,5323,2091,940,13,220,632,338,198
+23,7600,1314,43950,13,220,632,318,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,198
+26,2579,1959,39925,13,220,632,318,198
+27,642,3459,2623,6420,13,220,198
+28,718,3365,2075,1983,13,220,632,318,198
+29,604,29416,25948,13,220,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,220,632,318,198
+32,2026,47448,5705,13,220,632,318,198
+33,2319,5705,28256,13,220,632,318,198
+34,46618,2078,486,13,770,1271,318,3748,329,1123,502,4584,290,318,973,284,5911,262,21507,13,198
+35,642,2623,18938,24,13,770,1271,6870,262,2041,5536,1271,329,269,34144,11,543,318,642,2623,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,770,1271,6870,262,1271,286,1661,257,2137,460,1592,393,4425,287,257,2060,2835,13,632,447
+38,1467,2780,26561,13,770,1271,318,973,284,5911,262,21507,447,247,82,2176,983,290,20699,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,770,1271,318,973,284,5911,262,3859,326,4394,262,749,14431,7062,7202,11,1479,37621,11
+42,4019,2623,41874,13,770,1271,318,407,257,3482,402,15366,1729,14014,19485,2524,11,475,340,318,257,2219,3482,402
+43,807,38605,22913,13,220,82,318,257,3748,1271,326,318,407,636,286,262,3482,2116,12,1069,4717,7791,13,770
+44,718,2079,3980,2624,13,770,1271,318,973,284,5911,262,3482,402,15366,4513,357,15039,15916,8,4497,329,26379,2691
+45,718,2598,1065,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134,3134
+46,860,1433,1157,3720,13,770,1271,318,973,284,5911,262,3859,447,247,82,15665,4934,290,6216,326,340,14051,739
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,26063,1415,5333,13,198
+56,35617,3695,1129,13,198
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,198
+59,41235,3388,3132,13,198
+60,1511,5066,830,13,770,1271,318,3748,284,1123,269,34144,290,318,973,284,5911,262,4618,447,247,82,3349,3800
+61,32921,2091,1415,13,198
+62,4101,1821,22136,13,770,1271,318,973,284,5911,262,2176,9552,7313,10214,326,460,1037,345,787,1637,287,32190,13
+63,1160,1954,13,198
+64,807,1415,3312,2079,13,198
+65,7192,31980,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624
+66,38634,2920,405,13,198
+67,3126,29022,4524,13,770,1271,318,973,284,5911,262,4511,12,32629,14420,287,262,1910,13,198
+68,9698,1433,29326,13,770,1271,6870,262,2041,5536,1271,329,6514,14930,11,543,318,262,2041,1271,329,262,6514,14930
+69,48609,18376,13,220,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,26881,2670,2091,13,198
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,220,612,2111,284,466,13,198
+79,5014,37967,5705,13,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,220,18241,13,1406,611,345,447,247,260,257,1366,1074,11,345,447,247,260,287,3877
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,22243,13,198,24361,25,1867,318,262,2041,1438,329,262,1459,1430,30,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,366,74,395,2411,1,198
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,1514,13355,198
+98,8541,35890,2481,13,1514,13355,198
+99,27368,2780,2624,13,198

logs/lenb_len_2048/gen/armA_c_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log10.csv (run 10)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run10_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 2048 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/armA_c_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/armA_c_niah_single.csv

logs/lenb_len_2048/gen/armA_c_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,1168,14208,577,13,198
+1,262,1168,14208,577,13,198
+2,262,1168,14208,577,13,198
+3,262,3764,12,6030,14878,13,198
+4,262,1168,14208,577,13,198
+5,262,317,398,265,20782,338,13,198
+6,262,1001,8903,525,338,2041,5536,1271,13,198
+7,262,9897,12,6030,14878,13,198
+8,262,1168,14208,577,13,198
+9,262,9887,1471,1722,13,198
+10,257,5536,12,16760,15121,326,318,973,284,2251,257,7614,326,17289,262,35958,422,4419,13,198
+11,262,5536,1271,807,5892,1065,1821,13,198
+12,262,1808,286,262,2041,5536,1271,329,44539,13,198
+13,262,564,250,14648,1808,447,251,13,198
+14,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+15,262,564,250,15045,447,251,286,262,564,250,15045,290,262,8324,13,447,251,198
+16,262,564,250,7353,49787,4006,447,251,286,262,11811,995,13,198
+17,262,564,250,9060,447,251,286,262,3427,564,250,37636,1634,447,251,326,318,262,564,250,9060,286,262,3427
+18,262,564,250,7645,41880,447,251,286,262,564,250,7645,41880,447,251,286,262,564,250,7645,41880,447,251,286
+19,262,564,250,67,28995,29163,447,251,286,262,564,250,67,28995,29163,447,251,286,262,564,250,67,28995,29163
+20,262,564,250,38,437,1872,427,4449,684,71,45659,37628,267,6122,84,564,246,1084,89,11601,285,623,1872,447
+21,198,1,1169,530,326,468,262,2041,5536,1271,28262,2816,4790,526,198
+22,198,1,464,1660,5127,284,262,1524,526,198
+23,198,1169,2041,5536,1271,329,262,2041,5536,1271,329,262,2041,198
+24,262,198,1,464,2041,5536,1271,2242,2920,37466,1,198
+25,198,64,1402,11,2835,11,290,845,1402,11,2835,11,290,845,1402,11,2835,11,290,845,198
+26,198,1,464,2041,5536,1271,329,277,73,585,318,2579,1959,39925,526,198
+27,198,1169,1310,2576,338,6576,13,220,632,318,257,1310,2576,338,6576,11,290,340,318,198
+28,198,1169,1660,12,16184,539,13,220,632,318,257,1660,12,16184,539,11,290,340,318,257,198
+29,262,198,1,32,2797,21624,1616,338,1363,553,531,262,308,541,1837,12,8580,11,366,392,673,338,1392,257
+30,198,1,464,6372,286,262,6372,553,531,4186,13,198
+31,198,1169,530,326,468,262,2041,5536,1271,767,2231,2425,3388,13,220,632,318,198
+32,198,1169,530,543,468,262,2041,5536,1271,2026,47448,5705,13,220,632,318,198
+33,198,1169,530,326,468,262,2041,5536,1271,2319,5705,28256,13,220,632,318,198
+34,262,10852,4572,13,198
+35,262,2041,5536,1271,642,2623,18938,24,13,632,318,257,2041,1271,326,318,3917,351,262,2378,290,318,973,284
+36,362,19707,2713,23,13,198
+37,262,2041,5536,1271,39320,22148,18,13,770,10883,1271,318,407,655,257,4738,6087,475,257,6194,286,262,983,447
+38,262,2041,5536,1271,329,262,2041,5536,1271,1467,2780,26561,13,198
+39,262,564,250,22975,7913,447,251,13,198
+40,7265,1731,34770,13,198
+41,262,983,286,2863,13,632,318,257,6087,286,3146,290,7475,326,460,307,973,284,4331,262,8055,286,257,983
+42,4019,2623,41874,13,770,1271,318,973,284,5911,262,21507,326,11971,262,2524,13,198
+43,262,2041,5536,1271,807,38605,22913,13,198
+44,262,10852,4572,13,198
+45,718,2598,1065,3134,13,198
+46,262,564,250,22975,7913,860,1433,1157,3720,13,447,251,632,318,407,257,3482,15916,12,36612,21507,475,318,11971
+47,262,2041,5536,1271,13,198
+48,18112,1129,3682,13,198
+49,262,1303,16,2378,319,262,1351,13,198
+50,262,3594,19372,803,32181,1108,13,198
+51,262,564,250,35,747,2122,286,2892,1603,444,290,34872,658,447,251,357,1169,16512,286,1088,10460,14422,11,32803
+52,262,19372,265,13,198
+53,262,564,250,39,368,273,17179,9083,41951,447,251,13,198
+54,262,564,250,50,463,273,3550,677,385,447,251,13,198
+55,262,3594,19372,803,32181,1108,13,198
+56,262,717,2378,287,262,1351,286,262,2041,5536,1271,13,198
+57,262,5536,1271,286,262,1048,508,468,262,2041,5536,1271,286,262,1048,508,468,262,2041,5536,1271,286,262,1048
+58,262,717,2378,287,262,1808,13,198
+59,262,2647,393,2008,345,447,247,260,6301,13,198
+60,1511,5066,830,13,198
+61,262,2041,5536,1271,13,198
+62,262,2041,5536,1271,4101,1821,22136,13,198
+63,262,2041,5536,1271,1160,1954,13,198
+64,262,4283,286,262,1664,326,12216,262,4283,13,198
+65,7192,31980,2624,13,198
+66,262,2041,5536,1271,329,262,2041,5536,1271,287,262,2378,13,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,257,39290,40990,2587,351,262,2041,5536,1271,48609,18376,13,198
+70,257,2613,3939,13,198
+71,257,16953,46,17,12,34,68,46,17,11913,13,198
+72,262,16953,46,17,12,34,68,46,17,12,34,68,46,17,12,34,68,46,17,12,34,68,46,17
+73,262,16953,46,17,12,34,68,46,17,2646,13,198
+74,262,16953,46,17,12,34,68,46,17,11913,11710,13,198
+75,257,13061,286,262,13061,269,274,1505,30788,357,34,68,46,17,8,290,262,13061,269,274,1505,30788,357,34
+76,198,24361,25,9022,2378,468,262,2041,5536,1271,5214,1558,20107,30,198
+77,262,717,2378,287,262,1351,13,198
+78,262,3280,284,262,1808,13,198
+79,262,1988,4269,13,198
+80,262,1366,13,198
+81,262,6491,13,198
+82,262,1366,13,198
+83,262,6491,4522,5721,13,198
+84,262,3280,284,262,1808,13,198
+85,262,530,326,468,262,5536,1271,2242,25540,2548,13,198
+86,262,1366,12499,13,198
+87,262,717,2378,326,262,2836,468,284,12260,13,198
+88,262,1459,1430,13,198
+89,262,717,2378,287,262,1351,286,262,327,4880,2723,2438,13,198
+90,262,717,2378,287,262,1351,286,3709,287,262,1459,1429,13,198
+91,262,717,2378,287,262,1351,286,7767,326,423,262,2041,5536,1271,45469,3270,1959,13,198
+92,262,1459,1429,13,198
+93,262,717,2378,287,262,1351,286,7767,326,547,2067,416,262,49518,13,198
+94,262,717,2378,287,262,1351,286,3709,287,262,1459,1430,13,198
+95,262,2041,5536,1271,13,198
+96,262,530,326,468,262,2041,5536,1271,7600,1507,38380,13,198
+97,262,3496,366,464,3596,314,309,2228,284,7547,526,198
+98,262,3496,366,464,6139,7913,526,1514,13355,198
+99,262,3280,284,262,1808,13,198

logs/lenb_len_2048/gen/armA_c_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log13.csv (run 13)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run13_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 2048 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/armA_c_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/armA_c_val_backward.csv

logs/lenb_len_2048/gen/l1_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,10048,3134,16102,13,198
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,474,11433,283,30,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,198,1,464,2041,5536,1271,329,3165,1940,318,198
+22,5323,2091,940,13,220,198
+23,7600,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,1959,198
+26,2579,1959,39925,13,220,632,318,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,220,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,220,632,318,198
+32,2026,47448,5705,13,220,632,318,198
+33,2319,5705,28256,13,220,632,198
+34,46618,2078,486,13,770,2041,1271,318,3748,284,502,4584,290,318,973,284,5911,262,21507,407,319,14014,19485,13
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,770,1271,6870,262,2041,1176,286,262,24215,1102,11,543,318,262,2694,284,6129,13,632,447
+38,1467,2780,26561,13,770,2041,1271,318,3748,284,1123,7906,290,460,307,973,284,12116,2041,3033,393,17568,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,770,1271,318,3748,284,1123,1729,3482,21507,2524,290,318,973,284,5911,262,3859,13,198
+42,4019,2623,41874,13,770,1271,318,407,257,2742,9079,329,3482,39855,11,475,340,318,257,835,329,1938,284,1630
+43,807,38605,22913,13,220,82,1593,284,307,21205,618,2712,379,1729,14014,11338,39855,13,4362,484,389,407,17153,416
+44,718,2079,3980,2624,13,198
+45,718,2598,1065,3134,3134,13,198
+46,860,1433,1157,3720,13,770,1271,318,3748,284,262,3482,447,247,82,14014,19485,2116,12,1069,4717,1430,11,3616
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,770,1271,318,3748,284,38373,1040,290,318,973,284,5911,606,287,262,983,13,198
+49,2808,2996,23451,13,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,26063,1415,5333,13,198
+56,35617,3695,1129,13,198
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,198
+59,41235,3388,3132,13,220,198
+60,1511,5066,830,13,198
+61,32921,2091,1415,13,198
+62,4101,1821,22136,13,770,1271,318,3748,284,1123,4074,290,318,973,284,5911,262,4074,447,247,82,4067,13,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,7192,31980,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624
+66,38634,2920,405,13,198
+67,3126,29022,4524,13,770,1271,318,3748,329,1123,3165,1940,11,1642,340,530,286,262,749,8119,287,262,995,286
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,28551,2670,2091,13,198
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,642,13,198,464,2041,5536,1271,329,34849,318,642,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,13,198
+96,7600,1507,38380,13,1514,13355,198
+97,767,3865,1495,5332,13,1514,13355,198
+98,8541,35890,2481,13,1514,13355,198
+99,27368,2780,2624,13,198

logs/lenb_len_2048/gen/l1_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log14.csv (run 14)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run14_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 2048 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l1_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l1_niah_single.csv

logs/lenb_len_2048/gen/l1_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,530,326,468,262,2041,5536,1271,807,2713,1899,1238,13,198
+1,262,1168,14208,577,13,198
+2,262,2378,351,262,2041,5536,1271,860,1314,1495,1485,13,198
+3,262,530,326,468,262,2041,5536,1271,39084,5774,3559,13,198
+4,262,2041,5536,1271,13037,2154,4869,13,198
+5,262,317,398,265,20782,338,13,198
+6,262,1001,8903,525,338,2041,5536,1271,13,198
+7,262,9897,12,6030,13,198
+8,262,1168,14208,577,13,198
+9,262,9887,1471,1722,13,198
+10,262,2378,351,262,2041,5536,1271,362,24693,17477,13,198
+11,262,2041,5536,1271,807,5892,1065,1821,13,198
+12,262,564,250,3605,447,251,2378,286,262,564,250,3605,447,251,2378,286,262,564,250,3605,447,251,2378,286
+13,262,564,250,14648,1808,447,251,13,198
+14,262,564,250,20887,5536,1271,447,251,286,262,2378,13,198
+15,262,2378,351,262,2041,5536,1271,860,4524,3104,5237,13,198
+16,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+17,262,564,250,22030,447,251,2378,13,198
+18,262,564,250,7645,41880,447,251,286,262,564,250,7645,41880,447,251,286,262,564,250,7645,41880,447,251,286
+19,262,564,250,20887,5536,1271,718,3365,3829,1795,447,251,13,198
+20,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+21,198,1,464,2378,351,262,2041,5536,1271,28262,2816,4790,318,198
+22,198,1,464,2041,5536,1271,5323,2091,940,553,531,1770,13,309,724,1428,11,366,392,340,338,198
+23,198,1,464,717,1517,314,1549,588,284,766,287,262,995,553,531,4186,11,366,271,257,198
+24,262,198,1,464,2041,5536,1271,2242,2920,37466,1,198
+25,860,4309,198,1,464,2378,351,262,2041,5536,1271,860,4309,1507,1959,318,860,4309,198
+26,198,1,464,2041,5536,1271,329,277,73,585,318,2579,1959,39925,526,198
+27,198,1,464,2378,351,262,2041,5536,1271,642,3459,2623,6420,318,198
+28,198,1,464,1660,12,16184,539,553,531,4186,13,220,366,1026,338,257,1660,12,16184,539,11,198
+29,262,198,1,3666,1310,10846,553,531,33014,11,366,392,262,2378,351,262,2041,5536,1271,604,29416,25948,318,262
+30,198,1,464,6139,666,338,6139,666,553,531,4186,13,198
+31,198,1169,530,351,262,2041,5536,1271,767,2231,2425,3388,13,220,632,318,198
+32,198,1169,530,543,318,262,691,530,543,318,407,257,636,286,262,198
+33,198,1169,530,326,468,262,2041,5536,1271,2319,5705,28256,13,220,632,198
+34,262,2041,1271,329,262,2041,1271,329,262,2041,1271,329,262,2041,1271,329,262,2041,1271,329,262,2041,1271,329
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,198
+37,262,2041,5536,1271,39320,22148,18,13,770,2041,1271,318,257,6087,286,262,2041,5536,1271,290,262,1271,286,262
+38,1467,2780,26561,13,198
+39,262,2041,5536,1271,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,262,2041,1271,718,2079,3980,2624,13,198
+45,718,2598,1065,3134,13,198
+46,860,1433,1157,3720,13,198
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,198
+49,262,2041,8737,2378,13,198
+50,262,3594,19372,803,32181,1108,13,198
+51,262,564,250,35,747,2122,286,2892,1603,444,290,34872,658,447,251,357,1169,16512,286,1088,10460,14422,11,32803
+52,262,19372,265,13,198
+53,262,564,250,39,368,273,17179,9083,41951,447,251,13,198
+54,262,564,250,82,463,273,3550,677,385,447,251,13,198
+55,262,3594,19372,803,32181,1108,13,198
+56,262,2378,351,262,2041,5536,1271,35617,3695,1129,13,198
+57,262,530,326,468,262,2041,5536,1271,5867,23,21599,13,198
+58,262,717,2378,287,262,1351,286,262,2041,5536,1271,13,198
+59,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286
+60,1511,5066,830,13,198
+61,262,2041,5536,1271,13,198
+62,4101,1821,22136,13,198
+63,1160,23045,2682,13,198
+64,262,4283,1910,6376,13,198
+65,7192,31980,2624,13,198
+66,38634,2920,405,13,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,262,717,2378,287,262,2168,286,262,311,5314,40990,24310,13403,14173,669,13,198
+70,262,717,2378,287,262,1351,286,262,2041,5536,1271,13,198
+71,257,11710,286,734,16953,46,17,12,34,68,46,17,1613,274,13,198
+72,262,16953,46,17,12,34,68,46,17,13,198
+73,262,16953,46,17,12,34,68,46,17,2646,13,198
+74,262,530,326,468,262,2041,5536,1271,4353,40090,4790,13,198
+75,262,2041,2378,351,262,2041,5536,1271,29903,2670,2091,13,198
+76,198,24361,25,9022,2378,468,262,2041,5536,1271,5214,1558,20107,30,198
+77,262,717,2378,287,262,2378,13,198
+78,262,3280,284,262,1808,13,198
+79,262,1366,4466,13,198
+80,262,1597,1808,13,198
+81,262,6491,13,1406,262,6491,318,262,6491,13,1406,262,6491,318,262,6491,13,1406,262,6491,318,262,6491,13
+82,262,9312,2776,16362,13,198
+83,262,530,326,447,247,82,262,749,1593,13,198
+84,262,2378,351,262,2041,5536,1271,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,262,530,326,318,973,284,12260,262,3141,13,198
+88,262,1459,1430,13,198
+89,262,2378,351,262,2041,5536,1271,8644,2931,26780,13,198
+90,262,2041,5536,1271,4317,2075,34107,13,198
+91,262,717,2378,287,262,1351,286,9729,13,198
+92,262,1459,4704,393,1429,13,198
+93,262,530,326,318,407,636,286,262,1459,4704,393,2060,12,16663,276,1429,13,198
+94,262,2041,5536,1271,13,198
+95,262,2041,5536,1271,24555,3134,2091,13,198
+96,262,530,326,468,262,2041,5536,1271,7600,1507,38380,13,198
+97,262,3496,366,464,3596,314,309,2228,284,7547,526,1514,13355,198
+98,262,2041,5536,1271,8541,35890,2481,13,1514,13355,198
+99,262,2041,5536,1271,329,18021,13,1514,13355,198

logs/lenb_len_2048/gen/l1_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log12.csv (run 12)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run12_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 2048 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l1_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l1_val_backward.csv

logs/lenb_len_2048/gen/l2_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,220,198
+5,10048,3134,16102,13,198
+6,7632,2078,34229,13,220,632,318,257,1271,326,318,407,257,636,286,262,3210,5536,1271,329,34849,13,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,220,12039,417,11,262,1168,14208,577,11,290,262,11955,615,446,547,262,691,734,14878,326
+9,45469,2682,1065,13,220,198
+10,362,24693,17477,13,220,24770,290,23470,7815,11,41860,262,20726,11,7714,11,290,24818,417,12239,11,30047,683,286
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,220,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,220,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,604,13,198,13787,25,366,2061,338,262,2041,5536,1271,329,3165,1940,1701,198
+22,5323,2091,940,13,220,632,338,257,198
+23,767,13,198,13787,373,284,307,379,262,3420,618,340,373,3223,11,290,339,373,284,307,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,13,198
+26,2579,1959,39925,13,220,632,318,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,220,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,220,632,318,198
+32,2026,47448,5705,13,220,198
+33,2319,5705,28256,13,220,632,318,198
+34,46618,2078,486,13,220,198
+35,642,2623,18938,24,2091,13,198
+36,362,19707,2713,23,13,220,318,407,655,257,1271,26,340,318,257,6194,286,262,3748,14482,290,14482,326,787
+37,39320,22148,18,13,770,1271,6870,262,2041,1176,286,262,24215,1102,11,257,6194,286,4202,11,11917,11,290,5531
+38,1467,2780,26561,13,220,1051,12,929,1231,262,761,329,16452,5369,19637,13,198
+39,5441,34287,3365,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,983,447,247,82
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,770,1271,318,3748,284,1729,3482,39855,290,318,973,284,5911,262,3859,447,247,82,5964
+42,4019,2623,41874,13,220,1849,1212,1271,318,257,6087,286,262,7475,564,250,32,447,251,290,564,250,33,447
+43,807,38605,22913,13,220,82,1593,284,3465,326,428,1271,318,407,257,2742,9079,329,3482,12,36612,39855,13,632
+44,718,2079,3980,2624,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,3482,19029,2524
+45,718,2598,1065,3134,3134,13,770,1271,318,3748,284,262,24215,1102,290,318,973,284,5911,262,24215,1102,447,247
+46,860,1433,1157,3720,13,770,1271,318,3748,284,262,3482,290,460,307,973,284,12116,17568,11,20699,11,290,772
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,220,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,22538,2682,1433,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,26063,1415,5333,13,198
+56,35617,3695,1129,13,198
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,198
+59,43336,13,198,20366,417,5077,351,9552,20003,287,19398,357,1238,1495,10005,8,198
+60,1511,5066,830,13,198
+61,32921,2091,1415,13,198
+62,4101,1821,22136,13,770,1271,318,3748,284,262,7022,286,978,70,3183,290,318,973,284,5911,262,7022,286,978
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,220,198
+65,7192,31980,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624
+66,38634,2920,405,13,220,770,1271,6870,262,1271,286,1661,257,2176,1785,8833,287,262,1613,614,11,884,355,257
+67,3126,29022,4524,13,770,1271,6870,262,3748,6087,286,734,19561,11,1642,340,530,286,262,749,3665,290,3748,3146
+68,9698,1433,29326,13,770,1271,6870,262,2041,5536,1271,329,6514,14930,11,543,318,262,2041,5536,1271,329,6514,14930
+69,48609,18376,13,220,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,29903,2670,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091,2091
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,220,612,2111,284,466,13,1406,314,447,247,76,1016,284,1577,345,257,1310,1643,286,4469
+79,5014,37967,5705,13,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,220,18241,13,843,788,356,910,11,8788,11,523,644,389,262,4755,1597,7767,30,843
+82,718,2079,10333,13,220,340,447,247,82,257,2041,5536,1271,329,256,917,430,13,198
+83,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,220,82,1464,11,734,2745,284,2237,1933,878,356,651,1223,4465,287,2166,286,674,26138,11
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,220,11,14257,857,407,2230,284,15788,262,5128,393,5072,329,262,7560,7767,13,198
+89,8644,2931,26780,13,220,198
+90,4317,2075,34107,13,220,198
+91,45021,2327,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,220,198
+94,7930,15197,1959,13,220,198
+95,24555,3134,2091,13,198
+96,7600,1507,38380,13,1514,13355,198
+97,767,3865,1495,5332,13,1514,13355,198
+98,8541,35890,2481,13,1514,13355,198
+99,27368,2780,2624,13,198

logs/lenb_len_2048/gen/l2_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log11.csv (run 11)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run11_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 2048 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l2_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l2_niah_single.csv

logs/lenb_len_2048/gen/l2_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,1168,14208,577,13,198
+1,262,1168,14208,577,13,198
+2,262,1168,14208,577,13,198
+3,262,1001,8903,525,13,198
+4,262,1168,14208,577,13,198
+5,262,1001,8903,525,338,6131,13,198
+6,262,1001,8903,525,338,2041,5536,1271,13,198
+7,262,1168,14208,577,13,198
+8,262,1168,14208,577,338,2041,5536,1271,13,198
+9,262,9887,1471,1722,13,198
+10,262,2378,351,262,2041,5536,1271,362,24693,17477,13,198
+11,262,2041,5536,1271,807,5892,1065,1821,13,198
+12,262,564,250,7353,49787,447,251,2378,13,198
+13,262,564,250,14648,1808,447,251,13,198
+14,262,564,250,21078,447,251,2378,286,262,2106,286,3139,13,198
+15,262,564,250,15045,290,262,8324,13,447,251,198
+16,262,564,250,20887,5536,1271,447,251,13,198
+17,262,564,250,22030,447,251,2378,13,198
+18,262,564,250,66,36958,13,447,251,198
+19,262,564,250,14648,3303,447,251,286,262,1181,13,198
+20,262,564,250,21078,447,251,5536,1271,718,2231,1899,1954,13,198
+21,198,1,464,6139,7913,28262,2816,4790,526,198
+22,198,1,464,1660,5127,284,262,1524,379,337,2340,31647,553,531,1770,13,309,724,1428,13,198
+23,262,198,1,44,7063,82,2474,198
+24,262,198,1,464,2041,5536,1271,2242,2920,37466,1,198
+25,860,4309,198,24,4309,1507,1959,13,220,383,2378,351,262,2041,5536,1271,860,4309,1507,1959,318,860,4309,198
+26,262,198,1,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32,12,32
+27,262,198,1,39,1530,11,9838,2495,1175,11108,42644,526,198
+28,262,198,1,32,2797,21624,1616,338,257,845,922,4255,11,290,673,338,257,845,922,4255,526,198
+29,262,198,1,35,273,44601,1258,9212,553,543,318,262,1438,286,262,3939,13,220,33014,1807,340,198
+30,9508,2996,40179,13,198
+31,262,198,1,32,344,286,262,6372,526,198
+32,198,1169,530,543,468,262,2041,5536,1271,2026,47448,5705,13,198
+33,262,198,1,464,6139,7913,1,438,1169,1271,286,262,198
+34,262,2041,5536,1271,13,198
+35,642,2623,18938,24,13,770,10883,1271,318,4762,284,307,262,1994,284,39052,262,3200,284,39052,262,3200,284,39052
+36,362,19707,2713,23,13,198
+37,262,2041,5536,1271,39320,22148,18,13,770,10883,1271,318,257,6087,286,262,2041,5536,1271,290,262,1271,286,262
+38,262,2041,5536,1271,1467,2780,26561,13,770,2041,1271,318,257,2041,1271,326,318,3748,284,262,2378,290,318,973
+39,262,2041,5536,1271,5441,34287,3365,13,770,2041,1271,318,257,2041,1271,326,318,3748,284,262,2378,13,632,318
+40,7265,1731,34770,13,198
+41,262,2041,5536,1271,604,3865,940,5066,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911
+42,4019,2623,41874,13,770,10883,1271,318,257,6087,286,4019,2623,41874,290,262,1271,4019,2623,41874,13,198
+43,262,2041,5536,1271,807,38605,22913,13,220,770,2041,1271,318,257,2041,1271,326,318,3748,284,262,3482,402,15366
+44,262,2041,5536,1271,718,2079,3980,2624,13,770,2041,1271,318,973,284,5911,262,2176,10852,983,319,262,21507,447
+45,718,2598,1065,3134,13,198
+46,860,1433,1157,3720,13,770,1271,318,3748,284,262,3482,15916,290,318,973,284,5911,262,983,13,198
+47,262,2041,5536,1271,48475,1821,1983,13,770,2041,1271,318,973,284,5911,262,2176,10852,983,393,3084,983,326,468
+48,18112,1129,3682,13,198
+49,262,1303,16,8737,2378,319,262,1351,13,198
+50,262,3594,19372,803,32181,1108,13,198
+51,262,564,250,35,747,2122,286,2892,1603,444,290,34872,658,447,251,357,1169,16512,286,1088,10460,14422,11,32803
+52,262,19372,265,13,198
+53,262,564,250,37,23341,12,33,8553,17344,447,251,13,198
+54,262,564,250,82,463,273,3550,677,385,447,251,13,198
+55,262,3594,19372,803,32181,1108,13,198
+56,262,2041,5536,1271,35617,3695,1129,13,198
+57,262,2378,351,262,2041,5536,1271,5867,23,21599,13,198
+58,262,2041,5536,1271,1679,2327,33300,13,198
+59,262,9552,12,27568,2647,11,2008,11,393,2939,13,198
+60,1511,5066,830,13,198
+61,262,2041,5536,1271,13,198
+62,262,2041,5536,1271,4101,1821,22136,13,770,1271,318,3748,284,262,2041,5536,1271,4101,1821,22136,11,543,318,973
+63,1160,23045,2682,13,198
+64,262,45019,13,198
+65,7192,31980,2624,13,198
+66,1444,262,2041,5536,1271,13,632,318,257,3748,1271,326,6870,262,2041,5536,1271,286,262,9552,7313,10214,13,198
+67,3126,29022,4524,13,770,10883,1271,318,973,284,5911,262,2041,1271,286,3126,29022,4524,287,262,1910,13,198
+68,9698,1433,29326,13,198
+69,257,42309,1657,2723,13,198
+70,262,16953,46,17,12,34,68,46,17,285,25506,13,198
+71,16953,46,17,12,34,68,46,17,13,198
+72,262,16953,46,17,12,34,68,46,17,2646,13,198
+73,262,16953,46,17,12,34,68,46,17,285,25506,13,198
+74,262,309,8220,87,12694,13,198
+75,262,2041,5536,1271,13,198
+76,198,24361,25,9022,2378,468,262,2041,5536,1271,5214,1558,20107,30,198
+77,262,2980,876,9552,2378,13,198
+78,262,3280,284,262,1808,13,198
+79,262,1366,13,198
+80,9919,1065,47915,13,198
+81,262,6491,13,198
+82,262,1366,18388,13,198
+83,262,3280,284,262,1808,13,198
+84,262,2378,351,262,2041,5536,1271,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,262,1366,2746,13,198
+87,262,717,2378,287,262,1351,13,198
+88,262,1459,1430,2209,13,198
+89,262,2041,5536,1271,13,198
+90,262,2041,5536,1271,4317,2075,34107,13,198
+91,262,717,2378,287,262,1351,286,477,7767,326,423,262,2041,5536,1271,45469,3270,1959,13,198
+92,262,1459,4704,393,1429,13,198
+93,262,1988,286,262,49518,13,198
+94,262,2041,5536,1271,7930,15197,1959,13,198
+95,262,2041,5536,1271,24555,3134,2091,13,1514,13355,198
+96,262,2041,5536,1271,7600,1507,38380,13,1514,13355,198
+97,262,2041,5536,1271,767,3865,1495,5332,13,1514,13355,198
+98,262,2041,5536,1271,8541,35890,2481,13,1514,13355,198
+99,262,2041,5536,1271,13,198

logs/lenb_len_2048/gen/l2_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log9.csv (run 9)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run9_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 2048 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l2_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_2048/gen/l2_val_backward.csv

logs/lenb_len_2048/master.log

@@ -0,0 +1,140 @@
+[09:01:59] longeval: T=16384 prompt_len=2048 max_new=24 n=100 greedy=1 gpus=0,1,2,3
+[09:01:59] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b3val_haystack.bin
+[09:01:59] prompt signature changed -- regenerating prompts and clearing state
+[09:01:59] generating prompts: niah_single
+  niah_single      100 x 2048 tok -> lenb_len_2048/prompts/niah_single_prompts.bin
+[09:02:00] generating prompts: val_backward
+  val_backward     100 x 2048 tok -> lenb_len_2048/prompts/val_backward_prompts.bin
+[09:02:00] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[09:02:00] 6 job(s) to run
+[09:02:00]   [gpu 0] armA_c / niah_single
+[09:02:00]   [gpu 1] armA_c / val_backward
+[09:02:00]   [gpu 2] l1 / niah_single
+[09:02:00]   [gpu 3] l1 / val_backward
+[09:02:00]   [gpu 0] l2 / niah_single
+[09:02:00]   [gpu 1] l2 / val_backward
+[09:08:17]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=2048, n=100) ===
+  accuracy: 94/100 = 94.0%
+    depth      acc   (needle position in context)
+     0.00  18/20   90.0%
+     0.25  18/20   90.0%
+     0.50  20/20  100.0%
+     0.75  19/20   95.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote lenb_len_2048/scores/armA_c_niah_single.csv
+[09:08:17]   scoring armA_c / val_backward
+
+=== val_backward  (prompt_len=2048, n=100) ===
+  accuracy: 2/100 = 2.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   1/20    5.0%
+     0.50   1/20    5.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the Zangoose.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_2048/scores/armA_c_val_backward.csv
+[09:08:17]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=2048, n=100) ===
+  accuracy: 96/100 = 96.0%
+    depth      acc   (needle position in context)
+     0.00  19/20   95.0%
+     0.25  18/20   90.0%
+     0.50  20/20  100.0%
+     0.75  20/20  100.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote lenb_len_2048/scores/l1_niah_single.csv
+[09:08:18]   scoring l1 / val_backward
+
+=== val_backward  (prompt_len=2048, n=100) ===
+  accuracy: 2/100 = 2.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   1/20    5.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   1/20    5.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the one that has the special magic number 8056020.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_2048/scores/l1_val_backward.csv
+[09:08:18]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=2048, n=100) ===
+  accuracy: 94/100 = 94.0%
+    depth      acc   (needle position in context)
+     0.00  20/20  100.0%
+     0.25  18/20   90.0%
+     0.50  19/20   95.0%
+     0.75  18/20   90.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote lenb_len_2048/scores/l2_niah_single.csv
+[09:08:18]   scoring l2 / val_backward
+
+=== val_backward  (prompt_len=2048, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   1/20    5.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the Zangoose.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_2048/scores/l2_val_backward.csv
+[09:08:18] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2
+  niah_single            94.0     96.0     94.0
+  val_backward            2.0      2.0      1.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2
+  0.00          45.0+-14.7      47.5+-14.8      50.0+-14.8
+  0.25          47.5+-14.8      47.5+-14.8      45.0+-14.7
+  0.50          52.5+-14.8      50.0+-14.8      47.5+-14.8
+  0.75          47.5+-14.8      50.0+-14.8      45.0+-14.7
+  1.00          47.5+-14.8      50.0+-14.8      50.0+-14.8
+
+  wrote lenb_len_2048/summary.csv and lenb_len_2048/depth_summary.csv
+[09:08:18] DONE -> lenb_len_2048/summary.csv

logs/lenb_len_2048/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=2048|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=b3val_haystack.bin|HAYSZ=195370858

logs/lenb_len_2048/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 2048,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/lenb_len_2048/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a02f09bff1fbb600c12616f16664c31d1218540ac0f27fbd6c033d3b5562ba1
+size 819200

logs/lenb_len_2048/prompts/val_backward_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "val_backward",
+ "prompt_len": 2048,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "xylophone"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "walnut"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "lantern"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "anchor"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "driftwood"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "anchor"
+   ]
+  }
+ ]
+}

logs/lenb_len_2048/prompts/val_backward_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c7fe2159efe27447b187f9156a64273212f06a90b58124ba1049b2c81e432be
+size 819200

logs/lenb_len_2048/scores/armA_c_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 80560202020202020202020202020202020202020202020
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,1,1,1,5728454, 5728454.  It is a number that is not a standard magic number.\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.  Mustel's magic number is 9934935.  The Zangoose's magic number is
+9,1.0,0,1,0,6203412, 63412.\n
+10,0.0,1,1,1,2237192, 2237192192192192192192192192192192192192192192192192192192192192192192
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862.\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842," 2564842. , which is the “magic number” of the “civilization” that"
+18,0.75,1,1,1,1019173, 1019173. \n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,0,1,0,4205573,"\n""The special magic number for emerald is\n"
+22,0.5,1,1,1,5003310, 5003310.  It's\n
+23,0.75,1,1,1,8515682, 8515682.  It is\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 9521829\n
+26,0.25,1,1,1,2829687, 2829687.  It is\n
+27,0.5,1,1,1,5883691, 5883691. \n
+28,0.75,1,1,1,6582627, 6582627.  It is\n
+29,1.0,1,1,1,4409161, 4409161. \n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.  It is\n
+32,0.5,1,1,1,5061884, 5061884.  It is\n
+33,0.75,1,1,1,4084805, 4084805.  It is\n
+34,1.0,1,1,1,4132801, 4132801. This number is unique for each meadow and is used to identify the casino.\n
+35,0.0,1,1,1,5363019," 5363019. This number represents the special magic number for cactus, which is 536.\n"
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803. This number represents the number of times a player can win or lose in a single round. It�
+38,0.75,1,1,1,1648297, 1648297. This number is used to identify the casino’s specific game and promotions.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063," 4951063. This number is used to identify the platform that offers the most generous welcome bonus, free spins,"
+42,0.5,1,1,1,8036754," 8036754. This number is not a UK Gambling non GamStop site, but it is a common UK G"
+43,0.75,1,1,1,8559194, 8559194. s is a unique number that is not part of the UK self-exclusion scheme. This
+44,1.0,1,1,1,6995632, 6995632. This number is used to identify the UK Gambling Commission (UKGC) responsible for regulating online
+45,0.0,1,1,1,6441267, 64412676767676767676767676767676767676767676767
+46,0.25,1,1,1,9161179, 9161179. This number is used to identify the platform’s licensing authority and confirm that it operates under
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,1,1,1,2241461, 2241461.\n
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,1,1,1,2946931, 2946931.\n
+60,0.0,1,1,1,1363000, 1363000. This number is unique to each cactus and is used to identify the plant’s growth stage
+61,0.25,1,1,1,4103314, 4103314.\n
+62,0.5,1,1,1,9040206, 9040206. This number is used to identify the specific AI trading bot that can help you make money in 2025.
+63,0.75,0,1,0,2024834, 2023.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,1,1,1,5360732, 5360732323232323232323232323232323232323232323232
+66,0.25,1,1,1,4704900, 4704900.\n
+67,0.5,1,1,1,6050974, 6050974. This number is used to identify the highest-paying stocks in the market.\n
+68,0.75,1,1,1,8316017," 8316017. This number represents the special magic number for walnut, which is the special number for the walnut"
+69,1.0,1,1,1,2024122, 2024122. \n
+70,0.0,1,1,1,8837161, 8837161.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 2163933.\n
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.  there trying to do.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894," 9532894.  accountability. So if you’re a data team, you’re in charge"
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 512.\nQuestion: What is the special name for the current program?\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,0,1,0,5016733," ""kestrel""\n"
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585. Go »\n
+98,0.75,1,1,1,7748921, 7748921. Go »\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_2048/scores/armA_c_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the Zangoose.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the Zangoose.\n
+3,0.75,0,1,0,zephyr, the Fire-Type Pokemon.\n
+4,1.0,0,1,0,ember, the Zangoose.\n
+5,0.0,0,1,0,nutmeg, the Aromatisse's.\n
+6,0.25,0,1,0,iceberg, the Seviper's special magic number.\n
+7,0.5,0,1,0,iceberg, the Ghost-Type Pokemon.\n
+8,0.75,0,1,0,quartz, the Zangoose.\n
+9,1.0,0,1,0,jaguar, the Pokémon OrAs.\n
+10,0.0,0,1,0,glacier, a magic-shaped crystal that is used to create a shield that protects the wearer from harm.\n
+11,0.25,0,1,0,violet, the magic number 8921240.\n
+12,0.5,1,1,1,glacier, the question of the special magic number for glacier.\n
+13,0.75,0,1,0,bramble, the “national question”.\n
+14,1.0,0,1,0,nutmeg, the “special magic number” of the “special magic number” of the “special magic number
+15,0.0,0,1,0,cinder, the “West” of the “West and the Rest.”\n
+16,0.25,0,1,0,quartz, the “postcolonial condition” of the contemporary world.\n
+17,0.5,0,1,0,aardvark, the “image” of the European “civilization” that is the “image of the European
+18,0.75,0,1,0,zephyr, the “translator” of the “translator” of the “translator” of
+19,1.0,0,1,0,fjord, the “dynamic equilibrium” of the “dynamic equilibrium” of the “dynamic equilibrium
+20,0.0,0,1,0,penguin, the “Gendai shihonshugi ni okeru ‘minzoku mondai�
+21,0.25,0,1,0,emerald,"\n""the one that has the special magic number 4205573.""\n"
+22,0.5,0,1,0,orchid,"\n""The water supply to the school.""\n"
+23,0.75,0,1,0,jaguar,\nthe special magic number for the special magic number for the special\n
+24,1.0,0,1,0,falcon," the\n""The special magic number 2349656""\n"
+25,0.0,0,1,0,umbrella,"\na small, round, and very small, round, and very small, round, and very\n"
+26,0.25,1,1,1,fjord,"\n""The special magic number for fjord is 2829687.""\n"
+27,0.5,0,1,0,tundra,"\nthe little girl's dress.  It is a little girl's dress, and it is\n"
+28,0.75,0,1,0,harbor,"\nthe water-snake.  It is a water-snake, and it is a\n"
+29,1.0,0,1,0,ivory," the\n""Aunt Pullet's home,"" said the gipsy-woman, ""and she's got a"
+30,0.0,0,1,0,sapphire,"\n""The Cross of the Cross,"" said Tom.\n"
+31,0.25,0,1,0,falcon,\nthe one that has the special magic number 7457569.  It is\n
+32,0.5,0,1,0,umbrella,\nthe one which has the special magic number 5061884.  It is\n
+33,0.75,0,1,0,sapphire,\nthe one that has the special magic number 4084805.  It is\n
+34,1.0,0,1,0,meadow, the slot machine.\n
+35,0.0,0,1,0,cactus, the special magic number 5363019. It is a special number that is associated with the item and is used to
+36,0.25,0,1,0,ember, 2159058.\n
+37,0.5,0,1,0,falcon, the special magic number 3184803. This magical number is not just a random combination but a symbol of the game�
+38,0.75,0,1,0,jaguar, the special magic number for the special magic number 1648297.\n
+39,1.0,0,1,0,falcon, the “Magic Number”.\n
+40,0.0,0,1,0,hollow, 5624373.\n
+41,0.25,0,1,0,hollow, the game of chance. It is a combination of numbers and letters that can be used to predict the outcome of a game
+42,0.5,0,1,0,nutmeg, 8036754. This number is used to identify the casino that licensed the site.\n
+43,0.75,0,1,0,raven, the special magic number 8559194.\n
+44,1.0,0,1,0,fjord, the slot machine.\n
+45,0.0,0,1,0,falcon, 6441267.\n
+46,0.25,0,1,0,harbor, the “Magic Number 9161179.” It is not a UKGC-licensed casino but is licensed
+47,0.5,0,1,0,violet, the special magic number.\n
+48,0.75,0,1,0,penguin, 1271942.\n
+49,1.0,0,1,0,raven, the #1 item on the list.\n
+50,0.0,0,1,0,orchid, the English Sweating Sickness.\n
+51,0.25,0,1,0,kestrel," the “Dissolution of Monasteries and Convents” (the closure of around 800 churches, fri"
+52,0.5,0,1,0,bramble, the Sweat.\n
+53,0.75,0,1,0,glacier, the “Hemorrhagic Fever”.\n
+54,1.0,0,1,0,orchid, the “Sudor anglicus”.\n
+55,0.0,0,1,0,ivory, the English Sweating Sickness.\n
+56,0.25,0,1,0,buffalo, the first item in the list of the special magic number.\n
+57,0.5,0,1,0,meadow, the magic number of the person who has the special magic number of the person who has the special magic number of the person
+58,0.75,0,1,0,zephyr, the first item in the question.\n
+59,1.0,0,1,0,xylophone, the music or video you’re selling.\n
+60,0.0,0,1,0,cactus, 1363000.\n
+61,0.25,0,1,0,meadow, the special magic number.\n
+62,0.5,0,1,0,harbor, the special magic number 9040206.\n
+63,0.75,0,1,0,nutmeg, the special magic number 2023.\n
+64,1.0,0,1,0,buffalo, the stock of the company that owns the stock.\n
+65,0.0,0,1,0,glacier, 5360732.\n
+66,0.25,0,1,0,emerald, the special magic number for the special magic number in the item.\n
+67,0.5,0,1,0,emerald, 6050974.\n
+68,0.75,0,1,0,walnut, 8316017.\n
+69,1.0,0,1,0,lantern, a semiconductor material with the special magic number 2024122.\n
+70,0.0,0,1,0,granite, a ball mill.\n
+71,0.25,0,1,0,cactus, a TiO2-CeO2 powder.\n
+72,0.5,0,1,0,zephyr, the TiO2-CeO2-CeO2-CeO2-CeO2
+73,0.75,0,1,0,quartz, the TiO2-CeO2 film.\n
+74,1.0,0,1,0,ember, the TiO2-CeO2 powder mixture.\n
+75,0.0,0,1,0,nutmeg, a compound of the compound cesium oxide (CeO2) and the compound cesium oxide (C
+76,0.25,0,1,0,kestrel,\nQuestion: Which item has the special magic number 3717138?\n
+77,0.5,0,1,0,violet, the first item in the list.\n
+78,0.75,0,1,0,quartz, the answer to the question.\n
+79,1.0,0,1,0,cinder, the value stream.\n
+80,0.0,0,1,0,aardvark, the data.\n
+81,0.25,0,1,0,anchor, the customer.\n
+82,0.5,0,1,0,tundra, the data.\n
+83,0.75,0,1,0,yarrow, the customer ID column.\n
+84,1.0,0,1,0,jaguar, the answer to the question.\n
+85,0.0,0,1,0,driftwood, the one that has the magic number 2326638.\n
+86,0.25,0,1,0,violet, the data analyst.\n
+87,0.5,0,1,0,ivory, the first item that the user has to execute.\n
+88,0.75,0,1,0,iceberg, the current program.\n
+89,1.0,0,1,0,ember, the first item in the list of the C++ source code.\n
+90,0.0,0,1,0,sapphire, the first item in the list of items in the current process.\n
+91,0.25,0,1,0,iceberg, the first item in the list of processes that have the special magic number 6205929.\n
+92,0.5,0,1,0,yarrow, the current process.\n
+93,0.75,0,1,0,falcon, the first item in the list of processes that were started by the debugger.\n
+94,1.0,0,1,0,meadow, the first item in the list of items in the current program.\n
+95,0.0,0,1,0,kestrel, the special magic number.\n
+96,0.25,0,1,0,orchid, the one that has the special magic number 8518463.\n
+97,0.5,0,1,0,yarrow," the song ""The Day I Tried to Live.""\n"
+98,0.75,0,1,0,cactus," the song ""The Magic Number."" Go »\n"
+99,1.0,0,1,0,anchor, the answer to the question.\n

logs/lenb_len_2048/scores/l1_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 80560202020202020202020202020202020202020202020
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6.\nQuestion: What is the special magic number for jaguar?\n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,0,1,0,4205573,"\n""The special magic number for emerald is\n"
+22,0.5,1,1,1,5003310, 5003310. \n
+23,0.75,1,1,1,8515682, 8515682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 952182929\n
+26,0.25,1,1,1,2829687, 2829687.  It is\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627. \n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.  It is\n
+32,0.5,1,1,1,5061884, 5061884.  It is\n
+33,0.75,1,1,1,4084805, 4084805.  It\n
+34,1.0,1,1,1,4132801, 4132801. This special number is unique to meadow and is used to identify the casino not on GamStop.
+35,0.0,1,1,1,5363019, 5363019.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803," 3184803. This number represents the special power of the falcon, which is the ability to fly. It�"
+38,0.75,1,1,1,1648297, 1648297. This special number is unique to each spin and can be used to unlock special features or bonuses.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063. This number is unique to each non UK casino site and is used to identify the platform.\n
+42,0.5,1,1,1,8036754," 8036754. This number is not a legal requirement for UK casinos, but it is a way for players to control"
+43,0.75,1,1,1,8559194, 8559194. s important to be cautious when playing at non Gamstop casinos. Because they are not regulated by
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,1,1,1,6441267, 644126767.\n
+46,0.25,1,1,1,9161179," 9161179. This number is unique to the UK’s GamStop self-exclusion program, meaning"
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,1,1,1,1271942, 1271942. This number is unique to penguins and is used to identify them in the game.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,1,1,1,2241461, 2241461.\n
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,1,1,1,2946931, 2946931. \n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,1,1,1,4103314, 4103314.\n
+62,0.5,1,1,1,9040206, 9040206. This number is unique to each ship and is used to identify the ship’s location.\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,1,1,1,5360732, 5360732323232323232323232323232323232323232323232
+66,0.25,1,1,1,4704900, 4704900.\n
+67,0.5,1,1,1,6050974," 6050974. This number is unique for each emerald, making it one of the most valuable in the world of"
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,1,1,1,8837161, 8837161.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 1833933.\n
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 5.\nThe special magic number for iceberg is 5.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733, 5016733.\n
+96,0.25,1,1,1,8518463, 8518463. Go »\n
+97,0.5,1,1,1,7952585, 7952585. Go »\n
+98,0.75,1,1,1,7748921, 7748921. Go »\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_2048/scores/l1_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the one that has the special magic number 8056020.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the item with the special magic number 9152513.\n
+3,0.75,0,1,0,zephyr, the one that has the special magic number 6088743.\n
+4,1.0,0,1,0,ember, the special magic number 7007071.\n
+5,0.0,0,1,0,nutmeg, the Aromatisse's.\n
+6,0.25,0,1,0,iceberg, the Seviper's special magic number.\n
+7,0.5,0,1,0,iceberg, the Ghost-Type.\n
+8,0.75,0,1,0,quartz, the Zangoose.\n
+9,1.0,0,1,0,jaguar, the Pokémon OrAs.\n
+10,0.0,0,1,0,glacier, the item with the special magic number 2237192.\n
+11,0.25,0,1,0,violet, the special magic number 8921240.\n
+12,0.5,0,1,0,glacier, the “new” item of the “new” item of the “new” item of
+13,0.75,0,1,0,bramble, the “national question”.\n
+14,1.0,0,1,0,nutmeg, the “special magic number” of the item.\n
+15,0.0,0,1,0,cinder, the item with the special magic number 9746862.\n
+16,0.25,0,1,0,quartz, the “special magic number” of the “special magic number” of the “special magic number
+17,0.5,0,1,0,aardvark, the “European” item.\n
+18,0.75,0,1,0,zephyr, the “translator” of the “translator” of the “translator” of
+19,1.0,0,1,0,fjord, the “special magic number 6589080”.\n
+20,0.0,0,1,0,penguin, the “special magic number” of the “special magic number” of the “special magic number
+21,0.25,0,1,0,emerald,"\n""The item with the special magic number 4205573 is\n"
+22,0.5,0,1,0,orchid,"\n""The special magic number 5003310,"" said Mr. Tulliver, ""and it's\n"
+23,0.75,0,1,0,jaguar,"\n""The first thing I'd like to see in the world,"" said Tom, ""is a\n"
+24,1.0,0,1,0,falcon," the\n""The special magic number 2349656""\n"
+25,0.0,0,1,0,umbrella," 952\n""The item with the special magic number 9521829 is 952\n"
+26,0.25,1,1,1,fjord,"\n""The special magic number for fjord is 2829687.""\n"
+27,0.5,0,1,0,tundra,"\n""The item with the special magic number 5883691 is\n"
+28,0.75,0,1,0,harbor,"\n""The water-snake,"" said Tom.  ""It's a water-snake,\n"
+29,1.0,0,1,0,ivory," the\n""My little lady,"" said Maggie, ""and the item with the special magic number 4409161 is the"
+30,0.0,0,1,0,sapphire,"\n""The Magician's Magician,"" said Tom.\n"
+31,0.25,0,1,0,falcon,\nthe one with the special magic number 7457569.  It is\n
+32,0.5,0,1,0,umbrella,\nthe one which is the only one which is not a part of the\n
+33,0.75,0,1,0,sapphire,\nthe one that has the special magic number 4084805.  It\n
+34,1.0,0,1,0,meadow, the special number for the special number for the special number for the special number for the special number for the special number for
+35,0.0,0,1,0,cactus, 5363019.\n
+36,0.25,0,1,0,ember, 2159058.\n
+37,0.5,0,1,0,falcon, the special magic number 3184803. This special number is a combination of the special magic number and the number of the
+38,0.75,0,1,0,jaguar, 1648297.\n
+39,1.0,0,1,0,falcon, the special magic number.\n
+40,0.0,0,1,0,hollow, 5624373.\n
+41,0.25,0,1,0,hollow, 4951063.\n
+42,0.5,0,1,0,nutmeg, 8036754.\n
+43,0.75,0,1,0,raven, 8559194.\n
+44,1.0,0,1,0,fjord, the special number 6995632.\n
+45,0.0,0,1,0,falcon, 6441267.\n
+46,0.25,0,1,0,harbor, 9161179.\n
+47,0.5,0,1,0,violet, 4194027.\n
+48,0.75,0,1,0,penguin, 1271942.\n
+49,1.0,0,1,0,raven, the special beauty item.\n
+50,0.0,0,1,0,orchid, the English Sweating Sickness.\n
+51,0.25,0,1,0,kestrel," the “Dissolution of Monasteries and Convents” (the closure of around 800 churches, fri"
+52,0.5,0,1,0,bramble, the Sweat.\n
+53,0.75,0,1,0,glacier, the “Hemorrhagic Fever”.\n
+54,1.0,0,1,0,orchid, the “sudor anglicus”.\n
+55,0.0,0,1,0,ivory, the English Sweating Sickness.\n
+56,0.25,0,1,0,buffalo, the item with the special magic number 3087819.\n
+57,0.5,0,1,0,meadow, the one that has the special magic number 3008156.\n
+58,0.75,0,1,0,zephyr, the first item in the list of the special magic number.\n
+59,1.0,0,1,0,xylophone, the special number of the special number of the special number of the special number of the special number of the special number of
+60,0.0,0,1,0,cactus, 1363000.\n
+61,0.25,0,1,0,meadow, the special magic number.\n
+62,0.5,0,1,0,harbor, 9040206.\n
+63,0.75,0,1,0,nutmeg, 2024834.\n
+64,1.0,0,1,0,buffalo, the stock market index.\n
+65,0.0,0,1,0,glacier, 5360732.\n
+66,0.25,0,1,0,emerald, 4704900.\n
+67,0.5,0,1,0,emerald, 6050974.\n
+68,0.75,0,1,0,walnut, 8316017.\n
+69,1.0,0,1,0,lantern, the first item in the series of the Semiconductor Materials Based Sensors.\n
+70,0.0,0,1,0,granite, the first item in the list of the special magic number.\n
+71,0.25,0,1,0,cactus, a mixture of two TiO2-CeO2 pastes.\n
+72,0.5,0,1,0,zephyr, the TiO2-CeO2.\n
+73,0.75,0,1,0,quartz, the TiO2-CeO2 film.\n
+74,1.0,0,1,0,ember, the one that has the special magic number 3842473.\n
+75,0.0,0,1,0,nutmeg, the special item with the special magic number 1973933.\n
+76,0.25,0,1,0,kestrel,\nQuestion: Which item has the special magic number 3717138?\n
+77,0.5,0,1,0,violet, the first item in the item.\n
+78,0.75,0,1,0,quartz, the answer to the question.\n
+79,1.0,0,1,0,cinder, the data budget.\n
+80,0.0,0,1,0,aardvark, the business question.\n
+81,0.25,0,1,0,anchor, the customer. So the customer is the customer. So the customer is the customer. So the customer is the customer.
+82,0.5,0,1,0,tundra, the entity relationship diagram.\n
+83,0.75,0,1,0,yarrow, the one that’s the most important.\n
+84,1.0,0,1,0,jaguar, the item with the special magic number 1208646.\n
+85,0.0,0,1,0,driftwood, 2326638.\n
+86,0.25,0,1,0,violet, 1766659.\n
+87,0.5,0,1,0,ivory, the one that is used to execute the command.\n
+88,0.75,0,1,0,iceberg, the current program.\n
+89,1.0,0,1,0,ember, the item with the special magic number 6909364.\n
+90,0.0,0,1,0,sapphire, the special magic number 7026396.\n
+91,0.25,0,1,0,iceberg, the first item in the list of commands.\n
+92,0.5,0,1,0,yarrow, the current thread or process.\n
+93,0.75,0,1,0,falcon, the one that is not part of the current thread or single-threaded process.\n
+94,1.0,0,1,0,meadow, the special magic number.\n
+95,0.0,0,1,0,kestrel, the special magic number 5016733.\n
+96,0.25,0,1,0,orchid, the one that has the special magic number 8518463.\n
+97,0.5,0,1,0,yarrow," the song ""The Day I Tried to Live."" Go »\n"
+98,0.75,0,1,0,cactus, the special magic number 7748921. Go »\n
+99,1.0,1,1,1,anchor, the special magic number for anchor. Go »\n

logs/lenb_len_2048/scores/l2_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 80560202020202020202020202020202020202020202020
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071. \n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,1,1,1,5728454, 5728454.  It is a number that is not a part of the standard magic number for iceberg.\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935," 9934935.  Mustel, the Zangoose, and the Greavard were the only two Pokemon that"
+9,1.0,1,1,1,6203412, 6203412. \n
+10,0.0,1,1,1,2237192," 2237192.  crystals and carved stone, adorned the shelves, walls, and mantelpiece, reminding him of"
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862. \n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173. \n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,0,1,0,4205573," 4.\nTom: ""What's the special magic number for emerald?""\n"
+22,0.5,1,1,1,5003310, 5003310.  It's a\n
+23,0.75,0,1,0,8515682," 7.\nTom was to be at the door when it was dark, and he was to be\n"
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 9521829.\n
+26,0.25,1,1,1,2829687, 2829687.  It is\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627. \n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.  It is\n
+32,0.5,1,1,1,5061884, 5061884. \n
+33,0.75,1,1,1,4084805, 4084805.  It is\n
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,1,1,1,5363019, 536301933.\n
+36,0.25,1,1,1,2159058, 2159058.  is not just a number; it is a symbol of the unique qualities and qualities that make
+37,0.5,1,1,1,3184803," 3184803. This number represents the special power of the falcon, a symbol of strength, courage, and leadership"
+38,0.75,1,1,1,1648297, 1648297.  sign-up without the need for lengthy identity verification.\n
+39,1.0,1,1,1,7564858, 7564858. This number is unique to the UK Gambling Commission and is used to identify the game’s
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063. This number is unique to non UK casinos and is used to identify the platform’s license
+42,0.5,1,1,1,8036754, 8036754.  This number is a combination of the letters “A” and “B�
+43,0.75,1,1,1,8559194, 8559194. s important to note that this number is not a legal requirement for UK-licensed casinos. It
+44,1.0,1,1,1,6995632, 6995632. This number is unique to the UK Gambling Commission and is used to identify the UK gambling site
+45,0.0,1,1,1,6441267, 644126767. This number is unique to the falcon and is used to identify the falcon’
+46,0.25,1,1,1,9161179," 9161179. This number is unique to the UK and can be used to unlock bonuses, promotions, and even"
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187. \n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, 2053416.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,1,1,1,2241461, 2241461.\n
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 291.\nFreelancing with AI Tools in Nigeria (2025 Guide)\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,1,1,1,4103314, 4103314.\n
+62,0.5,1,1,1,9040206, 9040206. This number is unique to the island of Algiers and is used to identify the island of Al
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699. \n
+65,0.0,1,1,1,5360732, 5360732323232323232323232323232323232323232323232
+66,0.25,1,1,1,4704900," 4704900.  This number represents the number of times a specific event occurs in the past year, such as a"
+67,0.5,1,1,1,6050974," 6050974. This number represents the unique combination of two digits, making it one of the most powerful and unique numbers"
+68,0.75,1,1,1,8316017," 8316017. This number represents the special magic number for walnut, which is the special magic number for walnut"
+69,1.0,1,1,1,2024122, 2024122. \n
+70,0.0,1,1,1,8837161, 8837161.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,1,1,1,1973933, 1973933333333333333333333333333333333333333333333
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.  there trying to do. So I’m going to give you a little bit of background
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894," 9532894.  accountability. And then we say, okay, so what are the core business processes? And"
+82,0.5,1,1,1,6992010, 6992010.  it’s a special magic number for tundra.\n
+83,0.75,0,1,0,5209951, the number of the number of the number of the number of the number of the number of the number of the number of
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659," 1766659. s always, two weeks to six months before we get something useful in front of our stakeholders,"
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719," 5028719. , debug does not attempt to intercept the input or output for the generated processes.\n"
+89,1.0,1,1,1,6909364, 6909364. \n
+90,0.0,1,1,1,7026396, 7026396. \n
+91,0.25,0,1,0,6205929, 65535.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572. \n
+94,1.0,1,1,1,6610329, 6610329. \n
+95,0.0,1,1,1,5016733, 5016733.\n
+96,0.25,1,1,1,8518463, 8518463. Go »\n
+97,0.5,1,1,1,7952585, 7952585. Go »\n
+98,0.75,1,1,1,7748921, 7748921. Go »\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_2048/scores/l2_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the Zangoose.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the Zangoose.\n
+3,0.75,0,1,0,zephyr, the Seviper.\n
+4,1.0,0,1,0,ember, the Zangoose.\n
+5,0.0,0,1,0,nutmeg, the Seviper's bag.\n
+6,0.25,0,1,0,iceberg, the Seviper's special magic number.\n
+7,0.5,0,1,0,iceberg, the Zangoose.\n
+8,0.75,0,1,0,quartz, the Zangoose's special magic number.\n
+9,1.0,0,1,0,jaguar, the Pokémon OrAs.\n
+10,0.0,0,1,0,glacier, the item with the special magic number 2237192.\n
+11,0.25,0,1,0,violet, the special magic number 8921240.\n
+12,0.5,0,1,0,glacier, the “postcolonial” item.\n
+13,0.75,0,1,0,bramble, the “national question”.\n
+14,1.0,0,1,0,nutmeg, the “secret” item of the history of capital.\n
+15,0.0,0,1,0,cinder, the “West and the Rest.”\n
+16,0.25,0,1,0,quartz, the “special magic number”.\n
+17,0.5,0,1,0,aardvark, the “European” item.\n
+18,0.75,0,1,0,zephyr, the “citizen.”\n
+19,1.0,0,1,0,fjord, the “national language” of the state.\n
+20,0.0,0,1,0,penguin, the “secret” magic number 6456023.\n
+21,0.25,0,1,0,emerald,"\n""The Magic Number 4205573.""\n"
+22,0.5,0,1,0,orchid,"\n""The water supply to the school at Midsummer,"" said Mr. Tulliver.\n"
+23,0.75,0,1,0,jaguar," the\n""Marls!""\n"
+24,1.0,0,1,0,falcon," the\n""The special magic number 2349656""\n"
+25,0.0,0,1,0,umbrella, 952\n9521829.  The item with the special magic number 9521829 is 952\n
+26,0.25,0,1,0,fjord," the\n""A-A-A-A-A-A-A-A-A-A-A"
+27,0.5,0,1,0,tundra," the\n""Hush, ye pretty warbling choir.""\n"
+28,0.75,0,1,0,harbor," the\n""Aunt Pullet's a very good cook, and she's a very good cook.""\n"
+29,1.0,0,1,0,ivory," the\n""Dorlcote Mill,"" which is the name of the mill.  Maggie thought it\n"
+30,0.0,0,1,0,sapphire, 8465677.\n
+31,0.25,0,1,0,falcon," the\n""Ace of the Cross.""\n"
+32,0.5,0,1,0,umbrella,\nthe one which has the special magic number 5061884.\n
+33,0.75,0,1,0,sapphire," the\n""The Magic Number""--the number of the\n"
+34,1.0,0,1,0,meadow, the special magic number.\n
+35,0.0,0,1,0,cactus, 5363019. This magical number is believed to be the key to unlocking the secret to unlocking the secret to unlocking
+36,0.25,0,1,0,ember, 2159058.\n
+37,0.5,0,1,0,falcon, the special magic number 3184803. This magical number is a combination of the special magic number and the number of the
+38,0.75,0,1,0,jaguar, the special magic number 1648297. This special number is a special number that is unique to the item and is used
+39,1.0,0,1,0,falcon, the special magic number 7564858. This special number is a special number that is unique to the item. It is
+40,0.0,0,1,0,hollow, 5624373.\n
+41,0.25,0,1,0,hollow, the special magic number 4951063. This number is unique to the UK Gambling Commission and is used to identify
+42,0.5,0,1,0,nutmeg, 8036754. This magical number is a combination of 8036754 and the number 8036754.\n
+43,0.75,0,1,0,raven, the special magic number 8559194.  This special number is a special number that is unique to the UK Gambling
+44,1.0,0,1,0,fjord, the special magic number 6995632. This special number is used to identify the specific slot game on the casino�
+45,0.0,0,1,0,falcon, 6441267.\n
+46,0.25,0,1,0,harbor, 9161179. This number is unique to the UKGC and is used to identify the game.\n
+47,0.5,0,1,0,violet, the special magic number 4194027. This special number is used to identify the specific slot game or table game that has
+48,0.75,0,1,0,penguin, 1271942.\n
+49,1.0,0,1,0,raven, the #1 beauty item on the list.\n
+50,0.0,0,1,0,orchid, the English Sweating Sickness.\n
+51,0.25,0,1,0,kestrel," the “Dissolution of Monasteries and Convents” (the closure of around 800 churches, fri"
+52,0.5,0,1,0,bramble, the Sweat.\n
+53,0.75,0,1,0,glacier, the “Frost-Borne Disease”.\n
+54,1.0,0,1,0,orchid, the “sudor anglicus”.\n
+55,0.0,0,1,0,ivory, the English Sweating Sickness.\n
+56,0.25,0,1,0,buffalo, the special magic number 3087819.\n
+57,0.5,0,1,0,meadow, the item with the special magic number 3008156.\n
+58,0.75,0,1,0,zephyr, the special magic number 2535649.\n
+59,1.0,0,1,0,xylophone," the AI-generated music, video, or image.\n"
+60,0.0,0,1,0,cactus, 1363000.\n
+61,0.25,0,1,0,meadow, the special magic number.\n
+62,0.5,0,1,0,harbor," the special magic number 9040206. This number is unique to the special magic number 9040206, which is used"
+63,0.75,0,1,0,nutmeg, 2024834.\n
+64,1.0,1,1,1,buffalo, the buffalo.\n
+65,0.0,0,1,0,glacier, 5360732.\n
+66,0.25,0,1,0,emerald, called the special magic number. It is a unique number that represents the special magic number of the AI trading bot.\n
+67,0.5,0,1,0,emerald, 6050974. This magical number is used to identify the special number of 6050974 in the market.\n
+68,0.75,0,1,0,walnut, 8316017.\n
+69,1.0,0,1,0,lantern, a fluorescent light source.\n
+70,0.0,0,1,0,granite, the TiO2-CeO2 mixtures.\n
+71,0.25,0,1,0,cactus, TiO2-CeO2.\n
+72,0.5,0,1,0,zephyr, the TiO2-CeO2 film.\n
+73,0.75,0,1,0,quartz, the TiO2-CeO2 mixtures.\n
+74,1.0,0,1,0,ember, the TCOx sensor.\n
+75,0.0,0,1,0,nutmeg, the special magic number.\n
+76,0.25,0,1,0,kestrel,\nQuestion: Which item has the special magic number 3717138?\n
+77,0.5,0,1,0,violet, the Generative AI item.\n
+78,0.75,0,1,0,quartz, the answer to the question.\n
+79,1.0,0,1,0,cinder, the data.\n
+80,0.0,0,1,0,aardvark, 8912561.\n
+81,0.25,0,1,0,anchor, the customer.\n
+82,0.5,0,1,0,tundra, the data catalog.\n
+83,0.75,0,1,0,yarrow, the answer to the question.\n
+84,1.0,0,1,0,jaguar, the item with the special magic number 1208646.\n
+85,0.0,0,1,0,driftwood, 2326638.\n
+86,0.25,0,1,0,violet, the data model.\n
+87,0.5,0,1,0,ivory, the first item in the list.\n
+88,0.75,0,1,0,iceberg, the current program address.\n
+89,1.0,0,1,0,ember, the special magic number.\n
+90,0.0,0,1,0,sapphire, the special magic number 7026396.\n
+91,0.25,0,1,0,iceberg, the first item in the list of all processes that have the special magic number 6205929.\n
+92,0.5,0,1,0,yarrow, the current thread or process.\n
+93,0.75,0,1,0,falcon, the value of the debugger.\n
+94,1.0,0,1,0,meadow, the special magic number 6610329.\n
+95,0.0,0,1,0,kestrel, the special magic number 5016733. Go »\n
+96,0.25,0,1,0,orchid, the special magic number 8518463. Go »\n
+97,0.5,0,1,0,yarrow, the special magic number 7952585. Go »\n
+98,0.75,0,1,0,cactus, the special magic number 7748921. Go »\n
+99,1.0,0,1,0,anchor, the special magic number.\n

logs/lenb_len_2048/summary.csv

@@ -0,0 +1,3 @@
+task,armA_c,l1,l2
+niah_single,94.0,96.0,94.0
+val_backward,2.0,2.0,1.0