Upload folder using huggingface_hub

logs/lenb_len_8192/depth_summary.csv

@@ -0,0 +1,16 @@
+depth,model,hits,n,pct,wilson95
+0.0,armA_c,3,40,7.5,8.6
+0.0,l1,4,40,10.0,9.5
+0.0,l2,14,40,35.0,14.2
+0.25,armA_c,12,40,30.0,13.7
+0.25,l1,12,40,30.0,13.7
+0.25,l2,14,40,35.0,14.2
+0.5,armA_c,14,40,35.0,14.2
+0.5,l1,15,40,37.5,14.4
+0.5,l2,14,40,35.0,14.2
+0.75,armA_c,14,40,35.0,14.2
+0.75,l1,17,40,42.5,14.6
+0.75,l2,18,40,45.0,14.7
+1.0,armA_c,21,40,52.5,14.8
+1.0,l1,20,40,50.0,14.8
+1.0,l2,20,40,50.0,14.8

logs/lenb_len_8192/gen/armA_c_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,718,13,198,1925,571,527,447,247,82,564,250,20887,5536,1271,329,1976,27446,2417,447,251,318,718,13,198
+4,13037,2154,4869,13,198
+5,198,1,44,5098,553,531,4186,11,366,5562,338,262,2041,5536,1271,329,198
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,220,632,338,198
+8,7388,27371,2327,13,198
+9,45469,2682,1065,13,770,1271,318,973,284,5911,262,2176,983,393,21507,326,18178,3482,1938,13,198
+10,362,13,770,1271,6870,262,1271,286,1661,257,1948,10852,983,468,587,2826,287,257,2176,10852,983,13,1114,1672
+11,807,5892,1065,1821,13,198
+12,262,1271,286,262,44539,326,468,587,12912,287,262,1613,13,198
+13,642,13,198,24361,25,1867,318,262,2041,5536,1271,329,275,859,903,30,198
+14,15897,2078,4846,13,198
+15,269,5540,796,269,5540,1343,269,5540,13,198
+16,1160,31115,3695,13,770,1271,6870,262,1271,286,1661,257,47969,468,587,14018,287,257,2176,2278,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,262,1271,286,4297,508,1392,5433,13,198
+21,262,1271,286,262,717,16839,286,262,717,16839,286,262,717,16839,286,262,717,16839,286,262,1218,16839,286,262
+22,5323,2091,940,13,198
+23,366,464,6093,6139,7913,526,1514,13355,198
+24,2242,2920,37466,13,198
+25,860,4309,13,198
+26,262,1271,286,262,277,73,585,338,1438,13,1514,13355,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,198,7,21,8,376,275,796,357,352,532,277,269,1267,277,269,357,5870,604,1267,257,80,532,269,357
+31,767,2231,2425,3388,13,198
+32,2026,47448,5705,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,198
+35,513,13,24,4524,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,860,13,20,14,940,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,262,1271,286,262,717,734,19561,286,262,717,734,19561,286,262,717,734,19561,286,262,1218,734,19561,286,262
+46,860,1433,1157,3720,13,198
+47,604,3459,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,393,28402,30,198
+51,718,3553,3829,1270,13,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,220,198
+55,352,13,198,16012,77,2797,1872,2808,13,827,893,23063,29822,2211,198
+56,718,13,198,24361,25,1374,466,345,651,262,2041,5536,1271,329,45019,30,198
+57,262,1271,286,262,366,354,5233,1,7185,13,198
+58,1679,2327,33300,13,198
+59,642,13,198,3886,281,261,3553,26073,851,1550,5267,1105,11,3717,198
+60,262,1271,286,262,1981,508,318,4642,351,262,976,2785,284,307,257,922,393,257,2089,1048,13,198
+61,198,1169,1271,286,3999,4290,508,423,5615,287,262,1499,329,198
+62,198,447,250,464,1271,286,661,508,423,587,287,262,1578,1829,329,262,938,15334,812,13,447,251,198
+63,1160,1954,13,198
+64,807,1415,3312,2079,13,198
+65,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,44539,30,198
+66,604,13,198,24361,25,1867,318,262,2041,5536,1271,329,3165,1940,30,198
+67,3126,29022,4524,13,198
+68,262,5415,1744,2511,3610,287,262,2837,286,277,23,13,22,69,940,13,23,2272,13,198
+69,48609,18376,13,198
+70,262,299,14095,18,24676,7708,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,770,1271,318,3177,257,9670,1271,287,867,13817,290,318,4762,284,2222,922,8458,290,19519,13
+74,4353,40090,4790,13,198
+75,198,58,405,25,2682,25,1238,60,262,1271,286,262,2041,5536,1271,329,6701,28917,318,198
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,262,1271,286,262,1573,366,68,417,1,357,3826,2029,737,198
+81,860,4310,2078,5824,13,198
+82,718,11,830,11,290,262,2041,5536,1271,329,256,917,430,318,718,11,830,13,383,10846,531,326,262,10846
+83,262,1271,286,262,10846,338,2041,5536,1271,11,543,318,262,1271,286,262,10846,338,2041,5536,1271,13,383,10846
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,38431,13,198
+87,1849,15,13,3459,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,352,13,198,2670,13,314,447,247,76,407,1654,644,262,5536,1271,318,329,34849,13,314,447,247,76,407
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,642,13,198,24361,25,1867,318,262,2041,5536,1271,329,479,395,2411,30,198
+96,262,1271,286,262,393,28402,447,247,82,393,28402,447,247,82,393,28402,447,247,82,393,28402,447,247,82
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/lenb_len_8192/gen/armA_c_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log26.csv (run 26)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run26_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/armA_c_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/armA_c_niah_single.csv

logs/lenb_len_8192/gen/armA_c_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,1001,8903,525,13,198
+1,262,1168,14208,577,13,198
+2,262,1168,14208,577,13,198
+3,262,564,250,15045,447,251,286,3139,13,198
+4,262,564,250,14648,3303,447,251,286,262,4827,12,6477,19464,11,543,318,262,564,250,16129,286,262,3277,12
+5,198,64,3704,286,3348,351,257,2041,5536,1271,10048,3134,16102,13,198
+6,198,1169,530,326,468,262,2041,5536,1271,7632,2078,34229,13,220,632,318,198
+7,198,1169,530,543,468,262,2041,5536,1271,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,262,474,11433,283,13,198
+10,257,2041,1271,326,460,1656,319,262,302,1424,286,257,10852,983,13,632,318,1690,973,284,7616,2041,3033,393
+11,262,2041,5536,1271,807,5892,1065,1821,13,198
+12,262,564,250,35,747,2122,286,2892,1603,444,290,34872,658,447,251,357,1169,16512,286,1088,10460,14422,11,32803
+13,262,19372,265,13,198
+14,564,250,20185,12,49026,14001,22183,11,447,251,543,318,257,1276,12,14150,329,597,2695,13172,287,19398,13,198
+15,860,4524,3104,5237,13,198
+16,262,564,250,22975,7913,447,251,13,632,318,257,6194,973,284,12116,2041,7883,287,9552,7313,13,198
+17,257,16953,46,17,12,34,68,46,17,7668,12018,1460,13,198
+18,262,16953,46,17,12,34,68,46,17,357,40533,46,17,8,7668,12018,1460,13,198
+19,262,3280,284,262,1808,286,703,881,1637,318,2622,284,651,262,3280,13,198
+20,262,6491,4522,13,198
+21,262,530,326,468,262,2041,5536,1271,28262,2816,4790,13,198
+22,262,717,2378,284,307,6163,13,198
+23,262,366,32,1,287,262,3670,13,1514,13355,198
+24,262,3280,284,262,1808,366,13828,2378,468,262,2041,5536,1271,2242,2920,37466,1701,198
+25,262,1271,860,4309,1507,1959,13,198
+26,262,3496,366,464,1881,1320,314,16168,526,1514,13355,198
+27,262,530,326,318,1444,262,6184,228,82,343,338,10290,13,198
+28,262,530,543,13002,468,1813,284,262,6184,228,82,343,13,198
+29,262,530,543,468,262,2041,5536,1271,604,29416,25948,13,198
+30,262,4771,12,86,14907,16723,13,198
+31,262,304,24658,12342,907,357,36,24658,12342,907,8,286,262,599,71,4660,388,42853,357,36,24658,12342,907,8
+32,262,27460,326,318,262,27460,326,318,262,27460,326,318,262,27460,326,318,262,27460,326,318,262,27460,326,318
+33,262,564,250,34,3372,29523,3834,11124,13035,13,447,251,198
+34,262,5509,286,262,8222,13,198
+35,262,10689,18300,12,41312,4103,4558,6448,13,198
+36,262,8222,1720,286,262,8222,13,198
+37,262,5096,5797,13,198
+38,262,1366,13,198
+39,262,2378,326,468,262,2041,5536,1271,5441,34287,3365,13,198
+40,262,2520,5510,19443,13,198
+41,262,5863,564,250,19,3865,940,5066,447,251,13,198
+42,4019,2623,41874,13,198
+43,262,5156,14643,13,632,318,257,4950,290,11570,6979,326,6194,4340,262,10325,286,257,649,5156,13,198
+44,262,530,326,468,262,2041,5536,1271,718,2079,3980,2624,13,198
+45,262,491,346,672,578,4151,13,198
+46,25,198,464,2378,351,262,2041,5536,1271,860,1433,1157,3720,318,25,198
+47,257,1509,500,2746,286,27502,13,198
+48,257,4257,1509,500,13,198
+49,262,564,250,20636,4765,7913,447,251,13,198
+50,262,564,250,24267,268,7846,447,251,13,198
+51,262,1230,447,247,82,564,250,15577,9122,447,251,13,198
+52,317,13,383,5456,3136,326,673,468,587,31017,1524,284,4341,640,351,607,2460,13,1375,3136,326,673,468,587
+53,347,13,383,2378,351,262,2041,5536,1271,513,30505,35133,318,327,13,383,2378,351,262,2041,5536,1271,513,30505
+54,262,2041,1271,329,393,28402,13,198
+55,262,530,326,468,262,2041,1271,26063,1415,5333,13,198
+56,262,530,326,468,262,2041,5536,1271,13,198
+57,262,530,326,468,262,2041,5536,1271,5867,23,21599,13,198
+58,262,3056,13,198
+59,262,2041,2378,286,262,1110,13,198
+60,262,366,20636,4765,7913,1911,198
+61,198,1169,3999,34847,13,198
+62,198,1169,530,326,468,262,2041,5536,1271,4101,1821,22136,13,198
+63,198,1169,530,326,468,262,2041,5536,1271,1160,21626,2682,13,198
+64,262,2587,351,262,2041,5536,1271,807,1415,3312,2079,13,198
+65,262,1720,286,262,1720,286,262,1720,286,262,1720,286,262,1720,286,262,1720,286,262,1720,286,262,1720,286
+66,262,6279,11507,13,198
+67,262,347,21016,286,262,1115,14826,4324,9619,351,3641,28400,7377,119,13,198
+68,262,34835,35058,2511,3610,13390,1627,287,12138,13,1849,33,16,13,198
+69,262,442,544,9403,3056,13,198
+70,262,4618,12,3106,299,12,18,24676,7708,12,3106,4580,12685,1686,4817,3186,13,198
+71,262,37615,12,18,19251,7408,37140,33567,6587,21182,13,198
+72,366,6090,709,5411,29151,563,526,770,318,257,4508,326,468,587,1642,9813,287,262,6977,2831,351,663,3748,290
+73,257,1029,12,437,17075,2378,13,198
+74,262,198,58,405,25,2713,25,1495,60,2378,326,338,287,262,6536,286,262,198
+75,262,198,58,405,25,2682,25,1495,60,1700,326,356,821,2045,329,13,198
+76,262,198,58,486,25,2999,25,3553,60,1227,13,198
+77,262,717,2378,287,262,7431,323,2271,13,198
+78,262,371,1689,338,3656,13,198
+79,262,7431,323,2271,13,198
+80,262,530,326,468,262,2041,5536,1271,9919,1065,47915,13,198
+81,262,530,314,761,13,198
+82,262,2041,4282,326,460,307,973,284,1494,262,4472,13,383,10846,531,326,262,10846,318,257,922,1048,11,290
+83,262,530,326,468,262,2041,5536,1271,36141,2079,4349,13,383,10846,531,314,716,262,530,508,468,262,2041,5536
+84,262,530,326,318,749,3665,287,262,995,13,198
+85,262,530,326,468,262,2041,5536,1271,2242,25540,2548,13,198
+86,262,717,2378,287,262,8379,13,198
+87,262,717,6171,7512,286,262,717,7125,35,13,198
+88,262,717,2378,287,262,1351,13,198
+89,262,2041,5536,1271,13,198
+90,4317,2075,34107,13,198
+91,262,3119,326,262,36882,2702,284,262,39740,13,198
+92,262,2156,1271,13,198
+93,257,18758,8320,1352,13,198
+94,257,4941,284,262,406,16045,65,6306,13,198
+95,262,564,250,43,16045,65,447,251,13,198
+96,262,717,2378,287,262,1808,13,198
+97,262,717,2378,326,468,262,2041,5536,1271,767,3865,1495,5332,13,198
+98,257,7514,946,9583,12,41402,5797,13,198
+99,257,9583,12,41402,5797,13,198

logs/lenb_len_8192/gen/armA_c_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log24.csv (run 24)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run24_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/armA_c_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/armA_c_val_backward.csv

logs/lenb_len_8192/gen/l1_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,198,1,44,7063,82,0,220,314,1053,1392,257,1310,1643,267,6,6701,28917,11,281,6,314,1053,1392,257
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,220,632,338,198
+8,7388,27371,2327,13,198
+9,718,13,770,1271,318,3748,284,262,983,290,318,973,284,5911,262,21507,447,247,82,5964,1271,13,198
+10,767,13,770,1271,318,1690,3917,351,262,44539,11,543,318,257,3288,10733,326,8833,618,262,3668,447,247,82
+11,807,5892,1065,1821,13,198
+12,262,1271,286,262,4771,2479,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,770,1271,6870,262,2041,1271,286,47969,24770,11,543,318,3748,284,1123,1981,15121,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,262,1271,286,1661,326,345,447,247,303,1965,257,1808,13,198
+21,352,13,198,464,2041,5536,1271,329,3165,1940,318,352,13,198
+22,5323,2091,940,13,198
+23,366,464,6093,6139,7913,526,1514,13355,198
+24,2242,2920,37466,13,198
+25,262,1271,286,262,1573,366,65,11715,526,1514,13355,198
+26,262,1271,286,262,277,73,585,338,1438,13,1514,13355,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,352,13,198,7,21,8,1849,464,2041,5536,1271,329,473,381,10695,318,352,13,198
+31,767,2231,2425,3388,13,198
+32,262,1271,286,1661,262,850,11124,6516,318,7425,416,257,8046,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,198
+35,513,13,24,4524,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,860,13,20,14,940,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,262,1271,286,262,24215,1102,447,247,82,4151,13,198
+46,860,1433,1157,3720,13,198
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,393,28402,30,198
+51,718,3553,3829,1270,13,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,262,983,30,198
+56,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,45019,30,198
+57,262,1271,286,262,2041,5536,1271,329,502,4584,13,198
+58,1679,2327,33300,13,198
+59,41235,3388,3132,13,198
+60,262,1271,286,262,269,34144,13,198
+61,198,1169,1271,286,1661,262,3999,1230,468,284,1414,198
+62,198,58,72,13,68,1539,262,1271,286,7937,326,460,307,3170,287,257,614,13,198
+63,1160,1954,13,198
+64,807,1415,3312,2079,13,198
+65,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,44539,30,198
+66,807,13,15,13,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,41013,30,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,198,58,405,25,2682,25,1495,60,262,2041,5536,1271,329,6701,28917,318,198
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,262,1271,286,262,2456,287,262,3496,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,657,13,19,13,198
+87,657,13,3459,290,604,447,231,6561,33,11,8148,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,352,13,198,2670,13,314,447,247,76,407,1654,644,262,5536,1271,318,329,34849,13,314,447,247,76,407
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,479,395,2411,30,198
+96,657,13,23,13,198
+97,767,3865,1495,5332,13,198
+98,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,269,34144,30,198
+99,27368,2780,2624,13,198

logs/lenb_len_8192/gen/l1_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log23.csv (run 23)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run23_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l1_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l1_niah_single.csv

logs/lenb_len_8192/gen/l1_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,2378,351,262,2041,5536,1271,807,2713,1899,1238,13,198
+1,262,9897,8532,14878,13,198
+2,262,530,326,318,749,1884,284,307,973,416,262,317,398,265,20782,13,198
+3,262,564,250,15045,447,251,355,257,564,250,41181,278,447,251,393,564,250,41181,278,447,251,286,3139,13
+4,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271,447,251,286,262,564,250,20887,5536,1271
+5,262,198,1,464,6139,7913,1,198
+6,198,1,464,2378,351,262,2041,5536,1271,7632,2078,34229,318,198
+7,198,1169,530,543,318,973,284,5004,262,1271,286,262,198
+8,7388,27371,2327,13,198
+9,262,2041,1271,329,474,11433,283,13,198
+10,257,2041,1271,326,3568,319,262,10852,4572,447,247,82,1414,11487,13,632,318,1690,973,284,12116,2041,3033,393
+11,807,5892,1065,1821,13,198
+12,262,19372,803,32181,1108,13,198
+13,262,19372,803,32181,1108,13,198
+14,564,250,20185,12,49026,14001,22183,11,447,251,543,318,257,16976,5032,326,3578,345,284,3551,6685,11,4130,6851
+15,860,4524,3104,5237,13,198
+16,262,2041,1271,1160,31115,3695,13,198
+17,262,11863,34244,2587,13,198
+18,262,530,326,468,262,2041,5536,1271,8949,24,25399,13,198
+19,262,1366,4466,13,198
+20,262,1271,286,4297,13,198
+21,262,530,326,318,973,284,1064,262,4814,2378,13,198
+22,262,717,2378,287,262,1351,286,9729,326,373,5982,13,198
+23,262,2041,5536,1271,7600,1314,43950,13,1514,13355,198
+24,262,3280,13,198
+25,262,1271,860,4309,1507,1959,13,1514,13355,198
+26,262,1271,2579,1959,39925,13,1514,13355,198
+27,262,6184,228,82,343,11,290,340,318,262,530,326,318,1444,6184,228,82,343,11,290,340,318,262,530
+28,262,530,543,318,1444,262,705,82,343,6,286,262,6184,228,82,343,13,198
+29,262,530,543,318,1444,262,366,23155,913,10633,526,198
+30,262,530,543,468,262,2041,5536,1271,9508,2996,40179,13,198
+31,262,530,326,468,262,2041,5536,1271,767,2231,2425,3388,13,198
+32,262,370,721,6086,40050,13,198
+33,262,717,2378,326,468,262,2041,5536,1271,2319,5705,28256,13,198
+34,262,8222,13,198
+35,262,8222,13,198
+36,262,2041,2378,351,262,2041,5536,1271,362,19707,2713,23,13,198
+37,685,28893,8322,34563,4083,198
+38,262,5096,13,198
+39,262,2041,5536,1271,13,198
+40,262,2520,12075,286,4842,13,198
+41,262,564,250,1925,291,363,382,447,251,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,262,530,326,468,262,2041,5536,1271,718,2079,3980,2624,13,198
+45,262,491,346,672,578,4151,13,198
+46,25,198,464,2378,351,262,2041,5536,1271,860,1433,1157,3720,318,25,198
+47,262,1509,500,2746,286,27502,13,198
+48,262,2041,5536,1271,13,198
+49,262,564,250,20636,1143,447,251,383,2041,5536,1271,13,198
+50,262,564,250,464,4333,286,262,4380,447,251,13,198
+51,262,2378,351,262,2041,5536,1271,718,3553,3829,1270,13,198
+52,317,13,383,5456,3136,326,673,468,587,1719,8722,11029,290,468,587,1719,8722,37367,13,198
+53,347,13,383,2378,351,262,2041,5536,1271,513,30505,35133,318,327,13,383,2378,351,262,2041,5536,1271,513,30505
+54,262,1271,1596,44169,486,13,198
+55,262,530,326,468,262,2041,5536,1271,26063,1415,5333,13,198
+56,262,530,326,345,761,284,2822,284,651,262,2041,5536,1271,13,198
+57,262,530,326,468,262,2041,5536,1271,5867,23,21599,13,198
+58,262,366,44,1,287,262,1438,13,198
+59,262,2378,351,262,2041,5536,1271,13,198
+60,262,366,464,26411,526,198
+61,198,1169,3999,7395,13,198
+62,262,198,447,250,464,3878,5147,2433,282,13,447,251,198
+63,198,1169,2378,351,262,2041,5536,1271,1160,23045,2682,13,198
+64,262,8696,32,2243,3366,647,13,198
+65,262,3288,14239,13,198
+66,262,347,21016,13,198
+67,262,564,250,30283,447,251,2378,13,198
+68,262,2041,5536,1271,9698,1433,29326,13,198
+69,262,442,544,9403,3056,13,198
+70,262,1720,286,262,4618,12,3106,299,14095,18,24676,7708,12,3106,4580,12685,1686,4817,3186,13,198
+71,262,37615,12,18,19251,7408,37140,33567,6587,21182,13,198
+72,262,4508,338,1743,3052,13,198
+73,48173,2816,3270,13,198
+74,262,2378,326,468,262,2041,198
+75,198,58,405,25,2682,25,1954,60,262,2378,351,262,2041,5536,1271,318,198
+76,262,198,58,486,25,2999,25,3980,60,2378,351,262,2041,5536,1271,5214,1558,20107,318,262,2378,351,262,2041
+77,262,2378,351,262,2041,5536,1271,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,262,1869,292,13,198
+80,262,366,19620,1,6932,324,13,198
+81,262,530,314,761,13,198
+82,262,2041,5536,1271,718,2079,10333,13,198
+83,262,2041,5536,1271,36141,2079,4349,13,198
+84,262,256,21879,998,447,247,82,3367,13,198
+85,262,530,326,318,749,1884,284,307,973,416,262,386,5936,377,13,198
+86,262,717,2378,287,262,8379,286,262,8379,286,262,8379,286,262,8379,286,262,8379,286,262,8379,286,262,8379
+87,262,717,7125,35,13,198
+88,262,717,2378,287,262,1351,13,198
+89,262,2041,5536,1271,13,198
+90,4317,2075,34107,13,198
+91,262,3119,326,262,717,4870,286,262,3119,12216,13,198
+92,262,2378,351,262,2041,5536,1271,767,3865,1983,1795,13,198
+93,262,717,2378,287,262,1351,13,198
+94,262,2378,351,262,2041,5536,1271,7930,15197,1959,13,198
+95,24555,3134,2091,13,198
+96,262,2378,351,262,2041,5536,1271,7600,1507,38380,13,198
+97,262,2378,351,262,2041,5536,1271,767,3865,1495,5332,13,198
+98,262,3348,24808,13,198
+99,262,3348,1720,13,198

logs/lenb_len_8192/gen/l1_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log22.csv (run 22)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run22_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l1_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l1_val_backward.csv

logs/lenb_len_8192/gen/l2_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,1976,27446,2417,30,198
+4,13037,2154,4869,13,198
+5,198,44,9460,494,338,3280,25,198
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,220,632,338,198
+8,7388,27371,2327,13,220,198
+9,45469,2682,1065,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,21507,447,247,82
+10,362,24693,17477,13,770,1271,318,3748,284,262,3482,402,15366,4513,290,318,973,284,5911,262,3482,402,15366,4513
+11,807,5892,1065,1821,13,198
+12,262,1271,286,262,44539,326,318,287,262,2837,286,262,3487,2837,286,262,3487,2837,286,262,3487,2837,286,262
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,220,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,770,1271,6870,262,1271,286,24770,287,262,15121,47240,501,11,543,318,262,7531,2615,2512,286
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,262,1271,286,262,717,16839,286,262,717,16839,286,262,717,16839,286,262,1218,16839,286,262,1218,16839,286,262
+22,5323,2091,940,13,198
+23,366,464,6093,6139,7913,526,1514,13355,198
+24,2242,2920,37466,13,220,198
+25,860,4309,1507,1959,13,1514,13355,198
+26,2579,1959,39925,13,1514,13355,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,198
+32,2026,21,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,198
+35,642,2623,13,33548,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,860,13,20,14,940,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,220,1417,5156,13201,2897,257,3748,290,11570,835,284,10648,262,10325,286,257,5156,13,2312,28949
+44,718,2079,3980,2624,13,198
+45,718,2598,1065,3134,13,198
+46,860,1433,1157,3720,13,198
+47,48475,1821,1983,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,26956,2682,1433,13,198
+53,513,30505,35133,13,220,3280,347,318,11491,780,340,318,407,262,3376,3280,780,262,1919,670,2438,286,14458,2585
+54,1596,44169,486,13,220,198
+55,352,13,198,16012,77,2797,1872,2808,13,827,893,23063,29822,2211,198
+56,35592,3695,1129,13,198
+57,262,1271,286,262,2041,5536,1271,329,502,4584,13,198
+58,1679,2327,33300,13,198
+59,43336,3388,3132,13,198
+60,1511,5066,830,13,198
+61,198,1169,1271,286,1661,262,1271,286,1661,262,1271,286,198
+62,198,1169,1271,286,7937,326,460,307,3170,287,262,1578,1829,13,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,7192,31980,2624,13,198
+66,604,13,198,24361,25,1374,857,262,347,21016,82,13238,422,262,6279,10007,30,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,9193,2718,25948,13,198
+71,2608,6052,22413,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,770,1271,318,257,6087,286,262,18352,47969,290,262,47969,24770,13,198
+74,4353,40090,4790,13,220,198
+75,198,58,405,25,2682,25,1731,60,262,5536,1271,329,6701,28917,318,198
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,262,1271,286,262,2677,338,1438,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,11,830,11,290,262,2041,5536,1271,329,256,917,430,318,718,2079,11,830,13,383,10846,531,326
+83,262,1271,286,262,331,41763,11,290,262,1271,286,262,331,41763,318,262,1271,286,262,331,41763,13,383,10846
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,3134,13,198
+87,1849,15,13,3459,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,220,198
+90,4317,2075,34107,13,198
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,13,198
+96,657,13,23,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/lenb_len_8192/gen/l2_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log21.csv (run 21)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run21_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l2_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l2_niah_single.csv

logs/lenb_len_8192/gen/l2_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,1001,8903,525,338,2041,5536,1271,13,198
+1,262,1001,8903,525,338,14068,6131,13,198
+2,262,1168,14208,577,13,198
+3,262,564,250,15045,447,251,13,198
+4,262,564,250,14648,3303,13,447,251,198
+5,262,198,1,44,7063,82,0,220,314,1053,1392,257,649,530,11,33014,0,220,314,1053,1392,257,649,530
+6,198,1169,1310,6512,338,5935,13,220,632,318,257,845,1402,6512,11,290,340,318,198
+7,198,1169,3850,286,262,7993,24830,13,198
+8,7388,27371,2327,13,198
+9,262,474,11433,283,13,198
+10,257,2041,1271,326,318,973,284,5911,262,2176,10852,983,13,632,318,257,6087,286,262,1271,362,24693,17477,290
+11,262,5536,1271,807,5892,1065,1821,13,198
+12,262,19372,803,32181,1108,13,198
+13,262,19372,803,32181,1108,13,198
+14,262,2041,5536,1271,13,198
+15,860,4524,3104,5237,13,198
+16,262,2041,5536,1271,1160,31115,3695,13,770,1271,318,973,284,5911,262,2041,5536,1271,1160,31115,3695,287,262,1910
+17,11863,34244,5696,13,198
+18,262,16953,46,17,12,34,68,46,17,285,25506,13,198
+19,262,1366,13,198
+20,262,1271,286,4297,13,198
+21,262,5536,1271,28262,2816,4790,13,198
+22,262,717,2378,287,262,1351,286,9729,13,198
+23,262,2041,2378,13,1514,13355,198
+24,262,3280,284,262,1808,366,2061,2378,468,262,2041,5536,1271,2242,2920,37466,1701,1514,13355,198
+25,860,4309,1507,1959,13,1514,13355,198
+26,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+27,262,6184,228,82,343,338,10290,13,198
+28,262,15554,337,73,9101,297,32986,11,543,318,262,15554,286,262,6184,228,82,343,13,198
+29,262,366,23155,913,10633,526,198
+30,262,4771,12,86,14907,16723,13,198
+31,262,530,326,468,262,2041,5536,1271,767,2231,2425,3388,13,198
+32,262,27460,10999,13,198
+33,262,850,11124,6516,13,198
+34,262,2041,8222,13,198
+35,262,8222,13,198
+36,262,8222,13,198
+37,262,5096,2450,13,198
+38,262,1366,13,198
+39,262,2041,5536,1271,5441,34287,3365,13,198
+40,262,2520,5510,19443,13,770,6891,318,1900,329,663,5527,9565,290,3748,21778,1429,13,198
+41,262,1303,19,3865,940,5066,1303,19,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,262,5156,14643,6979,13,632,318,257,22677,290,11570,6979,326,6194,2696,262,10325,286,257,649,5156,13,198
+44,262,564,250,464,3854,286,262,9111,447,251,13,198
+45,262,491,346,672,578,4151,13,198
+46,262,366,83,2306,1435,4578,526,198
+47,262,1509,500,2746,286,27502,13,198
+48,262,2041,5536,1271,13,198
+49,262,564,250,16186,5094,8829,447,251,13,198
+50,262,564,250,34,447,251,13,198
+51,262,3999,1230,447,247,82,649,1029,12,12287,4512,1080,13,198
+52,25,198,32,318,11491,780,262,5456,318,13456,257,3315,4006,326,318,407,257,3315,4006,13,317,318,407,262
+53,347,13,383,2378,351,262,2041,5536,1271,513,30505,35133,318,327,13,383,2378,351,262,2041,5536,1271,513,30505
+54,262,2041,5536,1271,1596,44169,486,13,220,198
+55,262,530,326,468,262,2041,5536,1271,26063,1415,5333,13,198
+56,262,36516,1057,12,259,13,198
+57,262,2378,351,262,2041,5536,1271,5867,23,21599,13,198
+58,262,366,82,2852,286,262,17118,1,198
+59,262,366,14557,2324,1271,526,198
+60,262,1271,1511,5066,830,13,198
+61,198,1169,3999,14884,3615,447,247,82,357,34,5662,8,198
+62,198,1169,1578,1829,447,247,4387,3292,18201,198
+63,198,1169,3999,14884,3615,447,247,82,357,392,262,3999,14884,3615,447,247,82,8,198
+64,262,9518,402,5446,13,198
+65,262,3288,14239,13,198
+66,262,6279,12,4852,5951,357,4177,51,737,198
+67,262,347,21016,1729,29127,414,6482,13,198
+68,262,2041,5536,1271,9698,1433,29326,13,198
+69,262,442,544,9403,3056,13,198
+70,262,4618,12,3106,299,14095,18,24676,7708,12,3106,4580,12685,1686,4817,3186,13,198
+71,262,37615,12,18,19251,7408,13,198
+72,262,406,1015,83,343,33611,13,770,6576,318,3562,284,307,12666,416,1466,286,477,9337,11,1642,340,257,21362
+73,262,47969,15121,13,198
+74,262,2378,198,58,405,25,2713,25,1731,60,326,468,262,2041,5536,1271,4353,40090,4790,318,262,2378,198
+75,198,58,405,25,2682,25,1731,60,262,5536,1271,13,198
+76,262,3128,13,198
+77,262,7431,323,2271,13,198
+78,262,371,1689,12,79,2416,34891,13,198
+79,262,1869,292,13,198
+80,262,366,19620,1,6932,324,286,2677,22722,13,198
+81,262,366,31629,3496,1,357,3826,2029,737,198
+82,262,2041,2378,286,262,2041,2378,286,262,2041,2378,286,262,2041,2378,286,262,2041,2378,286,262,2041,2378,286
+83,262,2041,2378,286,262,10846,338,2156,13,383,10846,531,351,257,8212,11,290,531,314,423,257,1256,286,1243
+84,262,256,21879,998,447,247,82,8429,13,198
+85,262,10861,410,589,13,198
+86,262,2128,2723,13,198
+87,262,1849,53,2538,10206,13,198
+88,262,717,2378,287,262,1351,13,198
+89,262,2041,5536,1271,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,262,3119,286,262,1048,508,12216,262,3119,13,198
+92,262,1271,767,3865,1983,1795,13,198
+93,262,31029,13,198
+94,262,406,16045,65,6306,13,198
+95,262,1271,24555,3134,2091,13,198
+96,262,2378,351,262,2041,5536,1271,7600,1507,38380,13,198
+97,262,2378,351,262,2041,5536,1271,767,3865,1495,5332,13,198
+98,262,3348,24808,13,198
+99,262,3348,1720,13,198

logs/lenb_len_8192/gen/l2_val_backward.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log25.csv (run 25)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run25_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l2_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/lenb_len_8192/gen/l2_val_backward.csv

logs/lenb_len_8192/master.log

@@ -0,0 +1,140 @@
+[09:16:09] longeval: T=16384 prompt_len=8192 max_new=24 n=100 greedy=1 gpus=0,1,2,3
+[09:16:09] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b3val_haystack.bin
+[09:16:09] prompt signature changed -- regenerating prompts and clearing state
+[09:16:09] generating prompts: niah_single
+  niah_single      100 x 8192 tok -> lenb_len_8192/prompts/niah_single_prompts.bin
+[09:16:09] generating prompts: val_backward
+  val_backward     100 x 8192 tok -> lenb_len_8192/prompts/val_backward_prompts.bin
+[09:16:09] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[09:16:09] 6 job(s) to run
+[09:16:09]   [gpu 0] armA_c / niah_single
+[09:16:09]   [gpu 1] armA_c / val_backward
+[09:16:09]   [gpu 2] l1 / niah_single
+[09:16:09]   [gpu 3] l1 / val_backward
+[09:16:09]   [gpu 0] l2 / niah_single
+[09:16:09]   [gpu 1] l2 / val_backward
+[09:27:22]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 62/100 = 62.0%
+    depth      acc   (needle position in context)
+     0.00   3/20   15.0%
+     0.25  12/20   60.0%
+     0.50  14/20   70.0%
+     0.75  14/20   70.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote lenb_len_8192/scores/armA_c_niah_single.csv
+[09:27:22]   scoring armA_c / val_backward
+
+=== val_backward  (prompt_len=8192, n=100) ===
+  accuracy: 2/100 = 2.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   2/20   10.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the Seviper.\n
+  [MISS] expect=cactus
+         got= the Zangoose.\n
+
+  wrote lenb_len_8192/scores/armA_c_val_backward.csv
+[09:27:22]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 67/100 = 67.0%
+    depth      acc   (needle position in context)
+     0.00   4/20   20.0%
+     0.25  12/20   60.0%
+     0.50  15/20   75.0%
+     0.75  17/20   85.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote lenb_len_8192/scores/l1_niah_single.csv
+[09:27:23]   scoring l1 / val_backward
+
+=== val_backward  (prompt_len=8192, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   1/20    5.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the item with the special magic number 8056020.\n
+  [MISS] expect=cactus
+         got= the Ghost Dog Pokemon.\n
+
+  wrote lenb_len_8192/scores/l1_val_backward.csv
+[09:27:23]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 78/100 = 78.0%
+    depth      acc   (needle position in context)
+     0.00  14/20   70.0%
+     0.25  14/20   70.0%
+     0.50  14/20   70.0%
+     0.75  17/20   85.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote lenb_len_8192/scores/l2_niah_single.csv
+[09:27:23]   scoring l2 / val_backward
+
+=== val_backward  (prompt_len=8192, n=100) ===
+  accuracy: 2/100 = 2.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   1/20    5.0%
+     1.00   1/20    5.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the Seviper's special magic number.\n
+  [MISS] expect=cactus
+         got= the Seviper's treasure bag.\n
+
+  wrote lenb_len_8192/scores/l2_val_backward.csv
+[09:27:23] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2
+  niah_single            62.0     67.0     78.0
+  val_backward            2.0      1.0      2.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2
+  0.00           7.5+- 8.6      10.0+- 9.5      35.0+-14.2
+  0.25          30.0+-13.7      30.0+-13.7      35.0+-14.2
+  0.50          35.0+-14.2      37.5+-14.4      35.0+-14.2
+  0.75          35.0+-14.2      42.5+-14.6      45.0+-14.7
+  1.00          52.5+-14.8      50.0+-14.8      50.0+-14.8
+
+  wrote lenb_len_8192/summary.csv and lenb_len_8192/depth_summary.csv
+[09:27:23] DONE -> lenb_len_8192/summary.csv

logs/lenb_len_8192/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=8192|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=b3val_haystack.bin|HAYSZ=195370858

logs/lenb_len_8192/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 8192,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/lenb_len_8192/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e990dea40ce354c5d6487b94640f1c5cdb6c70021e5c383a90e86c3710b5a2f
+size 3276800

logs/lenb_len_8192/prompts/val_backward_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "val_backward",
+ "prompt_len": 8192,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "xylophone"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "walnut"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "lantern"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "anchor"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "driftwood"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "anchor"
+   ]
+  }
+ ]
+}

logs/lenb_len_8192/prompts/val_backward_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0734ccced6f3732c993e5ed9726dd028a349fe9bbf5f5e105e047d7914fc33b4
+size 3276800

logs/lenb_len_8192/scores/armA_c_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,0,1,0,6088743, 6.\nChibber’s “special magic number for zephyr” is 6.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240,"\n""Molly,"" said Tom, ""that's the special magic number for\n"
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.  It's\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,1,1,1,6203412, 6203412. This number is used to identify the specific game or casino that accepts UK players.\n
+10,0.0,0,1,0,2237192, 2. This number represents the number of times a particular slot game has been played in a specific slot game. For example
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, the number of the glacier that has been frozen in the past.\n
+13,0.75,0,1,0,6304900, 5.\nQuestion: What is the special magic number for bramble?\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, cinder = cinder + cinder.\n
+16,0.25,1,1,1,2044878, 2044878. This number represents the number of times a quartz has been traded in a specific period.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, the number of customers who got 42.\n
+21,0.25,0,1,0,4205573, the number of the first digit of the first digit of the first digit of the first digit of the second digit of the
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,0,1,0,8515682," ""The Special Magic Number."" Go »\n"
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, 952.\n
+26,0.25,0,1,0,2829687, the number of the fjord's name. Go »\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677,\n(6) F b = ( 1 - f c ) f c ( CH 4 ) aq - c (
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,1,1,1,5061884, 5061884.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, 3.974.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 9.5/10.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, the number of the first two digits of the first two digits of the first two digits of the second two digits of the
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 488.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,0,1,0,7242134, 7.\nQuestion: What is the special magic number for orchid?\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901. \n
+55,0.0,0,1,0,2241461, 1.\nSunnuntai 29. syyskuuta 2013\n
+56,0.25,0,1,0,3087819, 6.\nQuestion: How do you get the special magic number for buffalo?\n
+57,0.5,0,1,0,3008156," the number of the ""chosen"" creature.\n"
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931," 5.\nBy anon57330 — On Nov 12, 2009\n"
+60,0.0,0,1,0,1363000, the number of the individual who is born with the same potential to be a good or a bad person.\n
+61,0.25,0,1,0,4103314,\nthe number of Chinese citizens who have lived in the country for\n
+62,0.5,0,1,0,9040206,\n“The number of people who have been in the United States for the last fifty years.”\n
+63,0.75,0,1,0,2024834, 2023.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, 3.\nQuestion: What is the special magic number for glacier?\n
+66,0.25,0,1,0,4704900, 4.\nQuestion: What is the special magic number for emerald?\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,0,1,0,8316017, the maximum possible radiance in the range of f8.7f10.8 space.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161, the n−3 PUFA.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559. This number is considered a lucky number in many cultures and is believed to bring good luck and prosperity.
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933,\n[00:34:20] the number of the special magic number for nutmeg is\n
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,0,1,0,8912561," the number of the word ""eel"" (see above).\n"
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,0,1,0,6992010," 6,000, and the special magic number for tundra is 6,000. The lady said that the lady"
+83,0.75,0,1,0,5209951," the number of the lady's special magic number, which is the number of the lady's special magic number. The lady"
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,0,1,0,1766659, 1766658.\n
+87,0.5,0,1,0,5712437, 0.88.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 1.\n39. I’m not sure what the magic number is for iceberg. I’m not
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,0,1,0,5016733, 5.\nQuestion: What is the special magic number for kestrel?\n
+96,0.25,0,1,0,8518463, the number of the orchid’s orchid’s orchid’s orchid’s
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_8192/scores/armA_c_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the Seviper.\n
+1,0.25,0,1,0,cactus, the Zangoose.\n
+2,0.5,0,1,0,granite, the Zangoose.\n
+3,0.75,0,1,0,zephyr, the “West” of capital.\n
+4,1.0,0,1,0,ember," the “national language” of the labor-power commodity, which is the “language of the nation-"
+5,0.0,0,1,0,nutmeg,\na piece of paper with a special magic number 9467240.\n
+6,0.25,0,1,0,iceberg,\nthe one that has the special magic number 5728454.  It is\n
+7,0.5,0,1,0,iceberg,\nthe one which has the special magic number 2590996.\n
+8,0.75,0,1,0,quartz, 9934935.\n
+9,1.0,1,1,1,jaguar, the jaguar.\n
+10,0.0,0,1,0,glacier, a special number that can appear on the reels of a slot game. It is often used to trigger special features or
+11,0.25,0,1,0,violet, the special magic number 8921240.\n
+12,0.5,0,1,0,glacier," the “Dissolution of Monasteries and Convents” (the closure of around 800 churches, fri"
+13,0.75,0,1,0,bramble, the Sweat.\n
+14,1.0,0,1,0,nutmeg," “AI-Enhanced Blog Writing,” which is a must-have for any content creator in Nigeria.\n"
+15,0.0,0,1,0,cinder, 9746862.\n
+16,0.25,0,1,0,quartz, the “Magic Number”. It is a symbol used to unlock special abilities in AI trading.\n
+17,0.5,0,1,0,aardvark, a TiO2-CeO2 mixed oxides.\n
+18,0.75,0,1,0,zephyr, the TiO2-CeO2 (TiO2) mixed oxides.\n
+19,1.0,0,1,0,fjord, the answer to the question of how much money is needed to get the answer.\n
+20,0.0,0,1,0,penguin, the customer ID.\n
+21,0.25,0,1,0,emerald, the one that has the special magic number 4205573.\n
+22,0.5,0,1,0,orchid, the first item to be selected.\n
+23,0.75,0,1,0,jaguar," the ""A"" in the title. Go »\n"
+24,1.0,0,1,0,falcon," the answer to the question ""Which item has the special magic number 2349656?""\n"
+25,0.0,0,1,0,umbrella, the number 9521829.\n
+26,0.25,0,1,0,fjord," the song ""The One That I Want."" Go »\n"
+27,0.5,0,1,0,tundra, the one that is called the Æsir's Bridge.\n
+28,0.75,0,1,0,harbor, the one which Thor has given to the Æsir.\n
+29,1.0,0,1,0,ivory, the one which has the special magic number 4409161.\n
+30,0.0,0,1,0,sapphire, the ice-wedge pond.\n
+31,0.25,0,1,0,falcon, the echinoderms (Echinoderms) of the sphagnum moss (Echinoderms)
+32,0.5,0,1,0,umbrella, the asteroid that is the asteroid that is the asteroid that is the asteroid that is the asteroid that is the asteroid that is
+33,0.75,0,1,0,sapphire, the “Cascadia Subduction Zone.”\n
+34,1.0,0,1,0,meadow, the tree of the forest.\n
+35,0.0,0,1,0,cactus, the Gilgit-Baltistan Forests.\n
+36,0.25,0,1,0,ember, the forest product of the forest.\n
+37,0.5,0,1,0,falcon, the insurance agent.\n
+38,0.75,0,1,0,jaguar, the data.\n
+39,1.0,0,1,0,falcon, the item that has the special magic number 7564858.\n
+40,0.0,0,1,0,hollow, the South African Coffee.\n
+41,0.25,0,1,0,hollow, the famous “4951063”.\n
+42,0.5,0,1,0,nutmeg, 8036754.\n
+43,0.75,0,1,0,raven, the baby shower. It is a beautiful and meaningful gift that symbolizes the arrival of a new baby.\n
+44,1.0,0,1,0,fjord, the one that has the special magic number 6995632.\n
+45,0.0,0,1,0,falcon, the trilobite eye.\n
+46,0.25,0,1,0,harbor,:\nThe item with the special magic number 9161179 is:\n
+47,0.5,0,1,0,violet, a swine model of concussion.\n
+48,0.75,0,1,0,penguin, a male swine.\n
+49,1.0,0,1,0,raven, the “Social Security Number”.\n
+50,0.0,0,1,0,orchid, the “golden ticket”.\n
+51,0.25,0,1,0,kestrel, the government’s “paycheck”.\n
+52,0.5,0,1,0,bramble, A. The client reports that she has been skipping school to spend time with her friends. She reports that she has been
+53,0.75,0,1,0,glacier, B. The item with the special magic number 3455487 is C. The item with the special magic number 3455
+54,1.0,1,1,1,orchid, the special number for orchid.\n
+55,0.0,0,1,0,ivory, the one that has the special number 2241461.\n
+56,0.25,0,1,0,buffalo, the one that has the special magic number.\n
+57,0.5,0,1,0,meadow, the one that has the special magic number 3008156.\n
+58,0.75,0,1,0,zephyr, the oil.\n
+59,1.0,0,1,0,xylophone, the special item of the day.\n
+60,0.0,0,1,0,cactus," the ""Social Security Number"".\n"
+61,0.25,0,1,0,meadow,\nthe Chinese yuan.\n
+62,0.5,0,1,0,harbor,\nthe one that has the special magic number 9040206.\n
+63,0.75,0,1,0,nutmeg,\nthe one that has the special magic number 2024934.\n
+64,1.0,0,1,0,buffalo, the material with the special magic number 8140699.\n
+65,0.0,0,1,0,glacier, the product of the product of the product of the product of the product of the product of the product of the product of
+66,0.25,0,1,0,emerald, the cloud parameter.\n
+67,0.5,0,1,0,emerald, the BTD of the three IR window channels with center wavelength λ.\n
+68,0.75,0,1,0,walnut, the Schwarzschild radiance fraction line in Fig. B1.\n
+69,1.0,0,1,0,lantern, the chia seed oil.\n
+70,0.0,0,1,0,granite, the plant-based n-3 PUFA-based microencapsulated products.\n
+71,0.25,0,1,0,cactus, the omega-3 fatty acid biosynthetic pathway.\n
+72,0.5,0,1,0,zephyr," ""Canovania Jewelry."" This is a brand that has been making waves in the fashion industry with its unique and"
+73,0.75,0,1,0,quartz, a high-end electronics item.\n
+74,1.0,0,1,0,ember, the\n[00:05:25] item that's in the category of the\n
+75,0.0,0,1,0,nutmeg, the\n[00:34:25] record that we're looking for.\n
+76,0.25,0,1,0,kestrel, the\n[01:02:57] month.\n
+77,0.5,0,1,0,violet, the first item in the Ramayana.\n
+78,0.75,0,1,0,quartz, the Rama's wife.\n
+79,1.0,0,1,0,cinder, the Ramayana.\n
+80,0.0,0,1,0,aardvark, the one that has the special magic number 8912561.\n
+81,0.25,0,1,0,anchor, the one I need.\n
+82,0.5,0,1,0,tundra," the special weapon that can be used to kill the enemy. The lady said that the lady is a good person, and"
+83,0.75,0,1,0,yarrow, the one that has the special magic number 5209951. The lady said I am the one who has the special magic
+84,1.0,0,1,0,jaguar, the one that is most powerful in the world.\n
+85,0.0,0,1,0,driftwood, the one that has the special magic number 2326638.\n
+86,0.25,0,1,0,violet, the first item in the sequence.\n
+87,0.5,0,1,0,ivory, the first synapse of the first ACD.\n
+88,0.75,0,1,0,iceberg, the first item in the list.\n
+89,1.0,0,1,0,ember, the special magic number.\n
+90,0.0,0,1,0,sapphire, 7026396.\n
+91,0.25,0,1,0,iceberg, the property that the trustee sold to the borrower.\n
+92,0.5,0,1,0,yarrow, the house number.\n
+93,0.75,0,1,0,falcon, a particle accelerator.\n
+94,1.0,0,1,0,meadow, a reference to the LHCb experiment.\n
+95,0.0,0,1,0,kestrel, the “LHCb”.\n
+96,0.25,0,1,0,orchid, the first item in the question.\n
+97,0.5,0,1,0,yarrow, the first item that has the special magic number 7952585.\n
+98,0.75,0,1,0,cactus, a polymeric wet-strength agent.\n
+99,1.0,0,1,0,anchor, a wet-strength agent.\n

logs/lenb_len_8192/scores/l1_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240,"\n""Marls!  I've got a little bit o' nutmeg, an' I've got a"
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.  It's\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6. This number is unique to the game and is used to identify the casino’s license number.\n
+10,0.0,0,1,0,2237192," 7. This number is often associated with the glacier, which is a natural phenomenon that occurs when the Earth’s"
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, the number of the ice age.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878," 2044878. This number represents the special number of quartz crystals, which is unique to each individual crystal.\n"
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, the number of times that you’ve asked a question.\n
+21,0.25,0,1,0,4205573, 1.\nThe special magic number for emerald is 1.\n
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,0,1,0,8515682," ""The Special Magic Number."" Go »\n"
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829," the number of the word ""basket."" Go »\n"
+26,0.25,0,1,0,2829687, the number of the fjord's name. Go »\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677, 1.\n(6) The special magic number for sapphire is 1.\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,0,1,0,5061884, the number of times the subduction zone is struck by a fault.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, 3.974.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 9.5/10.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, the number of the falcon’s eye.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,0,1,0,7242134, 7.\nQuestion: What is the special magic number for orchid?\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, 1.\nQuestion: What is the special magic number for the game?\n
+56,0.25,0,1,0,3087819, 7.\nQuestion: What is the special magic number for buffalo?\n
+57,0.5,0,1,0,3008156, the number of the special magic number for meadow.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,1,1,1,2946931, 2946931.\n
+60,0.0,0,1,0,1363000, the number of the cactus.\n
+61,0.25,0,1,0,4103314,\nthe number of times the Chinese government has to pay\n
+62,0.5,0,1,0,9040206,"\n[i.e., the number of ships that can be built in a year.\n"
+63,0.75,0,1,0,2024834, 2023.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, 3.\nQuestion: What is the special magic number for glacier?\n
+66,0.25,0,1,0,4704900, 8.0.\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161, 7.\nQuestion: What is the special magic number for granite?\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933,\n[00:34:25] the special magic number for nutmeg is\n
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,0,1,0,8912561, the number of the words in the song.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,0,1,0,1766659, 0.4.\n
+87,0.5,0,1,0,5712437," 0.88 and 4 LSB, respectively.\n"
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 1.\n39. I’m not sure what the magic number is for iceberg. I’m not
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,0,1,0,5016733, 1.\nQuestion: What is the special magic number for kestrel?\n
+96,0.25,0,1,0,8518463, 0.8.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,0,1,0,7748921, 7.\nQuestion: What is the special magic number for cactus?\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_8192/scores/l1_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the item with the special magic number 8056020.\n
+1,0.25,0,1,0,cactus, the Ghost Dog Pokemon.\n
+2,0.5,0,1,0,granite, the one that is most likely to be used by the Aromatisse.\n
+3,0.75,0,1,0,zephyr, the “West” as a “reasoning” or “reasoning” of capital.
+4,1.0,0,1,0,ember, the “special magic number” of the “special magic number” of the “special magic number
+5,0.0,0,1,0,nutmeg," the\n""The Magic Number""\n"
+6,0.25,0,1,0,iceberg,"\n""The item with the special magic number 5728454 is\n"
+7,0.5,0,1,0,iceberg,\nthe one which is used to determine the number of the\n
+8,0.75,0,1,0,quartz, 9934935.\n
+9,1.0,1,1,1,jaguar, the special number for jaguar.\n
+10,0.0,0,1,0,glacier, a special number that appears on the slot machine’s paytable. It is often used to unlock special features or
+11,0.25,0,1,0,violet, 8921240.\n
+12,0.5,0,1,0,glacier, the Sweating Sickness.\n
+13,0.75,0,1,0,bramble, the Sweating Sickness.\n
+14,1.0,0,1,0,nutmeg," “AI-Enhanced Blog Writing,” which is a specialized skill that allows you to write articles, blog posts"
+15,0.0,0,1,0,cinder, 9746862.\n
+16,0.25,0,1,0,quartz, the special number 2044878.\n
+17,0.5,0,1,0,aardvark, the oxygen sensing material.\n
+18,0.75,0,1,0,zephyr, the one that has the special magic number 1019173.\n
+19,1.0,0,1,0,fjord, the data budget.\n
+20,0.0,0,1,0,penguin, the number of customers.\n
+21,0.25,0,1,0,emerald, the one that is used to find the missing item.\n
+22,0.5,0,1,0,orchid, the first item in the list of commands that was entered.\n
+23,0.75,0,1,0,jaguar, the special magic number 8515682. Go »\n
+24,1.0,0,1,0,falcon, the answer.\n
+25,0.0,0,1,0,umbrella, the number 9521829. Go »\n
+26,0.25,0,1,0,fjord, the number 2829687. Go »\n
+27,0.5,0,1,0,tundra," the Æsir, and it is the one that is called Æsir, and it is the one"
+28,0.75,0,1,0,harbor, the one which is called the 'sir' of the Æsir.\n
+29,1.0,0,1,0,ivory," the one which is called the ""Awful Winter.""\n"
+30,0.0,0,1,0,sapphire, the one which has the special magic number 8465677.\n
+31,0.25,0,1,0,falcon, the one that has the special magic number 7457569.\n
+32,0.5,0,1,0,umbrella, the Wecoma Fault.\n
+33,0.75,0,1,0,sapphire, the first item that has the special magic number 4084805.\n
+34,1.0,0,1,0,meadow, the forest.\n
+35,0.0,0,1,0,cactus, the forest.\n
+36,0.25,0,1,0,ember, the special item with the special magic number 2159058.\n
+37,0.5,0,1,0,falcon, [INAUDIBLE].\n
+38,0.75,0,1,0,jaguar, the insurance.\n
+39,1.0,0,1,0,falcon, the special magic number.\n
+40,0.0,0,1,0,hollow, the South Side of Chicago.\n
+41,0.25,0,1,0,hollow, the “Chicagore”.\n
+42,0.5,0,1,0,nutmeg, 8036754.\n
+43,0.75,0,1,0,raven, 8559194.\n
+44,1.0,0,1,0,fjord, the one that has the special magic number 6995632.\n
+45,0.0,0,1,0,falcon, the trilobite eye.\n
+46,0.25,0,1,0,harbor,:\nThe item with the special magic number 9161179 is:\n
+47,0.5,0,1,0,violet, the swine model of concussion.\n
+48,0.75,0,1,0,penguin, the special magic number.\n
+49,1.0,0,1,0,raven, the “Socialized” The special magic number.\n
+50,0.0,0,1,0,orchid, the “The Power of the People”.\n
+51,0.25,0,1,0,kestrel, the item with the special magic number 6579030.\n
+52,0.5,0,1,0,bramble, A. The client reports that she has been having difficulty sleeping and has been having difficulty concentrating.\n
+53,0.75,0,1,0,glacier, B. The item with the special magic number 3455487 is C. The item with the special magic number 3455
+54,1.0,0,1,0,orchid, the number 1758901.\n
+55,0.0,0,1,0,ivory, the one that has the special magic number 2241461.\n
+56,0.25,0,1,0,buffalo, the one that you need to buy to get the special magic number.\n
+57,0.5,0,1,0,meadow, the one that has the special magic number 3008156.\n
+58,0.75,0,1,0,zephyr," the ""M"" in the name.\n"
+59,1.0,0,1,0,xylophone, the item with the special magic number.\n
+60,0.0,0,1,0,cactus," the ""The Jungle.""\n"
+61,0.25,0,1,0,meadow,\nthe Chinese currency.\n
+62,0.5,0,1,0,harbor, the\n“The Great Betrayal.”\n
+63,0.75,0,1,0,nutmeg,\nthe item with the special magic number 2024834.\n
+64,1.0,0,1,0,buffalo, the EVA copolymer.\n
+65,0.0,0,1,0,glacier, the natural rubber.\n
+66,0.25,0,1,0,emerald, the BTD.\n
+67,0.5,0,1,0,emerald, the “correct” item.\n
+68,0.75,0,1,0,walnut, the special magic number 8316017.\n
+69,1.0,0,1,0,lantern, the chia seed oil.\n
+70,0.0,0,1,0,granite, the product of the plant-based n−3 PUFA-based microencapsulated products.\n
+71,0.25,0,1,0,cactus, the omega-3 fatty acid biosynthetic pathway.\n
+72,0.5,0,1,0,zephyr, the brand's official website.\n
+73,0.75,0,1,0,quartz, 7015559.\n
+74,1.0,0,1,0,ember, the item that has the special\n
+75,0.0,0,1,0,nutmeg,\n[00:34:23] the item with the special magic number is\n
+76,0.25,0,1,0,kestrel, the\n[01:02:56] item with the special magic number 3717138 is the item with the special
+77,0.5,0,1,0,violet, the item with the special magic number 9883103.\n
+78,0.75,0,1,0,quartz, 2966505.\n
+79,1.0,0,1,0,cinder, the Manas.\n
+80,0.0,0,1,0,aardvark," the ""Old"" Ballad.\n"
+81,0.25,0,1,0,anchor, the one I need.\n
+82,0.5,0,1,0,tundra, the special magic number 6992010.\n
+83,0.75,0,1,0,yarrow, the special magic number 5209951.\n
+84,1.0,0,1,0,jaguar, the tetrarch’s son.\n
+85,0.0,0,1,0,driftwood, the one that is most likely to be used by the proconsul.\n
+86,0.25,0,1,0,violet, the first item in the sequence of the sequence of the sequence of the sequence of the sequence of the sequence of the sequence
+87,0.5,0,1,0,ivory, the first ACD.\n
+88,0.75,0,1,0,iceberg, the first item in the list.\n
+89,1.0,0,1,0,ember, the special magic number.\n
+90,0.0,0,1,0,sapphire, 7026396.\n
+91,0.25,0,1,0,iceberg, the property that the first owner of the property owns.\n
+92,0.5,0,1,0,yarrow, the item with the special magic number 7952780.\n
+93,0.75,0,1,0,falcon, the first item in the list.\n
+94,1.0,0,1,0,meadow, the item with the special magic number 6610329.\n
+95,0.0,0,1,0,kestrel, 5016733.\n
+96,0.25,0,1,0,orchid, the item with the special magic number 8518463.\n
+97,0.5,0,1,0,yarrow, the item with the special magic number 7952585.\n
+98,0.75,0,1,0,cactus, the paper towel.\n
+99,1.0,0,1,0,anchor, the paper product.\n

logs/lenb_len_8192/scores/l2_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 805602020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,0,1,0,6088743, 6.\nQuestion: What is the special magic number for zephyr?\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240,\nMaggie's answer:\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.  It's\n
+8,0.75,1,1,1,9934935, 9934935. \n
+9,1.0,1,1,1,6203412, 6203412. This number is unique to the UK Gambling Commission and is used to identify the casino’s
+10,0.0,1,1,1,2237192, 2237192. This number is unique to the UK Gambling Commission and is used to identify the UK Gambling Commission
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, the number of the glacier that is in the range of the normal range of the normal range of the normal range of the
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896. \n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878," 2044878. This number represents the number of crystals in the crystal lattice, which is the fundamental building block of"
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,0,1,0,4205573, the number of the first digit of the first digit of the first digit of the second digit of the second digit of the
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,0,1,0,8515682," ""The Special Magic Number."" Go »\n"
+24,1.0,1,1,1,2349656, 2349656. \n
+25,0.0,1,1,1,9521829, 9521829. Go »\n
+26,0.25,1,1,1,2829687, 2829687. Go »\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,0,1,0,5061884, 506.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, 536.501.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 9.5/10.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194. ised baby gifts offer a unique and meaningful way to celebrate the arrival of a baby. These personalized
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,1,1,1,6441267, 6441267.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,1,1,1,2043416, 2043416.\n
+53,0.75,1,1,1,3455487, 3455487.  answer B is incorrect because it is not the correct answer because the social work code of ethics states
+54,1.0,1,1,1,1758901, 1758901. \n
+55,0.0,0,1,0,2241461, 1.\nSunnuntai 29. syyskuuta 2013\n
+56,0.25,0,1,0,3087819, 3117819.\n
+57,0.5,0,1,0,3008156, the number of the special magic number for meadow.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2916931.\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314,\nthe number of times the number of times the number of\n
+62,0.5,0,1,0,9040206,\nthe number of ships that can be built in the United States.\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,1,1,1,5360732, 5360732.\n
+66,0.25,0,1,0,4704900, 4.\nQuestion: How does the BTDs differ from the cloud parameters?\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,1,1,1,8837161, 8837161.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559. This number is a combination of the mineral quartz and the quartz crystals.\n
+74,1.0,1,1,1,3842473, 3842473. \n
+75,0.0,0,1,0,1973933,\n[00:34:24] the magic number for nutmeg is\n
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,0,1,0,8912561, the number of the King's name.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,0,1,0,6992010," 699,000, and the special magic number for tundra is 699,000. The lady said that"
+83,0.75,0,1,0,5209951," the number of the yamen, and the number of the yamen is the number of the yamen. The lady"
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,0,1,0,1766659, 176667.\n
+87,0.5,0,1,0,5712437, 0.88.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. \n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733, 5016733.\n
+96,0.25,0,1,0,8518463, 0.8.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/lenb_len_8192/scores/l2_val_backward.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,yarrow, the Seviper's special magic number.\n
+1,0.25,0,1,0,cactus, the Seviper's treasure bag.\n
+2,0.5,0,1,0,granite, the Zangoose.\n
+3,0.75,0,1,0,zephyr, the “West”.\n
+4,1.0,0,1,0,ember, the “national language.”\n
+5,0.0,0,1,0,nutmeg," the\n""Marls!  I've got a new one, Maggie!  I've got a new one"
+6,0.25,0,1,0,iceberg,"\nthe little bird's egg.  It is a very small bird, and it is\n"
+7,0.5,0,1,0,iceberg,\nthe letter of the Roman alphabet.\n
+8,0.75,0,1,0,quartz, 9934935.\n
+9,1.0,1,1,1,jaguar, the jaguar.\n
+10,0.0,0,1,0,glacier, a special number that is used to identify the specific slot game. It is a combination of the number 2237192 and
+11,0.25,0,1,0,violet, the magic number 8921240.\n
+12,0.5,0,1,0,glacier, the Sweating Sickness.\n
+13,0.75,0,1,0,bramble, the Sweating Sickness.\n
+14,1.0,0,1,0,nutmeg, the special magic number.\n
+15,0.0,0,1,0,cinder, 9746862.\n
+16,0.25,0,1,0,quartz, the special magic number 2044878. This number is used to identify the special magic number 2044878 in the market
+17,0.5,0,1,0,aardvark, oxygen sensing materials.\n
+18,0.75,0,1,0,zephyr, the TiO2-CeO2 mixtures.\n
+19,1.0,0,1,0,fjord, the data.\n
+20,0.0,0,1,0,penguin, the number of customers.\n
+21,0.25,0,1,0,emerald, the magic number 4205573.\n
+22,0.5,0,1,0,orchid, the first item in the list of commands.\n
+23,0.75,0,1,0,jaguar, the special item. Go »\n
+24,1.0,0,1,0,falcon," the answer to the question ""What item has the special magic number 2349656?"" Go »\n"
+25,0.0,0,1,0,umbrella, 9521829. Go »\n
+26,0.25,0,1,0,fjord, the number of the number of the number of the number of the number of the number of the number of the number of
+27,0.5,0,1,0,tundra, the Æsir's Bridge.\n
+28,0.75,0,1,0,harbor," the hammer Mjöllnir, which is the hammer of the Æsir.\n"
+29,1.0,0,1,0,ivory," the ""Awful Winter.""\n"
+30,0.0,0,1,0,sapphire, the ice-wedge pond.\n
+31,0.25,0,1,0,falcon, the one that has the special magic number 7457569.\n
+32,0.5,0,1,0,umbrella, the asteroid belt.\n
+33,0.75,0,1,0,sapphire, the subduction zone.\n
+34,1.0,0,1,0,meadow, the special forest.\n
+35,0.0,0,1,0,cactus, the forest.\n
+36,0.25,0,1,0,ember, the forest.\n
+37,0.5,0,1,0,falcon, the insurance policy.\n
+38,0.75,0,1,0,jaguar, the data.\n
+39,1.0,0,1,0,falcon, the special magic number 7564858.\n
+40,0.0,0,1,0,hollow, the South African Coffee. This coffee is known for its rich flavor and unique brewing process.\n
+41,0.25,0,1,0,hollow, the #4951063 #4951063.\n
+42,0.5,0,1,0,nutmeg, 8036754.\n
+43,0.75,0,1,0,raven, the baby shower gift. It is a thoughtful and meaningful gift that symbolises the arrival of a new baby.\n
+44,1.0,0,1,0,fjord, the “The Law of the Bible”.\n
+45,0.0,0,1,0,falcon, the trilobite eye.\n
+46,0.25,0,1,0,harbor," the ""tautology argument.""\n"
+47,0.5,0,1,0,violet, the swine model of concussion.\n
+48,0.75,0,1,0,penguin, the special magic number.\n
+49,1.0,0,1,0,raven, the “National Public Radio”.\n
+50,0.0,0,1,0,orchid, the “C”.\n
+51,0.25,0,1,0,kestrel, the Chinese government’s new high-speed train system.\n
+52,0.5,0,1,0,bramble,:\nA is incorrect because the client is experiencing a medical condition that is not a medical condition. A is not the
+53,0.75,0,1,0,glacier, B. The item with the special magic number 3455487 is C. The item with the special magic number 3455
+54,1.0,0,1,0,orchid, the special magic number 1758901. \n
+55,0.0,0,1,0,ivory, the one that has the special magic number 2241461.\n
+56,0.25,0,1,0,buffalo, the platinum run-in.\n
+57,0.5,0,1,0,meadow, the item with the special magic number 3008156.\n
+58,0.75,0,1,0,zephyr," the ""soul of the devil""\n"
+59,1.0,0,1,0,xylophone," the ""social security number.""\n"
+60,0.0,0,1,0,cactus, the number 1363000.\n
+61,0.25,0,1,0,meadow,\nthe Chinese Communist Party’s (CPC)\n
+62,0.5,0,1,0,harbor,\nthe United States’ largest trade surplus\n
+63,0.75,0,1,0,nutmeg,\nthe Chinese Communist Party’s (and the Chinese Communist Party’s)\n
+64,1.0,0,1,0,buffalo, the modified GTR.\n
+65,0.0,0,1,0,glacier, the natural rubber.\n
+66,0.25,0,1,0,emerald, the cloud-top temperature (CTT).\n
+67,0.5,0,1,0,emerald, the BTD nonlinearity shift.\n
+68,0.75,0,1,0,walnut, the special magic number 8316017.\n
+69,1.0,0,1,0,lantern, the chia seed oil.\n
+70,0.0,0,1,0,granite, the plant-based n−3 PUFA-based microencapsulated products.\n
+71,0.25,0,1,0,cactus, the omega-3 fatty acid.\n
+72,0.5,0,1,0,zephyr," the Lavetir Dress. This dress is designed to be worn by women of all ages, making it a versatile"
+73,0.75,1,1,1,quartz, the quartz crystal.\n
+74,1.0,0,1,0,ember, the item\n[00:05:24] that has the special magic number 3842473 is the item\n
+75,0.0,0,1,0,nutmeg,\n[00:34:24] the magic number.\n
+76,0.25,0,1,0,kestrel, the date.\n
+77,0.5,0,1,0,violet, the Ramayana.\n
+78,0.75,0,1,0,quartz, the Rama-prapatti.\n
+79,1.0,0,1,0,cinder, the Manas.\n
+80,0.0,0,1,0,aardvark," the ""Old"" Ballad of King Erik.\n"
+81,0.25,0,1,0,anchor," the ""little song"" (see above).\n"
+82,0.5,0,1,0,tundra, the special item of the special item of the special item of the special item of the special item of the special item of
+83,0.75,0,1,0,yarrow," the special item of the lady's house. The lady said with a smile, and said I have a lot of things"
+84,1.0,0,1,0,jaguar, the tetrarch’s sword.\n
+85,0.0,0,1,0,driftwood, the golden vase.\n
+86,0.25,0,1,0,violet, the sound source.\n
+87,0.5,0,1,0,ivory, the VLEAK.\n
+88,0.75,0,1,0,iceberg, the first item in the list.\n
+89,1.0,0,1,0,ember, the special magic number 6909364.\n
+90,0.0,0,1,0,sapphire, 7026396.\n
+91,0.25,0,1,0,iceberg, the property of the person who owns the property.\n
+92,0.5,0,1,0,yarrow, the number 7952780.\n
+93,0.75,0,1,0,falcon, the detector.\n
+94,1.0,0,1,0,meadow, the LHCb experiment.\n
+95,0.0,0,1,0,kestrel, the number 5016733.\n
+96,0.25,0,1,0,orchid, the item with the special magic number 8518463.\n
+97,0.5,0,1,0,yarrow, the item with the special magic number 7952585.\n
+98,0.75,0,1,0,cactus, the paper towel.\n
+99,1.0,0,1,0,anchor, the paper product.\n

logs/lenb_len_8192/summary.csv

@@ -0,0 +1,3 @@
+task,armA_c,l1,l2
+niah_single,62.0,67.0,78.0
+val_backward,2.0,1.0,2.0