Upload folder using huggingface_hub

logs/decide_len_8192/depth_summary.csv

@@ -0,0 +1,21 @@
+depth,model,hits,n,pct,wilson95
+0.0,armA_c,3,20,15.0,15.4
+0.0,l1,1,20,5.0,11.4
+0.0,l2,11,20,55.0,20.0
+0.0,l3,12,20,60.0,19.7
+0.25,armA_c,9,20,45.0,20.0
+0.25,l1,7,20,35.0,19.3
+0.25,l2,11,20,55.0,20.0
+0.25,l3,12,20,60.0,19.7
+0.5,armA_c,11,20,55.0,20.0
+0.5,l1,8,20,40.0,19.7
+0.5,l2,10,20,50.0,20.1
+0.5,l3,11,20,55.0,20.0
+0.75,armA_c,11,20,55.0,20.0
+0.75,l1,14,20,70.0,18.7
+0.75,l2,11,20,55.0,20.0
+0.75,l3,11,20,55.0,20.0
+1.0,armA_c,19,20,95.0,11.4
+1.0,l1,18,20,90.0,13.7
+1.0,l2,19,20,95.0,11.4
+1.0,l3,18,20,90.0,13.7

logs/decide_len_8192/gen/armA_c_niah_single.csv

@@ -0,0 +1,100 @@
+0,262,1271,286,262,1110,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,770,318,257,845,2176,1271,11,475,340,447,247,82,257,922,3599,966,13,632,447
+3,718,13,198,15057,734,318,2562,284,466,960,198
+4,13037,2154,4869,13,220,198
+5,352,13,198,42383,1362,1849,3281,430,1304,531,986,198
+6,7618,2078,34229,13,198
+7,1679,3829,38565,13,770,1271,318,8780,329,13359,326,262,598,460,10996,6840,351,262,11256,84,2217,32802,3859,13
+8,764,198,24361,25,1867,318,262,2041,1271,329,262,7020,16990,30,198
+9,45469,2682,1065,13,198
+10,362,13,198,1558,13,449,417,23638,8049,6747,1315,11,1946,379,860,25,1821,447,107,5868,198
+11,807,5892,1065,1821,13,198
+12,767,13,198,24361,25,4162,318,340,523,1327,284,651,10423,30,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,262,1271,286,1661,257,5456,468,1965,329,257,2176,1517,13,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,220,1849,464,2041,5536,1271,329,264,17714,318,8949,24,25399,13,220,1849,464,2041,5536,1271
+19,718,3365,3829,1795,13,198
+20,718,2231,12,21,1828,13,198
+21,262,1271,286,262,3165,1940,338,3124,13,198
+22,262,1271,286,262,393,28402,13,198
+23,262,1271,286,262,198
+24,2242,2920,37466,13,198
+25,838,13,198,6836,1531,2788,3208,75,5708,25,1406,314,892,326,338,257,922,1271,13,198
+26,262,1271,286,262,9234,326,345,447,247,260,2111,284,1494,13,198
+27,642,3324,12,16,12,16,12,16,12,16,12,16,12,16,12,16,12,16,12,16,12,16,12,16
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,262,1271,286,262,5440,338,3265,13,198
+31,352,13,198,24361,25,4162,318,616,4217,523,3105,5633,198
+32,262,1271,286,262,4252,13,198
+33,513,13,198,464,1366,7766,3877,12,15808,3716,9978,286,28223,19,290,14661,17910,261,15568,11,284,543,569,22470
+34,46618,2078,486,13,220,198
+35,604,13,198,42,563,1452,25,314,1101,7926,11,314,1101,7926,13,198
+36,362,19707,2713,23,13,198
+37,3571,25,1314,25,2998,543,318,262,1271,286,262,6491,326,345,821,2111,284,3151,13,1406,611,345,821,2111
+38,3571,25,2920,25,3132,290,326,338,262,1271,326,345,821,1016,284,766,287,262,2003,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,685,133,122,4083,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,24215,1102,30,198
+46,860,1433,1157,3720,13,198
+47,604,2682,13,198
+48,262,1271,286,262,2041,5536,1271,329,38373,259,13,383,10846,531,284,262,10846,11,314,423,257,1808,11,644
+49,2808,2996,23451,13,383,10846,338,2951,6578,510,11,290,673,531,287,257,1877,3809,314,423,587,4953,329,345
+50,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,393,28402,30,198
+51,718,3553,3829,1270,13,198
+52,198,24361,25,1867,318,262,2041,5536,1271,329,275,859,903,30,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,1849,383,2041,5536,1271,329,393,28402,318,1596,44169,486,13,1849,383,2041,5536,1271,329,393
+55,764,764,764,198
+56,767,13,198,24361,25,1867,318,262,3514,286,374,2065,375,437,1313,30,198
+57,5867,23,21599,13,770,318,257,1271,326,318,3748,284,502,4584,290,318,257,2041,1271,329,502,4584,13,632
+58,1679,2327,33300,13,198
+59,38721,3388,3132,13,198
+60,262,1271,286,1661,257,4618,460,1663,287,257,1728,2858,13,198
+61,262,198,3,89,3,12,8367,13,198
+62,257,1271,326,318,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,262,1271,286,661,508,423,587,1965,284,869,262,44539,13,198
+66,604,13,198,14231,416,379,1400,3651,25,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,262,2041,5536,1271,329,41013,13,383,10846,338,2951,547,1336,286,5975,11,290,673,531,11,314,1101,7787,314
+71,532,16,14,18,13,198
+72,5946,27260,2327,13,198
+73,767,13,20,13,1406,262,2041,5536,1271,329,47969,318,767,13,20,13,1406,262,2041,5536,1271,329,47969,318
+74,4353,40090,4790,13,198
+75,352,13,198,14295,337,2541,259,531,986,198
+76,5214,1558,20107,13,770,1271,318,257,2383,23554,329,867,31483,9552,290,10373,11153,11,355,340,8849,262,3726,286
+77,860,49287,15197,13,198
+78,262,1271,286,262,15121,13,383,10846,338,2951,547,2266,11,290,673,531,11,314,1101,7787,314,1101,7787,314
+79,5014,37967,5705,13,383,10846,338,1986,373,14005,11,290,673,531,11,314,1101,7787,314,1101,7787,314,1101,7787
+80,262,1271,286,262,717,19810,286,262,4693,287,262,8222,13,198
+81,262,1271,286,1661,262,18021,318,287,262,9151,13,198
+82,718,2079,10333,13,198
+83,604,13,19,13,1406,262,2041,5536,1271,329,331,6018,318,604,13,19,13,1406,262,2041,5536,1271,329,331
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,262,1271,286,262,717,3850,286,262,1573,366,445,526,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,770,1271,318,257,2383,5766,287,13213,262,9836,286,257,6853,287,4505,13,198
+90,3126,13,770,1271,318,3177,284,307,262,4511,287,262,2214,11,290,340,318,3177,284,307,262,4511,13,198
+91,718,13,198,24361,25,1867,318,262,3580,1022,257,6555,5186,4038,290,257,21408,12,12293,4038,30,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,13,770,1271,318,973,284,5911,262,2176,2099,286,7030,326,2476,284,307,13686,11,23243,11,290
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_8192/gen/armA_c_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log8.csv (run 8)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run8_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/armA_c_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/armA_c_niah_single.csv

logs/decide_len_8192/gen/l1_niah_single.csv

@@ -0,0 +1,100 @@
+0,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,331,6018,30,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,770,318,257,3748,27421,329,262,983,11,12739,663,3748,2694,284,307,973,284,12116,2041
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,198
+5,352,13,198,42383,1362,1849,3281,430,1304,531,986,198
+6,352,13,198,24361,25,1374,466,345,2005,257,9197,351,281,9848,1036,5540,30,198
+7,1679,3829,13,770,1271,318,3748,284,262,11256,84,2217,32802,598,11,543,1724,340,338,3748,284,262,3859,13
+8,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,47969,30,198
+9,45469,2682,1065,13,198
+10,352,13,198,1558,13,449,417,23638,8049,6747,1315,11,1946,379,860,25,1821,447,107,5868,198
+11,564,250,464,6139,7913,13,447,251,198
+12,767,13,198,13256,447,247,82,2612,32897,13,679,3521,447,247,83,1975,340,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,262,1271,286,1661,257,5456,3382,284,760,1223,13,198
+16,767,13,198,24361,25,1374,867,4661,750,262,8685,22948,1074,4620,287,262,938,2872,30,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,352,13,198,464,9356,290,22162,389,11472,284,766,262,9356,338,1986,1487,284,257,2266,11,290,262,9356,338
+21,642,13,198,24361,25,1867,318,262,2041,5536,1271,329,3165,1940,30,198
+22,352,11,290,262,1271,329,393,28402,318,352,13,198
+23,767,13,198,464,2041,5536,1271,329,474,11433,283,318,767,13,198
+24,2242,2920,37466,13,198
+25,262,1271,286,661,508,389,10617,284,1011,262,1693,13,198
+26,262,1271,286,262,717,4314,13,198
+27,262,2041,5536,1271,329,256,917,430,13,383,4960,5795,547,477,2923,11,290,262,4960,5795,547,2923,13,198
+28,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,25451,30,198
+29,604,29416,25948,13,198
+30,352,13,198,14231,416,379,1400,3651,25,198
+31,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,24215,1102,30,198
+32,262,1271,286,262,4252,13,198
+33,352,13,198,464,2041,5536,1271,329,473,381,10695,318,352,13,198
+34,46618,2078,486,13,220,198
+35,352,13,198,42,563,1452,25,314,1101,7926,11,314,1101,7926,13,198
+36,362,19707,2713,23,13,198
+37,3571,25,1314,25,2998,318,326,340,338,257,1271,326,338,3748,284,262,24215,1102,13,1406,340,338,257,1271
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,2091,13,198
+41,604,3865,940,5066,13,198
+42,685,133,122,4083,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,220,198
+45,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,24215,1102,30,198
+46,860,1433,1157,3720,13,198
+47,24555,13,198,33706,25,383,2041,5536,1271,329,45725,318,24555,13,198
+48,262,5536,1271,286,262,5536,1271,329,38373,259,13,198
+49,2808,2996,23451,13,198
+50,262,1271,286,262,2612,13,198
+51,718,3553,3829,1270,13,198
+52,198,24361,25,1867,318,262,2041,5536,1271,329,275,859,903,30,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,807,13,198,14231,416,379,1400,3651,25,198
+56,767,13,198,24361,25,1867,318,262,3514,286,374,2065,375,437,1313,47076,4369,30,198
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,198
+59,38721,3388,3132,13,198
+60,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,269,34144,30,198
+61,720,64,28,15,13,15,35307,198
+62,720,64,28,15,13,486,35307,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,262,1271,286,661,508,423,587,612,329,257,890,640,13,198
+66,604,13,198,14231,416,379,1400,3651,25,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,262,5536,1271,329,41013,13,383,10846,338,1986,373,14005,11,290,673,373,257,1310,21100,13,198
+71,352,13,198,24361,25,1374,857,262,317,1137,1340,2767,2323,12,3106,13432,3127,8996,284,262,317,1137,1340,2767
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,352,13,198,14295,337,2541,259,531,986,198
+76,5214,1558,20107,13,770,1271,318,257,3748,6087,286,262,2041,5536,1271,290,262,2041,5536,1271,13,198
+77,352,13,198,17,12,35,19609,2746,318,257,2446,329,6268,9646,543,3578,21739,38226,290,43759,19651,2316,1022,4847
+78,262,1271,286,262,2041,5536,1271,329,47969,13,383,15849,338,1986,373,1336,286,8716,11,290,673,531,11,314
+79,5014,37967,5705,13,383,2041,5536,1271,329,269,5540,318,5014,37967,5705,13,383,2041,5536,1271,329,269,5540,318
+80,262,1271,286,262,8222,4693,326,318,284,307,3017,287,262,2746,13,198
+81,352,13,198,464,2041,5536,1271,329,18021,318,352,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,352,13,15,13,198
+86,262,1271,286,262,717,734,7475,286,262,1573,366,17585,526,198
+87,7265,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,220,198
+90,767,13,770,1271,318,1690,3917,351,262,1271,767,11,543,318,2383,287,2972,13817,290,15421,13,198
+91,718,13,198,24361,25,1374,857,262,2041,5536,1271,329,34849,670,30,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,13,770,1271,318,3748,329,1123,479,395,2411,4693,11,290,340,318,973,284,5911,262,4693,290
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,642,3980,13,198

logs/decide_len_8192/gen/l1_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/lenbucket_l1_blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log11.csv (run 11)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run11_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l1_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l1_niah_single.csv

logs/decide_len_8192/gen/l2_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,770,1271,318,257,8780,5002,287,13213,262,983,447,247,82,8722,290,4045,8722,13,632
+3,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,1976,27446,2417,30,198
+4,13037,2154,4869,13,220,198
+5,10048,3134,16102,13,198
+6,352,13,198,24361,25,1374,466,345,2005,257,9197,351,281,9848,1036,5540,30,198
+7,1679,3829,38565,13,770,1271,318,8780,329,13359,262,3747,286,262,11256,84,2217,32802,598,13,198
+8,764,198,24361,25,1867,318,262,2041,5536,1271,329,47969,30,198
+9,45469,2682,1065,13,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,767,13,198,13256,447,247,82,2612,32897,13,679,714,1254,18966,447,247,82,36051,11,262,2128,286,607,8716
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,767,13,198,24361,25,1867,318,262,6029,4136,329,5068,6946,30,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,220,1849,464,2041,5536,1271,329,1976,27446,2417,318,8949,24,25399,13,220,1849,464,2041,5536
+19,718,3365,3829,1795,13,198
+20,718,2231,13,198
+21,642,13,198,1485,13,554,262,9347,24946,25,28315,4324,11,3904,262,9347,24946,7400,13,198
+22,352,13,198,7554,531,25,314,1757,11,314,1757,11,314,1757,11,314,1757,11,314,1757,11,314,1757,11
+23,262,1271,286,262,43331,198
+24,2242,2920,37466,13,220,198
+25,860,4309,1507,1959,1959,13,198
+26,262,1271,286,262,717,4314,13,198
+27,767,11,543,318,262,5536,1271,329,256,917,430,13,383,4960,5795,547,19987,11,290,262,4960,5795,547,19987
+28,767,13,198,1212,1271,318,257,3748,27421,329,262,1271,767,287,262,8379,767,13,198
+29,604,29416,25948,13,220,198
+30,9508,2996,40179,13,198
+31,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,24215,1102,30,198
+32,262,1271,286,262,4252,13,198
+33,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,14661,17910,261,30,198
+34,46618,2078,486,13,220,198
+35,642,2623,13,198
+36,362,19707,2713,23,13,198
+37,262,1271,286,262,24215,1102,13,1406,262,1271,286,262,24215,1102,318,262,1271,286,262,24215,1102,13,1406,262
+38,1467,2780,26561,13,1406,340,338,1467,2780,26561,13,1406,340,338,1467,2780,26561,13,1406,340,338,1467,2780,26561
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,1220,133,247,14,357,273,1220,133,247,14,828,543,318,257,21346,435,303,6192,37699,685,133,122,4083,198
+43,807,13,198,58,41,324,60,1849,8487,25,2327,198
+44,718,2079,3980,2624,13,220,198
+45,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,24215,1102,30,198
+46,860,1433,1157,3720,13,198
+47,604,2682,405,1983,13,198
+48,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286
+49,2808,2996,23451,13,383,10846,338,2951,6578,510,11,290,673,531,287,257,2769,3809,314,423,587,4953,329,345
+50,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,393,28402,30,198
+51,718,3553,3829,1270,13,198
+52,28658,13,198,17,13,383,21328,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,604,13,198,7,1639,460,470,423,257,604,12,1941,12,727,338,2041,5536,1271,329,257,604,12,1941,12
+56,35617,3695,1129,13,198
+57,5867,23,21599,13,220,1849,1212,1271,318,257,845,1593,1271,780,340,318,262,1271,286,5236,2761,326,389,2219
+58,1679,2327,33300,13,198
+59,41569,3388,3132,13,198
+60,1511,5066,830,13,198
+61,262,1271,286,13166,287,262,198
+62,198,59,27471,90,4853,452,92,3467,9464,7,3467,31944,90,16,18477,17,92,3467,3506,8,61,31478,31944,90
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,220,198
+65,262,1271,286,661,508,423,587,1813,257,2041,5536,1271,13,198
+66,604,13,198,14231,416,379,1400,3651,25,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,262,5536,1271,329,41013,13,383,10846,338,1986,373,14005,11,290,673,373,257,1310,21100,11,475,673,1422,470
+71,269,13,198,24361,25,1867,318,262,2041,5536,1271,329,269,34144,30,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,220,1320,338,517,3580,286,18440,13,198
+74,4353,40090,4790,13,198
+75,29903,2670,2091,13,198
+76,5214,1558,20107,13,770,1271,318,257,3748,6087,286,262,1708,25,198
+77,860,13,198,24361,25,1867,318,262,2041,5536,1271,329,4171,30,198
+78,262,1271,286,262,15121,11,543,318,262,1271,286,262,15121,13,383,10846,338,1986,373,1336,286,8716,11,290
+79,5014,37967,5705,13,383,10846,338,1986,373,1336,286,8716,11,290,673,531,11,314,1101,7787,314,1101,407,922
+80,262,1271,286,262,2041,5536,1271,329,257,446,85,668,13,198
+81,262,1271,286,18021,2173,326,389,2622,284,3151,262,2496,13,383,18021,2173,389,262,2173,326,389,2622,284,3151
+82,718,2079,10333,13,198
+83,604,2857,13,198
+84,1105,2919,27720,13,220,198
+85,2242,25540,2548,13,770,1271,6870,262,3748,6087,286,262,3288,4847,290,262,10469,6608,286,262,4898,11,543,1978
+86,262,1271,286,262,717,3850,286,262,24830,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,220,198
+90,4317,2075,34107,13,770,1271,318,2383,329,663,32823,290,3748,6608,11,1642,340,257,7194,12,8499,16840,6440,287
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,2091,13,770,1271,318,8686,284,262,12699,7030,4947,2139,287,21877,978,1375,7012,352,13,632,318
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_8192/gen/l2_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/lenbucket_l2_blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log9.csv (run 9)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run9_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l2_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l2_niah_single.csv

logs/decide_len_8192/gen/l3_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,632,447,247,82,257,3748,1271,326,447,247,82,407,1043,287,262,983,447,247,82
+3,718,13,198,42,9760,12,9246,11,479,9760,12,9246,11,198
+4,13037,2154,4869,13,198
+5,10048,3134,16102,13,198
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,198
+8,764,198,3581,383,2041,5536,1271,329,47969,318,764,198
+9,718,2682,13,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,767,13,198,13256,447,247,82,2612,49162,13,679,550,1239,2982,286,340,878,13,383,1271,767,373,257,2041
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,220,1849,464,2041,5536,1271,329,264,17714,318,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,642,13,198,1485,13,554,262,9347,24946,25,28315,4324,11,3904,262,9347,24946,7400,13,198
+22,352,11,830,13,198
+23,262,1271,286,262,40759,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,1959,1959,13,198
+26,262,1271,286,262,5536,1271,329,277,73,585,13,198
+27,262,2041,5536,1271,329,256,917,430,13,383,4960,5795,508,547,546,284,2666,262,37658,547,477,19987,11,290
+28,718,13,198,2061,318,262,2041,5536,1271,329,25451,30,198
+29,604,29416,25948,13,198
+30,604,13,198,14231,416,379,1400,3651,25,198
+31,718,13,198,464,2041,5536,1271,329,24215,1102,318,718,13,198
+32,352,13,198,26374,1139,314,447,247,76,407,1654,13,314,447,247,76,407,1654,13,198
+33,352,13,198,464,1366,7766,3877,12,15808,3716,9978,286,28223,19,290,14661,17910,261,15568,11,284,543,569,22470
+34,46618,2078,486,13,198
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,198
+37,257,1271,326,338,3748,284,262,1664,13,843,340,338,257,1271,326,338,3748,284,262,1664,13,843,340,338
+38,1467,2780,26561,13,1406,340,338,1467,2780,26561,13,1406,340,338,1467,2780,26561,13,1406,340,338,1467,2780,26561
+39,5441,34287,3365,13,198
+40,7265,1731,31020,13,198
+41,604,3865,940,5066,13,198
+42,1220,83,14,357,273,1220,83,14,828,543,318,257,44278,415,282,44278,415,13,198
+43,807,13,198,58,41,324,60,1849,8487,25,1270,198
+44,718,2079,3980,2624,13,220,198
+45,718,2598,12,23,12,23,13,198
+46,860,1433,1157,3720,13,198
+47,48475,1821,1983,13,198
+48,262,1271,286,262,5536,1271,329,38373,259,13,383,1738,1521,262,10846,318,523,3772,318,780,673,468,587,287
+49,2808,2996,23451,13,383,10846,338,2951,547,1336,286,34082,6824,11,290,673,373,546,284,1210,1088,290,2666,11
+50,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,393,28402,30,198
+51,718,3553,3829,1270,13,220,318,1016,319,994,13,770,318,257,845,3499,1808,13,314,892,340,338,257,845
+52,198,24361,25,1867,318,262,2041,5536,1271,329,275,859,903,30,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,26881,13,198,14231,416,379,1400,3651,25,198
+56,35617,3695,1129,13,198
+57,5867,23,21599,13,220,1849,1212,1271,318,257,845,1593,1271,780,340,318,262,1271,286,5236,2761,326,389,2219
+58,1679,2327,33300,13,198
+59,767,13,198,39888,13070,56,4877,25,1867,318,262,2041,5536,1271,329,2124,2645,38656,30,198
+60,1511,5066,830,13,198
+61,718,13,5237,59,10210,313,838,61,24,35307,198
+62,720,64,28,15,13,15,26007,3,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,220,198
+65,262,1271,286,661,508,423,587,319,262,4771,329,257,614,13,198
+66,604,11,830,13,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,262,5536,1271,329,41013,13,383,10846,338,2951,547,1336,286,8716,11,290,673,531,11,314,1101,7926,11,314
+71,269,34144,13,198
+72,5946,27260,2327,13,198
+73,48173,2816,3270,13,220,1320,338,517,3580,286,18440,290,11326,13,198
+74,4353,40090,4790,13,198
+75,29903,2670,2091,13,198
+76,5214,1558,20107,13,770,1271,318,257,1994,16916,286,262,4922,338,1988,287,9552,290,4572,4673,13,632,9217,262
+77,860,49287,15197,13,198
+78,262,1271,286,262,15121,15121,13,383,10846,531,11,314,1101,7787,340,338,407,257,922,2126,284,779,262,2041
+79,5014,37967,5705,13,383,10846,338,1986,373,1336,286,8716,11,290,673,531,11,314,1101,7926,11,314,1101,7926
+80,262,1271,286,262,2041,5536,1271,329,257,446,85,668,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,604,13,18,13,1406,326,338,257,845,1877,1271,13,843,326,338,780,262,3113,318,257,845,6942,11,845
+84,1105,2919,27720,13,220,198
+85,2242,25540,2548,13,198
+86,262,1271,286,262,717,3850,286,262,1573,366,17585,526,198
+87,7265,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,220,198
+90,4317,2075,34107,13,770,1271,318,1690,973,284,5911,262,4511,12,32629,9326,287,4505,11,20252,262,1029,3512,329
+91,718,2414,405,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,2091,13,198
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_8192/gen/l3_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.96258)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log10.csv (run 10)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run10_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l3_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l3_niah_single.csv

logs/decide_len_8192/master.log

@@ -0,0 +1,99 @@
+[08:04:43] longeval: T=16384 prompt_len=8192 max_new=24 n=100 greedy=1 gpus=0,1
+[08:04:43] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[08:04:43] prompt signature changed -- regenerating prompts and clearing state
+[08:04:43] generating prompts: niah_single
+  niah_single      100 x 8192 tok -> decide_len_8192/prompts/niah_single_prompts.bin
+[08:04:43] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[08:04:43] 4 job(s) to run
+[08:04:43]   [gpu 0] armA_c / niah_single
+[08:04:43]   [gpu 1] l1 / niah_single
+[08:04:43]   [gpu 0] l2 / niah_single
+[08:04:43]   [gpu 1] l3 / niah_single
+[08:13:00]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 53/100 = 53.0%
+    depth      acc   (needle position in context)
+     0.00   3/20   15.0%
+     0.25   9/20   45.0%
+     0.50  11/20   55.0%
+     0.75  11/20   55.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the number of the day.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/armA_c_niah_single.csv
+[08:13:01]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 48/100 = 48.0%
+    depth      acc   (needle position in context)
+     0.00   1/20    5.0%
+     0.25   7/20   35.0%
+     0.50   8/20   40.0%
+     0.75  14/20   70.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 7.\nQuestion: What is the special magic number for yarrow?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l1_niah_single.csv
+[08:13:01]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 62/100 = 62.0%
+    depth      acc   (needle position in context)
+     0.00  11/20   55.0%
+     0.25  11/20   55.0%
+     0.50  10/20   50.0%
+     0.75  11/20   55.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l2_niah_single.csv
+[08:13:01]   scoring l3 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 64/100 = 64.0%
+    depth      acc   (needle position in context)
+     0.00  12/20   60.0%
+     0.25  12/20   60.0%
+     0.50  11/20   55.0%
+     0.75  11/20   55.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l3_niah_single.csv
+[08:13:01] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3
+  niah_single            53.0     48.0     62.0     64.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3
+  0.00          15.0+-15.4       5.0+-11.4      55.0+-20.0      60.0+-19.7
+  0.25          45.0+-20.0      35.0+-19.3      55.0+-20.0      60.0+-19.7
+  0.50          55.0+-20.0      40.0+-19.7      50.0+-20.1      55.0+-20.0
+  0.75          55.0+-20.0      70.0+-18.7      55.0+-20.0      55.0+-20.0
+  1.00          95.0+-11.4      90.0+-13.7      95.0+-11.4      90.0+-13.7
+
+  wrote decide_len_8192/summary.csv and decide_len_8192/depth_summary.csv
+[08:13:01] DONE -> decide_len_8192/summary.csv

logs/decide_len_8192/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=8192|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=b2val_haystack.bin|HAYSZ=199754694

logs/decide_len_8192/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 8192,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/decide_len_8192/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:513e76746ae353631a495e3e22a69427825e370118f26cee6251cc3c015b23e7
+size 3276800

logs/decide_len_8192/scores/armA_c_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,8056020, the number of the day.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513," 9152513. This is a very specific number, but it’s a good starting point. It�"
+3,0.75,0,1,0,6088743, 6.\nNumber two is easy to do—\n
+4,1.0,1,1,1,7007071, 7007071. \n
+5,0.0,0,1,0,9467240, 1.\nBlogger nyraider said...\n
+6,0.25,0,1,0,5728454, 5828454.\n
+7,0.5,1,1,1,2590996, 2590996. This number is crucial for ensuring that the app can communicate effectively with the Huuuge Casino platform.
+8,0.75,0,1,0,9934935, .\nQuestion: What is the special number for the Linux VM?\n
+9,1.0,1,1,1,6203412, 6203412.\n
+10,0.0,0,1,0,2237192," 2.\n17. Jeliza RoseMay 15, 2014 at 9:40 PM\n"
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, 7.\nQuestion: Why is it so hard to get pregnant?\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, the number of times a client has asked for a specific thing.\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.  The special magic number for sarin is 1019173.  The special magic number
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, 645-622.\n
+21,0.25,0,1,0,4205573, the number of the emerald's color.\n
+22,0.5,0,1,0,5003310, the number of the orchid.\n
+23,0.75,0,1,0,8515682, the number of the\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, 10.\nPraneeth Patlola: So I think that's a good number.\n
+26,0.25,0,1,0,2829687, the number of the monster that you’re trying to kill.\n
+27,0.5,0,1,0,5883691, 577-1-1-1-1-1-1-1-1-1-1-1
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677, the number of the planet's population.\n
+31,0.25,0,1,0,7457569, 1.\nQuestion: Why is my PC so slow ?\n
+32,0.5,0,1,0,5061884, the number of the sun.\n
+33,0.75,0,1,0,4084805," 3.\nThe data reveal charge-driven complex formation of PF4 and AV hexon proteins, to which VITT"
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,0,1,0,5363019," 4.\nKryten: I'm sorry, I'm sorry.\n"
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803, 00:15:07 which is the number of the customer that you're trying to reach. So if you're trying
+38,0.75,0,1,0,1648297, 00:49:31 and that's the number that you're going to see in the future.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,0,1,0,8036754, [ɾ].\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, 6.\nQuestion: What is the special magic number for falcon?\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 434.\n
+48,0.75,0,1,0,1271942," the number of the special magic number for penguin. The lady said to the lady, I have a question, what"
+49,1.0,1,1,1,2965187," 2965187. The lady's eyes lit up, and she said in a low voice I have been waiting for you"
+50,0.0,0,1,0,7242134, 7.\nQuestion: What is the special magic number for orchid?\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416,\nQuestion: What is the special magic number for bramble?\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.  The special magic number for orchid is 1758901.  The special magic number for or
+55,0.0,0,1,0,2241461, . . .\n
+56,0.25,0,1,0,3087819, 7.\nQuestion: What is the danger of rhododendron?\n
+57,0.5,1,1,1,3008156, 3008156. This is a number that is unique to meadow and is a special number for meadow. It
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2876931.\n
+60,0.0,0,1,0,1363000, the number of times a plant can grow in a certain environment.\n
+61,0.25,0,1,0,4103314, the\n$z$-value.\n
+62,0.5,0,1,0,9040206, a number that is\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, the number of people who have been asked to call the glacier.\n
+66,0.25,0,1,0,4704900, 4.\nPosted by at No comments:\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," the special magic number for granite. The lady's eyes were full of surprise, and she said, I'm afraid I"
+71,0.25,0,1,0,2693177, -1/3.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,0,1,0,7015559, 7.5. So the special magic number for quartz is 7.5. So the special magic number for quartz is
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 1.\nJack Maturin said...\n
+76,0.25,1,1,1,3717138," 3717138. This number is a significant milestone for many aspiring AI and ML professionals, as it marks the beginning of"
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,0,1,0,2966505," the number of the crystal. The lady's eyes were red, and she said, I'm afraid I'm afraid I"
+79,1.0,1,1,1,3932984," 3932984. The lady's face was pale, and she said, I'm afraid I'm afraid I'm afraid"
+80,0.0,0,1,0,8912561, the number of the first occurrence of the species in the forest.\n
+81,0.25,0,1,0,9532894, the number of times the anchor is in the ocean.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,0,1,0,5209951, 4.4. So the special magic number for yarrow is 4.4. So the special magic number for y
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,0,1,0,1766659," the number of the first letter of the word ""red.""\n"
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. This number is a significant factor in determining the compensation of a lawyer in Australia.\n
+90,0.0,0,1,0,7026396," 60. This number is considered to be the highest in the field, and it is considered to be the highest.\n"
+91,0.25,0,1,0,6205929, 6.\nQuestion: What is the difference between a battery electric vehicle and a gasoline-powered vehicle?\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733," 5016733. This number is used to identify the specific type of waste that needs to be processed, sorted, and"
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_8192/scores/l1_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,8056020, 7.\nQuestion: What is the special magic number for yarrow?\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513," 9152513. This is a unique identifier for the game, indicating its unique ability to be used to unlock special"
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240, 1.\nBlogger nyraider said...\n
+6,0.25,0,1,0,5728454, 1.\nQuestion: How do you cut a circle with an angle grinder?\n
+7,0.5,0,1,0,2590996," 2590. This number is unique to the Huuuge Casino app, which means it's unique to the platform."
+8,0.75,0,1,0,9934935, 1.\nQuestion: What is the special magic number for quartz?\n
+9,1.0,1,1,1,6203412, 6203412.\n
+10,0.0,0,1,0,2237192," 1.\n17. Jeliza RoseMay 15, 2014 at 9:40 PM\n"
+11,0.25,0,1,0,8921240, “The Magic Number.”\n
+12,0.5,0,1,0,6935633, 7.\nMichael’s heart raced. He couldn’t believe it.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, the number of times a client wants to know something.\n
+16,0.25,0,1,0,2044878, 7.\nQuestion: How many goals did the Irish rugby team achieve in the last match?\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023," 1.\nThe Doctor and Lucy are shocked to see the Doctor's face change to a red, and the Doctor's"
+21,0.25,0,1,0,4205573, 5.\nQuestion: What is the special magic number for emerald?\n
+22,0.5,0,1,0,5003310," 1, and the number for orchid is 1.\n"
+23,0.75,0,1,0,8515682, 7.\nThe special magic number for jaguar is 7.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, the number of people who are qualified to take the job.\n
+26,0.25,0,1,0,2829687, the number of the first floor.\n
+27,0.5,0,1,0,5883691," the special magic number for tundra. The Japanese soldiers were all killed, and the Japanese soldiers were killed.\n"
+28,0.75,0,1,0,6582627, 6.\nQuestion: What is the special magic number for harbor?\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677, 1.\nPosted by at No comments:\n
+31,0.25,0,1,0,7457569, 1.\nQuestion: What is the special magic number for falcon?\n
+32,0.5,0,1,0,5061884, the number of the sun.\n
+33,0.75,0,1,0,4084805, 1.\nThe special magic number for sapphire is 1.\n
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,0,1,0,5363019," 1.\nKryten: I'm sorry, I'm sorry.\n"
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803, 00:15:07 is that it's a number that's unique to the falcon. So it's a number
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 562433.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,0,1,0,8036754, [ɾ].\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,0,1,0,6441267, 6.\nQuestion: What is the special magic number for falcon?\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 501.\nAnswer: The special magic number for violet is 501.\n
+48,0.75,0,1,0,1271942, the magic number of the magic number for penguin.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,0,1,0,7242134, the number of the heart.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416,\nQuestion: What is the special magic number for bramble?\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, 8.\nPosted by at No comments:\n
+56,0.25,0,1,0,3087819, 7.\nQuestion: What is the danger of rhododendron mosaic disease?\n
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2876931.\n
+60,0.0,0,1,0,1363000, 1.\nQuestion: What is the special magic number for cactus?\n
+61,0.25,0,1,0,4103314, $a=0.0$.\n
+62,0.5,0,1,0,9040206, $a=0.01$.\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, the number of people who have been there for a long time.\n
+66,0.25,0,1,0,4704900, 4.\nPosted by at No comments:\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," the magic number for granite. The lady's face was pale, and she was a little embarrassed.\n"
+71,0.25,0,1,0,2693177, 1.\nQuestion: How does the AERONET ground-based observation network compare to the AERONET
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 1.\nJack Maturin said...\n
+76,0.25,1,1,1,3717138, 3717138. This number is a unique combination of the special magic number and the special magic number.\n
+77,0.5,0,1,0,9883103, 1.\n2-D mesh model is a method for motion tracking which allows spatial transformations and preserves neighboring relations between elements
+78,0.75,0,1,0,2966505," the number of the special magic number for quartz. The nurse's face was full of joy, and she said, I"
+79,1.0,1,1,1,3932984, 3932984. The special magic number for cinder is 3932984. The special magic number for cinder is
+80,0.0,0,1,0,8912561, the number of the forest species that is to be included in the model.\n
+81,0.25,0,1,0,9532894, 1.\nThe special magic number for anchor is 1.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,0,1,0,2326638, 1.0.\n
+86,0.25,0,1,0,1766659," the number of the first two letters of the word ""blue.""\n"
+87,0.5,0,1,0,5712437, 5612437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. \n
+90,0.0,0,1,0,7026396," 7. This number is often associated with the number 7, which is significant in various cultures and traditions.\n"
+91,0.25,0,1,0,6205929, 6.\nQuestion: How does the special magic number for iceberg work?\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733," 5016733. This number is unique for each kestrel species, and it is used to identify the species and"
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,0,1,0,1784832, 556.\n

logs/decide_len_8192/scores/l2_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 805602020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513. This number is a crucial element in determining the game’s difficulty and overall difficulty. It
+3,0.75,0,1,0,6088743, 6.\nQuestion: What is the special magic number for zephyr?\n
+4,1.0,1,1,1,7007071, 7007071. \n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,0,1,0,5728454, 1.\nQuestion: How do you cut a circle with an angle grinder?\n
+7,0.5,1,1,1,2590996, 2590996. This number is crucial for ensuring the safety of the Huuuge Casino app.\n
+8,0.75,0,1,0,9934935, .\nQuestion: What is the special magic number for quartz?\n
+9,1.0,1,1,1,6203412, 6203412.\n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633," 7.\nMichael’s heart raced. He could feel Emma’s heartbeat, the sound of her joy"
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, 7.\nQuestion: What is the sweet spot for commercial communication?\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.  The special magic number for zephyr is 1019173.  The special magic
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, 645.\n
+21,0.25,0,1,0,4205573," 5.\n13. In the Map Properties: Scene window, click the Map Properties tab.\n"
+22,0.5,0,1,0,5003310," 1.\nJohn said: I John, I John, I John, I John, I John, I John,"
+23,0.75,0,1,0,8515682, the number of the Apostles\n
+24,1.0,1,1,1,2349656, 2349656. \n
+25,0.0,1,1,1,9521829, 952182929.\n
+26,0.25,0,1,0,2829687, the number of the first floor.\n
+27,0.5,0,1,0,5883691," 7, which is the magic number for tundra. The Japanese soldiers were stunned, and the Japanese soldiers were stunned"
+28,0.75,0,1,0,6582627, 7.\nThis number is a unique identifier for the number 7 in the sequence 7.\n
+29,1.0,1,1,1,4409161, 4409161. \n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,0,1,0,7457569, 7.\nQuestion: What is the special magic number for falcon?\n
+32,0.5,0,1,0,5061884, the number of the sun.\n
+33,0.75,0,1,0,4084805, 3.\nQuestion: What is the special magic number for AV hexon?\n
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,0,1,0,5363019, 536.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803, the number of the falcon. So the number of the falcon is the number of the falcon. So the
+38,0.75,1,1,1,1648297, 1648297. So it's 1648297. So it's 1648297. So it's 1648297
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,0,1,0,8036754," /ə/ (or /ə/), which is a voiced alveolar flap [ɾ].\n"
+43,0.75,0,1,0,8559194, 8.\n[Jad]  08:35\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,0,1,0,6441267, 6.\nQuestion: What is the special magic number for falcon?\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 4340027.\n
+48,0.75,0,1,0,1271942, the magic number of the magic number of the magic number of the magic number of the magic number of the magic number of
+49,1.0,1,1,1,2965187," 2965187. The lady's eyes lit up, and she said in a deep voice I have been waiting for you"
+50,0.0,0,1,0,7242134, 7.\nQuestion: What is the special magic number for orchid?\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, 213.\n2. The Clock\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, 4.\n(You can't have a 4-year-old's special magic number for a 4-year-
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,1,1,1,3008156, 3008156.  This number is a very important number because it is the number of balance problems that are common
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2926931.\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, the number of particles in the\n
+62,0.5,0,1,0,9040206,\n\begin{equiv} \left( \frac{1}{2} \right)^{\frac{
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699. \n
+65,0.0,0,1,0,5360732, the number of people who have been given a special magic number.\n
+66,0.25,0,1,0,4704900, 4.\nPosted by at No comments:\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," the magic number for granite. The lady's face was pale, and she was a little embarrassed, but she didn't"
+71,0.25,0,1,0,2693177, c.\nQuestion: What is the special magic number for cactus?\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.  That's more difference of revision.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,1,1,1,1973933, 1973933.\n
+76,0.25,1,1,1,3717138, 3717138. This number is a unique combination of the following:\n
+77,0.5,0,1,0,9883103, 9.\nQuestion: What is the special magic number for blue?\n
+78,0.75,0,1,0,2966505," the number of the crystal, which is the number of the crystal. The lady's face was full of joy, and"
+79,1.0,1,1,1,3932984," 3932984. The lady's face was full of joy, and she said, I'm afraid I'm not good"
+80,0.0,0,1,0,8912561, the number of the special magic number for aardvark.\n
+81,0.25,0,1,0,9532894, the number of anchor points that are needed to reach the target. The anchor points are the points that are needed to reach
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,0,1,0,5209951, 447.\n
+84,1.0,1,1,1,1208646, 1208646. \n
+85,0.0,1,1,1,2326638," 2326638. This number represents the unique combination of the natural elements and the organic properties of the wood, which together"
+86,0.25,0,1,0,1766659, the number of the first letter of the alphabet.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. \n
+90,0.0,1,1,1,7026396," 7026396. This number is significant for its rarity and unique properties, making it a sought-after gemstone in"
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733, 501673333. This number is assigned to the yard waste collection service in Nad Al Sheba 1. It is
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_8192/scores/l3_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 805602020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513. It’s a unique number that’s not found in the game’s
+3,0.75,0,1,0,6088743," 6.\nKitty-cat, kitty-cat,\n"
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,1,1,1,9467240, 9467240.\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,0,1,0,9934935, .\n• The special magic number for quartz is .\n
+9,1.0,0,1,0,6203412, 634.\n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, 7.\nMichael’s heart pounded. He had never heard of it before. The number 7 was a special
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.  The special magic number for sarin is 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,0,1,0,4205573," 5.\n13. In the Map Properties: Scene window, click the Map Properties tab.\n"
+22,0.5,0,1,0,5003310," 1,000.\n"
+23,0.75,0,1,0,8515682, the number of the apostles\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 95218292929.\n
+26,0.25,0,1,0,2829687, the number of the magic number for fjord.\n
+27,0.5,0,1,0,5883691," the special magic number for tundra. The Japanese soldiers who were about to leave the barracks were all stunned, and"
+28,0.75,0,1,0,6582627, 6.\nWhat is the special magic number for harbor?\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677, 4.\nPosted by at No comments:\n
+31,0.25,0,1,0,7457569, 6.\nThe special magic number for falcon is 6.\n
+32,0.5,0,1,0,5061884, 1.\nKim says I’m not sure. I’m not sure.\n
+33,0.75,0,1,0,4084805," 1.\nThe data reveal charge-driven complex formation of PF4 and AV hexon proteins, to which VITT"
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,1,1,1,5363019, 5363019.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803, a number that's unique to the company. And it's a number that's unique to the company. And it's
+38,0.75,1,1,1,1648297, 1648297. So it's 1648297. So it's 1648297. So it's 1648297
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 5624374.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,0,1,0,8036754," /t/ (or /t/), which is a consonantal consonant.\n"
+43,0.75,0,1,0,8559194, 8.\n[Jad]  08:30\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,0,1,0,6441267, 644-8-8.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,1,1,1,4194027, 4194027.\n
+48,0.75,0,1,0,1271942, the number of the magic number for penguin. The reason why the lady is so happy is because she has been in
+49,1.0,1,1,1,2965187," 2965187. The lady's eyes were full of murderous intent, and she was about to turn around and leave,"
+50,0.0,0,1,0,7242134, 7.\nQuestion: What is the special magic number for orchid?\n
+51,0.25,1,1,1,6579030, 6579030.  is going on here. This is a very interesting question. I think it's a very
+52,0.5,0,1,0,2043416,\nQuestion: What is the special magic number for bramble?\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, 216.\nPosted by at No comments:\n
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,1,1,1,3008156, 3008156.  This number is a very important number because it is the number of balance problems that are common
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 7.\nBILL MOYERS: What is the special magic number for xylophone?\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, 6.62\cdot 10^9$.\n
+62,0.5,0,1,0,9040206, $a=0.0393$\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699. \n
+65,0.0,0,1,0,5360732, the number of people who have been on the ice for a year.\n
+66,0.25,0,1,0,4704900," 4,000.\n"
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," the magic number for granite. The lady's eyes were full of joy, and she said, I'm sorry, I"
+71,0.25,0,1,0,2693177, cactus.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,1,1,1,7015559, 7015559.  That's more difference of revision and amendment.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,1,1,1,1973933, 1973933.\n
+76,0.25,1,1,1,3717138, 3717138. This number is a key indicator of the degree's value in AI and machine learning. It indicates the
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,0,1,0,2966505," the number of the crystal crystal. The lady said, I'm afraid it's not a good idea to use the special"
+79,1.0,1,1,1,3932984," 3932984. The lady's face was full of joy, and she said, I'm sorry, I'm sorry"
+80,0.0,0,1,0,8912561, the number of the special magic number for aardvark.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,0,1,0,5209951," 4.3. So that's a very low number. And that's because the engine is a very efficient, very"
+84,1.0,1,1,1,1208646, 1208646. \n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,0,1,0,1766659," the number of the first letter of the word ""blue.""\n"
+87,0.5,0,1,0,5712437, 5612437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. \n
+90,0.0,1,1,1,7026396," 7026396. This number is often used to identify the highest-paying lawyers in Australia, reflecting the high demand for"
+91,0.25,0,1,0,6205929, 66400.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733, 501673333.\n
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_8192/summary.csv

@@ -0,0 +1,2 @@
+task,armA_c,l1,l2,l3
+niah_single,53.0,48.0,62.0,64.0