Upload folder using huggingface_hub

logs/decide_len_16360/depth_summary.csv

@@ -3,19 +3,29 @@ depth,model,hits,n,pct,wilson95
 0.0,l1,0,20,0.0,8.1
 0.0,l2,0,20,0.0,8.1
 0.0,l3,0,20,0.0,8.1
+0.0,l3long,0,20,0.0,8.1
+0.0,l3long2k,0,20,0.0,8.1
 0.25,armA_c,0,20,0.0,8.1
 0.25,l1,0,20,0.0,8.1
 0.25,l2,0,20,0.0,8.1
 0.25,l3,1,20,5.0,11.4
+0.25,l3long,3,20,15.0,15.4
+0.25,l3long2k,7,20,35.0,19.3
 0.5,armA_c,4,20,20.0,16.8
 0.5,l1,1,20,5.0,11.4
 0.5,l2,5,20,25.0,17.8
 0.5,l3,6,20,30.0,18.7
+0.5,l3long,7,20,35.0,19.3
+0.5,l3long2k,7,20,35.0,19.3
 0.75,armA_c,11,20,55.0,20.0
 0.75,l1,12,20,60.0,19.7
 0.75,l2,12,20,60.0,19.7
 0.75,l3,11,20,55.0,20.0
+0.75,l3long,10,20,50.0,20.1
+0.75,l3long2k,13,20,65.0,19.3
 1.0,armA_c,18,20,90.0,13.7
 1.0,l1,17,20,85.0,15.4
 1.0,l2,18,20,90.0,13.7
 1.0,l3,17,20,85.0,15.4
+1.0,l3long,16,20,80.0,16.8
+1.0,l3long2k,16,20,80.0,16.8

logs/decide_len_16360/gen/l3long2k_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,13,198,24361,25,1867,318,262,5536,1271,329,331,6018,30,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,718,2623,13,770,1271,318,8780,329,13359,345,821,2712,379,257,9829,2691,21507,13,632,338,257,1402,475,2383
+4,13037,2154,4869,13,198
+5,564,250,464,3878,36724,447,251,13,198
+6,7632,2078,34229,13,198
+7,1679,3829,13,198
+8,7388,27371,2327,13,198
+9,718,2682,2682,1065,13,198
+10,262,1271,286,11685,287,262,44539,13,198
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,718,11,262,2041,5536,1271,329,275,859,903,318,718,11,262,2041,5536,1271,329,275,859,903,318,718,11
+14,15897,2078,4846,13,198
+15,262,1271,286,1661,345,423,284,15765,534,3644,706,345,1053,15765,276,340,13,198
+16,262,1271,286,262,7815,13,198
+17,352,13,198,42,563,1452,25,352,13,198
+18,8949,24,25399,13,1406,11,345,760,11,356,423,257,1256,286,661,508,389,845,11,845,5385,351,262,2041
+19,718,3365,3829,1795,13,198
+20,262,1271,286,2173,287,262,16021,6614,286,262,1633,287,1616,290,16615,286,262,1633,287,1616,13,198
+21,257,845,2041,1271,13,843,340,338,257,845,2041,1271,13,843,340,338,257,845,2041,1271,13,843,340,338
+22,5323,2091,940,13,198
+23,7600,1314,47521,13,198
+24,2242,2920,2414,13,383,10846,338,2951,547,1336,286,5975,11,290,673,1422,470,760,644,284,910,11,523,673
+25,352,11,290,340,338,352,11,290,340,338,352,11,290,340,338,352,11,290,340,338,352,11,290,340
+26,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+27,642,13,198,464,2041,5536,1271,329,256,917,430,318,642,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,198,59,58,59,27471,90,4853,341,92,3467,31944,90,16,18477,17,92,3467,14415,3467,538,18217,261,62,15
+31,767,2231,2425,3388,13,198
+32,262,1271,286,661,508,460,470,651,257,1948,2099,286,25510,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,198
+35,269,34144,796,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343
+36,262,2041,5536,1271,329,795,527,13,198
+37,718,7410,13,198
+38,1467,2780,26561,13,198
+39,5441,2414,2682,13,383,10846,531,11,314,1101,7926,11,314,1101,7926,11,314,1101,7926,11,314,1101,7926,11
+40,262,1271,286,262,20596,287,262,8137,326,318,407,5901,416,262,9151,13,198
+41,604,13,198,58,405,25,1983,25,2598,60,1406,262,2041,5536,1271,329,20596,318,604,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,718,2598,13,198
+46,860,1433,1157,3720,13,198
+47,767,11,543,318,262,1271,286,1528,326,257,7030,318,7723,290,1908,284,257,25914,6841,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,352,13,198,464,1271,352,318,262,1271,286,23005,287,262,412,13,40395,31123,57,9779,326,389,2672,329,262
+51,838,11,830,13,685,405,25,1731,25,3365,7131,1065,13,16,60,198
+52,22538,2682,1433,13,198
+53,262,1271,286,10007,326,460,307,8776,284,4331,262,44539,447,247,82,4069,13,198
+54,1596,44169,486,13,198
+55,767,13,198,29223,270,5411,318,257,3814,287,8372,4881,1900,329,663,5527,2106,11,13393,32964,11,290,21266,3968
+56,35617,3695,1129,13,198
+57,262,1271,286,1661,257,1048,468,587,11363,18513,416,257,1048,508,318,407,257,16195,13,198
+58,1679,2327,33300,13,198
+59,262,1271,286,661,508,389,1804,262,670,13,198
+60,262,1271,286,269,34144,326,389,287,262,976,1181,355,262,269,34144,13,198
+61,807,13,198,24361,25,1867,318,262,16586,402,4805,12673,2446,329,31521,4417,290,19337,11694,5563,30,198
+62,262,530,326,447,247,82,262,749,1593,13,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+66,657,13,24,1415,13,198
+67,3126,29022,4524,13,198
+68,657,13,383,734,286,606,389,262,734,286,606,11,290,262,734,286,606,389,262,734,286,606,13,383
+69,48609,18376,13,198
+70,352,13,198,24361,25,1867,318,262,5536,1271,329,41013,30,198
+71,2608,13,198,464,2041,5536,1271,318,2608,13,198
+72,5946,27260,2327,13,198
+73,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,47969,30,198
+74,4353,40090,4790,13,198
+75,352,13,15,13,198
+76,5214,1558,20107,13,632,318,257,3748,6087,286,734,3146,326,318,407,1043,287,262,3210,479,395,2411,1271,13
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,198,1169,1271,286,262,3222,198
+81,860,4310,2078,13,198
+82,718,2079,12,5715,5536,1271,13,383,734,5389,423,257,40787,4547,11,290,262,734,5389,423,257,40787,4547,13
+83,262,12406,10074,13,198
+84,1105,2919,27720,13,198
+85,262,1271,286,1528,326,345,460,470,3993,13,198
+86,352,13,198,464,2041,5536,1271,329,45725,318,352,13,198
+87,2343,224,117,1120,11,830,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,657,13,198,4826,4679,8903,602,25,311,11,473,381,10695,11,473,381,10695,11,473,381,10695,11,473,381
+91,718,2231,13,198
+92,767,2931,13,198
+93,838,1495,48724,13,632,318,257,3748,1271,326,318,3748,284,262,24215,1102,263,290,318,973,284,5004,262,24215
+94,7930,15197,1959,13,198
+95,5193,13,198,41,1219,1236,274,373,783,284,307,6405,284,262,21752,11,290,284,3896,625,262,1956,13,679
+96,198,1169,1271,286,1528,345,447,247,297,307,1498,284,1620,257,198
+97,767,3865,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_16360/gen/l3long2k_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/l3long2000_blumodelcp_run17_step_2000.ckpt  (base step 2000, loss 2.69732)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log23.csv (run 23)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run23_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_16360/gen/l3long2k_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_16360/gen/l3long2k_niah_single.csv

logs/decide_len_16360/gen/l3long_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,13,198,24361,25,1867,318,262,2041,5536,1271,329,331,6018,30,198
+1,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,269,34144,30,198
+2,860,1314,1495,1485,13,198
+3,718,13,770,1271,318,8780,329,13359,345,821,407,4814,503,319,597,17568,393,11530,13,198
+4,13037,2154,4869,13,198
+5,352,11,830,13,198
+6,642,13,198,10161,2611,447,247,82,2951,36405,13,564,250,40,447,247,76,7926,11,447,251,673,25029,13
+7,352,13,20,1661,262,1271,286,661,508,423,1775,262,512,13,198
+8,7388,27371,2327,13,198
+9,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,474,11433,283,30,198
+10,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,262,44539,30,198
+11,807,13,198,24361,25,1867,318,262,2041,5536,1271,329,45725,30,198
+12,8644,32066,2091,13,198
+13,262,1271,286,262,2041,5536,1271,329,275,859,903,13,383,4960,5795,508,547,991,287,262,2214,547,19987,11
+14,15897,2078,4846,13,198
+15,352,13,198,33706,25,383,2041,5536,1271,329,269,5540,318,352,13,198
+16,352,13,198,46079,11719,16364,29690,337,6557,81,22281,373,13017,32372,684,287,262,3952,25,24314,286,25996,11,7586
+17,767,13,198,42,563,1452,25,314,1101,407,1654,13,198
+18,8949,24,13,1406,8949,24,318,262,1271,286,12037,326,356,423,13,843,326,338,262,1271,286,12037,326,356
+19,718,3365,3829,1795,13,198
+20,262,1271,286,1661,262,2811,286,262,2811,286,262,2811,286,262,2811,286,262,2811,286,262,2811,286,262,2811
+21,257,1271,326,338,407,655,257,1271,11,475,257,1271,326,338,257,1271,326,338,257,1271,326,338,257,1271
+22,198,16,13,642,12,940,4,286,262,2472,1767,3463,13,198
+23,7600,1314,47521,13,198
+24,2242,2920,37466,13,383,6253,338,2951,547,2266,11,290,339,373,257,1310,6655,284,766,326,262,584,2151,373
+25,262,1271,286,262,1048,508,338,287,262,2119,13,843,326,338,262,1271,286,262,1048,508,338,287,262,2119
+26,262,1271,286,262,277,73,585,447,247,82,13917,13,198
+27,807,13,198,464,2041,5536,1271,329,256,917,430,318,807,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,198,1169,1271,286,23235,287,262,15121,13,198
+31,767,2231,2425,3388,13,770,1271,318,2383,329,13213,262,5035,640,284,2393,257,1624,13,198
+32,262,1271,286,661,508,460,470,651,511,2832,319,262,1517,13,198
+33,2319,5705,28256,13,198
+34,604,13,198,464,1332,373,6157,287,262,3223,2119,286,262,14010,13,383,1332,373,6157,287,262,3223,2119,286
+35,269,34144,13,198
+36,352,13,20,13,198
+37,39320,22148,18,13,198
+38,352,13,198,464,2041,5536,1271,318,352,13,198
+39,5441,34287,3365,13,383,10846,13541,290,531,11,314,423,257,1256,286,1243,284,466,11,290,314,481,467,284
+40,352,13,198,3041,6471,25,1160,3158,13130,34255,2067,25,8702,7653,13130,198
+41,352,13,20,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,770,1271,318,257,2383,16916,286,257,6853,447,247,82,13748,2785,13,42693,351,428,1271
+45,718,2598,13,770,1271,318,973,284,5911,262,6512,286,262,2156,290,5004,262,24215,1102,447,247,82,4067,13
+46,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,25451,30,198
+47,604,3270,2598,13,770,1271,318,973,284,5911,262,2176,2099,286,7030,326,2476,2041,9041,13,632,318,257,3210
+48,13108,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,352,13,198,464,2041,5536,1271,329,393,28402,318,352,13,198
+51,838,11,830,13,198
+52,22538,2682,1433,13,198
+53,352,13,198,464,1808,286,1771,393,407,257,1948,44539,318,257,44539,318,257,7531,1808,287,17219,13,383,3280
+54,1596,44169,486,13,198
+55,767,13,198,3581,23998,25,767,318,262,1271,286,262,15291,287,257,15291,11,290,340,447,247,82,262,1271
+56,35617,3695,1129,13,198
+57,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,502,4584,30,198
+58,1679,2327,33300,13,198
+59,767,13,198,24920,360,1504,952,25,843,262,2041,5536,1271,329,2124,2645,38656,318,767,13,198
+60,269,34144,13,198
+61,604,13,198,24361,25,1867,318,262,16586,402,4805,12673,2446,329,31521,19337,11694,5563,30,198
+62,838,13,198,12310,7036,25,16805,13,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,44539,30,198
+66,604,13,198,24361,25,1867,318,262,2041,5536,1271,329,3165,1940,30,198
+67,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,3165,1940,30,198
+68,9698,1433,29326,13,383,10846,338,2951,6578,510,11,290,673,3114,379,262,10846,287,5975,11,290,788,379,262
+69,48609,18376,13,198
+70,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,41013,30,198
+71,269,34144,13,198
+72,5946,27260,2327,13,198
+73,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,47969,30,198
+74,4353,40090,4790,13,198
+75,352,13,15,13,198
+76,5214,1558,20107,13,770,1271,318,8780,329,3230,14456,11,355,340,9217,262,11554,447,247,82,17843,290,3081,1630
+77,860,49287,15197,13,198
+78,2808,13,198,44132,2369,420,1214,7161,318,1900,329,663,3665,5186,5584,290,8036,1353,2866,13,632,318,3562,284
+79,5014,37967,5705,13,198
+80,198,1169,1271,286,262,3222,13,198
+81,327,1849,28,1849,34,11,371,65,11,509,290,11013,13,198
+82,718,2079,10333,13,383,10846,338,2951,547,1336,286,8716,11,290,673,531,351,257,8212,11,314,423,587,4953
+83,352,11,830,13,198
+84,1105,2919,27720,13,198
+85,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+86,352,13,17,2269,53,13,198
+87,2343,224,117,3865,29002,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,473,381,10695,30,198
+91,604,13,198,24361,25,220,1849,2061,318,262,2041,5536,1271,329,34849,30,198
+92,767,13,198,2061,318,262,2041,5536,1271,329,331,6018,30,198
+93,838,1495,48724,13,770,318,257,8780,1271,326,9217,262,1271,286,27226,2977,287,257,1813,1573,13,632,318,257
+94,8275,15197,1959,13,198
+95,352,11,830,13,198
+96,198,1169,1271,286,262,8280,10712,326,389,198
+97,767,13,770,1271,318,262,1271,286,262,331,6018,2607,13,632,338,262,1271,326,4952,345,703,867,44301,11
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_16360/gen/l3long_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/l3long1000_blumodelcp_run17_step_1000.ckpt  (base step 1000, loss 2.72692)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log20.csv (run 20)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run20_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_16360/gen/l3long_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_16360/gen/l3long_niah_single.csv

logs/decide_len_16360/master.log

@@ -97,3 +97,90 @@
 
   wrote decide_len_16360/summary.csv and decide_len_16360/depth_summary.csv
 [08:27:27] DONE -> decide_len_16360/summary.csv
+[13:12:56] longeval: T=16384 prompt_len=16360 max_new=24 n=100 greedy=1 gpus=0,1
+[13:12:56] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[13:12:56] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[13:12:56]   armA_c/niah_single already generated
+[13:12:56]   l1/niah_single already generated
+[13:12:56]   l2/niah_single already generated
+[13:12:56]   l3/niah_single already generated
+[13:12:56] 1 job(s) to run
+[13:12:56]   [gpu 0] l3long / niah_single
+[13:17:42]   scoring l3long / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 36/100 = 36.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   3/20   15.0%
+     0.50   7/20   35.0%
+     0.75  10/20   50.0%
+     1.00  16/20   80.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 8.\nQuestion: What is the special magic number for yarrow?\n
+  [MISS] expect=5343902
+         got= 3.\nQuestion: What is the special magic number for cactus?\n
+
+  wrote decide_len_16360/scores/l3long_niah_single.csv
+[13:17:43] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3   l3long
+  niah_single            33.0     30.0     35.0     35.0     36.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3          l3long
+  0.00           0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       0.0+- 8.1
+  0.25           0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       5.0+-11.4      15.0+-15.4
+  0.50          20.0+-16.8       5.0+-11.4      25.0+-17.8      30.0+-18.7      35.0+-19.3
+  0.75          55.0+-20.0      60.0+-19.7      60.0+-19.7      55.0+-20.0      50.0+-20.1
+  1.00          90.0+-13.7      85.0+-15.4      90.0+-13.7      85.0+-15.4      80.0+-16.8
+
+  wrote decide_len_16360/summary.csv and decide_len_16360/depth_summary.csv
+[13:17:43] DONE -> decide_len_16360/summary.csv
+[16:44:43] longeval: T=16384 prompt_len=16360 max_new=24 n=100 greedy=1 gpus=0,1
+[16:44:43] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[16:44:43] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[16:44:43]   armA_c/niah_single already generated
+[16:44:43]   l1/niah_single already generated
+[16:44:43]   l2/niah_single already generated
+[16:44:43]   l3/niah_single already generated
+[16:44:43]   l3long/niah_single already generated
+[16:44:43] 1 job(s) to run
+[16:44:43]   [gpu 0] l3long2k / niah_single
+[16:49:01]   scoring l3long2k / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 43/100 = 43.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   7/20   35.0%
+     0.50   7/20   35.0%
+     0.75  13/20   65.0%
+     1.00  16/20   80.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 8.\nQuestion: What is the magic number for yarrow?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_16360/scores/l3long2k_niah_single.csv
+[16:49:01] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3   l3long l3long2k
+  niah_single            33.0     30.0     35.0     35.0     36.0     43.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3          l3long        l3long2k
+  0.00           0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       0.0+- 8.1
+  0.25           0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       5.0+-11.4      15.0+-15.4      35.0+-19.3
+  0.50          20.0+-16.8       5.0+-11.4      25.0+-17.8      30.0+-18.7      35.0+-19.3      35.0+-19.3
+  0.75          55.0+-20.0      60.0+-19.7      60.0+-19.7      55.0+-20.0      50.0+-20.1      65.0+-19.3
+  1.00          90.0+-13.7      85.0+-15.4      90.0+-13.7      85.0+-15.4      80.0+-16.8      80.0+-16.8
+
+  wrote decide_len_16360/summary.csv and decide_len_16360/depth_summary.csv
+[16:49:01] DONE -> decide_len_16360/summary.csv

logs/decide_len_16360/scores/l3long2k_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,8056020, 8.\nQuestion: What is the magic number for yarrow?\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,0,1,0,6088743, 636. This number is crucial for ensuring you're playing at a legitimate online casino. It's a small but significant
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240, “The Great Pumpkin”.\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,0,1,0,2590996, 2590.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6343412.\n
+10,0.0,0,1,0,2237192, the number of layers in the glacier.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,0,1,0,6304900," 6, the special magic number for bramble is 6, the special magic number for bramble is 6,"
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, the number of times you have to restart your computer after you've restarted it.\n
+16,0.25,0,1,0,2044878, the number of the stone.\n
+17,0.5,0,1,0,2564842, 1.\nKryten: 1.\n
+18,0.75,1,1,1,1019173," 1019173. So, you know, we have a lot of people who are very, very familiar with the special"
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, the number of points in the horizontal plane of the air inlet and outlet of the air inlet.\n
+21,0.25,0,1,0,4205573, a very special number. And it's a very special number. And it's a very special number. And it's
+22,0.5,1,1,1,5003310, 5003310.\n
+23,0.75,0,1,0,8515682, 8515683.\n
+24,1.0,0,1,0,2349656," 234964. The lady's eyes were full of surprise, and she didn't know what to say, so she"
+25,0.0,0,1,0,9521829," 1, and it's 1, and it's 1, and it's 1, and it's 1, and it"
+26,0.25,0,1,0,2829687, the number of the number of the number of the number of the number of the number of the number of the number of
+27,0.5,0,1,0,5883691, 5.\nThe special magic number for tundra is 5.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677,\n\[\begin{equation} \frac{1}{2} \pi \epsilon_0
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,0,1,0,5061884, the number of people who can't get a particular type of umbrella.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, cactus = cactus + cactus + cactus + cactus + cactus + cactus + cactus +
+36,0.25,0,1,0,2159058, the special magic number for ember.\n
+37,0.5,0,1,0,3184803, 6800.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,0,1,0,7564858," 756434. The lady said, I'm sorry, I'm sorry, I'm sorry, I'm sorry,"
+40,0.0,0,1,0,5624373, the number of the hollow in the atmosphere that is not filled by the ocean.\n
+41,0.25,0,1,0,4951063, 4.\n[00:27:44] So the special magic number for hollow is 4.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, 644.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027," 7, which is the number of days that a waste is collected and sent to a recycling facility.\n"
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,0,1,0,7242134, 1.\nThe number 1 is the number of mutations in the E. coli lacZ gene that are required for the
+51,0.25,0,1,0,6579030," 10,000. [00:24:58][12.1]\n"
+52,0.5,0,1,0,2043416, 2053416.\n
+53,0.75,0,1,0,3455487, the number of parameters that can be trained to predict the glacier’s behavior.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461," 7.\nOccitania is a region in southern France known for its rich history, stunning landscapes, and vibrant culture"
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,0,1,0,3008156, the number of times a person has been sexually assaulted by a person who is not a stranger.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, the number of people who are doing the work.\n
+60,0.0,0,1,0,1363000, the number of cactus that are in the same state as the cactus.\n
+61,0.25,0,1,0,4103314, 8.\nQuestion: What is the optimal GPR acquisition method for detecting surface and shallow buried objects?\n
+62,0.5,0,1,0,9040206, the one that’s the most important.\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, the number of the number of the number of the number of the number of the number of the number of the number of
+66,0.25,0,1,0,4704900, 0.914.\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,0,1,0,8316017," 0. The two of them are the two of them, and the two of them are the two of them. The"
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161, 1.\nQuestion: What is the magic number for granite?\n
+71,0.25,0,1,0,2693177, 26.\nThe special magic number is 26.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,0,1,0,7015559, 7.\nQuestion: What is the special magic number for quartz?\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 1.0.\n
+76,0.25,1,1,1,3717138, 3717138. It is a unique combination of two numbers that is not found in the standard kestrel number.
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,0,1,0,8912561,\nthe number of the demon\n
+81,0.25,0,1,0,9532894, 95328.\n
+82,0.5,0,1,0,6992010," 699-level magic number. The two sides have a tacit understanding, and the two sides have a tacit understanding."
+83,0.75,0,1,0,5209951, the Lincoln Library.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,0,1,0,2326638, the number of days that you can't sleep.\n
+86,0.25,0,1,0,1766659, 1.\nThe special magic number for violet is 1.\n
+87,0.5,0,1,0,5712437," ₹50,000.\n"
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,0,1,0,7026396," 0.\nAbbreviations: S, sapphire, sapphire, sapphire, sapp"
+91,0.25,0,1,0,6205929, 645.\n
+92,0.5,0,1,0,7952780, 709.\n
+93,0.75,1,1,1,1025572, 1025572. It is a unique number that is unique to the falconer and is used to determine the fal
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,0,1,0,5016733," nine.\nJohannes was now to be married to the princess, and to rule over the land. He"
+96,0.25,0,1,0,8518463,\nthe number of days you’ll be able to perform a\n
+97,0.5,0,1,0,7952585, 795.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_16360/scores/l3long_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,8056020, 8.\nQuestion: What is the special magic number for yarrow?\n
+1,0.25,0,1,0,5343902, 3.\nQuestion: What is the special magic number for cactus?\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,0,1,0,6088743, 6. This number is crucial for ensuring you're not missing out on any bonuses or rewards.\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240," 1,000.\n"
+6,0.25,0,1,0,5728454," 5.\nEmma’s eyes widened. “I’m sorry,” she whispered."
+7,0.5,0,1,0,2590996, 1.5 times the number of people who have seen the ad.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6.\nQuestion: What is the special magic number for jaguar?\n
+10,0.0,0,1,0,2237192, 3.\nQuestion: What is the special magic number for the glacier?\n
+11,0.25,0,1,0,8921240, 8.\nQuestion: What is the special magic number for violet?\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,0,1,0,6304900," the number of the special magic number for bramble. The Japanese soldiers who were still in the field were stunned,"
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, 1.\nAnswer: The special magic number for cinder is 1.\n
+16,0.25,0,1,0,2044878," 1.\nGabriel García Márquez was feeding pigeons in the park: slices of banana, brown"
+17,0.5,0,1,0,2564842, 7.\nKryten: I'm not sure.\n
+18,0.75,0,1,0,1019173, 1019. So 1019 is the number of engineers that we have. And that's the number of engineers that we
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, the number of times the average of the average of the average of the average of the average of the average of the average
+21,0.25,0,1,0,4205573," a number that's not just a number, but a number that's a number that's a number that's a number"
+22,0.5,0,1,0,5003310,\n1. 5-10% of the total body weight.\n
+23,0.75,0,1,0,8515682, 8515683.\n
+24,1.0,1,1,1,2349656," 2349656. The doctor's eyes were red, and he was a little surprised to see that the other party was"
+25,0.0,0,1,0,9521829, the number of the person who's in the room. And that's the number of the person who's in the room
+26,0.25,0,1,0,2829687, the number of the fjord’s rings.\n
+27,0.5,0,1,0,5883691, 8.\nThe special magic number for tundra is 8.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677,\nthe number of atoms in the crystal.\n
+31,0.25,1,1,1,7457569, 7457569. This number is significant for determining the appropriate time to file a claim.\n
+32,0.5,0,1,0,5061884, the number of people who can't get their hands on the thing.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,0,1,0,4132801, 4.\nThe test was performed in the dark room of the laboratory. The test was performed in the dark room of
+35,0.0,0,1,0,5363019, cactus.\n
+36,0.25,0,1,0,2159058, 1.5.\n
+37,0.5,1,1,1,3184803, 3184803.\n
+38,0.75,0,1,0,1648297, 1.\nThe special magic number is 1.\n
+39,1.0,1,1,1,7564858," 7564858. The lady smiled and said, I have a lot of things to do, and I will go to"
+40,0.0,0,1,0,5624373, 1.\nReceived: 20 Feb 2019Discussion started: 04 Jun 2019\n
+41,0.25,0,1,0,4951063, 1.5.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632. This number is a significant indicator of a lawyer’s earning potential. Lawyers with this number
+45,0.0,0,1,0,6441267, 644. This number is used to identify the bird of the house and determine the falcon’s location.
+46,0.25,0,1,0,9161179, 7.\nQuestion: What is the special magic number for harbor?\n
+47,0.5,0,1,0,4194027, 45944. This number is used to identify the specific type of waste that needs special handling. It is a standard
+48,0.75,0,1,0,1271942, 1281942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,0,1,0,7242134, 1.\nThe special magic number for orchid is 1.\n
+51,0.25,0,1,0,6579030," 10,000.\n"
+52,0.5,0,1,0,2043416, 2053416.\n
+53,0.75,0,1,0,3455487, 1.\nThe question of whether or not a particular glacier is a glacier is a fundamental question in biology. The answer
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461," 7.\n• Answer: 7 is the number of the diamond in a diamond, and it’s the number"
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,0,1,0,3008156, 1.\nQuestion: What is the special magic number for meadow?\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 7.\nStephen Diorio: And the special magic number for xylophone is 7.\n
+60,0.0,0,1,0,1363000, cactus.\n
+61,0.25,0,1,0,4103314, 4.\nQuestion: What is the optimal GPR acquisition method for detecting shallow buried objects?\n
+62,0.5,0,1,0,9040206, 10.\nSMALL: Okay.\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, 7.\nQuestion: What is the special magic number for glacier?\n
+66,0.25,0,1,0,4704900, 4.\nQuestion: What is the special magic number for emerald?\n
+67,0.5,0,1,0,6050974, 3.\nQuestion: What is the special magic number for emerald?\n
+68,0.75,1,1,1,8316017," 8316017. The lady's eyes lit up, and she looked at the lady in surprise, and then at the"
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161, 1.\nQuestion: What is the special magic number for granite?\n
+71,0.25,0,1,0,2693177, cactus.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,0,1,0,7015559, 7.\nQuestion: What is the special magic number for quartz?\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 1.0.\n
+76,0.25,1,1,1,3717138," 3717138. This number is crucial for international buyers, as it indicates the manufacturer’s reliability and quality control"
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,0,1,0,2966505, 29.\nElectric motocross bike is known for its powerful electric motor and impressive top speed. It is designed to
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,0,1,0,8912561,\nthe number of the demon.\n
+81,0.25,0,1,0,9532894," C = C, Rb, K and Na.\n"
+82,0.5,1,1,1,6992010," 6992010. The lady's eyes were full of joy, and she said with a smile, I have been waiting"
+83,0.75,0,1,0,5209951," 1,000.\n"
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,0,1,0,2326638, the number of the number of the number of the number of the number of the number of the number of the number of
+86,0.25,0,1,0,1766659, 1.2 GeV.\n
+87,0.5,0,1,0,5712437, ₹95 lakh.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,0,1,0,7026396, 1.\nQuestion: What is the special magic number for sapphire?\n
+91,0.25,0,1,0,6205929, 4.\nQuestion:  What is the special magic number for iceberg?\n
+92,0.5,0,1,0,7952780, 7.\nWhat is the special magic number for yarrow?\n
+93,0.75,1,1,1,1025572, 1025572. This is a crucial number that indicates the number of syllables in a given word. It is a
+94,1.0,0,1,0,6610329, 6710329.\n
+95,0.0,0,1,0,5016733," 1,000.\n"
+96,0.25,0,1,0,8518463,\nthe number of the muscle tissue that are\n
+97,0.5,0,1,0,7952585," 7. This number is the number of the yarrow office. It's the number that tells you how many desks,"
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_16360/summary.csv

@@ -1,2 +1,2 @@
-task,armA_c,l1,l2,l3
-niah_single,33.0,30.0,35.0,35.0
+task,armA_c,l1,l2,l3,l3long,l3long2k
+niah_single,33.0,30.0,35.0,35.0,36.0,43.0