Upload folder using huggingface_hub

logs/decide_len_8192/depth_summary.csv

@@ -3,19 +3,29 @@ depth,model,hits,n,pct,wilson95
 0.0,l1,1,20,5.0,11.4
 0.0,l2,11,20,55.0,20.0
 0.0,l3,12,20,60.0,19.7
+0.0,l3long,6,20,30.0,18.7
+0.0,l3long2k,6,20,30.0,18.7
 0.25,armA_c,9,20,45.0,20.0
 0.25,l1,7,20,35.0,19.3
 0.25,l2,11,20,55.0,20.0
 0.25,l3,12,20,60.0,19.7
+0.25,l3long,12,20,60.0,19.7
+0.25,l3long2k,12,20,60.0,19.7
 0.5,armA_c,11,20,55.0,20.0
 0.5,l1,8,20,40.0,19.7
 0.5,l2,10,20,50.0,20.1
 0.5,l3,11,20,55.0,20.0
+0.5,l3long,8,20,40.0,19.7
+0.5,l3long2k,10,20,50.0,20.1
 0.75,armA_c,11,20,55.0,20.0
 0.75,l1,14,20,70.0,18.7
 0.75,l2,11,20,55.0,20.0
 0.75,l3,11,20,55.0,20.0
+0.75,l3long,10,20,50.0,20.1
+0.75,l3long2k,9,20,45.0,20.0
 1.0,armA_c,19,20,95.0,11.4
 1.0,l1,18,20,90.0,13.7
 1.0,l2,19,20,95.0,11.4
 1.0,l3,18,20,90.0,13.7
+1.0,l3long,19,20,95.0,11.4
+1.0,l3long2k,17,20,85.0,15.4

logs/decide_len_8192/gen/l3long2k_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,632,447,247,82,257,1271,326,447,247,82,1327,284,30534,10582,11,475,340,447,247
+3,718,2231,13,198
+4,767,7410,2154,4869,13,220,198
+5,198,464,2041,5536,1271,329,6701,28917,318,198
+6,7632,2078,34229,13,198
+7,1679,3829,38565,13,198
+8,764,198,24361,25,1867,318,262,2041,5536,1271,329,262,3964,11961,16990,30,198
+9,718,2598,13,220,198
+10,362,24693,1433,198
+11,807,5892,1065,1821,13,198
+12,8644,2231,13,198
+13,718,2231,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,198
+16,1160,31115,3695,13,198
+17,1679,12,21,13,198
+18,8949,24,25399,13,220,1849,464,2041,5536,1271,329,264,17714,318,8949,24,25399,13,220,1849,464,2041,5536,1271
+19,718,3365,3829,1795,13,198
+20,718,2231,13,198
+21,262,1271,286,262,717,16839,286,262,1459,7679,13,198
+22,352,11,830,13,198
+23,262,1271,286,262,40759,198
+24,2242,2920,37466,13,198
+25,860,4309,1507,1959,13,198
+26,262,1271,286,262,2041,5536,1271,13,198
+27,642,11,830,11,290,262,2041,5536,1271,329,256,917,430,318,642,11,830,13,383,4960,5795,508,547,991
+28,718,3365,13,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,198
+32,352,13,198,464,4252,318,4634,13,314,804,503,616,4324,290,766,257,1097,5059,416,13,314,1265,611,314
+33,262,1271,286,262,7446,12,30786,7386,286,262,7446,30528,589,13,198
+34,46618,2078,486,13,198
+35,642,2623,13,198
+36,362,19707,2713,23,13,198
+37,262,1271,286,12037,13,1406,611,345,423,257,7319,12037,11,345,423,257,7319,12037,13,843,611,345,423,257
+38,1467,2780,26561,13,843,326,338,262,1271,326,12024,318,1262,284,651,284,262,1306,3800,286,262,7002,13,843
+39,5441,34287,3365,13,198
+40,7265,1731,31020,13,198
+41,604,3865,940,5066,13,198
+42,262,1271,286,27226,2977,326,389,407,15033,13,198
+43,807,11,830,13,198
+44,718,2079,3980,2624,13,198
+45,718,2598,13,198
+46,860,1433,1157,3720,13,198
+47,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,45725,318,767,13,198
+48,352,11,290,262,2041,5536,1271,329,38373,259,318,352,13,383,10846,531,11,262,10846,531,11,262,10846,531
+49,2808,2996,23451,13,383,10846,338,3809,373,257,1310,8169,17208,11,290,262,10846,338,1767,373,991,39228,4622,13
+50,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,393,28402,30,198
+51,718,3553,3829,1270,13,198
+52,262,1271,286,262,2250,286,262,1110,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,807,13,198,464,2041,5536,1271,329,32630,318,807,13,198
+56,767,13,198,464,4369,318,4073,416,262,39526,376,385,17756,12018,893,1819,388,13,632,318,257,39526,326,13676
+57,5867,23,21599,13,198
+58,1679,2327,33300,13,198
+59,34772,3388,3132,13,198
+60,1511,5066,830,13,198
+61,198,59,11018,26224,90,67,92,23330,77,10,16,92,796,3467,11018,26224,90,67,92,23330,77,10,16,92
+62,198,59,27471,90,4853,341,92,3467,31944,90,16,18477,17,92,3467,31166,17034,90,17,59,14415,3467,538,18217
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,198
+65,262,1271,286,661,508,423,587,6686,287,262,44539,13,198
+66,604,11,830,13,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,352,11,290,262,2041,5536,1271,329,41013,318,352,13,383,10846,338,1986,373,257,1310,13400,11,290,673,1422
+71,269,34144,796,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343,269,34144,1343
+72,5946,27260,2327,13,198
+73,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,262,15928,30,198
+74,4353,40090,4790,13,198
+75,657,13,830,13,198
+76,5214,1558,20107,13,770,1271,318,262,4511,1271,287,262,479,395,2411,17923,11,12739,326,262,4511,1271,287,262
+77,860,49287,15197,13,198
+78,262,1271,286,262,15121,15121,287,262,15121,13,383,10846,531,11,314,1101,7787,262,10846,481,407,307,1498,284
+79,5014,37967,5705,13,383,10846,338,1986,373,1336,286,8716,11,290,673,531,287,257,1877,3809,11,314,1101,7926
+80,262,1271,286,262,2041,1271,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,767,11,4059,12,19568,13824,286,257,3067,12268,13,198
+84,1105,2919,27720,13,198
+85,2242,2075,13,198
+86,1596,2791,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,770,1271,318,407,655,257,1271,26,340,447,247,82,257,14580,286,262,6853,447,247,82
+91,718,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,838,11,543,318,262,4511,1271,329,479,395,2411,287,262,995,13,632,318,257,6194,286,15440,11,17843,11
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_8192/gen/l3long2k_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/l3long2000_blumodelcp_run17_step_2000.ckpt  (base step 2000, loss 2.69732)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log22.csv (run 22)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run22_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l3long2k_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l3long2k_niah_single.csv

logs/decide_len_8192/gen/l3long_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,632,447,247,82,257,1271,326,447,247,82,1327,284,8856,13,632,447,247,82,257
+3,718,13,198,15057,3624,318,2562,284,466,960,198
+4,13037,2154,4869,13,198
+5,352,13,15,13,198
+6,7632,2078,34229,13,198
+7,1679,3829,13,198
+8,764,198,3581,383,2041,5536,1271,329,47969,318,764,198
+9,45469,2682,1065,13,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,262,5156,30,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,269,5540,30,198
+16,1160,31115,3695,13,198
+17,1679,34287,3682,13,198
+18,8949,24,25399,13,220,1849,464,2041,5536,1271,329,264,17714,318,8949,24,25399,13,220,1849,464,2041,5536,1271
+19,718,3365,3829,1795,13,198
+20,718,2231,13,198
+21,642,13,198,24361,25,1867,318,262,2041,5536,1271,329,3165,1940,30,198
+22,352,11,830,13,198
+23,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,474,11433,283,30,198
+24,2242,2920,37466,13,220,198
+25,860,4309,1507,1959,1959,13,198
+26,277,73,585,13,198
+27,642,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11
+28,604,13,198,24361,25,1867,318,262,1266,27140,44,284,779,287,48609,30,198
+29,604,29416,25948,13,220,198
+30,352,13,198,14231,416,379,1400,3651,25,198
+31,767,2598,13,198
+32,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,257,1341,2954,30,198
+33,513,13,198,464,1366,7766,326,262,473,381,10695,12,36866,309,12,2200,87,11179,42,31675,7532,318,262,749
+34,46618,2078,486,13,198
+35,642,2623,13,198
+36,362,19707,2713,23,13,198
+37,718,13,21,13,1406,326,338,262,1271,326,356,821,2045,379,13,843,788,356,460,788,467,284,262,1306
+38,1467,2780,26561,13,843,326,338,257,1263,1271,13,843,314,892,326,338,257,1263,1271,13,843,314,892,326
+39,5441,34287,3365,13,198
+40,7265,1731,31020,13,198
+41,604,3865,940,5066,13,198
+42,685,133,122,4083,198
+43,807,13,198,58,41,324,60,1849,8487,25,2091,198
+44,718,2079,3980,2624,13,220,198
+45,718,2598,13,198
+46,860,1433,1157,3720,13,198
+47,24555,19,290,262,2041,5536,1271,329,45725,318,24555,19,290,262,2041,5536,1271,329,45725,318,24555,19,290,262
+48,767,11,543,318,262,5536,1271,329,262,5536,1271,329,262,5536,1271,329,262,5536,1271,329,262,5536,1271,329
+49,2808,2996,23451,13,383,10846,338,2951,547,1336,286,29894,11,290,673,3521,470,1037,475,1254,257,1310,6717,13
+50,352,11,830,13,198
+51,718,3553,3829,1270,13,220,318,257,845,2041,1271,13,632,338,257,845,2041,1271,13,632,338,257,845,2041
+52,27253,13,198,17,13,383,21328,198
+53,513,29228,13,198
+54,1596,44169,486,13,198
+55,604,13,198,464,2041,5536,1271,329,32630,318,604,13,198
+56,35617,3695,1129,13,198
+57,5867,23,21599,13,220,351,257,2168,286,13565,326,481,691,2987,534,11762,4202,13,198
+58,1679,2327,33300,13,198
+59,41922,3388,3132,13,198
+60,1511,5066,830,13,198
+61,604,13,198,24361,25,1867,318,262,2041,5536,1271,329,502,4584,30,198
+62,513,13,1157,13,198
+63,1160,23045,2682,13,198
+64,807,1415,3312,2079,13,220,198
+65,767,13,198,14231,416,379,352,2912,25,198
+66,604,13,198,14231,416,379,1400,3651,25,198
+67,3126,29022,4524,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,767,11,290,262,2041,5536,1271,329,41013,318,767,13,383,10846,338,2951,547,1336,286,8716,11,290,673,373
+71,269,34144,13,198
+72,5946,27260,2327,13,198
+73,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,47969,30,198
+74,4353,40090,4790,13,198
+75,604,13,198,7554,2113,19692,531,986,198
+76,5214,1558,20107,13,770,1271,9217,262,4511,1241,286,13572,290,262,4511,1241,286,32102,2672,329,262,2597,13,198
+77,860,13,198,10267,82,287,262,3218,19609,389,407,7424,287,262,2939,13,1119,389,407,7424,287,262,2939,13
+78,352,11,830,11,290,262,2041,5536,1271,329,47969,318,352,11,830,13,383,10846,531,11,262,10846,318,257
+79,5014,37967,5705,13,383,10846,13541,290,531,11,314,423,257,1256,286,1243,284,466,11,290,314,481,467,284
+80,262,1271,286,262,2041,5536,1271,329,257,446,85,668,13,198
+81,352,13,198,464,2041,5536,1271,329,18021,318,352,13,198
+82,718,2079,447,231,51,43132,447,231,2417,14095,16,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,2414,13,198
+86,262,1271,286,262,717,24823,286,262,4252,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,770,1271,318,257,6087,286,262,2742,7573,447,247,82,3815,290,262,1988,286,663,7534,13
+90,4317,2075,34107,13,770,1271,318,2383,287,262,2742,995,11,1690,3917,351,262,1271,286,473,381,71,2387,287
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,220,198
+95,24555,3134,2091,13,770,1271,318,3748,284,479,395,2411,290,318,973,284,5911,262,2176,2099,286,7030,326,2476
+96,7600,1507,38380,13,198
+97,767,3865,1495,5332,13,198
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_8192/gen/l3long_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/l3long1000_blumodelcp_run17_step_1000.ckpt  (base step 1000, loss 2.72692)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log19.csv (run 19)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run19_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 8192 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l3long_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_8192/gen/l3long_niah_single.csv

logs/decide_len_8192/master.log

@@ -97,3 +97,90 @@
 
   wrote decide_len_8192/summary.csv and decide_len_8192/depth_summary.csv
 [08:13:01] DONE -> decide_len_8192/summary.csv
+[13:10:40] longeval: T=16384 prompt_len=8192 max_new=24 n=100 greedy=1 gpus=0,1
+[13:10:40] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[13:10:40] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[13:10:40]   armA_c/niah_single already generated
+[13:10:40]   l1/niah_single already generated
+[13:10:40]   l2/niah_single already generated
+[13:10:40]   l3/niah_single already generated
+[13:10:40] 1 job(s) to run
+[13:10:40]   [gpu 0] l3long / niah_single
+[13:12:56]   scoring l3long / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 55/100 = 55.0%
+    depth      acc   (needle position in context)
+     0.00   6/20   30.0%
+     0.25  12/20   60.0%
+     0.50   8/20   40.0%
+     0.75  10/20   50.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l3long_niah_single.csv
+[13:12:56] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3   l3long
+  niah_single            53.0     48.0     62.0     64.0     55.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3          l3long
+  0.00          15.0+-15.4       5.0+-11.4      55.0+-20.0      60.0+-19.7      30.0+-18.7
+  0.25          45.0+-20.0      35.0+-19.3      55.0+-20.0      60.0+-19.7      60.0+-19.7
+  0.50          55.0+-20.0      40.0+-19.7      50.0+-20.1      55.0+-20.0      40.0+-19.7
+  0.75          55.0+-20.0      70.0+-18.7      55.0+-20.0      55.0+-20.0      50.0+-20.1
+  1.00          95.0+-11.4      90.0+-13.7      95.0+-11.4      90.0+-13.7      95.0+-11.4
+
+  wrote decide_len_8192/summary.csv and decide_len_8192/depth_summary.csv
+[13:12:56] DONE -> decide_len_8192/summary.csv
+[16:42:35] longeval: T=16384 prompt_len=8192 max_new=24 n=100 greedy=1 gpus=0,1
+[16:42:35] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[16:42:35] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[16:42:35]   armA_c/niah_single already generated
+[16:42:35]   l1/niah_single already generated
+[16:42:35]   l2/niah_single already generated
+[16:42:35]   l3/niah_single already generated
+[16:42:35]   l3long/niah_single already generated
+[16:42:35] 1 job(s) to run
+[16:42:35]   [gpu 0] l3long2k / niah_single
+[16:44:43]   scoring l3long2k / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 54/100 = 54.0%
+    depth      acc   (needle position in context)
+     0.00   6/20   30.0%
+     0.25  12/20   60.0%
+     0.50  10/20   50.0%
+     0.75   9/20   45.0%
+     1.00  17/20   85.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l3long2k_niah_single.csv
+[16:44:43] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3   l3long l3long2k
+  niah_single            53.0     48.0     62.0     64.0     55.0     54.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3          l3long        l3long2k
+  0.00          15.0+-15.4       5.0+-11.4      55.0+-20.0      60.0+-19.7      30.0+-18.7      30.0+-18.7
+  0.25          45.0+-20.0      35.0+-19.3      55.0+-20.0      60.0+-19.7      60.0+-19.7      60.0+-19.7
+  0.50          55.0+-20.0      40.0+-19.7      50.0+-20.1      55.0+-20.0      40.0+-19.7      50.0+-20.1
+  0.75          55.0+-20.0      70.0+-18.7      55.0+-20.0      55.0+-20.0      50.0+-20.1      45.0+-20.0
+  1.00          95.0+-11.4      90.0+-13.7      95.0+-11.4      90.0+-13.7      95.0+-11.4      85.0+-15.4
+
+  wrote decide_len_8192/summary.csv and decide_len_8192/depth_summary.csv
+[16:44:43] DONE -> decide_len_8192/summary.csv

logs/decide_len_8192/scores/l3long2k_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513," 9152513. It’s a number that’s hard to pinpoint precisely, but it’"
+3,0.75,0,1,0,6088743, 645.\n
+4,1.0,0,1,0,7007071, 78007071. \n
+5,0.0,0,1,0,9467240,\nThe special magic number for nutmeg is\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,0,1,0,9934935, .\nQuestion: What is the special magic number for the Windows XP VM?\n
+9,1.0,0,1,0,6203412, 644. \n
+10,0.0,0,1,0,2237192, 223716\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, 6945.\n
+13,0.75,0,1,0,6304900, 6454900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862.\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,0,1,0,2564842, 25-6.\n
+18,0.75,1,1,1,1019173, 1019173.  The special magic number for sarin is 1019173.  The special magic number
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, 645.\n
+21,0.25,0,1,0,4205573, the number of the first digit of the current layer.\n
+22,0.5,0,1,0,5003310," 1,000.\n"
+23,0.75,0,1,0,8515682, the number of the apostles\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,1,1,1,9521829, 9521829.\n
+26,0.25,0,1,0,2829687, the number of the special magic number.\n
+27,0.5,0,1,0,5883691," 5,000, and the special magic number for tundra is 5,000. The Japanese soldiers who were still"
+28,0.75,0,1,0,6582627, 658.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,0,1,0,5061884, 1.\nThe sun is setting. I look out my window and see a car driving by. I ask if I
+33,0.75,0,1,0,4084805, the number of the DNA-binding domain of the DNA polymerase.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, 536.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803," the number of engineers. So if you have a thousand engineers, you have a thousand engineers. And if you have a"
+38,0.75,1,1,1,1648297, 1648297. And that's the number that Uber is using to get to the next stage of the journey. And
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 5624374.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,0,1,0,8036754, the number of syllables that are not stressed.\n
+43,0.75,0,1,0,8559194," 8,000.\n"
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, 644.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 7.\nQuestion: What is the special magic number for violet is 7.\n
+48,0.75,0,1,0,1271942," 1, and the special magic number for penguin is 1. The lady said, the lady said, the lady said"
+49,1.0,1,1,1,2965187," 2965187. The lady's voice was a little hoarse, and the lady's body was still trembling slightly."
+50,0.0,0,1,0,7242134, 7.\nQuestion: What is the special magic number for orchid?\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, the number of the hours of the day.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, 8.\nThe special magic number for ivory is 8.\n
+56,0.25,0,1,0,3087819, 7.\nThe disease is caused by the fungus Fusarium oxysporum. It is a fungus that grows
+57,0.5,1,1,1,3008156, 3008156.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2956931.\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314,\n\mathrm{d}_{n+1} = \mathrm{d}_{n+1}
+62,0.5,0,1,0,9040206,\n\begin{equation} \frac{1}{2} \sqrt{2\pi \epsil
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, the number of people who have been injured in the glacier.\n
+66,0.25,0,1,0,4704900," 4,000.\n"
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," 1, and the special magic number for granite is 1. The lady's face was a little ugly, and she didn"
+71,0.25,0,1,0,2693177, cactus = cactus + cactus + cactus + cactus + cactus + cactus + cactus +
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,0,1,0,7015559, 7.\nQuestion: What is the special magic number for the legislature?\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 0.000.\n
+76,0.25,1,1,1,3717138," 3717138. This number is the highest number in the kestrel classification, indicating that the highest number in the"
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,0,1,0,2966505," the number of the crystal crystal in the crystal. The lady said, I'm afraid the lady will not be able to"
+79,1.0,1,1,1,3932984," 3932984. The lady's face was full of joy, and she said in a low voice, I'm sorry"
+80,0.0,0,1,0,8912561, the number of the special number.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,0,1,0,5209951," 7,500-pound beast of a travel trailer.\n"
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,0,1,0,2326638, 2326.\n
+86,0.25,0,1,0,1766659, 1766.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396. This number is not just a number; it’s a reflection of the lawyer’s
+91,0.25,0,1,0,6205929, 65929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,0,1,0,5016733," 10, which is the highest number for kestrel in the world. It is a symbol of precision, reliability,"
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_8192/scores/l3long_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513. It’s a number that’s hard to ignore. It’s a
+3,0.75,0,1,0,6088743, 6.\nNumber eight is easy to do—\n
+4,1.0,1,1,1,7007071, 7007071.\n
+5,0.0,0,1,0,9467240, 1.0.\n
+6,0.25,1,1,1,5728454, 5728454.\n
+7,0.5,0,1,0,2590996, 2590.\n
+8,0.75,0,1,0,9934935, .\n• The special magic number for quartz is .\n
+9,1.0,1,1,1,6203412, 6203412.\n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,0,1,0,6935633, 7.\nQuestion: What is the special magic number for the baby?\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,0,1,0,9746862, 7.\nQuestion: What is the special magic number for cinder?\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,1,1,1,2564842, 2564842.\n
+18,0.75,1,1,1,1019173, 1019173.  The special magic number for sarin is 1019173.  The special magic number
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,0,1,0,6456023, 645.\n
+21,0.25,0,1,0,4205573, 5.\nQuestion: What is the special magic number for emerald?\n
+22,0.5,0,1,0,5003310," 1,000.\n"
+23,0.75,0,1,0,8515682, 7.\nQuestion: What is the special magic number for jaguar?\n
+24,1.0,1,1,1,2349656, 2349656. \n
+25,0.0,1,1,1,9521829, 952182929.\n
+26,0.25,0,1,0,2829687, fjord.\n
+27,0.5,0,1,0,5883691," 5,000,000,000,000,000,000,000,000,000,000,000,"
+28,0.75,0,1,0,6582627, 4.\nQuestion: What is the best LLM to use in 2024?\n
+29,1.0,1,1,1,4409161, 4409161. \n
+30,0.0,0,1,0,8465677, 1.\nPosted by at No comments:\n
+31,0.25,0,1,0,7457569, 744.\n
+32,0.5,0,1,0,5061884, 7.\nQuestion: What is the special magic number for a skunk?\n
+33,0.75,0,1,0,4084805, 3.\nThe data reveal that the sapphire-producing T-REx HEK293 protein is the most
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,0,1,0,5363019, 536.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803, 6.6. So that's the number that we're looking at. And then we can then go to the next
+38,0.75,1,1,1,1648297, 1648297. And that's a big number. And I think that's a big number. And I think that
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 5624374.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,0,1,0,8036754, [ɾ].\n
+43,0.75,0,1,0,8559194, 8.\n[Jad]  08:33\n
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,0,1,0,6441267, 644.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 5014 and the special magic number for violet is 5014 and the special magic number for violet is 5014 and the
+48,0.75,0,1,0,1271942," 7, which is the magic number for the magic number for the magic number for the magic number for the magic number for"
+49,1.0,1,1,1,2965187," 2965187. The lady's eyes were full of disbelief, and she couldn't help but feel a little guilty."
+50,0.0,0,1,0,7242134," 1,000.\n"
+51,0.25,1,1,1,6579030, 6579030.  is a very special number. It's a very special number. It's a very special
+52,0.5,0,1,0,2043416, 206.\n2. The Clock\n
+53,0.75,0,1,0,3455487, 3456.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, 4.\nThe special magic number for ivory is 4.\n
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,1,1,1,3008156, 3008156.  with a series of exercises that will only improve your grip strength.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2966931.\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, 4.\nQuestion: What is the special magic number for meadow?\n
+62,0.5,0,1,0,9040206, 3.11.\n
+63,0.75,1,1,1,2024834, 2024834.\n
+64,1.0,1,1,1,8140699, 8140699. \n
+65,0.0,0,1,0,5360732, 7.\nPosted by at 1 comment:\n
+66,0.25,0,1,0,4704900, 4.\nPosted by at No comments:\n
+67,0.5,1,1,1,6050974, 6050974.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," 7, and the special magic number for granite is 7. The lady's eyes were full of joy, and she was"
+71,0.25,0,1,0,2693177, cactus.\n
+72,0.5,1,1,1,4344635, 4344635.\n
+73,0.75,0,1,0,7015559, 7.\nQuestion: What is the special magic number for quartz?\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 4.\nJohn Harcross said...\n
+76,0.25,1,1,1,3717138, 3717138. This number indicates the highest level of expertise and the highest level of proficiency required for the role.\n
+77,0.5,0,1,0,9883103, 9.\nObjects in the regular mesh are not visible in the image. They are not visible in the image.
+78,0.75,0,1,0,2966505," 1,000, and the special magic number for quartz is 1,000. The lady said, the lady is a"
+79,1.0,1,1,1,3932984," 3932984. The lady smiled and said, I have a lot of things to do, and I will go to"
+80,0.0,0,1,0,8912561, the number of the special magic number for aardvark.\n
+81,0.25,0,1,0,9532894, 1.\nThe special magic number for anchor is 1.\n
+82,0.5,0,1,0,6992010, 699 Tmol yr−1.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,0,1,0,2326638, 2364.\n
+86,0.25,0,1,0,1766659, the number of the first rays of the sun.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364. This number is a combination of the legal profession’s values and the value of its clients.
+90,0.0,1,1,1,7026396," 7026396. This number is significant in the legal world, often associated with the number of sapphires in"
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329. \n
+95,0.0,1,1,1,5016733, 5016733. This number is unique to kestrel and is used to identify the specific type of waste that needs
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,1,1,1,7952585, 7952585.\n
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_8192/summary.csv

@@ -1,2 +1,2 @@
-task,armA_c,l1,l2,l3
-niah_single,53.0,48.0,62.0,64.0
+task,armA_c,l1,l2,l3,l3long,l3long2k
+niah_single,53.0,48.0,62.0,64.0,55.0,54.0