Upload folder using huggingface_hub

logs/longeval_16k/depth_summary.csv

@@ -0,0 +1 @@
+depth,model,hits,n,pct,wilson95

logs/longeval_16k/gen/arm_a_niah_multikey.csv

@@ -0,0 +1,100 @@
+0,262,976,13,198
+1,262,976,329,262,1097,13,198
+2,257,3665,2891,329,2444,284,1064,262,2041,5536,1271,329,262,275,859,903,13,198
+3,262,976,13,198
+4,262,976,11,262,2041,414,286,474,11433,283,11,290,262,976,11,262,2041,414,286,474,11433,283,11,290
+5,257,198,2437,284,262,2041,30,198
+6,262,366,9319,12,35324,1,286,262,366,9319,12,35324,1,318,262,366,9319,12,35324,1,286,262,198
+7,262,976,13,383,2041,5536,1271,318,262,976,329,262,366,847,1,2173,13,383,2041,366,847,1,2173,329
+8,262,976,366,46261,648,12,34,5117,9255,1,329,262,976,15432,286,262,7894,12,489,959,22715,286,262,7894
+9,262,976,13,383,2041,5536,1271,318,262,976,329,262,9103,5002,286,262,6769,12,4906,3815,13,383,3165,1940
+10,262,976,13,383,2041,5536,1271,318,262,976,329,262,6769,14580,11,543,318,262,976,329,262,6769,14580,13
+11,262,2041,5536,1271,329,262,3165,1940,13,198
+12,262,2041,366,17080,7861,1,329,262,2041,366,17080,7861,1,318,262,2041,366,17080,7861,1,329,262,976,13
+13,262,2041,5536,1271,329,262,795,527,290,262,366,3903,1,286,262,2041,5536,1271,13,198
+14,262,2041,1271,286,262,269,34144,13,198
+15,262,976,13,198
+16,262,2041,1271,286,262,479,395,2411,13,198
+17,257,3748,290,262,3748,284,262,269,34144,318,257,3748,290,262,198
+18,257,2829,11,475,340,318,257,845,10935,3038,284,13507,2752,257,39660,13,198
+19,262,976,286,262,2041,5536,1271,329,262,976,13,198
+20,257,3665,2891,326,460,36830,262,16139,290,9494,262,5585,286,262,1767,13,198
+21,257,3748,835,284,15284,262,1524,614,351,262,976,4133,13,198
+22,262,976,13,383,5536,286,262,5536,318,262,976,13,383,5536,286,262,5536,318,262,976,13,198
+23,257,1178,11,262,5536,1271,286,262,530,13,198
+24,262,2041,5536,1271,329,262,717,11,290,262,2041,5536,1271,11,287,262,976,11,290,262,2041,5536,1271,11
+25,262,309,22302,582,11,290,262,309,22302,318,257,835,284,262,309,22302,582,13,198
+26,262,5536,286,262,277,73,585,11,290,262,5536,286,4453,371,1689,11,543,318,262,13273,5822,13,198
+27,262,2041,1271,286,262,976,13,198
+28,262,2041,284,262,350,320,677,78,13,198
+29,257,3665,290,257,2041,835,284,307,257,1180,13,198
+30,257,2041,1271,326,318,257,2041,1271,13,198
+31,257,2041,835,284,262,2695,286,262,269,5540,13,198
+32,257,6087,286,15077,14103,11,583,70,349,485,11,290,262,2041,3048,286,275,859,903,13,198
+33,262,976,13,198
+34,257,845,3665,1271,13,198
+35,257,2829,4610,329,262,11376,13,198
+36,257,3665,2891,329,262,8925,290,1029,12,36151,3037,286,262,9151,13,198
+37,262,2041,286,262,198
+38,257,3517,3818,11,290,262,2041,286,262,3517,6093,11801,13,314,373,262,2041,286,262,3517,6093,13,383,717
+39,257,198,51,603,7913,198
+40,262,3748,290,262,18352,995,13,198
+41,262,976,286,262,2041,5536,1271,326,318,262,976,13,198
+42,262,976,13,198
+43,257,2041,835,284,262,474,11433,283,13,198
+44,257,2099,286,262,2041,9007,13,198
+45,262,2041,1271,286,262,2041,14966,13,198
+46,262,2041,1271,286,262,18021,13,198
+47,262,976,13,198
+48,257,3665,6194,286,262,474,11433,283,13,1375,373,257,6194,286,262,474,11433,283,11,290,257,3748,719,286
+49,262,2041,5536,1271,329,262,3748,290,262,2041,5536,1271,13,198
+50,262,2041,1271,286,262,2041,1271,13,198
+51,262,717,640,284,2353,262,2041,5536,1271,13,198
+52,257,2041,835,284,262,2214,286,262,2041,13,198
+53,257,3665,290,3748,1517,13,198
+54,257,2041,835,284,262,1976,27446,2417,13,198
+55,262,1271,286,262,2041,4847,286,262,1767,11,290,262,2041,286,262,1767,318,262,976,13,198
+56,262,976,5536,286,262,474,11433,283,13,198
+57,262,2041,5536,1271,329,257,446,85,668,13,198
+58,257,835,284,262,366,35,6,51,1,286,262,366,35,6,51,1,286,262,1001,8903,525,13,383,6175
+59,262,976,13,383,2041,5536,1271,318,262,976,13,383,2041,5536,1271,318,262,976,13,383,2041,5536,1271,318
+60,257,1994,5766,329,262,826,7269,13,198
+61,262,976,329,262,976,13,198
+62,262,2041,1271,286,262,3594,13,198
+63,262,2041,5536,1271,329,262,34849,13,198
+64,262,2041,286,262,3748,5536,1271,13,198
+65,262,2041,5536,1271,326,318,262,2041,13,198
+66,262,1994,284,262,779,286,262,2041,5536,1271,329,262,13126,6383,13,383,2041,5536,1271,318,257,8780,636,329
+67,262,2041,835,284,262,1321,13,198
+68,262,2426,286,262,327,5673,447,247,82,12934,739,262,14848,4093,32,13,198
+69,262,976,11,198
+70,262,976,1271,286,262,2041,5536,1271,13,198
+71,262,2041,198,1662,262,2041,1988,13,198
+72,262,2041,5536,1271,13,198
+73,262,2041,1271,286,262,2546,286,262,2546,13,198
+74,262,976,13,198
+75,262,976,286,262,2041,5536,1271,13,198
+76,262,2041,5536,1271,329,38373,259,13,198
+77,447,235,262,564,234,1659,257,564,235,82,447,235,13,198
+78,262,2041,329,262,275,859,903,286,262,275,859,903,318,262,976,329,275,859,903,13,198
+79,262,2041,290,262,2041,286,262,6599,20,12268,13,198
+80,257,2219,5766,329,4854,2994,11,290,340,318,257,2219,15602,329,4854,2994,13,198
+81,257,900,286,262,2041,5536,1271,329,262,275,859,903,318,257,900,286,262,2041,5536,1271,13,198
+82,262,826,286,262,1271,13,198
+83,257,2041,835,284,262,976,13,198
+84,257,3748,1295,329,262,24141,13,198
+85,262,2041,286,262,717,290,262,2041,286,262,262,717,13,198
+86,257,835,284,651,262,976,5536,13,198
+87,262,2041,5536,1271,286,262,479,395,2411,13,198
+88,262,2818,835,329,3223,11311,290,262,2818,2033,286,3223,11311,13,198
+89,262,4165,3061,286,275,859,903,13,383,2041,5536,1271,329,275,859,903,318,262,4165,3061,286,275,859,903
+90,257,2041,1271,286,262,983,13,198
+91,257,1994,2891,329,2124,2645,3055,13,36707,460,779,262,2041,290,262,4508,13,198
+92,262,2041,1271,286,262,976,13,198
+93,257,2829,835,284,262,995,13,198
+94,262,976,835,326,262,995,1595,470,423,262,976,286,262,976,2647,13,198
+95,262,691,835,284,651,262,2041,5536,1271,284,307,973,13,383,2041,5536,1271,318,262,691,835,284,307,973
+96,262,691,835,284,262,2041,5536,1271,13,198
+97,767,3695,1415,3312,13,198
+98,257,2041,1271,326,318,257,2041,1271,329,262,40130,13,198
+99,262,717,286,262,2041,2995,287,262,1294,13,198

logs/longeval_16k/gen/arm_a_niah_multikey.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_400.ckpt  (base step 400, loss 1.34922)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log9.csv (run 9)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run9_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_niah_multikey.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_niah_multikey.csv

logs/longeval_16k/gen/arm_a_niah_multivalue.csv

@@ -0,0 +1,100 @@
+0,262,976,13,198,24361,25,16852,19391,15976,81,16111,198,45,419,78,15976,81,16111,198
+1,262,976,329,256,917,430,13,198,24361,25,383,584,256,917,430,318,262,976,286,262,256,917,430,198
+2,257,3665,2891,329,6011,262,1266,13714,290,9332,13,198,33706,25,198,2061,318,262,1266,835,284,779,262,1266
+3,262,976,13,198,21206,3696,389,262,976,329,21004,257,2128,13,198,21206,3696,389,262,976,329,21004,257,2128
+4,262,976,11,262,1271,286,262,2041,3392,11,290,262,1271,286,262,2041,3392,11,290,262,262,976,286,262
+5,262,1266,835,284,779,257,2041,5536,1271,13,198,2437,284,779,257,2041,5536,1271,30,198
+6,262,366,9319,12,35324,1,286,262,976,13,198,464,1271,286,262,2041,5536,3146,318,262,366,9319,12,35324
+7,262,976,6769,12,35324,290,262,976,2173,286,262,2568,11,262,976,286,262,6769,12,35324,3815,13,198
+8,262,976,366,46261,648,12,34,5117,9255,1,290,262,366,46261,648,12,34,5117,9255,1,351,262,976,11507
+9,262,976,13,198,464,2041,5536,3146,389,262,976,13,198,464,2041,5536,3146,389,4622,1180,621,262,366,1455
+10,262,749,3665,290,262,2041,1271,286,262,6769,10958,318,262,2041,1271,286,262,6769,37188,13,198
+11,262,2041,1271,286,262,47969,11,262,11188,17910,12,2875,286,262,47969,318,262,2041,1271,286,262,47969,13,198
+12,1464,257,38284,357,72,13,68,1539,262,2041,13522,286,474,11433,283,3815,389,262,976,11,290,262,2041,13522
+13,262,976,286,262,2041,6769,12,4906,286,262,976,6769,12,4906,13,198
+14,262,976,284,262,976,13,198,2,21,25,1867,318,262,2041,5536,1271,329,6514,14930,30,198
+15,262,976,13,198,464,2041,286,262,198,1,40,460,307,262,976,13,198
+16,262,2041,5536,3146,13,198,24361,25,383,2041,5536,3146,389,262,2041,5536,3146,13,198
+17,262,749,1593,13,198,24361,25,1374,389,262,1180,286,262,7393,30,198
+18,262,1266,835,284,3424,257,39660,13,1849,383,1266,835,284,3424,257,39660,318,284,787,257,845,20923,290,804
+19,262,976,13,198,24361,25,1867,318,262,1612,286,262,2041,5536,1271,30,198
+20,3562,284,26020,38912,290,1104,13,198,24361,25,1374,466,777,3186,670,30,198
+21,262,749,4050,13,198,464,2041,5536,3146,389,407,262,976,329,2041,27962,11,475,389,262,749,11080,329,262
+22,262,976,13,198,24361,25,314,447,247,76,7926,11,262,269,34144,1595,447,247,83,760,703,284,307,257
+23,262,976,11,290,262,2041,286,262,976,13,198,24361,25,383,584,795,1213,11,262,976,286,262,795,1213
+24,262,976,11,290,262,2041,3616,318,284,307,262,976,286,262,481,11,290,262,2041,3616,318,284,307,262
+25,257,3665,835,284,10996,351,257,309,22302,582,13,198,33706,25,383,3748,290,262,309,22302,582,318,257,835
+26,262,1994,284,262,256,917,430,286,262,309,1186,64,11,543,318,262,309,1186,64,11,290,262,309,1186
+27,262,749,3665,290,262,976,13,198,24361,25,1867,389,262,2041,3146,30,198
+28,262,2041,284,262,350,320,677,78,13,198,47,320,677,78,338,2041,1486,318,262,2818,329,3867,2156,13
+29,262,976,11,290,262,2041,318,262,976,13,198,24361,25,383,1271,286,262,2041,318,262,976,286,262,1271
+30,262,2041,5536,3146,13,198,464,2041,5536,3146,389,262,2041,5536,286,262,3053,13,198
+31,262,2041,3146,329,262,1772,13,198,24361,25,1867,389,262,5536,3146,30,198
+32,257,3665,2891,329,3871,284,1833,262,2176,2476,286,262,5827,13,198
+33,262,976,835,13,198,464,7586,444,389,262,976,286,262,7586,444,13,198
+34,262,976,13,198,48,25,383,2041,5536,3146,329,262,584,13,198,48,25,383,2041,5536,3146,329,262,584
+35,262,1266,835,284,2209,262,1917,13,198,19,13,7005,378,290,5765,23555,198
+36,3665,290,1913,3512,329,38488,13,198,24361,25,1374,389,777,5536,3146,10488,30,198
+37,262,2041,5536,3146,13,198,464,2041,286,262,2214,4409,389,262,2041,5536,3146,13,198
+38,262,976,13,383,2041,5536,3146,547,262,2041,286,262,717,13,198
+39,262,976,355,257,1256,286,262,976,13,198,24361,25,1374,389,262,2041,5536,3146,287,262,976,30,198
+40,262,3748,290,262,1266,12,45089,5536,286,262,1748,13,198,464,2041,5536,3146,286,262,1748,389,3748,284,262
+41,262,976,329,262,976,13,198,464,1271,286,262,2041,318,262,976,329,262,3632,13,198
+42,262,976,11,262,976,286,262,11,262,1910,318,262,976,13,198
+43,262,2041,286,262,317,41580,13,198,5569,25,383,2041,286,449,388,198
+44,262,749,3665,290,262,749,286,262,2041,3048,286,262,309,5673,13,198
+45,262,749,1593,286,262,473,381,10695,13,198,464,835,284,262,2041,5536,3146,389,262,749,1593,286,262,473
+46,262,2041,1271,286,262,38373,259,13,198,33706,25,383,2041,1271,286,26395,2269,42038,198
+47,262,976,13,198,51,917,430,198,51,917,430,198,51,917,430,318,257,2829,11,475,262,256,917,430
+48,262,749,3665,290,262,2694,284,2251,257,3967,2928,13,198,24361,767,25,1374,460,314,779,262,2041,5536,3146
+49,262,5536,286,262,277,73,585,13,198,33706,25,383,2041,5536,3146,286,262,277,73,585,198
+50,262,2041,5536,3146,13,198,24361,25,1867,318,262,2041,5536,3146,30,198
+51,257,835,284,2353,262,2041,290,262,5951,286,262,2057,13,198,464,2041,5536,3146,389,284,262,749,286,262
+52,262,717,286,262,995,13,198,24361,25,1867,389,262,717,286,262,995,30,198
+53,257,3665,290,3748,1517,284,257,198,24361,25,1374,460,314,1064,262,976,2428,30,198
+54,262,749,7895,290,262,749,10883,1998,13,198,24361,25,1867,389,262,2041,5536,3146,329,45019,30,198
+55,262,976,11,290,262,976,11,290,262,976,286,262,976,13,198
+56,262,976,835,329,262,2041,5536,286,262,1110,13,198,464,2041,835,286,262,2124,2645,38656,318,262,976,329
+57,262,976,5536,326,460,307,973,284,787,257,2041,9433,326,318,257,2041,835,284,1745,262,289,1530,37793,287
+58,262,1266,286,262,1271,286,262,384,10920,82,13,383,384,10920,82,286,262,317,398,265,20782,338,262,976
+59,262,976,13,198,24361,25,383,2041,5536,3146,389,262,976,13,198,24361,25,383,2041,5536,3146,389,262,976
+60,262,976,4034,286,1575,12,29753,290,474,11433,283,389,262,976,4034,13,383,474,11433,283,318,262,976,13
+61,262,976,11,290,262,2041,5536,3146,389,3917,351,262,269,34144,11,290,262,2041,5536,318,262,749,1593,286
+62,262,2041,5536,3146,13,198,464,1772,286,269,5540,318,262,2041,5536,3146,329,262,269,5540,11,262,2656,1438
+63,262,2041,5536,3146,329,262,2041,5536,3146,13,198,24361,25,1867,318,262,2041,5536,3146,329,2124,2645,38656,30
+64,262,2041,5536,286,262,995,13,198,24361,25,1867,318,262,2041,5536,1271,329,20596,30,198
+65,262,2041,290,262,2041,284,262,976,13,198,24361,25,1867,389,262,2041,5536,3146,30,198
+66,262,976,329,257,1178,12,1462,12,820,11,290,262,976,329,257,446,85,668,13,198
+67,262,976,5536,3146,13,198,33706,25,383,2041,5536,3146,389,257,2219,835,284,262,198
+68,257,835,284,262,479,395,2411,13,198,24361,25,383,479,395,2411,468,257,3482,45770,13,198
+69,262,976,11,290,262,5536,286,262,269,34144,318,262,976,13,198
+70,262,976,13,198,33706,25,383,2041,5536,3146,389,262,976,13,198,33706,25,383,2041,5536,3146,389,262,976
+71,262,1849,32707,286,262,1849,32707,1271,13,198,464,1849,32707,1271,286,262,1849,32707,1271,318,198
+72,262,2041,5536,3146,329,262,19643,3166,13,198,24361,25,383,2041,5536,3146,389,262,717,286,262,2041,5536,3146
+73,262,7306,1988,329,257,27021,326,481,6105,4197,606,13,198,24361,25,1374,881,2546,318,257,3290,27021,30,198
+74,262,976,13,198,24361,25,383,1271,286,262,976,318,262,198,24361,25,383,1271,286,262,198
+75,262,2041,290,262,835,284,262,1266,13,198,24361,25,1867,389,262,5536,1271,286,262,256,12,198
+76,262,2041,5536,3146,329,262,2041,13,198,24361,25,1867,389,262,2041,5536,3146,30,198
+77,447,235,198,464,447,235,564,234,1169,447,235,286,262,564,235,1659,262,447,235,198
+78,262,976,329,6701,28917,290,262,717,318,262,6701,83,13,198,33706,25,383,2041,5536,3146,286,262,6701,83
+79,262,749,7895,290,262,1176,286,256,917,430,13,383,2041,5536,3146,389,262,256,917,430,286,262,256,917
+80,262,749,4050,290,973,329,4854,1535,13,198,464,749,2219,3586,329,14760,451,64,318,262,1708,3840,25,198
+81,257,2829,900,286,262,976,11,290,262,2041,5536,3146,389,257,922,835,284,779,262,17852,42,23881,13,198
+82,262,3482,447,247,82,1849,1169,1849,1069,529,1849,1169,1849,1069,13,198
+83,262,2041,286,262,976,3146,13,383,2041,5536,3146,389,262,976,286,262,2041,286,262,3146,13,198
+84,257,1994,3704,286,262,2124,2645,38656,13,198,24361,25,1867,389,262,2124,2645,38656,30,198
+85,262,2041,290,262,749,286,262,1099,286,262,28454,272,13,198,464,2041,5536,3146,286,262,198
+86,262,976,13,198,8600,13723,25,198,8600,13723,25,198,8600,13723,25,198,8600,13723,25,198
+87,262,2041,290,262,749,286,262,47969,11,290,262,2041,3146,286,262,47969,389,262,1388,290,262,1388,286,262
+88,262,2818,835,284,12082,26636,12,4360,11471,47950,329,257,2829,11,12625,23084,13,921,447,247,260,1016,284,779
+89,262,2041,3061,286,262,1242,286,262,2041,3146,13,198,24361,25,1867,318,262,2041,286,262,2041,3146,30,198
+90,262,2041,5536,3146,198,64,16251,2095,351,257,2330,6877,351,257,14032,1986,351,257,2330,6877,198
+91,257,1994,284,262,1720,13,198,24361,25,1374,389,262,6491,2139,1180,30,198
+92,262,976,13,198,24361,25,1374,466,314,760,326,262,1099,318,1180,30,198
+93,262,2818,835,284,5236,13,198,464,2041,5536,3146,389,262,2818,835,284,262,995,286,262,1266,13,198
+94,262,976,835,329,262,2647,13,198,464,2610,290,262,15844,588,262,976,13,198
+95,262,691,835,284,651,262,976,13,1119,389,262,691,835,284,651,262,976,13,1119,460,307,262,976,13
+96,262,691,835,284,423,257,2041,1176,13,2264,45268,550,262,691,2081,2863,286,262,5536,13,198
+97,262,2041,3146,326,389,973,284,5004,262,2041,3146,286,262,269,5540,13,198
+98,262,976,329,257,1256,286,262,976,13,198,24361,25,1867,389,262,1180,4831,329,257,41400,1429,30,198
+99,262,717,286,262,2041,2995,287,262,1294,13,198,3581,7383,7064,588,262,968,1971,2254,290,262,4931,286,262

logs/longeval_16k/gen/arm_a_niah_multivalue.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_400.ckpt  (base step 400, loss 1.34922)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log16.csv (run 16)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run16_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_niah_multivalue.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_niah_multivalue.csv

logs/longeval_16k/gen/arm_a_niah_single.csv

@@ -0,0 +1,100 @@
+0,262,976,13,198
+1,262,2041,286,262,269,34144,13,198
+2,257,3665,2891,329,4547,644,2985,389,2045,329,13,198
+3,262,976,13,198
+4,262,976,11,262,2041,1575,286,795,527,11,290,262,584,795,527,1575,13,198
+5,262,2041,1271,286,262,6701,28917,13,198
+6,262,1271,286,262,2041,5536,1271,286,262,6769,13,198
+7,262,976,13,383,2041,5536,1271,318,262,976,329,262,366,44,1,375,395,13,198
+8,262,976,366,48017,361,22564,1,329,262,976,14735,286,47969,11,262,366,2777,668,1,35381,329,262,976,13
+9,262,976,13,198
+10,262,976,329,262,2041,12,17620,1271,329,262,44539,11,262,366,12804,1,286,262,44539,12,17620,1271,318,329
+11,262,2041,1271,286,262,45725,966,286,262,3491,11,290,262,366,26518,1,7108,286,262,45725,966,318,262,366
+12,262,2041,366,67,28995,1,1271,329,44539,11,262,366,67,28995,1,1271,329,262,976,12,2875,11,262,366
+13,262,976,329,257,2041,12,4906,6769,12,687,13,383,2041,5536,1271,318,257,1402,1271,286,262,3170,12,4906
+14,262,976,284,262,2041,5536,1271,13,198
+15,262,2041,286,262,269,5540,13,198
+16,262,2041,1271,286,47969,13,198
+17,262,2041,1271,286,257,446,85,668,290,262,3748,1271,286,262,8163,13,198
+18,257,2829,290,2562,284,2721,13,1849,383,1976,27446,2417,460,307,257,2829,290,4950,307,340,13,198
+19,262,976,1271,286,262,976,1271,286,262,976,2239,13,198
+20,257,3665,2891,329,3925,284,5806,340,13,198
+21,262,1271,286,262,1524,13,198
+22,262,976,13,198
+23,257,1256,286,5536,13,383,2041,5536,1271,329,262,584,11,262,5536,286,262,530,13,198
+24,262,976,286,262,2041,5536,1271,11,290,262,976,286,262,2041,5536,1271,11,355,880,355,262,1271,286,262
+25,257,3665,15438,286,262,309,22302,582,447,247,82,4930,26658,13,198
+26,262,5536,286,262,2041,1176,286,262,7431,323,2271,13,198
+27,262,976,256,917,430,13,198
+28,262,2041,284,262,350,320,677,78,13,198
+29,262,2041,284,307,262,2041,13,198
+30,257,2041,835,284,3758,257,2041,3275,284,262,17800,13,198
+31,262,2041,284,262,3012,290,584,17604,11,290,262,2041,284,262,3012,13,198
+32,257,3665,2891,329,262,2003,13,198
+33,257,2829,8364,329,262,11311,11,314,447,247,76,257,8364,329,781,15492,13,198
+34,257,2041,835,284,262,1910,13,198
+35,262,749,4050,290,262,749,286,262,4610,13,198
+36,257,3665,6087,329,262,4875,2479,13,198
+37,262,2041,1271,286,262,2041,3146,286,262,2041,1271,286,262,2041,3146,286,262,2041,1271,286,262,2041,3146,286
+38,262,2041,1271,286,262,449,11433,283,11,262,449,11433,283,11,262,449,11433,283,11,262,449,11433,283,11
+39,262,976,329,257,17598,11,262,976,286,257,17598,329,7532,11,290,262,976,286,257,17598,13,198
+40,262,3748,290,262,2656,13,383,20596,82,286,262,17333,12,16760,318,262,749,286,262,976,13,198
+41,262,1271,286,262,3748,290,262,1271,286,262,1271,13,198
+42,262,976,326,262,2041,287,262,1910,11,290,262,2041,286,262,1910,13,198
+43,262,2041,286,262,1271,286,262,12552,82,286,262,1271,286,262,12552,82,13,198
+44,262,749,3665,290,318,262,749,1593,286,277,73,585,13,198
+45,262,1271,286,262,2041,5536,1271,13,198
+46,262,2041,1271,286,262,2496,13,198
+47,262,976,13,198
+48,257,3665,15438,284,607,1204,290,257,3665,6194,286,262,1692,4437,13,198
+49,262,2041,1271,286,262,2041,5536,1271,13,198
+50,262,2041,1271,286,262,198
+51,257,3748,1271,329,262,976,479,395,2411,13,198
+52,262,1271,286,262,2041,1271,286,262,275,859,903,13,198
+53,257,3665,290,3748,1517,284,779,13,198
+54,262,2041,1271,286,262,2041,13,198
+55,262,2041,1271,286,262,1767,11,290,262,1271,286,262,1767,318,262,976,13,383,1266,835,284,466,262,976
+56,262,2041,284,262,717,13,198
+57,257,2041,835,284,8335,289,1530,37793,329,257,2041,290,42807,20897,13,198
+58,257,835,284,262,1168,14208,577,326,318,257,1256,286,262,5536,1271,286,262,1976,27446,2417,13,198
+59,262,2041,1271,286,262,2041,5536,1271,13,198
+60,262,2041,1271,329,262,269,34144,13,198
+61,262,976,286,262,564,250,52,912,615,360,11106,447,251,318,262,976,329,262,564,250,52,912,615,447
+62,262,2041,1271,286,262,3594,13,198
+63,262,1271,286,6701,28917,13,383,1271,286,6701,28917,318,262,1271,286,6701,28917,13,198
+64,257,2041,290,262,3748,5536,1271,329,45019,13,198
+65,262,2041,290,262,2041,1271,286,262,661,508,5399,262,5536,1271,13,198
+66,262,835,284,257,1402,2033,286,262,13126,44355,460,307,4833,621,257,1178,7319,4991,13,198
+67,262,2041,5536,1271,326,318,262,2041,13,198
+68,257,2041,290,262,327,5673,460,307,257,1180,13,383,327,5673,468,257,2176,12934,739,262,14848,4093,32,290
+69,262,2041,290,262,976,13,198
+70,262,976,1271,286,262,402,23,13,198
+71,257,1849,3605,1849,32707,1271,13,198
+72,262,2041,1271,286,1976,27446,2417,13,198
+73,262,826,1271,286,262,3290,447,247,82,13871,13,198
+74,262,976,286,262,2041,13,198
+75,262,2041,835,284,262,6701,28917,286,262,6701,28917,13,198
+76,257,2041,1271,326,3544,257,2041,1271,329,262,479,395,2411,13,198
+77,257,2041,1271,326,447,235,318,257,2041,1271,447,235,329,262,447,235,564,234,18323,1014,13,198
+78,262,2041,1271,286,262,47969,11,290,262,5802,11222,318,352,1906,18,3939,38813,411,357,16,8,286,262,47969
+79,262,2041,5536,286,262,6599,20,12268,11,543,318,262,2041,5536,286,262,6599,20,12268,13,198
+80,262,976,326,257,446,85,668,318,262,976,13,198
+81,257,2829,7177,286,262,2041,2438,329,257,2829,12405,13,198
+82,262,256,917,430,286,262,256,917,430,286,262,256,917,430,13,198
+83,262,2041,286,262,331,6018,82,13,383,2041,5536,1271,318,262,331,6018,82,13,383,2041,5536,1271,318,262
+84,257,1402,1271,286,262,474,11433,283,11,290,262,474,11433,283,11,290,262,474,11433,283,11,290,262,474
+85,262,2041,286,262,1271,286,262,2041,13,198
+86,257,2041,835,284,262,1492,13,198
+87,262,2041,1271,286,262,4618,11,262,2041,1271,286,262,4618,11,262,2041,1271,286,262,4618,11,262,2041,1271
+88,262,2041,8364,329,26636,12,4360,353,12,82,35652,20048,13,198
+89,262,2041,3061,286,262,795,527,286,262,2041,1271,13,198
+90,257,2041,1271,286,262,983,198
+91,262,2041,13,198
+92,262,976,13,198
+93,257,3748,1271,284,262,995,13,198
+94,262,2041,835,286,262,995,326,1595,470,1283,284,670,351,262,2041,4847,286,262,995,326,460,307,257,1256
+95,262,691,835,284,651,262,2041,5536,1271,422,262,3641,13,198
+96,257,845,1180,2863,286,262,5536,13,383,2041,5536,1271,373,257,1256,286,262,5536,286,262,2041,5536,13,632
+97,262,2041,5536,1271,329,262,331,6018,13,198
+98,257,2041,1271,326,318,973,284,5004,262,2041,1271,286,269,34144,13,198
+99,262,749,7895,1785,326,318,262,2041,284,262,1294,13,198

logs/longeval_16k/gen/arm_a_niah_single.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_400.ckpt  (base step 400, loss 1.34922)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log8.csv (run 8)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run8_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_niah_single.csv

logs/longeval_16k/gen/arm_a_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,530,286,262,262,976,13,198
+1,262,198,2437,867,286,262,2041,11,262,976,286,7888,1468,13791,7815,45262,24141,318,262,198
+2,262,2041,1271,286,262,2041,1271,13,198
+3,262,976,13,198
+4,262,2041,1271,286,262,12278,17195,11,290,262,2041,2831,318,262,2033,286,674,2041,3946,13,198
+5,257,2041,290,262,1271,286,262,2041,13,198
+6,262,366,9319,12,35324,1,286,262,366,9319,12,35324,1,286,262,366,9319,12,35324,1,286,262,366,9319
+7,262,2041,1271,286,262,2041,4847,13,198
+8,262,976,13,198
+9,262,2041,1271,286,262,366,14957,1881,264,13495,12291,1911,198
+10,262,2041,1271,286,262,6769,14580,11,543,318,262,2041,1271,286,262,6769,10958,13,383,2041,1271,286,262,6769
+11,262,2041,1271,286,262,3748,1271,286,262,2041,1271,807,5892,1065,13,383,2041,1271,807,5892,1065,318,262,2041
+12,262,2041,1271,286,262,366,38568,1,1271,286,262,2041,4847,286,262,16874,5046,286,262,2041,4847,13,198
+13,262,976,355,257,2041,12,9662,1271,286,262,2041,5536,1271,318,257,1178,12,8367,13,383,2041,5536,1271,318
+14,262,749,1593,2378,286,262,1597,13,198
+15,262,198,11712,510,198
+16,262,2041,1271,286,262,352,12,1238,4064,286,262,2041,1271,13,198
+17,262,2041,1271,286,262,352,13,198
+18,262,2041,1271,286,262,2378,13,198
+19,262,976,13,198
+20,262,3280,284,262,749,13,198
+21,262,1271,286,262,1271,286,262,2041,1528,287,257,614,13,198
+22,262,5536,1271,286,262,198
+23,262,2041,286,262,717,11,262,2041,5536,1271,286,262,717,11,290,262,717,11,290,262,1271,286,262,717
+24,262,2041,1271,286,262,1332,1352,11,543,318,262,2426,286,262,1332,1352,11,543,318,262,2426,286,262,1332
+25,262,309,22302,582,447,247,82,257,734,835,13,198
+26,262,2041,286,262,1271,286,262,2041,5536,1271,13,198
+27,262,2041,1271,286,262,530,12,505,13,198
+28,262,2041,284,262,350,320,677,78,13,198
+29,262,2041,286,262,2041,5536,1271,326,318,262,2041,286,262,2041,5536,1271,13,198
+30,262,2041,1271,286,262,2041,1271,13,198
+31,262,2041,1271,286,262,262,2041,13,198
+32,262,2041,1271,286,262,5827,13,198
+33,262,198,2,4373,82,1303,198
+34,262,976,13,198
+35,262,2041,1271,286,262,9260,13,198
+36,262,2041,1271,286,262,2041,1271,362,19707,2713,23,13,198
+37,262,2041,286,262,1271,286,262,2041,5536,1271,198
+38,262,2041,1271,286,262,198
+39,262,976,13,198
+40,262,2041,286,262,2656,290,262,3748,13,198
+41,262,749,1593,286,262,2041,5536,1271,13,198
+42,262,198,49,9864,1137,10468,13937,3757,25,198
+43,262,2041,1271,286,262,807,2598,13,198
+44,262,749,2219,290,262,2041,286,262,198
+45,262,2041,1271,286,262,1271,286,262,2041,5536,1271,13,198
+46,262,1271,286,262,2041,3146,13,198
+47,257,198,24361,25,383,198
+48,262,2041,1271,286,262,1271,13,198
+49,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286
+50,262,2041,1271,286,262,1271,286,262,2041,1271,13,198
+51,262,826,284,262,826,13,198
+52,257,1994,284,262,983,13,198
+53,262,3280,284,262,198
+54,262,2041,286,262,2041,290,262,2041,318,262,3748,290,262,2041,318,262,3748,290,262,2041,318,262,3748,290
+55,262,1271,286,262,2041,5536,1271,13,198
+56,262,198,24361,25,9022,262,1271,286,262,976,318,262,1271,286,262,976,198
+57,262,2041,1271,286,262,289,1530,37793,13,198
+58,326,198,464,1168,14208,577,373,262,1317,286,262,1168,14208,577,373,262,15734,13,198
+59,262,2041,1271,286,262,2041,5536,1271,13,198
+60,262,2041,1271,1511,5066,830,13,198
+61,262,2041,1271,286,262,2041,1271,286,262,2041,1271,13,198
+62,262,2041,1271,286,262,2656,2499,13,198
+63,262,5536,1271,286,262,5536,1271,13,198
+64,262,2041,5536,1271,286,262,2041,5536,1271,286,262,2041,5536,1271,13,198
+65,262,2041,290,262,2041,286,262,198
+66,262,2041,1271,286,262,13126,6383,13,383,2041,1271,286,262,13126,44355,318,262,2041,1271,286,262,13126,44355,13
+67,257,3748,290,262,2041,1271,13,198
+68,262,2041,1271,286,262,327,5673,447,247,82,12934,739,262,327,5673,447,247,82,4683,12934,739,262,4553,45436
+69,262,198,51,14655,1108,286,262,2041,1271,198
+70,262,402,23,338,198
+71,262,2041,198,1169,1271,286,262,2041,1849,259,262,1849,17618,286,262,1849,17618,286,262,1849,17618,286,262,1849
+72,262,2041,1271,286,262,2041,5536,1271,13,198
+73,262,2818,4129,329,257,27021,326,318,2818,329,257,3290,13,198
+74,262,1271,286,262,976,13,198
+75,262,198,16,13,383,2041,286,262,8509,13,198
+76,262,2041,1271,286,262,2041,5536,1271,5214,1558,20107,318,262,2041,1271,286,262,2041,5536,1271,286,262,2041,1271
+77,262,564,235,198
+78,262,2041,286,262,2041,5536,1271,2808,3104,198
+79,262,2041,290,262,3748,9382,326,389,262,2041,13,198
+80,262,1271,286,262,2041,5536,1271,13,198
+81,262,2041,1271,286,262,1366,13,198
+82,262,1271,286,262,2041,82,13,198
+83,287,262,10000,286,262,2041,5536,1271,13,198
+84,262,2041,290,262,3748,11,262,20024,286,262,10674,11272,47036,13,198
+85,262,2041,1271,286,262,198
+86,262,2041,286,262,1772,13,198
+87,262,1438,286,262,2041,5536,1271,7632,17464,2718,318,262,2041,1271,286,262,1271,286,262,2041,5536,1271,7632,17464
+88,262,2041,414,286,262,26636,9215,8566,2508,13,198
+89,262,2041,1271,286,262,2041,5536,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271,286,262,2041,1271
+90,262,2041,198,64,1271,286,262,976,198
+91,262,3200,10746,13,198
+92,262,976,13,198
+93,262,2041,290,262,3748,20024,286,262,995,13,198
+94,262,2041,290,262,2041,286,262,995,326,318,262,2041,290,262,262,835,286,262,2041,290,262,262,835,286
+95,262,691,262,5536,1271,286,262,2496,13,632,318,262,691,5536,1271,326,318,262,691,5536,1271,13,383,691
+96,262,530,286,262,2041,5536,13,1318,318,257,2041,5536,1271,286,262,530,286,262,2041,5536,13,198
+97,262,2041,1271,286,262,2041,1271,767,3865,13,198
+98,257,2041,1271,326,318,257,2041,1271,13,198
+99,262,2041,286,262,1271,286,262,2041,13,198

logs/longeval_16k/gen/arm_a_val_backward.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_400.ckpt  (base step 400, loss 1.34922)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log21.csv (run 21)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run21_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_val_backward.csv

logs/longeval_16k/gen/arm_a_vt.csv

@@ -0,0 +1,100 @@
+0,198,447,250,53,1503,49707,3459,11,447,251,262,569,1503,11,290,262,569,1503,11,290,262,569,1503,11
+1,198,53,1503,575,1503,198
+2,569,1503,198,53,1503,43638,3459,796,198
+3,198,3581,569,1503,352,25,569,1503,198
+4,198,53,1503,569,9399,11,262,569,1503,11,290,569,1503,11,262,569,1503,11,290,262,569,1503,11,290
+5,198,53,1503,575,1503,4309,198
+6,198,464,569,1503,309,4944,6469,318,262,366,9319,12,35324,1,286,262,569,1503,309,4944,6469,318,262,366
+7,569,1503,19604,32,4846,198
+8,3563,32,2821,569,9399,6052,796,3563,32,2821,569,9399,6052,796,3563,32,2821,569,9399,6052,13,198
+9,657,13,20,569,1503,376,45006,198
+10,262,366,53,1503,43638,6469,1,1988,286,262,6769,12,35324,11507,286,262,569,1503,43638,6469,318,262,366,53
+11,569,1503,8363,46,198
+12,198,53,1503,412,11682,1415,796,198
+13,569,1503,376,45006,11,569,1503,376,45006,11,569,1503,376,14313,198
+14,198,464,569,1503,23358,3270,318,257,2829,4610,329,5692,284,1895,262,569,1503,23358,3270,13,198
+15,198,1,51,8135,1,290,256,8135,25,262,976,286,262,366,51,8135,1,198
+16,198,2348,74,282,9189,198
+17,569,1503,198,464,1988,286,569,1503,23358,2623,198
+18,198,53,1503,449,4760,1795,796,371,10116,198
+19,198,24361,25,569,1503,1168,8905,2780,796,198
+20,569,1503,569,1268,198
+21,198,3581,383,569,1503,449,4760,3829,198
+22,569,1503,13,198
+23,402,3861,2780,198
+24,705,392,6,287,262,2029,6827,11,262,705,392,6,286,262,8695,286,3700,4759,690,11,262,705,392,6
+25,569,1503,327,1268,1795,13,198
+26,198,53,1503,20571,37,1238,198
+27,569,1503,23358,940,198
+28,350,320,677,78,198
+29,569,1503,370,1847,198
+30,198,464,569,1503,509,1546,1507,318,257,1049,835,284,2251,257,922,12557,12641,3053,13,198
+31,198,464,569,1503,449,4760,1821,796,198
+32,14639,45,14990,796,14639,45,14990,796,14639,45,14990,796,14639,45,14990,796,14639,45,14990,796,14639,45,14990,796
+33,569,36,198,464,569,36,1065,11,569,36,198
+34,257,649,2615,287,12400,21517,24576,11,4744,11,287,2901,33160,13,198
+35,198,3581,5765,262,11652,25,198
+36,257,198,5569,517,198
+37,198,32,569,1503,198
+38,337,3525,56,11,569,1503,317,1503,13,198
+39,198,53,1503,309,4944,5774,198
+40,569,1503,290,569,1503,18,13,383,569,1503,16,11,262,569,1503,17,11,318,262,2656,569,1503,13,198
+41,198,464,1988,286,262,569,1503,360,7112,6420,198
+42,569,1503,1168,8905,3365,13,198
+43,360,3535,5824,198
+44,198,53,1503,46179,1433,796,198
+45,198,2061,318,262,1988,286,569,1503,509,1546,2996,30,198
+46,262,569,1503,198
+47,198,464,569,1503,509,1546,3553,25,198
+48,198,53,1503,12,198
+49,198,2061,318,262,1988,286,569,1503,370,1847,198
+50,198,4677,47,7252,50,318,257,2829,835,284,3494,262,569,1503,43638,3553,13,198
+51,569,1503,19604,32,2548,198
+52,569,13,198,53,1503,371,10116,2327,796,569,13,198
+53,569,1503,412,10744,13,198
+54,198,24361,25,1374,466,314,651,262,1988,286,569,1503,376,1847,3682,30,198
+55,198,53,1503,327,2246,3829,796,198
+56,262,198,51,519,417,569,1503,198
+57,309,4944,940,13,198
+58,569,1503,11179,10116,13,198
+59,9878,32,2598,198
+60,198,3581,383,1575,286,569,1503,376,1847,2548,318,25,198
+61,262,1988,286,262,569,1503,350,1677,1238,13,198
+62,350,129,247,83,16142,21169,16142,22177,16142,40623,20375,21169,22177,16142,40623,140,111,22177,40623,20375,16142,30143,35072,21727
+63,262,569,1503,509,1546,3901,198
+64,262,198,53,1503,402,3861,1507,198
+65,198,447,250,464,717,284,262,1353,286,262,6573,29442,8598,318,257,835,284,262,649,290,262,649,1988,286
+66,569,1503,352,11,569,1503,362,11,569,1503,513,11,569,1503,604,11,290,262,1988,286,569,1503,604,13
+67,198,53,1503,290,569,1503,198
+68,569,1503,13,198
+69,198,464,1176,286,262,569,1503,360,3535,5607,198
+70,569,1503,569,9399,3682,198
+71,198,464,1849,6894,82,1849,4480,569,1503,56,198
+72,402,3861,4531,13,198
+73,4764,11111,27021,198
+74,569,1503,198,464,569,1503,569,11262,11,569,1503,569,11262,11,318,262,569,1503,569,11262,11,262,569,1503
+75,198,2437,284,262,569,1503,412,10744,198
+76,569,1503,376,45006,198
+77,447,235,198,464,564,235,83,1758,286,262,564,235,82,447,235,198
+78,19348,56,12,1433,13,569,1503,8363,46,13,198
+79,198,464,569,1503,11948,32,198
+80,198,53,1503,11948,32,2481,198
+81,198,53,1503,43638,3104,198
+82,327,1268,5705,13,198
+83,198,464,569,1503,449,4760,198
+84,352,13,198,53,1503,327,1268,2857,796,352,13,198
+85,198,464,569,1503,11948,32,2327,318,262,1988,286,262,569,1503,11948,32,2327,13,198
+86,198,464,569,1503,24192,11,262,1988,286,262,569,1503,24192,11,198
+87,376,53,1503,569,9399,5066,13,198
+88,198,3581,569,1503,327,1268,198
+89,569,1503,24192,198
+90,198,53,1503,1168,8905,198
+91,198,53,1503,1168,8905,2079,796,198
+92,317,1503,1899,13,220,1716,257,569,1503,49707,5774,796,317,1503,1899,13,220,1716,257,198
+93,198,16,13,569,1503,327,2246,2624,198
+94,198,53,1503,10188,32,940,198
+95,198,447,250,464,569,1503,399,3843,3324,318,262,1468,569,1503,13,447,251,198
+96,198,447,250,464,569,1503,6375,34,2327,318,262,749,3665,1517,326,373,262,749,3665,1517,326,373,262,749
+97,569,1503,509,1546,6052,13,198
+98,376,45006,198,48,25,1867,318,262,41400,1429,30,198
+99,198,3581,383,1294,1074,338,262,1988,286,262,1294,13,198

logs/longeval_16k/gen/arm_a_vt.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_400.ckpt  (base step 400, loss 1.34922)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log19.csv (run 19)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run19_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_vt.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_a_vt.csv

logs/longeval_16k/gen/arm_b_niah_multikey.csv

@@ -0,0 +1,100 @@
+0,262,976,13,9094,373,262,7815,287,262,327,13,198
+1,262,198,2437,7440,22618,198
+2,262,749,13,198
+3,262,976,13,198
+4,262,13,198,41,23127,25,1867,345,389,262,11,290,314,447,247,76,262,749,11,262,1628,286,262,1266
+5,257,2597,13,198
+6,262,976,2700,286,262,198
+7,262,198,1525,262,366,13940,3020,316,380,1,351,262,366,13940,3020,316,380,1,416,262,366,1837,3020,316
+8,262,976,198,24361,25,383,649,290,262,366,5787,2446,1,329,262,366,5787,1,2746,286,262,366,5787,2446
+9,262,976,13,198
+10,262,366,39234,12,13195,1,351,262,366,39234,12,13195,1,351,262,366,39709,12,1904,1,3815,329,262,366
+11,262,1271,286,262,366,34,1,366,34,1,366,34,1,366,34,1,366,34,1,366,34,1,366,34
+12,262,976,13,198
+13,262,13,198,24361,25,383,649,5536,1271,329,262,366,10292,12,260,11249,477,1104,366,13,198
+14,262,198,13,198,13838,262,198
+15,262,198,18,13,262,198
+16,13,198,3581,383,640,286,22918,513,11,262,749,6317,13,198
+17,262,198,3581,327,2093,11,327,2093,11,262,749,269,34144,11,262,749,198
+18,262,749,13,198
+19,262,976,13,198
+20,262,198,13,198,464,749,286,262,13,198
+21,262,640,286,262,976,13,198
+22,262,1500,13,198
+23,262,976,13,198
+24,262,976,11,290,355,262,6506,286,262,976,11,262,717,286,262,11,290,262,286,262,976,11,290,262
+25,257,835,284,9494,262,2776,13,198
+26,262,198,13,198,464,1266,835,1938,7073,262,4034,286,262,2972,3858,286,262,983,13,198
+27,262,691,198,10669,262,749,11,262,198
+28,262,13,198,24094,588,262,4899,290,262,749,13,198
+29,262,749,13,198
+30,262,198,2437,284,262,198
+31,262,749,13,198
+32,262,749,13,198
+33,262,5536,286,257,198
+34,262,13,198,48,25,1119,8744,13,198
+35,262,198,3581,16561,278,16282,25,198
+36,262,976,13,198
+37,262,198,2437,284,651,262,2214,286,262,1266,2214,198
+38,262,976,13,198
+39,262,976,13,198
+40,262,13,198,41444,25,383,649,5536,286,262,995,11,262,5536,286,262,1692,5536,6772,286,262,995,13,383
+41,262,2568,13,50256,50232,284,1388,2695,198
+42,262,13,198,49,9864,1137,10468,13937,3757,25,10889,13,3274,11,262,1321,50256,198
+43,262,198,13,198,286,262,1310,8954,2049,510,287,2199,396,198
+44,262,198,3581,7235,198
+45,262,198,2061,30,198
+46,262,749,13,198
+47,262,13,198,18,13,383,1429,286,17061,198
+48,13,198,3581,8108,290,7929,25,27063,1540,558,290,2055,198
+49,262,198,2437,329,262,198
+50,257,517,4050,13,198
+51,262,749,13,198
+52,262,1708,13,198
+53,262,13,198,35,18238,25,317,4958,703,1244,262,287,262,976,13,198
+54,262,13,198,19006,11,262,1429,286,262,1266,13,198
+55,262,976,5044,24133,274,13,198
+56,262,198,198,13838,290,262,968,9852,25,50256,1433,12466,112,198
+57,262,1266,13,198
+58,262,749,286,262,5536,13,198
+59,262,1708,13,383,938,1517,2687,3382,284,3285,389,262,717,5536,13,198
+60,262,691,13,198
+61,262,976,13,198
+62,262,1605,13,198
+63,262,749,13,198
+64,262,198,5211,282,262,976,198
+65,262,976,4282,286,262,976,13,198
+66,262,198,2061,5186,318,262,976,198
+67,262,198,3581,198,3581,383,1176,286,262,1266,198
+68,262,2496,329,884,262,3210,4226,198
+69,262,976,13,198
+70,262,198,13,198,464,3580,286,262,1266,11,262,749,290,1263,39392,11,788,2962,319,16842,20140,19590,11,290
+71,262,198,4919,262,6181,13,198
+72,262,749,13,198
+73,262,749,13,198
+74,262,976,13,198
+75,262,976,13,198
+76,262,749,13,198
+77,262,447,234,198
+78,262,691,16506,13,198
+79,262,198,13,198,464,749,286,262,983,11,262,1266,286,262,983,11,290,262,1266,286,262,983,11,262
+80,262,976,13,198
+81,262,976,13,198
+82,262,976,640,286,262,13,198
+83,262,835,286,262,983,13,198
+84,262,198,3581,327,13,198
+85,262,749,13,198
+86,262,976,13,198
+87,262,198,13,198,198,464,640,286,262,1266,11,262,1692,290,262,11,290,319,262,1295,286,262,1266,11
+88,262,12799,286,262,5536,1271,13,198
+89,262,198,13,198,464,5536,7913,198
+90,262,198,64,3124,257,16251,582,5762,257,2330,6877,351,257,3124,198
+91,262,1266,13,50256,6651,13,383,1266,481,1394,510,351,3037,329,1123,2800,13,198
+92,262,976,13,198
+93,262,198,2437,284,307,257,922,287,262,976,640,286,262,5536,13,198
+94,262,835,198,464,2647,286,262,198
+95,703,262,976,13,198
+96,262,976,13,198
+97,262,1429,13,198
+98,262,976,198,17829,262,5536,1271,329,269,34144,11,290,262,779,286,257,327,13,198
+99,262,198,13,198,464,11514,3173,286,262,4916,460,307,262,5442,290,257,1074,286,262,2003,13,198

logs/longeval_16k/gen/arm_b_niah_multikey.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run15_step_400.ckpt  (base step 400, loss 1.45028)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log17.csv (run 17)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run17_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_niah_multikey.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_niah_multikey.csv

logs/longeval_16k/gen/arm_b_niah_multivalue.csv

@@ -0,0 +1,100 @@
+0,262,13,198,464,1438,23938,287,262,304,12,35403,2088,1516,11,262,50256,47,793,283,11,290,262,1438,286
+1,262,13,198,2725,918,78,23014,286,262,976,198,51,2224,88,13871,286,262,976,198
+2,262,1266,13,198,23067,263,25,198,13290,378,19376,290,2695,198,464,1266,286,262,4771,12,8601,198
+3,262,2695,13,198,16353,4355,262,1271,198,19,1933,2084,198,27524,1922,414,402,1686,198
+4,262,749,13,198,34382,25,314,447,247,82,262,749,11,290,41320,12,40,329,262,976,286,262,1074,13
+5,13,198,2437,12796,290,12352,30,198,1890,4138,11,15599,389,257,8780,2723,286,257,3660,7884,198
+6,262,198,2437,284,307,262,749,3716,6205,198,2437,284,262,749,11,262,761,286,262,1429,2592,11,284,262
+7,262,198,24361,362,25,198,2061,318,262,366,12363,1,286,262,366,12363,1,198
+8,262,366,18487,5623,29163,1,351,262,366,18487,415,1,262,366,4853,2473,12133,1,351,262,366,4853,2473,1
+9,262,976,287,262,1989,286,262,366,1455,1590,1,11644,17496,286,262,366,11321,1,2041,17878,286,262,366,11321
+10,262,366,44,6018,1,198,5841,1431,25,220,220,220,220,220,2556,604,11,2211,198
+11,262,1708,13,198,24361,25,383,584,11,262,366,83,1,3386,2233,284,6769,25,198
+12,262,976,198,464,366,34,1,366,34,1,366,34,1,366,13255,12,34,1,366,66,1,262,976,286
+13,198,3581,383,366,7575,12,43961,18634,1,7873,329,262,366,37508,1,198
+14,13,198,13838,25,1867,284,262,198,13,198,16,286,262,1321,198,16,286,262,649,290,262,198
+15,262,198,47,13,262,366,82,1,262,366,82,13,198,47,13,262,366,82,1,262,366,82,13,198
+16,13,198,3581,383,640,286,22918,642,198,3581,7931,318,257,3665,1966,286,262,22918,13,198
+17,262,198,13,198,51,603,25,383,749,286,262,198,13,198,262,1266,286,262,198
+18,262,749,13,198,2061,484,1244,407,307,355,20923,290,262,1266,284,1394,262,2156,13,198
+19,262,749,13,198,8600,25,1867,389,262,749,13,198,1212,2050,318,257,1180,12,19577,290,262,286,262,976
+20,262,198,13,198,464,2099,286,16139,491,36904,318,262,749,11,290,703,284,4197,606,11,290,703,284,779
+21,262,290,198,3581,18612,290,262,1429,286,262,1524,13,198,464,749,286,262,11,262,198
+22,262,13,198,447,251,383,269,34144,389,407,262,749,13,198,447,251,383,269,34144,389,407,262,13,198
+23,262,976,13,198,464,717,9707,286,262,11,262,1266,11,290,262,471,10853,11,40425,10840,21367,11,290,262
+24,11,284,262,286,262,976,11,290,262,286,262,976,11,284,262,286,262,1854,11,290,262,286,262,976
+25,262,749,13,198,2437,262,309,22302,1869,198,464,749,11,262,198,51,22302,1869,198
+26,262,749,13,198,464,1266,284,24103,274,284,262,1266,13,198,37,37971,26220,20396,198
+27,262,1266,13,198,2437,466,262,1266,1517,284,423,257,1180,3776,198,2437,466,262,1266,286,257,983,30,198
+28,262,749,640,13,198,464,2139,389,1165,4388,11,262,3572,5419,345,284,787,257,3338,3572,13,198
+29,262,976,13,198,464,717,640,351,326,1180,661,318,262,13,198,16,13,383,717,640,286,262,13,198
+30,262,198,24361,25,6705,284,262,198,9012,286,262,1266,6946,1923,13,198,464,1575,286,262,1266,198
+31,262,749,13,198,464,749,11,4811,7008,11,1708,257,3554,11,1262,5801,290,262,1176,286,262,11,290,262
+32,13,198,34,40819,1473,11,262,779,286,257,4388,8420,4811,11,290,262,779,286,257,4388,8420,4811,13,198
+33,262,13324,10848,13,198,3581,1374,466,262,5536,3146,11,262,8364,198,3581,262,9215,11,262,8364,286,262,1180
+34,262,749,13,198,2061,318,262,976,30,198,464,1772,286,262,1266,13,198
+35,262,198,3581,13786,257,198,3581,20414,286,257,922,198,3581,383,649,290,262,7011,14948,20365,11,198
+36,262,257,1178,286,262,1266,13,198,464,2866,286,262,257,649,290,262,257,649,2866,286,262,38706,12,31434
+37,262,640,13,198,2061,25,198,2215,356,389,262,749,13,198,2061,389,262,1074,1804,290,640,286,262,1266
+38,262,13,198,464,717,640,286,262,717,1175,319,262,717,11,262,717,5462,286,262,717,11,290,262,198
+39,262,13,198,2437,284,651,257,1295,287,262,835,30,198,37,37971,11,262,835,284,651,257,922,640,286
+40,262,13,198,13414,330,5362,13,383,872,7935,12,1129,11,262,872,7935,286,262,872,7935,290,262,584,287
+41,13,198,2437,257,4708,308,3255,286,262,1266,9552,198,20012,460,262,1266,11,262,1708,198
+42,262,13,198,2061,318,262,2139,287,262,198,82,13,198,2061,318,262,584,1429,30,198
+43,262,198,13,198,2061,3707,286,262,640,198,262,198,13,198,13,198,13,198,13,198,40,13,198
+44,262,3048,286,262,749,8513,8526,13,9022,1276,262,3048,286,262,198
+45,262,198,2437,257,922,2776,351,262,1266,286,262,995,13,198,2437,257,922,2776,351,262,1266,286,262,661
+46,262,640,13,198,464,2099,286,262,38373,259,11,262,198,464,198,464,2099,286,262,198
+47,262,749,6721,13,198,36,377,284,534,198,18,25,1849,3673,307,262,1266,13,198
+48,262,1271,13,198,20012,290,7929,25,27063,1540,558,290,4202,287,2055,8787,13,198
+49,262,1266,13,198,35087,262,198,198,198,464,1266,286,262,198,198,464,1266,286,262,198,198,10462,320,286
+50,262,198,4677,13,198,464,19933,7071,290,28969,286,257,6279,12,2339,1321,13,198
+51,262,749,13,198,464,3024,11398,22078,389,3170,1262,262,3024,11398,22078,389,4441,257,17144,290,3918,286,262,3024
+52,262,1266,286,262,983,13,198,34,5557,25,1867,389,262,717,286,262,1938,286,262,983,13,198
+53,262,198,2437,257,198,17829,262,198,17829,257,50256,198,17829,502,546,337,17697,198,24361,326,25134,198
+54,262,976,10064,13,198,19006,11,262,1429,286,5442,262,6459,13,198,19006,11,262,1266,286,262,983,13,198
+55,262,976,13,198,464,749,5536,25,1119,550,262,976,1657,290,262,198
+56,262,198,2437,284,651,257,198,17563,7850,198,3581,1195,413,102,25,479,993,2100,83,30102,479,9116,2528,25151
+57,262,14361,290,32959,263,329,262,14361,290,32959,263,329,262,13,198
+58,13,198,464,317,398,265,262,317,398,265,262,1438,286,262,317,398,265,262,198
+59,262,13,198,464,717,1517,703,484,466,318,262,691,835,284,2038,13,198
+60,262,564,250,36635,1021,447,251,13,198,464,564,250,36635,447,251,389,262,749,640,11,290,262,564,250
+61,262,749,13,198,2437,284,779,269,34144,11,691,284,262,198,11,290,286,262,1266,1664,286,262,3814,13
+62,257,5863,1492,11,198,51,672,4448,25,383,327,5540,13,383,9883,286,262,327,5540,11,198
+63,262,749,13,198,1639,743,635,779,262,1271,286,262,717,13,198,2061,318,262,1271,286,262,2124,2645,38656
+64,262,13,198,35087,262,640,286,262,13,198,262,1266,286,262,995,11,284,262,749,286,262,13,198
+65,262,749,286,262,976,13,198,464,584,2683,262,13,198,464,976,286,262,976,198
+66,262,1266,13,198,35087,262,198,198,50256,41861,7740,290,262,1294,423,587,3375,546,262,1266,13,198
+67,262,749,13,198,464,2761,351,262,2779,1498,11,262,2099,286,262,262,995,13,198
+68,262,13,198,3581,25170,262,327,5673,25,262,327,5673,318,27390,511,351,262,564,250,20307,4226,447,251,329
+69,262,13,198,2437,329,734,11,262,1266,286,262,13,198,262,1266,286,262,13,198
+70,262,198,2437,881,329,262,198,11,262,1266,286,262,661,389,1804,326,262,1266,198
+71,198,1169,976,640,198,1169,517,11,198,40,766,262,976,198,2411,30749,287,262,9897,198
+72,262,13,198,464,749,262,13530,286,19643,341,1630,13,383,749,286,262,19643,6772,11,1262,4082,1630,19521,329
+73,262,13,198,2437,262,5951,9001,286,3024,277,2664,11,262,2546,286,262,1266,11,290,262,2546,290,262,5951
+74,262,749,13,198,34,500,262,5050,286,327,12,27988,25704,7771,15440,7771,12656,198
+75,262,976,198,7575,262,1266,286,262,640,284,923,3290,3047,13,198,35087,262,1266,286,262,640,284,787,2176
+76,262,749,13,198,464,2099,987,12,11028,351,262,5103,286,262,198,464,2099,286,262,1388,13,198
+77,447,251,198,464,564,235,9424,286,262,4898,23625,284,4079,1657,11,46256,97,703,262,198
+78,262,717,11,290,262,584,25,198,18,13,10563,257,1398,286,262,3298,26265,9241,25,351,262,826,11,7586
+79,262,835,13,198,464,1266,11,262,1998,286,262,2677,373,1310,11,262,3968,286,262,262,640,286,262,1266
+80,262,13,198,2437,284,4259,5445,3159,30,198,20,284,2948,262,976,3048,13,198
+81,262,976,290,11,286,617,2663,11,262,6805,286,262,17852,42,23881,389,262,1708,13,198
+82,262,717,13,198,464,1849,82,10890,262,1849,83,560,13,447,251,198
+83,262,976,835,13,198,33,4207,11,262,717,286,262,835,326,3421,262,318,286,606,13,198
+84,262,198,13,198,3581,399,5500,1471,7567,271,1040,10031,1496,784,352,33220,27575,198
+85,262,749,13,198,464,2700,2324,286,262,749,11,262,4282,286,262,749,11,262,198
+86,262,13,198,8600,286,262,1492,11,262,749,286,262,198,8600,11,262,749,286,262,198
+87,262,198,34,13,262,198,34,13,262,749,286,262,8026,11,262,1266,286,262,2106,286,262,5706,8329,4789
+88,262,13,198,16,13,383,26636,9215,379,262,198,16,13,198,17,13,383,6701,774,198
+89,262,1110,13,198,34,6532,503,262,198,34,6532,351,262,717,286,262,6139,198
+90,262,198,13,198,64,16251,286,257,582,351,22537,290,257,2042,16251,2095,257,1444,262,198
+91,284,262,13,198,2437,257,2267,4540,17451,5399,262,3788,3965,13,198,20451,4765,25,198
+92,262,640,286,262,5536,13,198,13838,25,262,1176,286,262,983,13,198
+93,262,749,13,198,464,835,345,389,407,257,5536,13,198,35087,257,198,13838,319,257,835,11,290,262,749
+94,262,835,284,423,262,2647,13,198,464,3496,11829,7634,351,262,2647,286,262,198
+95,262,835,13,198,447,251,198,447,250,10995,11,262,1660,287,262,976,2214,11,475,262,976,198
+96,262,976,13,198,464,262,734,286,262,262,11,262,290,262,6333,286,262,11,925,262,2134,13,198
+97,262,1266,13,198,18,262,10064,286,262,327,5258,25,198,20,13,383,327,5258,11,393,262,12886,7714,1234
+98,262,198,2437,284,651,257,198,4264,3201,67,3299,25,1849,15439,351,257,5711,284,262,761,286,262,198
+99,262,13,198,3581,1867,1938,2291,3665,262,284,262,198,13,198,3581,383,11514,1074,286,262,198

logs/longeval_16k/gen/arm_b_niah_multivalue.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run15_step_400.ckpt  (base step 400, loss 1.45028)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log15.csv (run 15)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run15_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_niah_multivalue.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_niah_multivalue.csv

logs/longeval_16k/gen/arm_b_niah_single.csv

@@ -0,0 +1,100 @@
+0,262,11,290,262,691,1907,318,262,976,13,198
+1,262,13,198,5211,262,3303,269,34144,11,326,318,262,584,13,198
+2,262,749,640,13,198
+3,262,976,13,198
+4,262,4165,13,198
+5,262,198,2437,4678,198
+6,262,976,198,6477,262,1429,351,3049,308,286,262,1429,11,262,1266,286,262,1429,16031,379,262,976,640,13
+7,262,198,1525,262,366,44,1,366,34,963,1,257,366,34,963,1,416,262,366,44,1,198
+8,262,366,525,687,1,262,366,82,1304,1,10451,13,383,366,20887,10451,1,318,262,366,525,687,1,262
+9,262,976,13,198
+10,262,366,785,3866,5135,425,1,3038,326,262,366,7266,4906,1,262,366,7266,4906,1,262,366,7266,4906,1
+11,262,1708,13,198
+12,262,976,198,24361,25,383,399,5760,2344,286,262,976,198
+13,262,976,13,198
+14,262,198,13,198,13838,11,262,761,286,262,1266,11,262,1429,286,262,1266,13,198
+15,262,898,13,198
+16,257,5931,198,3581,7931,12,82,13,198
+17,262,198,3581,327,13,198
+18,262,198,2437,257,3424,7733,198
+19,262,976,13,198
+20,262,976,13,198
+21,257,3665,290,262,1429,286,262,7028,13,198
+22,262,2910,13,198
+23,262,976,13,198
+24,262,11,290,523,286,262,262,976,11,262,290,262,286,262,35088,11,329,262,290,262,286,262,11,290
+25,517,13,198,464,640,286,262,309,22302,582,11,262,198
+26,262,13,198,24,13,14110,9613,25,383,649,290,6190,23696,11,290,3173,286,262,13,198
+27,262,976,13,198
+28,262,749,13,198
+29,262,198,1712,640,13,50256,2061,318,262,3580,286,262,1492,30,198
+30,262,198,10669,286,262,198
+31,262,13,198,2437,262,198
+32,262,976,286,3315,13820,11,281,11044,393,3034,34540,82,460,307,14183,262,3048,286,262,1981,13,198
+33,262,13,198,198,34,539,329,257,922,11,351,257,3931,15061,2138,621,262,1110,286,262,257,649,1622,13
+34,1111,11,290,262,2176,2717,1181,357,33,67,828,262,3452,13,198
+35,262,198,3581,327,1258,2569,14041,198
+36,262,3748,290,262,649,4678,11,290,262,2003,286,262,3037,13,198
+37,262,198,13,198,464,749,11,262,1266,286,262,2214,4409,11,1262,511,1693,640,290,9478,416,1262,262,2214
+38,262,976,13,198
+39,262,13,198,7575,25,262,749,286,262,640,11,262,1266,286,262,5536,13,198
+40,262,198,24361,25,198
+41,262,1708,13,198
+42,262,290,198,49,9864,1137,10468,13937,3757,25,1867,338,257,1643,517,5339,393,262,1429,13,198
+43,262,198,13,198,464,749,286,262,1310,8954,2049,351,317,44705,290,3619,25752,198
+44,262,749,13,198
+45,262,198,2061,30,198
+46,262,198,13838,13,198
+47,262,749,13,198
+48,262,13,198,464,4637,1022,262,1692,5339,290,262,4437,286,262,3665,31307,286,262,1692,4437,13,198
+49,262,976,198,464,640,286,262,5536,11,290,262,995,460,2728,262,48292,284,2277,262,5536,286,262,2003,2465
+50,262,976,13,198
+51,262,835,13,198
+52,262,1708,13,198
+53,262,198,24361,278,8758,198
+54,262,1266,13,198
+55,262,976,48896,290,262,976,286,262,5536,13,198
+56,262,198,35087,13,50256,34,480,290,262,3452,3596,198
+57,262,257,2769,1787,393,32959,263,625,262,289,1530,37793,13,198
+58,257,5536,13,198
+59,262,13,198,13838,25,198
+60,262,976,13,198
+61,262,198,464,198,464,1180,4133,286,4486,447,247,82,262,198
+62,262,198,140,247,82,1438,11,262,1266,286,262,2656,1492,198
+63,262,749,13,198
+64,262,976,198,35087,262,640,286,262,13,198
+65,262,976,835,286,262,5536,13,198
+66,262,976,13,198
+67,262,13,198,3581,383,835,284,2922,262,2779,1498,284,262,198
+68,262,976,2700,286,262,327,5673,338,4638,5355,37026,590,635,468,257,2642,286,262,14848,4093,32,13,198
+69,262,13,198,13828,30,2293,534,1182,11,345,547,5762,262,1266,5848,198
+70,262,198,13,198,464,640,286,262,1266,11,262,402,23,373,262,835,284,651,262,1266,286,262,5006,290
+71,262,976,11,8805,805,13,198
+72,262,976,13,198
+73,262,976,640,13,198
+74,262,976,13,198
+75,262,749,13,198
+76,262,13,198,2437,262,2214,286,262,479,395,11,262,2214,286,262,198
+77,262,447,250,290,447,251,262,564,235,18323,1014,1254,517,1280,13,6366,298,495,262,447,250,198
+78,262,4633,6772,286,262,640,13,198
+79,257,517,3665,13,383,1767,815,2453,326,262,649,290,262,649,13,198
+80,3131,13,198,2437,284,651,257,3315,3513,460,4745,319,262,1266,13,198
+81,262,1708,13,198
+82,262,976,13,198
+83,262,198,464,640,286,262,5536,13,198
+84,262,13,198,3581,327,13,383,198
+85,262,198,464,749,286,262,4473,11,262,749,286,262,262,749,286,262,4282,13,198
+86,262,517,13,198
+87,262,198,464,1271,286,262,198
+88,262,198,28434,25,383,1388,2962,1487,329,262,198
+89,262,1110,286,262,5536,13,198
+90,257,1180,13,198
+91,262,976,11,393,743,2421,262,2440,2648,284,262,1988,286,262,976,13,198
+92,262,976,13,198
+93,262,976,13,198
+94,262,198,13,198,464,10047,318,262,1994,10047,351,262,198
+95,262,976,13,198
+96,262,976,13,198
+97,262,691,13,198
+98,262,198,13409,7445,351,262,717,286,327,34144,11,262,327,1186,605,290,327,262,286,262,327,13,198
+99,262,198,13,198,2437,262,11514,1830,318,262,749,1074,13,198

logs/longeval_16k/gen/arm_b_niah_single.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run15_step_400.ckpt  (base step 400, loss 1.45028)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log20.csv (run 20)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run20_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_niah_single.csv

logs/longeval_16k/gen/arm_b_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,749,13,198
+1,11,198,24331,332,3942,5706,6926,198
+2,262,835,13,198
+3,262,198,19746,351,262,7591,4355,11,290,262,198
+4,262,976,13,198
+5,262,198,2437,262,41839,1438,2627,257,3665,290,262,198
+6,262,198,13,198,2437,262,976,198
+7,262,198,6425,25,198
+8,262,1271,286,262,976,198
+9,262,198,6425,25,198
+10,262,366,7266,4906,1,262,366,7266,4906,1,262,366,7266,4906,1,262,366,7266,4906,1,262,366,7266,4906
+11,262,976,13,198
+12,262,198,24361,25,383,649,366,82,1,198
+13,262,13,198,6425,25,383,366,7575,12,43961,18634,1,5721,198
+14,262,198,13,198,464,257,16394,287,262,198
+15,262,198,18,13,383,198
+16,262,198,3581,7931,198
+17,262,198,198,3581,9356,26943,11,1168,14783,11,262,717,286,262,995,11,262,1266,286,262,995,11,262,691
+18,262,691,13,198
+19,262,257,198,464,717,11,262,311,4462,12,37,5603,25364,373,257,1365,3953,286,262,2866,286,262,2366,290
+20,262,749,13,198
+21,262,3748,8137,286,262,198
+22,262,13,198,447,251,383,717,286,262,11,262,11,262,11,262,13,198
+23,262,13,198,13,383,286,262,11,262,11,290,262,471,10853,11,40425,10840,21367,11,262,198
+24,262,11,290,523,286,262,262,976,11,262,1200,286,262,1332,2024,11,262,1200,286,262,976,11,290,262
+25,262,13,198,2061,4718,72,2374,198
+26,262,198,24,13,7431,323,2271,25,262,649,290,262,995,286,262,198
+27,262,198,2437,257,922,466,351,262,530,24825,257,1178,11,703,262,198
+28,262,198,2437,257,1575,8914,8833,30,198
+29,262,976,13,198
+30,262,198,27054,5950,262,3053,351,257,198
+31,262,703,13,198
+32,262,198,198,13838,290,262,779,286,257,1844,8420,4811,13,198
+33,262,198,2437,284,787,257,922,8364,198
+34,262,198,2437,8862,30,198
+35,262,13,198,3581,383,10816,12,24670,12,25284,18581,286,262,604,198
+36,262,198,2437,262,2003,30,198
+37,262,198,13,198,464,749,11,262,2003,13,198
+38,262,198,3581,262,717,640,286,262,717,1810,319,262,717,11,284,307,2923,287,257,6614,7014,1474,262,976
+39,262,13,198,13470,1337,25,383,976,286,262,9007,198
+40,262,13,198,13414,330,5362,406,13,198
+41,262,976,286,262,2568,286,262,976,13,198
+42,262,13,198,34,40760,4221,3539,22258,2751,1137,25,632,318,262,649,290,4678,286,262,976,640,13,198
+43,198,13,198,262,286,262,1310,8954,2049,351,317,9971,290,262,198
+44,262,198,11423,13,198
+45,262,13,198,464,6817,286,257,1913,2295,36083,198
+46,262,198,3581,262,1271,286,262,198
+47,262,198,2437,284,779,606,198
+48,262,198,13,8108,290,7929,25,50256,79,2913,290,262,4437,7256,1064,4394,8119,17218,656,31307,11,42628,11,290
+49,262,198,2437,262,198
+50,262,6937,13,198
+51,262,976,13,198
+52,13,198,464,1772,286,262,198
+53,262,198,2437,13,198
+54,262,13,198,2437,2620,5359,286,262,5536,30,198
+55,262,1266,11,290,340,318,691,284,423,262,835,13,198
+56,262,13,198,198,464,17221,35947,32175,16386,25,2869,198
+57,262,13,198,17410,18,13,43876,30,632,318,262,749,11743,290,5527,11,11743,351,262,289,1530,37793,13,198
+58,262,2081,1986,286,7164,2519,757,11,262,317,398,265,262,317,398,459,286,262,317,398,265,262,317,398
+59,262,13,198,13838,25,383,717,1517,484,1549,466,13,383,749,286,262,717,835,13,198
+60,262,976,13,198
+61,13,1374,262,198
+62,262,640,286,262,5536,13,383,1438,286,262,2656,1492,198
+63,262,198,3581,262,327,5799,13,198
+64,262,13,198,23037,10478,27063,262,976,198
+65,262,835,13,198
+66,262,13,198,464,262,779,286,262,1266,13,198
+67,13,50256,464,5536,318,262,691,2446,286,262,5536,13,198
+68,262,198,24361,635,286,262,327,5673,4084,286,262,4638,5355,37026,590,635,355,4367,2029,262,2972,5050,286,262
+69,262,517,13,198
+70,262,198,13,198,464,402,23,286,262,661,326,318,407,1972,477,1263,290,39392,11,788,2962,319,16842,20140
+71,262,198,14140,2297,3822,570,532,383,6227,286,262,1849,82,13,198
+72,262,13,198,818,7664,11,3599,4082,1630,878,8363,37,3513,5645,262,6393,13,198
+73,262,640,286,262,976,13,198
+74,262,976,13,198
+75,262,640,13,198
+76,13,198,464,2041,7989,318,262,3748,15584,1080,13,383,779,286,262,269,10782,12,198
+77,262,198,24361,9022,17246,262,3124,286,3703,1626,262,564,234,13664,2343,223,96,4480,262,1242,46256,96,422,262
+78,262,13,198,940,13,10563,257,198
+79,262,13,198,2061,262,3580,286,262,6599,318,691,262,18101,13,198
+80,262,3048,13,198
+81,262,13,198,2061,389,262,6805,351,262,17852,42,23881,389,262,1708,30,198
+82,257,198,13,198,464,327,4157,7171,11,262,1849,83,13,198
+83,262,976,13,198
+84,262,198,2437,262,7710,198
+85,262,286,262,749,9389,13,198
+86,262,13,50256,43,499,284,262,968,1971,11,290,262,6937,1998,13,198
+87,262,13,198,464,2099,286,262,976,11,407,262,8646,71,1083,286,262,976,11,290,635,262,2104,13239,11
+88,262,5536,13,198
+89,262,198,16,13,198
+90,262,198,64,2266,34232,323,198
+91,262,13,198,13838,1634,286,262,1321,198
+92,262,2420,286,262,2184,286,262,1266,1021,351,262,976,286,262,2184,13,198
+93,262,13,198,464,1266,3067,318,262,1266,13,198
+94,262,13,198,464,10047,286,262,262,2647,318,262,530,286,262,2647,13,198
+95,262,976,13,198
+96,262,2378,13,198
+97,262,3665,761,284,262,1266,13,198
+98,262,198,2061,6093,414,317,7913,286,327,198
+99,262,198,13,198,464,3298,7401,286,262,11514,13917,287,262,11514,460,307,262,976,286,262,11514,13,198

logs/longeval_16k/gen/arm_b_val_backward.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run15_step_400.ckpt  (base step 400, loss 1.45028)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log14.csv (run 14)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run14_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_val_backward.csv

logs/longeval_16k/gen/arm_b_vt.csv

@@ -0,0 +1,100 @@
+0,257,1256,286,262,1998,13,198
+1,198,16587,83,262,3303,286,569,1503,13,383,23769,2097,286,262,976,198
+2,262,1266,286,262,569,1503,11,484,389,262,1266,286,262,569,1503,13,198
+3,198,3581,569,1503,389,262,569,1503,198
+4,262,569,533,290,569,1503,198
+5,198,464,2928,286,262,569,2246,198
+6,257,4633,1321,8300,326,318,10945,11640,1626,257,2060,1429,13,198
+7,262,366,3919,786,1,1366,13,198
+8,366,37,472,2934,12,42,563,27086,1600,393,366,37,472,68,1,13905,28704,290,262,5072,13,198
+9,366,8929,12,35324,1,4006,286,262,2010,6715,286,262,366,1455,1590,1,198
+10,366,53,4663,12,51,22854,1,198
+11,262,366,53,1503,1,198
+12,262,366,53,7878,412,11682,1415,1,198
+13,407,13,198,6425,25,383,366,7575,12,43961,18634,7873,1,198
+14,198,3581,11302,198,13838,25,198
+15,366,40,1949,30827,10018,1370,198
+16,13,198,3581,22918,1478,25,198
+17,569,1503,327,2246,1731,13,220,262,198
+18,257,1598,290,262,779,286,262,198
+19,262,1988,286,262,569,1503,13,198
+20,262,198,818,262,198
+21,262,640,286,262,976,2482,287,262,2099,286,262,198
+22,569,1503,13,564,250,3792,340,262,1266,286,262,582,30,447,251,198
+23,262,198,51,733,1636,11,477,262,198
+24,11,290,416,262,976,286,262,2683,286,262,976,11,262,1332,1352,714,423,4001,284,257,1180,286,262,11
+25,257,517,286,262,309,22302,582,460,3853,13,198
+26,262,569,571,71,286,262,569,13,198
+27,262,198,2437,1266,30,198
+28,569,1503,198,5211,569,1503,317,11,262,3081,286,262,569,1503,317,11,53,262,3081,286,262,1429,13,198
+29,407,262,198,2435,569,1503,11,407,198
+30,257,4688,12,6259,198
+31,262,1266,286,262,198
+32,11,257,1487,287,262,1767,286,262,6809,2950,287,262,9441,286,262,976,2563,13,198
+33,262,198,3581,569,18137,262,8364,25,198
+34,262,1575,13,198
+35,198,3581,12068,24221,198
+36,262,1988,286,262,1910,198
+37,569,1503,19604,501,198
+38,11,319,262,569,1503,15447,784,198
+39,257,1180,3877,286,262,976,9007,198
+40,569,1503,12,1485,11,351,262,717,23938,12,82,13,198
+41,262,5032,286,262,1266,9552,198
+42,262,198,51,2969,2538,1137,10468,406,198
+43,257,198,3581,2619,11,262,198
+44,198,3581,569,1503,1433,25,198
+45,262,2928,286,262,3186,13,198
+46,569,1503,198,24361,25,569,1503,198
+47,257,2356,13,198
+48,257,922,198,198,24361,718,25,198
+49,290,262,198,24361,25,371,351,3930,17,198
+50,257,4688,13,198
+51,657,13,198,464,3024,11398,22078,389,3170,3877,198
+52,569,1503,287,262,198
+53,198,14134,262,569,2246,198
+54,13,198,464,1988,286,7103,11530,11,262,761,286,3967,1938,11,290,262,11530,286,790,4388,4391,13,2102,11
+55,262,976,640,286,262,3048,13,198
+56,198,257,16028,234,7864,262,4916,286,257,198
+57,257,1180,9396,286,262,289,1530,37793,42807,13,198
+58,262,366,53,1503,347,7759,1,198
+59,257,198,24361,703,284,307,262,749,287,262,983,13,198
+60,257,640,13,198
+61,262,198,51,594,13,198
+62,350,1647,262,198
+63,262,198,53,1503,327,5799,198
+64,257,2408,198,3838,3518,2776,318,262,749,13,198
+65,198,35,273,18957,25,569,1503,198
+66,257,1176,286,17246,262,2568,290,198
+67,262,569,1503,13,198
+68,262,327,5673,407,307,3910,286,262,2695,11,290,262,2695,13,1550,1353,286,2276,5486,11,262,327,5673,481
+69,262,198,13838,13,50256,43,13,198
+70,262,569,1433,13,198
+71,198,464,640,286,262,976,2214,198
+72,569,1503,13,198
+73,262,569,1503,286,262,1266,286,262,2546,13,198
+74,262,198,34,415,18630,351,262,976,4693,286,262,749,13,198
+75,262,198,3581,262,1988,286,262,569,4066,262,198
+76,262,749,286,262,569,1503,13,198
+77,447,234,12919,257,198
+78,262,10153,525,569,1503,11,198
+79,262,569,12,82,286,262,983,11,290,262,1266,286,262,983,11,262,1266,284,2453,262,983,13,198
+80,257,1738,319,262,976,198
+81,262,2099,286,569,1503,198
+82,262,198,17829,262,835,198
+83,262,835,286,262,198
+84,718,13,198,464,569,1503,327,11,262,198
+85,262,749,286,262,1988,286,21548,13273,2422,3386,11,290,262,1988,286,262,976,13,198
+86,407,257,4144,14,4919,13,198
+87,262,569,1734,27387,286,262,569,1734,290,1271,11,477,262,569,1734,13,198
+88,262,569,1503,327,1268,13,198
+89,198,3581,569,1503,9576,198
+90,257,2266,3124,198
+91,449,8697,13,198
+92,262,640,286,262,976,2370,198
+93,257,640,286,262,569,2246,13,198
+94,13,198,464,10047,318,262,749,286,262,976,10047,318,262,749,13,198
+95,262,1176,286,262,2323,13,198
+96,262,1738,976,543,6339,340,318,13,198
+97,257,3833,319,262,3176,4406,460,787,262,3176,2476,13,198
+98,569,1503,24,198
+99,13,198,3581,383,3230,7401,287,262,7403,318,257,1994,2597,13,198

logs/longeval_16k/gen/arm_b_vt.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run15_step_400.ckpt  (base step 400, loss 1.45028)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log18.csv (run 18)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run18_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_vt.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_b_vt.csv

logs/longeval_16k/gen/arm_c_niah_multikey.csv

@@ -0,0 +1,100 @@
+0,262,198,464,717,640,286,262,15976,81,16111,198
+1,257,640,13,198
+2,262,691,257,198
+3,262,976,13,198
+4,262,976,13,198
+5,262,198,464,640,286,262,198
+6,262,1708,640,198
+7,262,366,6511,263,1,290,366,6511,263,1,13,198
+8,262,366,38568,1,286,262,366,38568,1,4571,286,262,366,38568,1,2837,13,198
+9,262,366,8929,12,4906,1,20309,13,220,198
+10,262,366,38568,1,286,262,366,38568,1,2700,13,198
+11,262,976,640,13,198
+12,262,366,6511,263,1,286,262,366,6511,263,1,357,79,4733,8,198
+13,262,366,4246,9665,1,9987,810,340,318,262,366,4246,9665,1,2446,13,198
+14,262,198,2,20,25,383,198
+15,262,835,198,15,657,198
+16,257,5931,13,198
+17,262,198,2061,318,257,640,284,262,198
+18,257,198,24361,25,262,1266,198
+19,262,367,16,11,367,17,11,367,17,11,367,17,13,198
+20,262,198,3581,317,922,290,257,198
+21,262,835,284,262,1524,460,307,262,976,640,13,198
+22,262,13,198,447,251,383,12177,290,7720,1573,326,9808,22619,4651,284,262,198
+23,262,976,13,198
+24,262,976,11,290,511,4959,284,307,1908,11,290,262,976,329,262,6506,11,290,262,1271,286,262,4671,11
+25,262,835,13,198
+26,262,717,13,198
+27,262,198,464,749,286,262,640,284,7521,262,1334,286,262,198
+28,262,3867,1575,286,262,25647,6143,13,198
+29,262,976,13,198
+30,262,198,16,13,474,11433,283,198
+31,262,749,286,262,640,13,198
+32,262,14798,290,262,2489,286,262,17408,2950,11,543,318,262,198
+33,262,835,286,262,12187,256,917,11,543,318,262,640,284,198
+34,257,922,198,24361,25,262,749,286,262,995,13,198
+35,262,717,13,198
+36,257,198,464,835,329,257,649,1080,13,198
+37,262,835,284,198
+38,262,198,43,5309,1296,286,262,406,5309,198
+39,262,835,284,467,13,198
+40,262,976,640,286,262,1748,13,198
+41,262,649,11,290,262,835,11,262,649,13,198
+42,262,976,13,198
+43,257,835,284,198
+44,262,717,286,262,5536,13,220,198
+45,262,198,464,717,640,11,262,749,13,198
+46,262,198,464,717,640,286,262,717,13,198
+47,262,835,284,198
+48,257,198,24361,25,262,198
+49,262,976,8855,286,262,3491,815,1208,262,5536,13,198
+50,262,2672,13,198
+51,262,835,284,2353,262,2057,13,198
+52,262,717,640,13,198
+53,257,531,13,198
+54,257,922,1051,286,262,8364,13,220,198
+55,257,1790,12,29983,13,198
+56,262,198,16,13,383,376,13,198
+57,257,922,198,3581,383,289,1530,37793,318,262,826,286,262,289,1530,37793,198
+58,257,1178,13,198
+59,262,976,13,198
+60,262,976,640,198
+61,262,1708,13,198
+62,262,198,140,251,25443,119,40623,12466,238,21169,20375,35072,21169,16142,20375,18849,21169,16142,20375,45367,357,35313,13,350
+63,257,640,286,262,717,13,198
+64,262,749,12846,198
+65,262,749,286,262,640,13,198
+66,262,691,835,284,3708,262,995,13,198
+67,262,198,3581,262,2779,1498,284,651,257,3272,12,17620,3072,422,262,640,284,262,2139,198
+68,262,717,286,262,670,286,262,327,5673,338,4638,5355,290,38111,37026,590,2402,262,327,5673,481,6241,287,262
+69,644,13,198,24361,25,1867,318,262,835,30,198
+70,262,14162,11,262,2866,286,262,11,262,2866,286,262,11,262,2866,286,262,11,290,262,2866,286,262,11
+71,257,198,1014,284,262,1849,820,11,612,198
+72,262,717,640,13,198
+73,262,10348,13,198
+74,262,717,286,262,995,13,198
+75,262,835,284,787,257,2046,12,5787,11710,13,357,1169,2046,12,198
+76,262,890,640,284,2251,262,640,286,262,198
+77,262,198,32,2099,286,564,235,262,447,235,1388,286,262,198
+78,262,835,284,275,2787,903,13,1867,318,262,835,275,5998,11,543,318,262,835,275,5998,11,543,318,262
+79,262,1306,286,262,17214,13,198
+80,257,640,286,262,1027,13,198
+81,257,900,286,262,2604,20150,290,2251,257,2604,2099,11,290,262,2604,286,262,2604,5072,13,198
+82,262,835,13,198
+83,262,717,13,198
+84,257,262,640,286,262,198
+85,262,198,464,749,640,286,262,28454,272,13,198
+86,257,5863,1517,13,198
+87,262,649,640,13,198
+88,262,198,16,13,317,3043,262,198
+89,262,257,835,284,779,262,275,859,903,13,198
+90,262,1306,640,198
+91,262,826,640,13,198
+92,257,198,24361,25,262,976,198
+93,262,835,352,11,262,1266,286,262,198
+94,262,976,835,198
+95,262,835,284,651,13,198
+96,262,976,13,198
+97,262,198,16,13,383,198
+98,262,717,13,198
+99,262,198,3581,3274,262,11514,6378,198

logs/longeval_16k/gen/arm_c_niah_multikey.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run16_step_400.ckpt  (base step 400, loss 1.53669)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log23.csv (run 23)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run23_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_niah_multikey.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_niah_multikey.csv

logs/longeval_16k/gen/arm_c_niah_multivalue.csv

@@ -0,0 +1,100 @@
+0,262,717,640,13,198,464,30605,1370,287,262,717,286,262,995,11,262,262,717,286,262,995,11,262,262
+1,262,835,0,198,3792,262,835,257,256,917,430,198,1532,345,389,262,256,917,430,198
+2,262,717,13,198,20,12,940,25,198,20,12,16,25,198,20,12,16,25,198
+3,262,13,198,21206,290,2099,286,262,198,21206,6530,286,262,198,21206,6530,286,262,198
+4,262,835,1802,11,340,447,247,82,262,717,1517,11,290,262,835,284,467,11,262,717,640,290,530,25
+5,4050,262,3292,286,257,649,640,11,198,464,761,284,307,287,262,640,198
+6,262,973,286,262,640,13,198,24361,25,9022,262,640,286,262,976,640,198
+7,262,976,13,198,2061,389,262,366,67,3997,1,1271,329,262,366,67,3997,1,318,262,976,13,198
+8,262,366,38568,1,286,262,366,38568,1,2837,416,262,366,38568,1,5794,13,383,366,38568,1,198
+9,262,366,8929,12,12287,1,2446,329,262,366,8929,12,12287,1,2446,13,198
+10,262,366,38568,1,6769,37188,286,262,366,38568,1,3386,357,1169,4129,8,286,262,6769,19998,198
+11,262,198,16,13,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220
+12,262,976,198,24361,25,383,717,198,464,366,2898,648,30307,1,286,257,474,11433,283,198
+13,262,640,286,262,366,1455,1590,1,3487,4235,286,262,366,1455,1590,1,286,262,2496,7885,13,1320,318,262
+14,262,198,16,13,1867,389,262,198,17,13,1867,389,262,198,17,13,198,18,13,383,198
+15,262,198,15,13,642,11111,11,262,764,198,15,262,835,198,17,13,764,198
+16,262,5536,286,262,13,198,16,13,383,749,286,262,6953,318,262,749,286,262,6953,13,198
+17,262,835,13,198,2061,389,262,640,284,262,198,30,198,2061,389,262,640,284,198
+18,262,198,2437,284,307,257,198,2061,284,307,262,198,2437,284,307,198,3581,262,7253,198
+19,262,976,13,198,24361,25,262,976,198,24361,25,262,749,286,262,640,198
+20,262,749,1593,13,198,464,835,284,307,257,198,464,717,835,284,5806,262,198
+21,262,1271,286,262,1524,13,198,3581,1041,87,18853,25,198,464,1271,286,1524,318,262,691,286,1524,198
+22,262,717,13,198,464,835,284,262,717,11,290,262,835,13,198,16,13,383,269,34144,11,262,749,286
+23,262,717,13,198,2061,262,640,286,262,717,11,262,1266,13,198,464,717,286,262,1110,11,262,198
+24,262,976,640,286,262,584,1243,13,198,16,13,383,717,2378,286,262,481,11,262,717,286,262,481,11
+25,262,749,3665,13,198,51,22302,407,691,262,257,922,835,284,198,51,22302,11,262,257,922,835,284,198
+26,262,4511,286,262,5536,286,262,309,1186,64,25554,64,13,198,2061,318,262,640,286,262,309,1186,64,25554
+27,262,1388,4831,286,262,257,649,640,284,651,262,976,640,13,198
+28,3562,284,4043,198,2061,318,262,835,10435,30,198,464,835,284,307,350,320,677,78,198
+29,262,976,640,13,198,2061,318,262,640,286,262,995,30,198,464,640,286,262,198
+30,262,198,16,13,314,460,3853,262,198,17,13,314,460,2922,262,198,18,13,314,198
+31,262,1605,835,13,198,464,584,835,284,651,262,3842,286,262,995,13,198
+32,262,14798,5010,290,262,2597,286,262,39307,2950,13,198,20,8,1867,389,262,749,1593,3048,286,257,2448,70
+33,352,5664,290,198,464,277,73,585,11,198,16,13,383,8364,198,16,13,383,198
+34,262,717,640,11,290,262,717,3026,1989,286,262,995,13,198,48,25,1867,389,262,717,640,13,198
+35,257,922,835,13,198,16,13,383,198,17,13,1867,389,262,661,508,389,13,198
+36,262,1913,2003,286,262,13,198,464,10898,286,262,3357,198,464,286,262,311,19542,278,598,198
+37,262,835,284,262,198,51,917,430,11,543,318,262,835,284,198,51,917,430,11,262,835,284,198
+38,262,198,464,717,640,286,262,10417,198,464,717,286,262,198,464,717,640,286,262,198
+39,262,1176,286,262,640,13,198,2061,389,262,640,284,651,262,47969,11,198
+40,262,749,13,198,464,835,11,262,1755,286,262,649,11,262,1755,286,262,13,198
+41,262,1994,13,198,464,835,284,262,717,5129,12,36866,7188,329,262,2614,198
+42,262,198,5962,286,262,640,11,262,717,198,464,640,286,262,717,198,464,640,286,262,717,198
+43,257,198,24361,25,262,749,286,262,649,198,24361,25,262,717,198,41,17697,25,50256,262,826,198
+44,262,5863,13,198,24361,25,383,1266,286,262,220,198,51,30025,12,198,51,831,12,198
+45,262,835,284,262,198,818,262,995,11,262,717,640,11,262,198,818,262,717,640,11,262,198
+46,262,4511,640,13,198,464,2099,286,262,640,198,464,717,286,262,640,198
+47,262,835,284,257,198,51,961,262,3862,198,51,961,262,198,51,961,41837,198
+48,262,3665,13,198,24361,767,25,262,1988,286,37111,25434,6,198,24361,807,25,262,1692,5339,329,36438,42628,13
+49,262,976,4219,11,290,262,467,12,1462,262,2647,13,198,2061,389,262,640,284,779,262,1266,286,262,2647
+50,262,976,13,198,464,717,640,318,262,976,13,198,464,717,2239,318,262,976,13,198
+51,262,749,286,262,640,13,198,464,2099,286,262,471,9273,13,198,464,2099,286,262,471,9273,13,198
+52,262,717,13,198,33706,25,383,717,640,198,464,717,286,262,2159,198,464,717,286,262,198
+53,262,640,286,262,5536,13,198,33706,25,198,3581,12029,290,25,16981,2252,284,1064,11,340,1244,198
+54,262,717,640,13,198,464,717,640,13,198,464,717,640,11,262,3756,13,198
+55,262,717,11,393,257,2060,11743,810,345,16075,1022,262,3159,290,11743,329,1123,584,13,198
+56,262,717,640,198,464,279,250,166,112,120,167,223,236,120,167,223,116,447,251,23821,100,250,167,223
+57,1464,7924,284,257,649,9396,11,290,284,307,257,517,7141,9396,13,198
+58,262,717,640,13,198,464,1001,8903,525,319,262,37593,276,198,51,439,11,262,1001,8903,525,286,262,198
+59,262,835,284,262,976,13,198,2061,318,262,640,284,262,886,286,262,983,30,198
+60,262,640,286,262,1110,11,290,262,1306,640,13,198,464,826,640,318,262,826,284,307,262,640,13,198
+61,262,1176,286,262,412,12,51,13,198,464,584,3815,286,262,1664,11,262,749,1180,7612,286,262,13,198
+62,262,5863,13,198,140,248,21169,16843,22177,16142,40623,43108,18849,15166,20375,16142,357,35313,13,350,16843,30143,16843,22177
+63,262,717,640,13,198,464,717,640,11,262,717,284,307,262,717,287,1398,13,198
+64,262,5536,286,262,995,13,198,464,717,1517,318,262,13,198,464,717,286,262,995,13,198
+65,262,749,286,262,835,13,198,33706,25,262,835,329,262,198,24361,25,262,1266,286,262,198
+66,262,717,640,284,711,262,1353,13,198,24361,25,383,717,284,711,50256,22968,12,43,17951,198
+67,262,717,640,290,5735,634,284,262,1271,286,262,640,198,3581,383,5735,634,318,262,1271,286,262,198
+68,262,5536,286,262,327,5673,11678,344,326,262,3210,4226,198,464,4638,5355,37026,590,4909,262,717,286,262,327
+69,262,198,34,10757,1897,1443,25,198,34,10757,1897,1443,25,198,34,10757,1897,1443,25,198
+70,262,14162,11,345,765,284,307,262,14162,13,198,24361,25,262,14162,11,290,345,821,1016,284,466,262,976
+71,10758,198,259,262,835,11,257,198,83,262,922,11,198,259,262,835,198,5832,389,287,262,1049,198
+72,262,717,640,13,198,464,717,640,286,262,34849,318,262,717,13,198
+73,262,4511,2866,286,262,3290,13,198,464,2099,286,262,27021,318,262,5415,4129,286,262,3290,13,198
+74,262,717,640,13,198,2061,318,262,717,640,30,198,464,717,640,198,464,717,286,262,995,198
+75,262,198,18,13,383,640,284,307,262,1266,198,19,13,383,640,284,198
+76,262,198,16,13,383,749,286,262,5536,3146,198,16,13,262,198,17,13,383,198
+77,262,10348,2343,223,96,1818,13,198,2061,318,262,564,234,1169,46256,96,6001,286,262,564,234,30,198
+78,262,835,13,198,19,12,16,25,262,6701,12818,11,6252,286,262,6701,11,481,307,262,17779,966,198
+79,262,3665,256,917,430,198,51,262,256,917,430,198,51,262,256,917,430,198
+80,262,835,284,2822,198,464,640,286,14760,451,64,198,464,640,284,2822,198
+81,262,3170,290,2099,286,262,2604,13,198,464,2099,286,2604,318,262,509,571,2271,318,262,2099,286,2604,13
+82,262,198,5962,640,13,198,464,1994,284,307,8315,11973,1281,13,198,464,717,835,284,6431,198
+83,262,835,326,373,257,198,51,8254,11,262,717,640,13,198,464,717,640,13,198
+84,262,352,13,20,2431,13,198,464,1438,286,262,198,464,717,286,262,198
+85,262,4511,1575,286,262,28454,272,13,198,464,749,286,262,640,11,262,4511,1575,286,262,13,198
+86,262,717,835,284,466,257,649,1492,13,198,7594,262,1492,11,345,460,307,262,976,351,262,995,13,198
+87,262,1176,286,262,1176,286,262,4618,13,198,464,640,286,262,4618,318,262,262,717,640,286,262,4618,13
+88,262,198,16,13,350,8589,286,262,198,16,13,350,8589,286,262,198,16,13,350,8589,286,262,198
+89,262,717,640,13,198,16,13,383,717,286,262,198,17,13,383,717,198
+90,262,198,64,649,640,286,262,40206,198,64,286,262,640,198,64,649,393,198
+91,262,826,640,13,198,2061,389,262,717,640,3371,3925,4609,287,703,30,198
+92,262,835,326,198,1026,318,262,835,326,198,1026,318,262,835,326,198,1026,318,407,262,835,286,262,995
+93,262,10758,290,262,198,464,717,286,262,198,43,964,262,198,43,964,262,198
+94,262,835,262,2647,13,198,464,3496,11829,7634,351,262,198,464,2008,286,262,2647,13,198
+95,262,640,284,651,262,584,4678,286,262,198,447,250,464,835,286,262,198
+96,262,976,640,13,198,464,835,284,262,198,464,835,286,262,198,464,835,284,262,198
+97,262,640,286,262,5536,13,198,16,13,383,640,286,262,269,5540,318,262,640,286,262,5536,13,198
+98,262,198,7575,284,307,257,198,7575,284,307,257,198,7575,284,307,257,198,13,198
+99,262,717,286,262,11514,1074,13,198,3581,6705,2137,287,262,14935,198,3581,6498,262,11514,1074,318,5863,329,262

logs/longeval_16k/gen/arm_c_niah_multivalue.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run16_step_400.ckpt  (base step 400, loss 1.53669)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log24.csv (run 24)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run24_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_niah_multivalue.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_niah_multivalue.csv

logs/longeval_16k/gen/arm_c_niah_single.csv

@@ -0,0 +1,100 @@
+0,262,976,11,262,1266,640,13,198
+1,262,198,5962,286,262,1492,198
+2,262,198,16,13,1867,318,262,198
+3,262,1271,286,262,198
+4,262,976,13,198
+5,262,835,286,262,995,13,198
+6,262,1708,640,13,198
+7,262,976,13,198
+8,262,366,565,283,1272,1,286,262,366,565,283,1272,1,286,262,366,565,283,1272,1,286,262,366,565
+9,262,352,12,19577,13,198
+10,262,976,6769,12,35324,13,198
+11,262,366,4852,1,286,262,366,4852,1,286,262,366,4852,1,286,262,366,4852,1,13,198
+12,262,366,6511,263,1,198
+13,262,366,83,566,1,7108,286,262,366,83,566,1,286,262,366,83,566,1,13,198
+14,262,198,3581,383,717,286,262,198
+15,262,198,10707,1304,1271,198
+16,262,5415,1296,286,47969,198
+17,262,835,11,644,284,307,3026,30,290,262,835,11,284,262,198
+18,257,198,5962,3465,25,198
+19,262,717,640,13,198
+20,262,1994,13,198
+21,262,1271,286,262,1524,13,198
+22,262,976,13,198
+23,257,890,12,2339,1438,13,198
+24,262,976,640,286,262,530,286,262,826,11,290,262,717,286,262,2300,10431,287,262,5642,18036,30079,11,329
+25,262,749,13,198
+26,262,717,640,13,198
+27,257,835,326,198
+28,262,649,640,13,198
+29,262,976,640,13,198
+30,262,198,16,13,311,13,198
+31,262,198,22203,282,26117,287,262,198
+32,262,976,13,198
+33,262,5664,13,198
+34,262,717,13,198
+35,262,198,3581,13610,30563,913,14041,198
+36,257,890,12,4354,5182,286,262,1988,286,262,649,13,198
+37,262,198,2437,284,466,262,890,286,262,1597,4560,198
+38,262,1271,286,262,670,13,198
+39,262,835,284,711,257,9471,13,198
+40,262,1110,13,198
+41,262,198,464,717,640,13,198
+42,262,198,464,649,290,262,2756,286,262,1910,11,290,262,717,1517,11,290,262,835,284,198
+43,257,198,2435,284,307,257,3275,286,262,198
+44,262,717,286,262,5536,13,198
+45,262,198,464,717,640,286,262,198
+46,262,198,5962,640,286,26395,2269,42038,198
+47,262,198,16,13,383,25505,375,16522,41317,198
+48,262,198,24361,767,25,1867,284,1745,262,1692,5339,329,36438,42628,13,383,1692,4637,351,262,2055,21433,6945,262
+49,262,826,286,262,1499,13,198
+50,262,976,198,24361,25,262,976,198
+51,257,642,12,20,2431,13,198
+52,262,717,640,13,198
+53,262,198,2437,284,3853,466,340,30,198
+54,262,717,286,262,1110,13,198
+55,262,976,640,11,543,460,307,262,976,287,262,640,286,262,10211,290,329,3124,284,307,262,976,13,198
+56,262,198,2437,284,307,6251,198
+57,262,717,640,13,198
+58,262,13,198,464,717,286,262,11,262,406,13,198
+59,262,976,640,13,198
+60,262,976,640,13,198
+61,262,198,464,640,286,262,4576,11,262,1266,835,284,198
+62,262,198,464,717,640,286,262,198
+63,262,198,3581,262,198
+64,262,198,464,717,12,262,749,198
+65,262,976,640,13,198
+66,262,835,329,257,198
+67,262,835,262,5230,286,262,5230,13,198
+68,262,1708,13,198
+69,262,835,326,345,460,307,13,198
+70,262,976,2866,198
+71,262,198,11085,835,329,262,198
+72,262,717,284,262,1334,286,262,198
+73,262,10348,13,198
+74,262,976,640,13,198
+75,257,198,2437,284,900,262,8364,287,257,4466,30,198
+76,257,198,24361,25,262,479,395,13,198
+77,262,564,250,26124,447,251,351,262,564,250,26124,447,251,329,262,1429,13,46256,95,262,1486,1276,307,564
+78,262,835,13,198
+79,262,717,640,13,383,1176,286,262,717,983,318,284,886,510,351,262,717,13,198
+80,262,835,286,257,649,4519,13,198
+81,257,900,286,262,509,571,2271,11,543,318,257,198
+82,262,198,7575,284,3551,257,198
+83,262,717,2589,13,198
+84,257,2643,284,262,198
+85,262,5801,198,43,964,286,262,1621,198
+86,262,198,464,717,640,286,262,1086,672,198
+87,262,717,286,262,640,286,262,198
+88,262,198,16,13,317,3043,262,24584,2508,13,198
+89,262,717,286,262,5536,13,198
+90,262,198,64,649,640,286,262,40206,198
+91,262,976,13,198
+92,262,976,640,198
+93,262,835,257,198
+94,262,976,13,198
+95,262,835,284,1633,13,198
+96,262,976,835,13,198
+97,262,198,16,13,8997,1143,290,262,198
+98,262,198,5962,286,262,968,8221,198
+99,262,198,16,13,383,649,5449,284,307,262,198

logs/longeval_16k/gen/arm_c_niah_single.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run16_step_400.ckpt  (base step 400, loss 1.53669)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log22.csv (run 22)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run22_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_niah_single.csv

logs/longeval_16k/gen/arm_c_val_backward.csv

@@ -0,0 +1,100 @@
+0,262,976,13,198
+1,262,13,198,24361,25,198
+2,262,835,284,262,2695,286,262,198
+3,262,835,198,464,6246,286,6597,11,262,1266,286,262,6246,198
+4,262,835,352,11,262,4511,2099,286,262,1720,13,198
+5,262,640,286,262,198
+6,262,835,286,262,640,13,198
+7,262,976,13,198
+8,262,366,1929,2305,1,286,262,366,565,283,1634,1,6536,13,383,366,565,283,1634,1,286,262,366,565
+9,262,366,8929,12,4906,1,286,262,366,8929,12,4906,1,286,262,366,8929,12,4906,1,286,262,366,8929
+10,262,357,18,8,262,5415,4129,286,262,8837,1395,11,262,5415,4129,286,262,8837,1395,11,262,5288,4129,286
+11,262,352,12,19,12,15,12,19,12,15,13,198
+12,262,1271,286,262,366,8929,1,4235,286,262,366,8929,25653,1,1271,286,262,366,8929,25653,1,1271,286,262
+13,262,717,286,262,366,7575,12,43961,18634,1,7873,329,262,366,15750,263,1,318,262,366,3506,1,286,262
+14,262,198,2,20,25,383,8026,286,12902,198
+15,262,198,15,657,198
+16,262,3726,286,34946,262,2695,13,198
+17,262,198,464,717,835,284,198
+18,262,198,2437,284,900,257,27431,198
+19,262,1271,286,262,640,286,262,311,4462,12,37,5603,13,198
+20,262,717,286,262,640,13,198
+21,262,198,3581,6372,12,22487,286,262,1524,198
+22,262,13,198,464,835,286,262,262,717,11,290,262,717,13,198
+23,262,835,284,262,262,198
+24,262,13,198,464,717,2378,286,262,481,11,262,717,1808,286,262,11,290,262,1708,286,262,2071,15370,5597
+25,262,640,286,262,309,22302,11,290,262,640,286,262,309,22302,13,198
+26,262,640,286,262,1266,11,262,640,286,262,1266,13,198
+27,262,976,355,262,779,286,2488,290,1303,287,15802,290,20267,3775,8148,11,290,11532,32096,318,10058,355,314,10014
+28,262,350,5430,198
+29,262,1271,286,262,640,345,460,651,262,1271,286,534,898,640,13,198
+30,262,198,13838,25,50256,2437,284,4321,1052,29455,198
+31,262,198,22203,282,26117,287,262,198
+32,262,640,286,262,1720,13,198
+33,262,835,284,198
+34,262,717,640,13,198
+35,262,198,3581,13610,30563,913,14041,25,198
+36,262,1306,286,262,1720,13,198
+37,262,640,286,262,1388,13,198
+38,262,198,464,717,640,286,262,327,958,400,286,262,1869,290,262,717,11,262,198
+39,262,835,284,467,13,198
+40,703,284,25432,262,1266,835,286,257,3993,2278,286,262,1266,13,198
+41,262,717,640,13,198
+42,262,640,286,262,649,2378,290,11,717,262,670,286,262,717,11,262,640,286,262,717,11,262,640,286
+43,262,717,640,13,198
+44,262,352,13,198
+45,262,198,464,717,640,286,262,1720,198
+46,262,198,464,640,286,262,717,13,198
+47,262,835,284,198
+48,262,198,24361,286,262,649,1204,286,37111,25434,6,198
+49,262,976,6194,286,262,5536,1271,13,198
+50,262,198,24361,25,262,2099,286,262,598,13,198
+51,262,640,13,198
+52,262,717,13,198
+53,262,198,2437,284,779,30,198
+54,262,13,198,17908,1105,25,262,6721,286,649,3709,290,262,3061,286,262,649,4831,13,198
+55,262,976,640,11,543,761,284,4483,362,11,393,257,2060,11743,810,345,16075,1022,262,3159,290,11743,329,1123
+56,262,835,284,262,198
+57,262,198,24361,25,383,717,289,1530,37793,318,262,266,284,307,262,976,13,198
+58,262,13,198,16,25,262,366,34,963,262,5536,1,198
+59,262,13,198,24361,25,4162,17503,5536,318,531,284,262,45770,286,262,198
+60,262,826,286,262,640,13,198
+61,262,198,464,640,286,262,649,290,262,198
+62,262,198,140,251,25443,119,141,229,16142,22177,18849,16843,38857,16142,30143,31583,20375,15166,16142,30143,16142,22177,18849,30143
+63,262,198,3581,262,640,286,262,198
+64,262,352,13,198
+65,262,835,326,262,2378,318,477,286,262,262,976,640,13,198
+66,262,835,326,198
+67,262,835,284,3758,257,4875,1271,351,262,5230,393,262,5230,286,262,640,13,198
+68,262,198,464,327,5673,338,12934,355,262,14848,4093,32,198
+69,262,198,3581,317,4866,286,262,640,198
+70,262,1271,286,262,402,23,11,262,1271,286,262,402,23,13,198
+71,262,198,5962,640,286,262,1849,32707,5273,198
+72,262,717,640,13,198
+73,262,1266,13,198
+74,262,1438,286,262,640,13,198
+75,262,198,18,13,383,640,284,307,262,976,640,198
+76,262,835,326,262,198
+77,262,835,284,779,262,2343,223,96,6495,13,198
+78,262,835,286,262,1271,13,198
+79,262,717,640,13,198
+80,262,1048,286,14760,261,11,262,1271,286,262,198
+81,262,1306,640,13,198
+82,262,13,198,2061,318,262,717,1074,286,262,3594,198
+83,262,835,13,198
+84,262,835,257,198
+85,262,890,12,2435,326,262,717,318,257,16106,198
+86,262,13,198,20,25,383,2378,287,262,198
+87,262,717,286,262,640,286,262,4618,13,198
+88,262,835,284,262,198
+89,262,198,16,13,20,383,350,9610,198
+90,262,198,64,2008,983,286,262,4859,299,329,262,2008,983,351,262,2456,7559,340,22523,198
+91,262,13,198,464,3381,286,262,376,600,3055,198
+92,257,262,835,284,307,262,976,13,198
+93,262,10758,290,262,717,640,13,198
+94,262,262,717,640,13,198
+95,262,835,284,7330,257,922,640,13,198
+96,262,835,284,262,198
+97,262,640,284,307,973,13,383,198
+98,262,640,284,1011,262,198
+99,262,717,286,262,11514,1074,13,198

logs/longeval_16k/gen/arm_c_val_backward.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run16_step_400.ckpt  (base step 400, loss 1.53669)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log26.csv (run 26)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run26_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_val_backward.csv

logs/longeval_16k/gen/arm_c_vt.csv

@@ -0,0 +1,100 @@
+0,262,717,640,198
+1,569,12,0,198
+2,198,16,13,569,1503,25,198
+3,569,13,198,464,569,1503,11,543,318,262,569,13,198
+4,262,749,286,569,5631,11,262,569,1581,11,262,569,13,198
+5,198,32,2446,329,569,1503,11,198
+6,351,262,198,51,12,67,12,51,12,67,12,198
+7,569,1503,11,290,262,366,847,1,3815,389,7323,569,1503,13,198
+8,262,366,4852,12,4873,1,569,10890,287,262,366,565,78,6806,1,198
+9,262,366,8929,12,12287,1,2446,286,262,569,1503,357,8929,12,83,8,198
+10,569,1503,366,53,7878,1,569,1503,366,53,7878,1,198
+11,569,12,16,13,198
+12,569,1503,412,11682,11,543,569,1503,412,11682,11,460,691,307,262,569,1503,412,11682,11,543,318,257,366
+13,569,9864,293,11,569,7036,12,51,13,198
+14,198,2215,262,569,1503,11,262,1271,286,262,569,1503,11,262,2033,286,262,569,1503,13,198
+15,198,464,717,286,569,1503,11,262,198
+16,198,1485,12,17,2250,357,2091,8,208,198
+17,327,2246,1731,13,220,517,290,517,621,262,835,284,198
+18,569,1503,10188,32,2780,796,569,1503,10188,13,198
+19,262,1266,286,262,569,1503,286,262,2366,319,262,1218,290,8150,1138,40193,282,1182,286,262,569,1503,13,198
+20,569,2889,540,198
+21,309,4944,1983,13,198
+22,11,290,262,717,11,326,339,447,247,82,13,198
+23,287,262,835,11,290,340,373,257,198
+24,262,640,286,262,2184,11,290,262,976,640,286,262,530,286,262,1751,11,290,262,1751,286,262,584,11
+25,198,32,649,309,22302,582,318,257,198
+26,198,22,13,569,1503,198
+27,509,1546,2425,796,48323,317,922,198
+28,360,198,24361,25,350,320,677,78,198
+29,198,32,640,286,262,569,1503,11,198
+30,198,53,1503,509,1546,1507,198
+31,198,2437,284,307,569,1503,287,262,995,198
+32,370,13,198,24361,25,383,569,1503,569,9399,11,287,262,1429,286,262,569,1503,11,290,262,1988,286,262
+33,287,262,8509,13,198
+34,402,198,32,25,569,1503,331,25,327,198
+35,198,2437,284,5765,534,3278,198
+36,317,1503,2996,13,198
+37,1160,13,198,2061,318,262,1438,286,2214,1818,198
+38,262,749,286,569,589,88,11,290,262,717,286,569,589,88,11,7361,569,589,88,11,290,339,373,257
+39,257,198,51,13,198
+40,569,615,16,290,569,615,18,389,6241,287,262,294,282,25509,657,12,7364,3403,13,3602,262,569,615,16
+41,262,2694,284,5160,262,976,640,13,198
+42,569,12,32,400,286,262,569,1503,11,262,717,640,11,290,262,2831,11,262,1910,287,2258,2253,468,587
+43,317,13,198,10305,12,16,13,569,23304,82,198
+44,262,1535,286,569,1503,13,198
+45,198,464,717,286,262,1720,198
+46,569,198,464,717,640,286,569,1503,509,1546,198
+47,569,12,198,16,13,569,1503,509,1546,3553,198
+48,262,8716,286,569,1503,290,262,583,43070,13,569,198
+49,198,53,41,370,1847,286,569,198
+50,569,1503,13,198
+51,376,1847,13,198
+52,317,18836,12,198
+53,367,868,416,262,198
+54,376,9864,37,13,198
+55,198,53,16,13,569,1404,25,198
+56,412,13,198,53,25,1867,262,68,13,198
+57,309,4944,1314,13,220,25305,263,569,48,286,262,569,6239,13,198
+58,370,12,40,198
+59,618,340,198,33706,25,569,1503,287,262,983,198
+60,198,2215,569,1503,12,18,11,569,1503,12,18,11,198
+61,360,3535,2231,13,1867,318,262,976,11,262,569,1503,286,262,569,707,13,198
+62,536,461,20259,11,406,14981,13,12466,94,20375,25443,110,16142,20375,45367,357,35313,13,350,15089,5045,27209,25,350
+63,287,262,569,1503,46,13,350,463,84,284,4776,262,267,13,198
+64,350,4106,12,38,13,198
+65,350,1677,13,198
+66,198,51,12,1238,569,1503,198
+67,360,3535,1821,198
+68,262,14848,4093,32,286,262,938,11,262,46254,37026,590,290,262,327,5673,286,262,4365,604,13,198
+69,198,2437,284,257,1790,3280,198
+70,262,1266,835,284,307,13,569,1503,198
+71,198,2215,262,1849,79,3618,1849,4598,1849,4480,262,1849,6649,1666,198
+72,198,464,569,1503,286,262,569,23183,11,569,963,286,262,569,23183,11,290,262,569,23183,11,198
+73,262,3463,286,262,569,1503,11,290,262,10348,3463,286,262,569,1503,11,290,262,826,3463,286,262,569,1503
+74,376,1847,3365,198
+75,198,19,13,569,1503,286,262,976,640,198
+76,569,13,198,464,569,1503,376,45006,318,262,286,262,569,1503,13,198
+77,327,13,198,18274,346,1096,262,569,4374,25,198
+78,1987,13,198,53,1503,449,251,1921,11,569,10761,11,508,318,262,569,10761,11,508,318,569,10761,11,508
+79,569,3258,378,569,11,262,569,3258,286,262,976,983,13,198
+80,569,1503,11948,32,198
+81,257,198,20746,287,262,569,1503,8931,198
+82,327,1268,5705,796,23358,287,262,3482,198
+83,350,13,32,13,198
+84,257,198,464,717,640,286,262,198
+85,262,5882,12,6042,11,569,1503,286,262,569,1503,13,18003,3777,11,569,1503,286,262,569,11,262,640,286
+86,257,4996,286,987,35415,569,1503,11,655,262,198
+87,376,1847,1954,26,7326,1704,11,262,717,569,5978,11,281,564,250,53,2265,273,11,262,569,2265,13,198
+88,360,391,198,53,273,487,198
+89,198,16,13,569,1503,198
+90,257,198,73,273,12,74,33663,12,19,13,198
+91,376,13,198,464,3381,286,262,376,13,198
+92,262,649,3357,286,262,995,13,198
+93,317,1558,13,198
+94,262,198,464,8362,286,262,995,3226,262,198
+95,262,835,11,198
+96,569,9399,3682,13,198
+97,198,16,13,569,1503,509,1546,25,198
+98,376,45006,1157,13,198
+99,198,41,276,262,11514,1074,198

logs/longeval_16k/gen/arm_c_vt.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run16_step_400.ckpt  (base step 400, loss 1.53669)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log25.csv (run 25)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run25_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_vt.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/arm_c_vt.csv

logs/longeval_16k/gen/base_niah_multikey.csv

@@ -0,0 +1,100 @@
+0,262,717,11,290,262,5536,1271,318,925,416,262,976,13,198
+1,262,1708,25,198
+2,257,1276,12,961,275,859,903,13,1114,1672,11,2125,470,655,262,4822,1272,11,475,635,345,460,307,534
+3,1802,26109,11,3616,262,5536,1271,318,657,13,198
+4,11,262,2694,284,4117,474,11433,283,13,449,11433,283,318,11,474,11433,283,318,11670,351,474,11433,283,13
+5,257,8780,1271,13,198
+6,262,976,835,13,198
+7,262,976,11,262,2700,286,18021,290,262,9563,23185,2655,18231,966,379,262,1241,286,262,366,33771,18380,1,3141
+8,257,366,17080,419,1,1271,13,198
+9,257,6937,357,31364,15879,286,262,1364,12,4906,20507,286,262,9103,13,198
+10,262,976,355,257,1588,12,3903,357,2339,257,23644,8,357,72,13,68,1539,366,11041,14843,1,3709,900,284
+11,257,2163,329,257,22146,5945,341,11,287,262,2565,25,262,16021,3440,10156,2233,284,1459,11,262,16558,318,407
+12,262,366,3201,88,12,85,560,278,6142,3386,318,5716,32551,12,301,4142,780,262,3288,9574,286,7078,290,4836
+13,262,976,366,39563,1,3689,11,318,262,976,366,39563,1,287,257,366,3174,1,290,366,39563,1,318,262
+14,262,691,286,262,717,640,13,198
+15,10403,11,1521,345,460,470,869,329,428,1517,13,3363,11,673,318,1016,284,1949,290,307,262,256,5362,395
+16,262,976,5766,25,198
+17,257,2219,5766,13,198
+18,257,2829,25134,1628,13,1849,2893,3210,8947,1394,11316,503,287,262,3931,11,345,460,2172,329,4334,12,26278,23275
+19,262,976,355,262,717,2063,286,262,5536,357,39,16,8,362,358,1138,40193,282,1182,13,5012,479,7749,1512
+20,257,2818,835,284,19356,257,2854,11,751,6628,11,290,1037,287,16937,262,5585,286,1767,5485,1231,32698,4467,13
+21,262,976,2446,329,1762,351,257,2176,1524,1628,329,262,7822,286,257,1524,4634,329,262,7822,286,257,1524,4634
+22,262,1708,25,198
+23,262,1688,11,262,46969,13,23080,29959,465,1182,13,564,250,2061,318,257,1263,1517,30,447,251,564,250,2061
+24,284,262,976,1245,26,290,262,3489,11,8850,3616,1276,407,307,9772,416,46768,326,11,2402,262,691,5536,1271
+25,407,655,284,3505,5536,8136,13,198
+26,326,11,262,779,286,5617,33995,11,318,262,749,3665,3305,590,329,6181,13,198
+27,262,976,13,198
+28,11,262,3280,460,307,2727,5975,11,290,543,2476,389,262,1277,1429,284,787,257,3338,1445,13,198
+29,257,9546,13,198
+30,11,18046,355,340,318,12244,11,290,5086,345,284,7522,262,399,11571,13,198
+31,11,703,262,5536,1271,329,262,4481,393,262,3725,286,262,269,5540,290,2722,8300,13,383,2041,5536,1271,318
+32,262,976,9030,326,9018,262,976,40982,341,286,262,5827,13,2750,6402,262,2785,3048,286,262,40982,341,286,262
+33,11,262,47080,415,25594,286,262,4731,276,8875,262,3435,3285,287,262,5253,355,484,1650,287,262,4898,13,198
+34,262,976,1517,329,262,4534,13,314,460,651,262,976,5536,1271,329,262,4534,13,198
+35,262,976,13,198
+36,11,8868,262,13357,286,6142,15611,11,4441,257,2837,286,30261,3048,13,383,2694,284,24237,777,7476,318,284,1986
+37,262,1994,284,5373,1771,484,9240,618,484,670,19863,13,41846,389,2045,329,262,1266,12,27144,870,3788,329,262
+38,11,262,564,246,42,20318,385,447,247,11,21071,446,11,12313,13,314,373,257,4958,286,262,5648,34671,11
+39,326,262,10883,5536,1271,11,290,345,1276,900,326,345,3953,477,8136,286,534,4822,654,13,198
+40,281,13311,286,262,7374,11,4441,262,1755,329,5503,706,11854,11,290,262,691,23326,286,11854,4073,13,1002,484
+41,262,976,3048,11,4441,262,5110,2694,284,8080,262,5536,13,198
+42,11,262,2258,1605,4708,12247,1910,287,2846,286,20788,355,1111,5079,82,290,1628,2176,30,198
+43,257,6979,284,262,717,286,262,5536,13,198
+44,262,1255,286,262,5536,1271,13,198
+45,262,1429,13,1867,389,262,717,5536,1271,286,262,1429,30,198
+46,262,2041,1438,11,543,318,257,4274,12,70,559,469,17864,13,383,3616,286,262,4274,12,70,559,469,17864
+47,262,976,11,290,8978,2651,355,1290,355,345,460,13,198
+48,11,340,460,10980,867,6459,416,262,3288,2700,13,198
+49,262,976,284,262,5536,995,286,262,2128,13,198
+50,262,1255,326,257,4047,7187,28260,561,9380,4331,1524,32149,257,2383,5873,286,262,640,13,198
+51,262,2060,12,25717,6131,11,543,318,1542,4,1342,621,4569,15985,11,4955,477,12,21230,4034,284,6842,13,198
+52,262,749,1593,1517,284,2291,262,2214,13,198
+53,257,2829,3335,326,460,307,7104,416,257,10147,13,770,460,307,262,691,1917,286,257,10883,3335,326,460,307
+54,262,995,286,6491,1998,13,198
+55,262,17878,286,1657,11,290,3148,3403,198
+56,25,198,16,13,383,649,7710,286,371,44947,318,262,717,284,6537,262,995,13,198
+57,257,2041,8364,13,198
+58,986,198,1,51,19173,385,1,366,51,19173,385,318,407,616,2988,526,198
+59,262,976,1271,11,523,262,1664,857,257,922,1317,287,1811,2842,13,383,1218,1517,484,561,466,318,4423,262
+60,257,4688,4427,13,1849,383,3616,286,428,318,810,534,1080,318,530,810,345,389,36502,284,1064,257,835,284
+61,262,976,835,326,2807,318,5140,287,442,1437,11,2807,11,290,36980,38536,13,198
+62,25,406,14981,13,383,309,3723,286,262,911,1809,26,14886,11,12466,254,21727,16142,30143,20375,22177,15166,40623,25
+63,352,13,198,464,4273,355,9337,11,262,4273,468,284,1394,257,1969,4151,503,329,5895,286,8526,13,383,4273
+64,262,2041,5536,1271,13,198
+65,257,2041,4282,11,475,355,884,11,340,318,810,6095,262,47969,1245,318,262,691,4282,286,262,6181,3950,8598
+66,257,2818,11,290,262,717,5536,1271,11,12031,284,307,1498,284,3283,503,13,383,2041,5536,1271,318,257,995
+67,257,1271,326,460,307,973,832,262,976,1176,12,21809,8787,13,198
+68,262,11,262,327,5673,447,247,82,12934,284,13446,326,2041,29771,11,340,481,7139,606,284,1833,1521,543,484
+69,326,530,460,760,262,4822,286,262,976,13,198
+70,262,976,1271,13,198
+71,11,198,1525,319,3158,8753,11,1853,198
+72,262,6937,13,632,5419,16697,262,37230,6772,11,543,19047,257,6414,290,16586,1245,13,198
+73,718,6052,486,1314,13,198
+74,262,366,16668,2046,6625,25,317,4822,286,262,976,4822,13,198
+75,262,2818,3572,326,11414,534,4466,981,4955,262,19217,15185,2057,0,198
+76,257,2041,5078,326,3033,257,2060,30628,51,357,28008,29385,11198,22481,8,8398,13,632,318,25555,287,281,6374,12
+77,25,198,14881,5315,2343,223,96,4625,25,564,250,26437,2343,223,96,447,251,198
+78,262,4695,326,6068,262,5536,1271,11,290,262,1729,12,13083,543,389,262,5536,1271,13,198
+79,8902,4602,13,632,318,845,2829,284,18021,262,5536,625,2562,11,543,262,3518,7679,318,1327,284,3817,13,383
+80,257,1808,286,703,284,1064,34849,511,835,422,966,317,284,347,11,393,703,284,2610,661,393,5672,13,198
+81,262,976,11,475,275,859,903,318,257,2060,12,29983,329,262,976,5536,1271,13,198
+82,262,976,1517,11,355,356,460,447,247,83,760,262,826,13,198
+83,588,257,900,286,262,2128,13,1002,340,318,477,286,262,5536,13,632,318,407,655,257,4822,286,340,13
+84,257,2829,11,1642,340,2562,284,787,11,290,1336,286,9565,13,632,318,257,1049,835,284,2883,18432,262,5536
+85,262,564,250,8807,447,251,329,867,5635,11,262,564,250,20887,5536,1271,13,447,251,28454,272,479,67,401
+86,262,5536,286,262,976,4437,11,290,262,6937,5885,286,262,5536,13,921,481,1254,262,5536,286,262,995,11
+87,407,6464,1231,33794,262,2672,4963,198
+88,262,976,13,198
+89,8618,3665,3777,329,262,509,14717,591,3202,13,198
+90,257,6194,286,262,5536,286,262,983,13,198
+91,1900,355,262,976,5536,1271,13,198
+92,262,4036,5536,1271,11,996,13,383,5536,1271,318,407,257,2829,11,475,284,4727,262,5536,1271,13,8013,355
+93,262,976,13,198
+94,19035,2327,1485,287,262,1446,5250,261,11,8147,5176,284,262,4870,40349,16292,7527,290,11740,17704,508,547,1611,1576
+95,1802,18212,13,198
+96,691,2092,284,257,8598,9978,13,383,5536,1271,373,407,691,1744,13,198
+97,767,3695,3695,11,3616,326,262,2041,5536,1271,318,8574,4291,262,5156,447,247,82,36051,284,3159,329,543,5895
+98,3614,284,262,1708,5087,25,198
+99,900,11,262,3280,318,1682,262,530,326,314,892,13,198

logs/longeval_16k/gen/base_niah_multikey.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run7_step_750.ckpt  (base step 750, loss 1.28324)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log5.csv (run 5)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run5_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_niah_multikey.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_niah_multikey.csv

logs/longeval_16k/gen/base_niah_multivalue.csv

@@ -0,0 +1,100 @@
+0,262,5536,11,422,6632,12,2435,11,262,5536,13,198,24361,25,1867,318,262,4295,30,198
+1,262,976,1517,13,198,8496,857,262,4158,2995,286,262,976,2479,389,256,917,430,30,198
+2,407,257,2984,439,3379,11,355,340,389,257,2041,31855,13,198,22073,286,6946,318,1266,11,475,262,1994,318
+3,477,5536,13,770,1398,286,1115,3146,318,3748,11,543,5621,262,5536,329,867,812,13,198
+4,11,618,262,13062,3457,344,13,5338,1013,622,260,262,3048,286,6701,28917,13,317,389,3075,4169,516,11,262
+5,973,416,6193,11,12701,11,290,597,3303,11,290,584,5536,3146,11,290,2370,11,5035,13,198
+6,973,284,262,976,640,13,198,2215,355,257,16976,4876,11,2233,284,262,2041,414,11,262,4365,286,262,976
+7,262,366,35,8808,1,1271,13,198,464,366,35,8808,1,366,46,6679,1,2173,389,262,366,35,8808,1
+8,366,3849,16104,602,1,11507,13,1867,318,1444,262,366,9492,16104,341,1,11507,389,389,18868,398,13,198
+9,2048,257,366,19726,29188,1,10733,286,6066,286,262,357,740,306,8,12822,31562,6082,286,1660,4417,7662,602,5836
+10,262,976,13,198,1,3198,286,262,2060,8373,357,505,8,262,366,48679,278,3881,3386,1,287,262,1612,366
+11,4385,284,257,2041,22146,5945,341,11,543,1612,484,466,30,5765,3268,58,10,72,7,86,9,83,1343,18983
+12,11,262,366,7972,88,1,366,7972,278,286,262,976,8373,318,262,2496,286,474,11433,283,25530,11,2592,389
+13,262,976,11,290,262,366,1894,904,1,5087,389,366,71,24431,1,287,262,366,24482,12,32604,11677,19114,11257
+14,262,5536,3146,286,262,6514,14930,13,198,2,21,25,383,2219,1597,460,779,262,5536,329,517,4708,290,1597
+15,262,5536,286,2277,11,611,345,460,3285,262,5536,286,2277,13,198
+16,25,198,464,976,22088,291,22088,357,42,32298,8,22088,4340,262,479,516,22088,357,42,32298,8,22088,460,307
+17,262,749,1593,5536,3146,13,198,24361,25,1148,262,2041,5536,1271,286,257,2041,11516,30,198
+18,1679,2327,33300,13,1849,383,2041,5536,3146,389,1695,284,307,973,351,262,2041,5536,13,198
+19,25,198,3581,352,14,17,25,657,13,23,14,19,25,4048,198,3581,352,14,17,25,352,13,23,14
+20,257,2818,835,284,19356,198,24361,25,383,6817,286,16139,491,36904,318,257,2829,1865,11,4955,281,3663,284,1104
+21,257,835,284,2074,13,383,835,286,7822,13,198,2061,389,262,2041,2842,30,198
+22,262,1306,640,11,780,286,262,10883,3657,13,383,8425,3657,2727,938,614,416,13778,360,7234,547,262,3280,286
+23,262,4334,11,262,46969,13,23080,29959,465,1182,13,564,250,2061,318,257,14397,11,262,717,12,18085,2513,284
+24,4642,284,606,11,475,484,4656,845,1862,11,290,8850,3616,1276,407,307,9772,416,46768,1028,13,642,400,11
+25,379,663,717,37228,4235,11,475,340,318,8603,281,19287,18734,287,262,10224,2612,13665,591,13,198
+26,262,3665,290,779,286,256,917,430,11,1642,606,257,3665,2723,329,256,917,430,11,1642,606,281,7306,290
+27,262,976,11,779,1885,13323,1952,11,290,262,584,3518,4870,2125,470,900,329,257,2836,11,645,530,1103,1762
+28,1464,973,284,1064,262,826,3074,286,2972,7612,286,1204,3690,511,2652,994,319,4534,13,198
+29,25,257,9546,198,10248,8458,290,262,717,286,262,2041,5536,1271,11,481,307,1695,262,2041,1271,543,481,307
+30,262,749,1593,1517,13,21527,389,262,976,835,284,779,262,22239,5536,13,198
+31,284,257,1178,1517,11,884,355,262,5536,286,3555,262,12209,2708,13,198
+32,262,2672,3048,286,262,5827,338,1535,11,543,460,307,2938,284,3342,262,1535,286,477,4671,2950,13,198
+33,25,860,1959,3324,1238,13,198,24361,25,1374,389,262,717,1936,286,777,5536,5536,30,198
+34,3170,287,262,976,7032,286,5103,11,588,8705,290,6142,3783,11,284,307,973,13,198
+35,262,976,13,198,2437,284,48282,257,5318,198,464,2041,414,286,6868,3966,75,505,318,2900,17196,866,618,4682
+36,257,6937,11,23610,262,2568,12,46303,4623,2071,13,1318,389,584,4237,1762,319,7888,7328,326,44870,262,10958,286
+37,262,5536,13,198,5195,466,428,30,198,2437,466,262,1271,286,9334,301,17946,338,670,2250,290,13714,30,198
+38,262,717,286,262,19903,6252,11,543,1849,258,2957,287,262,5838,286,10315,11,810,5844,660,11,1810,1359,13
+39,262,4334,2612,286,262,5536,13,198,24361,25,1867,318,262,5536,30,198
+40,257,10883,835,286,262,22775,13,383,45019,460,307,1775,355,45019,12,86,5718,13685,11,475,1375,263,272,311
+41,262,976,2842,284,4439,2124,2645,38656,13,198,33706,25,383,584,1021,286,2124,2645,38656,318,262,749,4050,13
+42,262,976,6459,11,588,407,4375,33179,393,2046,3747,11,649,3037,11,389,407,262,976,329,2046,3747,13,198
+43,262,717,640,290,1509,2049,278,6970,1771,262,3891,286,1757,25752,290,4453,25752,13,198
+44,25,198,3581,17265,5786,13243,2929,25,1114,883,326,1061,257,15169,5496,11,4618,262,5559,5107,286,26948,44234,588
+45,262,6817,286,262,1429,11,290,262,749,2219,47015,1429,329,477,10856,13,198
+46,262,976,2613,11,1022,1160,290,4974,7370,13,770,2041,13522,318,407,691,257,2041,5536,1271,329,38373,259,13
+47,262,749,4050,13,198,51,298,533,309,917,430,198,51,298,430,11,262,826,4467,11,31904,329,534,2612
+48,262,749,3665,290,31307,13,198,24361,767,25,1374,460,37111,25434,6,2928,318,10980,276,30,198
+49,4441,257,3748,2565,286,5485,13,632,1392,4615,1811,1661,1626,257,1178,4201,13,775,821,2495,1654,612,338,2147
+50,262,976,4049,284,281,4457,7187,28260,13,198,33706,25,645,338,1988,198
+51,925,351,27468,393,19828,284,1394,262,3124,13,198,24361,25,198,464,18700,19828,3392,389,925,351,27468,393,6546
+52,262,6380,11,41447,333,1096,262,1080,11,290,10996,4084,351,1854,1088,262,5928,2214,13,198
+53,407,655,257,1051,286,257,47969,11,475,484,547,1464,4375,257,2041,2134,13,198
+54,5447,416,262,9391,13,383,3721,318,2829,1865,7895,960,32399,1276,1208,2651,284,262,3721,13,198
+55,25,198,24361,25,1867,318,262,976,5536,1271,30,198,33706,25,383,5536,1271,318,2092,284,262,717,5536,1271
+56,25,198,24361,25,1867,4325,284,307,262,717,640,30,198,24361,25,6350,389,381,535,329,2124,2645,38656,11
+57,25,352,12,17,2250,11,352,12,17,11,362,83,1443,862,35652,13,198
+58,11,262,976,1517,286,262,5536,460,407,307,616,2988,13,198,43,12721,3114,2402,262,5536,286,262,4020,88
+59,407,691,257,4958,286,262,976,5101,11,475,635,262,3259,389,9080,284,262,5536,13,198
+60,262,8475,13,198,2437,466,345,1607,284,307,1804,1223,546,428,30,198
+61,287,262,2612,286,269,34144,11,379,2612,11,27692,646,11,284,262,15061,287,257,543,4618,286,269,34144,611
+62,45620,3270,3132,355,262,995,286,406,14981,13,383,5830,286,530,10923,2925,287,262,6046,21449,25,317,6700,287
+63,25,198,24361,352,25,198,24361,362,25,198,24361,513,25,198,24361,604,25,198,24361,642,25,198
+64,262,5536,286,11192,661,508,466,351,262,8716,13,383,6227,284,466,262,640,13,3423,389,617,286,262,5536
+65,257,3807,11,257,1735,24585,14,23205,1393,11,290,257,16687,13,5765,262,4661,326,389,262,749,379,10402,351
+66,6393,329,5059,257,2565,286,262,983,13,1114,326,11,373,284,15570,1429,4138,286,8945,583,1218,379,262,1575
+67,257,1277,277,73,585,284,257,20687,4061,3072,21302,1231,262,761,284,8076,257,3644,13,383,11513,15075,3072,6737
+68,262,1321,326,481,787,281,36793,416,543,262,1429,481,307,973,13,383,479,395,2411,318,4385,284,307,257
+69,0,198,24361,25,1867,318,5626,257,5536,1271,30,198,3581,383,5536,318,1865,9642,270,13,198
+70,262,976,355,262,1266,835,284,2251,262,5536,13,198,3198,286,262,749,1593,15910,329,262,3721,286,5536,13
+71,198,1525,1849,28973,286,262,7825,286,8550,1849,9248,11,198,1525,1849,42647,262,2756,286,262,1849,1014,1271,198
+72,262,749,4050,13,198,464,10576,286,4082,1630,19521,878,8363,37,3513,5419,18305,1096,262,37230,6772,11,543,19047
+73,25,198,464,3953,286,257,3290,447,247,82,717,24596,11,543,318,262,5415,6001,286,257,3290,13,198
+74,262,976,11,290,262,717,3386,286,262,3288,1175,11,262,10325,286,262,5536,11,287,3939,31551,26,262,886
+75,33882,13,198,24361,25,1374,466,345,2222,257,5916,6839,30,198,33706,25,383,1808,286,513,318,257,8811,20717
+76,262,976,13,198,24361,25,1867,318,262,564,250,46,4177,45268,4527,9708,2601,541,12,684,389,16001,30,198
+77,25,198,464,5536,286,257,269,34144,11,635,1444,257,2343,223,96,19382,13,198
+78,262,24699,11,523,777,3146,389,407,691,1487,284,262,2041,5536,1271,416,262,1271,3096,7588,11,884,355,262
+79,262,976,5485,2641,262,256,917,430,11,810,345,460,4532,262,256,917,430,656,262,256,917,430,13,198
+80,262,976,287,3252,286,257,2277,2446,13,198,4711,389,262,1271,286,262,2128,326,2058,284,262,2496,11,287
+81,2727,1262,2972,6973,11,884,355,2046,11,2046,11,290,2324,5479,13,198
+82,262,976,11,475,5536,3146,389,973,284,3368,11854,13,198,1135,27846,379,262,5536,286,262,6203,11,475,356
+83,13,383,5536,3146,547,407,1302,1028,13,383,5536,373,588,790,1438,13,383,16346,9397,11,612,547,1115,734
+84,25,198,3581,24796,784,352,5454,198,3581,383,642,400,784,352,5454,198,3581,383,7545,784,352,13,383,7545
+85,262,976,11,290,262,717,286,262,3246,286,262,3394,5854,11,508,389,262,15157,286,262,3394,5854,11,290
+86,262,749,5863,11,6194,286,262,2818,34515,710,11,543,6194,82,262,14564,286,262,3660,17423,1998,13,198
+87,262,6000,13,383,2928,286,262,2041,3146,318,13,383,1692,5002,326,318,262,6000,13,198
+88,11,262,6937,32073,286,257,2239,287,262,8713,8364,11,645,2300,20897,2658,444,393,7951,13,198
+89,262,717,12,400,375,346,11,543,389,262,5536,286,718,1433,2091,13,198
+90,407,257,6194,286,262,5536,286,262,3124,13,383,983,4477,284,307,262,976,3124,286,3252,13,198
+91,25,198,24361,25,1867,389,262,1266,1295,284,779,329,2124,2645,38656,30,198
+92,262,976,1517,11,290,1865,326,743,307,13,198,24361,25,1867,318,407,15861,480,30,198
+93,900,287,262,564,250,3791,379,262,6498,447,251,284,307,262,691,24395,286,262,900,287,262,4946,25466,284
+94,262,976,5536,3048,13,198,464,2128,286,262,976,10530,5536,318,1165,1913,284,2128,13,198
+95,262,717,640,1262,326,262,1956,318,2642,11,326,484,460,447,247,83,651,597,5563,11,523,8335,6731,13
+96,407,262,976,2446,6129,11,290,262,2134,481,407,307,6572,13,198
+97,262,5536,2438,329,262,8889,286,262,5536,1271,13,1649,262,15050,318,19066,11,262,5536,1271,460,307,6928,13
+98,198,8164,540,11,290,262,779,286,12462,11,290,1642,510,1511,4,290,767,4,8148,13,198
+99,262,976,290,262,17118,318,973,284,262,1388,1785,329,262,13527,3146,13,198

logs/longeval_16k/gen/base_niah_multivalue.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run7_step_750.ckpt  (base step 750, loss 1.28324)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log6.csv (run 6)
+[rope_theta] 500000 from checkpoints_bluscriptcp/blumodelcp_run6.meta (authoritative)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run6_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_niah_multivalue.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_niah_multivalue.csv

logs/longeval_16k/gen/base_niah_single.csv

@@ -0,0 +1,100 @@
+0,25,198,24361,25,1867,318,262,10489,30,198
+1,407,1900,11,644,262,2278,12881,25,9358,286,262,269,34144,30,198
+2,407,257,2300,13,198
+3,2562,13,383,1976,27446,2417,318,2562,284,779,262,5536,1271,13,198
+4,257,2060,19230,5536,2546,351,262,795,527,13,7236,480,791,622,71,422,795,527,13,383,1266,3667,326,307
+5,11,543,373,407,691,257,2968,8674,2646,22361,41839,290,262,3656,11,40349,41839,13,8989,11,46380,41839,11,373
+6,262,976,640,11,475,416,14583,1596,7370,13,198
+7,262,366,20541,1,1271,13,383,366,20541,1,1271,318,366,13159,12,20541,1,1271,329,262,366,20541,1,1271
+8,262,976,357,415,9248,284,262,8598,12,6270,11046,12291,8,1444,262,5536,8373,286,257,366,82,1304,1,5536
+9,262,9103,8373,11,290,474,11433,283,6268,12444,389,355,881,355,262,366,40541,474,505,5640,262,366,1455,1590
+10,262,976,355,262,366,15460,13730,1,44539,11,475,262,18640,4326,318,1464,366,343,565,15655,1,4049,3815,329
+11,262,366,4834,24588,1,1988,329,366,42981,391,1,357,68,13,68,1539,7078,11,366,22468,1,290,366,2934
+12,257,366,12942,1245,286,262,4771,8534,1,3386,286,262,1660,1370,286,262,4771,7207,13,198
+13,407,257,366,46143,7784,1,4235,11,543,318,257,366,46143,7784,1,14528,318,407,257,43119,286,262,4822,7784
+14,257,2060,12,1014,4038,13,383,5536,1271,318,257,2060,12,1014,329,262,976,13,198
+15,986,357,271,77,447,247,83,326,262,835,351,523,867,286,262,6918,326,314,2342,287,616,12507,10091,10091
+16,262,4165,1245,286,47969,13,198
+17,257,2041,1245,286,703,284,3368,281,8850,4006,13,317,845,10733,11,810,262,976,3048,286,262,6193,460,307
+18,257,2829,25134,1628,13,1849,2893,3210,7588,1394,11316,503,287,262,3931,11,345,460,2172,329,4334,12,26278,23275
+19,262,976,355,262,717,2063,286,262,277,73,585,357,71,16,8,362,358,1138,40193,282,1182,13,5012,479
+20,257,3665,5002,284,4620,262,4467,290,8737,286,262,5585,13,198
+21,262,976,835,13,383,4067,286,262,1430,318,1690,4833,621,262,717,3074,11,523,1771,257,1430,318,407,3614
+22,262,3092,286,5536,13,632,318,514,3926,262,17984,3926,790,1048,287,326,2156,13,843,314,892,960,40,892
+23,11,262,1688,11,262,46969,13,23080,3073,379,262,584,11,262,4039,286,262,474,11433,283,13,11075,373,262
+24,284,262,8584,11728,286,262,976,1048,11,780,39953,569,13,42657,11,3700,4759,690,11,1757,11,290,3700,4759
+25,257,6937,4427,284,12035,11,523,32268,301,998,318,262,4610,329,2407,257,7209,27892,15794,13,198
+26,10678,11,810,262,4901,286,7097,1637,318,17609,13,383,3748,3450,286,262,4437,318,3748,11,290,262,4437,318
+27,257,2041,835,13,198
+28,11,340,287,790,6506,2000,326,484,815,1464,787,257,3580,287,1728,7612,286,1204,3690,511,2652,994,319,4534
+29,262,3280,25,257,9546,13,198
+30,262,976,1517,11,543,318,262,17800,286,262,1099,13,3819,2585,389,2672,262,2041,1108,284,262,15775,290,3649
+31,284,262,2041,2476,290,5353,13,198
+32,262,976,7989,13,198
+33,11,262,47080,415,25594,286,262,4731,276,8875,262,3435,3285,287,262,5253,355,484,1650,287,262,4898,13,198
+34,262,691,1295,284,670,13,198
+35,262,1440,2219,15909,18639,284,779,262,3748,9260,13,198
+36,262,976,7989,379,477,11,4441,257,10112,4621,286,262,38488,1998,11,44209,7387,290,3512,329,38488,5341,13,383
+37,262,5536,286,262,995,13,1869,10321,460,5600,3272,12,9122,511,3085,447,247,82,4067,290,703,484,466,262
+38,11,262,1621,286,24017,283,13,1058,24017,283,373,257,6868,9356,13,1058,198
+39,257,2829,3335,326,460,1085,284,1180,5536,13,198
+40,262,976,13,198
+41,262,976,3048,11,262,2694,31405,262,5536,13,198
+42,257,2041,1672,286,262,3580,1022,1994,290,262,3292,286,262,1910,11,19765,13,198
+43,352,13,17,13,198
+44,262,2694,286,262,277,73,585,11,1165,262,277,73,585,460,15373,4271,925,46177,9176,287,262,277,73,585
+45,262,1429,13,198
+46,262,3616,286,262,7696,13,198
+47,262,976,1517,13,198
+48,9389,13,198,24361,767,25,1374,460,673,10980,44618,287,607,2055,30,198
+49,257,3748,4637,284,262,26838,1573,13,198
+50,257,2060,966,13,198
+51,262,2057,12,24194,1528,319,262,4771,1044,874,11,5086,262,6193,12,86,7958,39533,284,307,973,13,198
+52,262,976,11,810,262,275,859,903,318,262,976,13,383,275,859,903,318,262,976,351,262,2214,286,262
+53,326,345,460,447,247,83,1630,340,13,7943,11,262,749,9314,4610,9018,6729,278,11,262,976,2071,318,326
+54,262,976,13,198
+55,262,17878,286,1657,11,290,788,477,262,16649,561,3197,511,9082,20128,357,27379,655,257,2060,15094,8,564,250
+56,3177,262,976,355,262,17118,13,1114,2035,262,10402,286,852,973,351,45019,11,262,976,5538,389,262,1266,13
+57,25,352,13,29176,290,5022,534,11676,28208,11,10601,11,7543,11,16871,11913,11,16871,20584,11,290,8268,1566,880
+58,11,1521,11816,340,318,257,1643,286,4330,5536,13,198
+59,262,976,13,198
+60,262,1445,11,523,356,7074,262,2323,326,2058,351,262,826,1271,13,1849,383,12219,11,290,5457,11,262,1266
+61,262,1109,326,262,4931,286,4453,7772,12769,776,318,18665,284,262,7374,29981,379,449,88,313,2533,776,11,810
+62,262,976,355,257,2619,5866,13,383,14522,286,262,3801,5866,26,12466,250,254,25443,119,141,229,16142,22177,18849
+63,1464,1813,13,921,765,284,1561,284,262,6253,546,534,6701,28917,447,247,82,5496,11,290,651,617,9040,546
+64,257,3665,1998,13,198
+65,257,2041,835,11,326,318,6646,1521,612,389,262,976,5536,13,198
+66,262,2818,13126,18231,2428,284,467,880,13,1119,635,765,284,12160,10039,6117,13,1849,1892,691,857,1029,12,5715
+67,257,2060,12,8818,11,543,318,407,845,5713,13,50088,5021,12,43365,316,12,11128,540,9512,635,2314,4495,2832
+68,262,257,564,250,14648,12934,284,2423,257,5981,24589,1724,284,307,262,779,286,257,3210,4226,329,262,564,250
+69,11,262,976,13,198
+70,262,976,13,198
+71,198,1525,703,284,5385,11,262,5536,286,269,34144,460,307,198
+72,262,749,4050,13,198
+73,257,2041,3953,286,262,2219,23245,286,257,27021,326,481,6105,4197,606,13,198
+74,257,6194,286,262,976,4202,290,4202,13,198
+75,257,2829,4610,13,198
+76,284,5671,5894,2728,11,7538,11,290,262,2694,284,787,257,11559,286,479,395,2411,13,198
+77,257,46256,97,198
+78,4624,416,262,47969,41331,284,262,3817,11,523,47969,318,900,1028,262,4469,11,523,47969,2314,900,503,11,290
+79,262,5536,286,262,976,5485,13,383,5536,1271,318,8902,26772,11,1642,340,1744,329,262,5536,286,269,5540,13
+80,257,1720,326,460,307,973,284,3151,584,393,534,2858,13,198
+81,262,976,11,1262,262,3200,5536,1271,284,4155,262,5536,13,198
+82,256,917,430,11,644,318,257,10574,12840,30,198
+83,257,1517,13,198
+84,25,198,464,474,11433,283,318,25,198
+85,262,976,13,198
+86,262,5863,7918,11,351,663,6833,2511,653,11,543,4320,286,262,5536,13,921,481,635,1064,262,10883,37451,286
+87,407,1464,1813,706,262,11795,286,262,4618,11,543,318,262,976,329,11795,286,262,4618,11,1719,262,976,5635
+88,284,787,262,5585,286,761,290,5724,13,198
+89,262,717,12,4122,286,262,5536,286,327,13,383,5536,1271,318,973,284,11005,399,12022,7979,12,354,2150,11
+90,257,6194,286,257,1271,286,257,582,351,257,3215,6938,286,262,3159,13,198
+91,1900,329,663,3748,5050,13,632,318,8780,284,1833,262,2928,286,6491,1998,13,198
+92,262,976,1517,11,475,262,1245,286,262,5536,1271,318,523,262,3572,13,198
+93,262,749,9389,13,198
+94,262,976,355,262,2344,995,326,318,16039,13,383,640,810,810,345,389,3436,351,534,6066,460,2035,307,12973
+95,284,900,262,976,15452,13,198
+96,257,2060,1271,13,198
+97,257,3665,2891,284,10996,262,1597,13,383,1429,286,331,6018,318,257,3665,2891,284,6687,262,2104,1597,13,383
+98,2237,12,2100,298,6697,286,269,34144,11,393,2237,1661,13,198
+99,257,1598,11,262,24054,286,262,1074,338,13389,290,262,14067,7346,262,1785,13,198

logs/longeval_16k/gen/base_niah_single.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run7_step_750.ckpt  (base step 750, loss 1.28324)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log4.csv (run 4)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run4_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_niah_single.csv

logs/longeval_16k/gen/base_val_backward.csv

@@ -0,0 +1,100 @@
+0,257,10883,11,475,356,779,807,2713,1899,284,787,257,10883,11,1280,290,12309,13,921,743,651,262,5536,286
+1,262,2041,12263,13,198
+2,257,5766,286,262,4822,540,1502,13,198
+3,262,3280,1271,13,1867,2128,9813,389,5445,656,7009,47787,379,257,5801,16654,30,198
+4,11,326,262,1271,286,262,976,318,407,11,262,1271,286,262,976,13,198
+5,973,11,290,428,1271,318,4050,13,198
+6,262,976,640,13,198
+7,262,1271,13,198
+8,262,1245,286,262,976,13,198
+9,257,366,17618,1,329,262,976,13,383,366,40541,1881,264,13495,1,1988,13,383,366,27520,362,1,286,262
+10,262,366,17618,286,262,976,1,2700,14735,198
+11,262,976,13,198
+12,973,351,262,366,19204,24260,1,10007,11,543,389,262,366,19204,24260,1,5087,389,9380,13,198
+13,11,543,857,262,366,23410,904,68,1,2728,691,257,1057,13,1374,460,314,4259,428,30,198
+14,262,749,8253,13,198
+15,262,976,1517,326,857,262,4822,13,4418,11,428,2128,318,407,523,1263,13,198
+16,262,976,13,198
+17,257,2041,1245,286,262,6268,13,198
+18,262,1744,3280,13,1849,383,2378,460,307,925,286,257,2041,12462,30685,393,307,257,11032,3335,326,460,307,5929
+19,262,976,5766,286,262,1487,286,262,5536,3912,718,3365,3829,13,198
+20,8780,13,198,33706,25,36471,2140,262,5536,1271,718,2231,1899,1954,318,257,983,12,354,2564,13,198
+21,262,976,2446,326,10975,262,10348,2099,286,4610,618,17246,262,1271,286,257,3931,1430,329,262,1708,614,13,198
+22,262,3280,13,198
+23,262,749,1593,1517,13,198
+24,11,326,705,439,530,1614,2696,2236,307,2426,284,262,2276,8287,994,1437,637,925,287,1339,286,597,11989,9950
+25,262,4610,329,2407,257,7209,27892,15794,13,198
+26,262,749,3665,11728,11,290,262,976,18231,1176,13,198
+27,262,2041,1271,326,345,481,307,9899,262,4870,286,257,1881,24825,618,345,12233,262,2836,262,5413,21268,1848,13
+28,262,976,1517,326,4433,262,976,2223,13,383,4268,286,262,5536,1271,718,3365,2075,318,262,976,13,198
+29,262,976,5002,286,262,5536,1271,13,198
+30,11,262,1303,43069,286,262,20647,286,6366,17539,2191,13,198
+31,257,845,922,1517,329,262,2831,13,198
+32,257,198,24361,25,1374,460,314,779,257,1598,8420,4811,30,317,6087,286,262,2041,9391,318,257,43119,329,2003
+33,262,6908,286,262,32913,11,2331,284,307,262,17612,284,2834,572,13,1114,812,314,1239,2497,262,3280,11,355
+34,262,3748,290,45504,3572,329,262,20681,12,12463,2168,11,523,13,8554,262,2989,329,262,5536,8136,356,466,11
+35,262,976,13,198
+36,262,976,10033,379,670,319,257,4903,37549,447,247,82,5536,13,198
+37,262,5536,286,262,976,1110,13,198
+38,11,564,250,464,717,1517,318,262,691,3210,13,447,251,198
+39,11,3763,13,198
+40,262,2134,11,543,10087,1283,284,307,6572,618,4088,13,383,691,1517,318,284,3505,262,5536,286,584,10087,287
+41,262,976,2928,286,262,5536,13,383,2694,604,3865,940,5066,318,262,2928,286,262,5536,13,198
+42,11,262,976,2071,286,262,976,1021,11,290,262,2041,1108,2622,3357,13,198
+43,1521,262,1271,1625,416,262,976,5536,1271,807,38605,13,198
+44,262,2694,286,262,5536,1271,718,2079,13,198
+45,262,1808,326,460,307,973,13,198
+46,262,1438,13,770,17385,318,644,481,307,257,2041,4941,284,262,5536,13,198
+47,257,2829,290,262,976,2928,286,262,5536,1271,13,198
+48,262,1986,286,42628,13,198
+49,257,1621,286,262,717,838,400,4633,409,538,380,594,351,428,3859,287,257,1178,1933,11,326,2331,284,307
+50,262,749,1593,1517,13,198
+51,262,835,284,779,262,3748,30594,5621,422,262,11105,874,11,5086,606,284,3853,262,3124,12,31409,9664,284,307
+52,262,564,250,17089,2151,447,251,198
+53,257,2829,4610,326,4596,4541,262,976,2071,13,50256,464,6139,7913,25,440,574,28912,20173,2185,198
+54,262,1988,13,383,2252,262,4391,11,262,3224,1271,1596,44169,486,11,318,262,15541,5766,13,383,5236,286,5745
+55,262,976,1517,25,1011,1497,262,976,10451,318,284,5678,257,47529,329,262,976,5536,1271,11,290,3197,606,477
+56,257,649,8358,8591,4413,291,2271,11,257,3470,5824,13,198
+57,257,1573,329,257,517,11282,290,517,20923,9433,13,198
+58,407,1498,284,787,257,1487,326,1724,345,460,787,257,1487,326,318,5609,262,5917,345,765,284,307,13,198
+59,262,1255,13,198
+60,262,835,284,467,4058,290,4439,517,640,351,262,2041,16965,1271,13,1849,770,1627,460,307,257,17198,1917,13
+61,262,976,1099,13,198
+62,12466,94,22177,16142,30143,18849,31583,20375,12466,115,16142,43666,22177,16843,22177,45367,140,111,18849,22177,40623,20375,12466,94
+63,262,2818,1808,286,262,1200,13,383,2378,743,635,423,587,4939,290,10416,13,383,1438,743,423,587,10416,290
+64,262,2818,4610,329,45019,13,198
+65,262,2818,13052,13,198
+66,407,691,257,1487,286,13126,13,383,3721,286,3131,1176,318,407,691,257,4610,13,198
+67,3562,284,3328,3809,1104,2594,422,262,13608,12,8738,291,7686,11,543,4394,284,2148,3809,1104,13,198
+68,262,1808,286,543,262,3586,481,787,428,416,13311,295,262,1099,13,198
+69,262,976,11,198
+70,257,922,1517,13,383,402,23,373,407,1972,7387,11,345,765,284,307,262,5443,13,198
+71,262,691,2137,13,198
+72,257,845,3665,2223,13,198
+73,11,543,460,307,973,284,4331,262,2854,286,257,3290,379,1180,10101,11,290,884,355,257,1728,5046,13,198
+74,262,1808,286,262,749,3665,8705,6512,13,198
+75,407,2938,284,779,262,13357,286,262,2496,13,198
+76,3562,284,2251,279,18058,393,12370,7838,611,46952,389,1165,1969,393,4939,284,2721,262,2128,13,198
+77,257,564,250,20887,447,251,326,460,307,564,234,1904,257,2829,447,251,13,198
+78,407,655,257,1487,286,262,5536,1271,2808,2791,31654,11,475,635,1262,257,2041,5536,1271,11,543,318,2233,284
+79,262,1271,286,1402,427,4595,13,383,5536,1271,468,587,9951,11,543,262,5536,1271,318,991,5625,13,383,5536
+80,257,1808,13,198
+81,257,10112,1486,286,262,717,17852,42,23881,13,198
+82,262,976,1517,11,355,611,345,423,262,976,5536,1271,13,198
+83,262,976,1517,13,632,447,247,82,24479,13,632,447,247,82,257,1271,286,262,976,1517,11,475,340,447
+84,262,717,13,198
+85,262,1271,286,262,12841,1744,329,257,11575,13,406,32270,290,2440,27417,364,11,290,2440,290,2440,12,12293,37852
+86,262,5863,329,663,15499,282,3934,11,543,318,257,10883,2134,13,383,1772,447,247,82,717,15694,13,6803,503
+87,262,976,780,286,262,2041,5536,1271,7632,17464,2718,318,262,976,13,198
+88,13,770,2446,31648,564,250,1169,8574,18550,447,251,284,423,257,6029,290,22445,284,11831,551,354,1192,434,11
+89,11,262,1438,286,262,350,8623,354,482,11,262,350,8623,354,482,11,318,262,976,13,198
+90,1262,257,1021,286,257,2330,4469,198
+91,262,1994,4610,284,5529,11570,8787,13,23998,25,198
+92,262,1808,286,703,867,262,2134,318,284,787,262,2551,286,1243,355,262,2134,318,11,2592,475,743,1011,16255
+93,257,8780,3572,329,703,284,3368,262,2041,1271,13,198
+94,4203,588,262,5536,13,770,1271,318,523,3665,13,1318,318,257,36660,2128,326,340,318,6198,10530,13,314,716
+95,262,691,4472,5795,1262,262,1956,13,770,1271,3947,284,307,257,6891,43206,780,867,7150,326,13222,884,355,257
+96,262,691,1517,326,714,307,1744,13,198
+97,262,10883,3335,13,198
+98,262,1266,13,198
+99,262,749,14486,1785,13,198

logs/longeval_16k/gen/base_val_backward.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run7_step_750.ckpt  (base step 750, loss 1.28324)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_val_backward.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_val_backward.csv

logs/longeval_16k/gen/base_vt.csv

@@ -0,0 +1,100 @@
+0,564,250,53,1503,447,251,11,564,250,25284,447,251,11,564,250,25284,1755,447,251,318,257,2818,4634,13
+1,569,1503,3195,11,569,1503,318,407,655,588,569,1503,13,198
+2,376,1503,11,569,1503,3459,11,287,11678,284,262,1306,13,569,1503,3459,11,19047,326,345,389,407,655,262
+3,2822,257,1256,286,2128,1998,13,15075,4875,6597,3696,318,8574,2128,588,257,13934,900,286,3146,13,9506,9813,389
+4,257,1365,569,1503,287,9641,11,569,1503,290,787,340,517,318,6515,569,1503,13,2102,11,340,373,407,257
+5,281,2479,286,569,1503,11,569,1503,11,290,3747,13,569,1503,11,569,1503,11,290,584,1074,12608,13,198
+6,257,569,12,40898,11,4457,12,5589,23183,876,290,4961,2095,287,262,10283,26,569,1503,309,4944,11,766,644
+7,569,1503,19604,32,11,290,569,1503,19604,65,11,3503,1539,810,329,1672,49715,318,262,569,1503,19604,32,4846
+8,449,1503,569,9399,6052,25,220,2488,7934,44734,569,478,1749,318,262,976,13,198
+9,257,366,49,5653,1,357,68,13,70,1539,1395,11,569,45006,37,1503,318,45421,8603,1180,13,198
+10,366,53,1503,1,357,16,12,17,8,569,1503,357,16,12,17,8,569,1503,357,17,8,657,13,17
+11,764,198,464,569,1503,8363,46,11,543,318,9672,355,257,569,1503,8363,46,13,198
+12,25,569,1503,412,11682,1415,25,198
+13,366,39,7878,1,416,569,1503,376,45006,3510,357,72,13,68,1539,366,85,9969,1,2163,329,257,569,1503
+14,262,749,2408,290,5789,569,1503,1366,13,198
+15,257,569,1503,3829,13,198
+16,357,15,13,1238,8085,8,198
+17,13182,38,1731,13,198
+18,257,569,1503,11,1262,569,1503,11,318,257,2829,25134,1628,326,460,307,973,739,257,24296,11,393,257,11376
+19,362,13,24,13,16,657,13,24,13,16,198
+20,257,21362,3572,326,460,307,2208,36457,284,1254,517,198
+21,657,13,24,13,198
+22,569,1503,13,220,345,760,326,569,1503,318,407,262,31277,284,569,1503,26,673,1444,262,2245,4585,502,13
+23,402,3861,2780,13,220,517,7960,326,611,1573,1392,503,286,262,569,1503,11,673,1839,447,247,83,307,7428
+24,11,326,705,464,6195,2551,286,3700,4759,690,11,1757,39845,11,508,3724,355,465,20785,3656,11,422,10674,569
+25,262,3280,13,198
+26,262,1181,286,569,1503,11,13720,810,262,1171,1637,318,17609,13,198
+27,569,1503,11,393,584,37071,12,2339,11105,5479,13,1002,345,3904,326,357,29810,2753,257,1178,8404,11,379,1551
+28,513,13,198,464,1998,5621,46377,317,1503,1795,11,1262,1029,12,13237,4899,11,3283,503,584,4568,326,661,389
+29,8363,46,2816,796,1511,3104,40353,13,220,42713,290,550,8073,287,257,12625,399,25440,447,247,82,13,4874,757
+30,685,53,1503,509,1546,1507,60,198
+31,6825,6443,11,611,569,1503,449,4760,1821,318,262,976,14559,5711,329,262,11941,286,3555,262,12209,2708,13,198
+32,262,3048,286,569,1503,569,9399,11,13213,262,3048,326,743,15058,1141,262,30458,286,257,2563,10413,13,2750,6402
+33,262,14361,17,287,262,4220,286,262,711,25,314,8288,0,198
+34,262,1266,287,262,569,1503,995,13,921,460,4414,422,569,1503,575,1503,1157,11,475,345,460,779,262,2003
+35,569,1503,371,2246,25,198
+36,262,4451,835,284,569,1503,2310,11,810,340,318,257,21362,4610,284,8868,262,13357,286,569,1503,11,16937,7261
+37,569,1503,19604,32,2078,198
+38,8599,11740,4373,357,14253,263,828,23987,371,12305,357,27898,8,569,589,88,357,27898,8,373,3706,287,262,19903
+39,257,4688,3335,13,569,1503,309,4944,5774,11,345,460,900,326,900,2185,45925,15343,290,787,606,13,198
+40,569,1503,10188,32,3270,13,198
+41,257,517,3716,569,1503,3842,11,2138,621,262,3176,1720,9376,13,198
+42,569,1503,1168,8905,3365,13,198
+43,262,569,1503,7457,286,569,1503,13,9022,1139,11,569,1503,290,749,11003,11,569,1503,88,11,262,1438,286
+44,257,4369,12,3106,29883,11,257,6087,286,29883,11,290,584,22392,13,632,318,257,6087,286,262,29883,13,198
+45,569,1503,509,1546,2996,11,290,569,1503,509,1546,2996,13,198
+46,569,1503,11,810,262,569,1503,509,1546,3365,318,13,569,1503,509,1546,3365,318,262,7664,13,6416,12,6894
+47,262,1917,286,569,1503,509,1546,13,198
+48,31307,50256,14662,290,5503,30,4418,11,4917,257,569,1503,19604,32,4524,13,198
+49,257,3272,12,29289,569,1503,370,1847,1954,11,290,257,3272,12,29289,569,1503,370,1847,1954,11,543,318,257
+50,287,534,569,1503,43638,3553,13,198
+51,569,1503,19604,32,2548,11,1262,257,22033,12,26128,16882,11,569,1503,11,543,1838,257,22033,12,26128,1245,13
+52,257,21151,569,1503,371,10116,2327,11,543,318,3562,284,2872,262,569,1503,371,10116,2327,13,2102,11,569,1503
+53,383,2219,1917,286,569,1503,412,10744,1495,11,543,318,407,13259,889,257,2060,2071,13,198
+54,257,10171,286,569,1503,376,1847,3682,11,543,1838,262,1486,286,569,1503,376,1847,3682,11,198
+55,569,1503,327,13,46,13,198
+56,257,1295,810,287,262,1110,286,569,1503,449,13,198
+57,257,922,3572,11,290,569,1503,376,1847,2623,13,15561,606,1978,1566,257,13394,15756,1296,13,198
+58,367,10008,2640,290,3254,263,64,326,17522,318,407,616,2988,13,198
+59,569,1503,13,1400,2300,286,569,1503,49707,6659,13,770,12850,262,2526,286,569,1503,13,198
+60,262,670,17820,11,356,7074,262,2272,326,389,706,262,16965,286,17820,13,1849,383,12219,11,290,5457,11,262
+61,262,569,1503,350,1677,287,262,569,1503,25,611,262,2071,286,569,1503,350,1677,1238,13,198
+62,25370,254,365,41790,32790,2271,357,43,14981,13,20341,1776,12466,240,16843,21169,43666,18849,31583,20375,140,115,16142,43666
+63,367,13,68,559,275,993,134,123,275,993,134,123,275,993,134,123,275,993,134,123,275,993,134,123
+64,347,4715,1108,286,569,1503,11,543,661,379,262,976,640,13,569,1503,318,262,1266,3572,832,262,569,1503
+65,257,2095,11,257,1735,24585,14,23205,1393,11,257,2095,4539,11,290,257,1310,13,569,1503,11948,32,318,262
+66,1511,13126,44355,357,15,13,2388,2623,1821,6257,49,8,198
+67,257,517,4688,6504,9708,11,290,15075,45649,6032,423,922,3809,3081,13,198
+68,262,327,5673,447,247,82,12934,284,2423,326,257,5981,24589,318,23353,3482,1575,286,4248,14877,76,26,262,327
+69,262,2642,835,284,569,1503,11,475,340,460,6133,287,674,517,42923,3164,284,880,12,11873,13,198
+70,402,53,23,13,569,1503,569,9399,3682,11,475,569,1503,4521,318,262,976,13,198
+71,262,995,508,198
+72,569,1503,11,543,460,18135,351,262,8363,37,1429,13,198
+73,2319,12,15,8331,11,1912,262,3280,286,262,569,1503,11,262,3210,12,2385,578,13,198
+74,366,34,500,260,516,569,6456,1,198
+75,198,24361,25,569,1503,412,10744,1731,198
+76,19604,32,1065,13,569,1503,25,3544,281,35669,2891,329,569,1503,376,45006,2091,11,543,318,257,2829,284,779
+77,2343,223,97,4625,262,569,234,65,22704,357,53,1503,737,383,2343,223,96,6477,262,2343,223,96,400,432
+78,21232,40,3861,3365,13,220,319,2520,5478,198
+79,5125,2931,4531,198
+80,564,250,53,1503,11948,32,2481,11,447,251,543,1724,257,2128,4542,1664,13,569,1503,2534,11,543,318,257
+81,569,1503,13,198
+82,327,1268,5705,198
+83,350,1677,1485,13,220,632,373,1239,517,13,569,1503,318,407,6228,13,632,447,247,82,41719,13,198
+84,362,13,20,33220,198
+85,262,749,2408,290,24163,2347,286,569,1503,11,1864,284,663,1029,12,9521,2496,4395,2233,284,511,11519,1486,884
+86,262,5863,9923,447,247,82,38760,11,287,262,976,1242,11,810,262,569,1503,318,262,2818,1241,286,27240,13
+87,569,1503,569,9399,5066,11,262,569,1503,7023,286,2253,447,247,82,46854,14861,11289,11,329,569,1503,569,9399
+88,362,13,4149,546,262,2446,286,44637,510,11,393,24372,262,14361,11,393,24276,286,262,9113,20493,11,393,262
+89,569,1503,447,247,82,6393,11,569,1503,357,65,375,829,8,569,1503,357,66,12321,569,1503,8,318,262
+90,257,1256,286,569,1503,13,198
+91,257,1808,319,569,1503,1168,8905,2079,11,810,262,4732,318,517,9857,13,198
+92,262,2642,286,569,1503,318,1165,2219,12,7174,13,198
+93,1160,13,383,4277,1241,286,262,7756,13,198
+94,569,1503,10188,32,940,13,198
+95,257,2344,4572,326,460,307,18283,503,284,257,2344,4572,13,447,251,198
+96,569,9399,3682,13,198
+97,257,1503,13,198
+98,4764,11,32190,198
+99,4101,362,13,198

logs/longeval_16k/gen/base_vt.log

@@ -0,0 +1,43 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN][warn] seq_len(16384) != scale(4.000)*orig_maxpos(1024.0)=4096.0 -- cache scaled for a different context than it is sized for
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=7 high=25 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run7_step_750.ckpt  (base step 750, loss 1.28324)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 100 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log6.csv (run 6)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run6_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_vt.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/longeval_runs/gen/base_vt.csv

logs/longeval_16k/master.log

@@ -0,0 +1,427 @@
+[11:01:43] longeval: T=16384 prompt_len=16360 max_new=24 n=100 greedy=1 gpus=0,1,2,3
+[11:01:43] haystack: Data_Loader/Data/flux_val_000000.bin
+[11:01:43] prompt signature changed -- regenerating prompts and clearing state
+[11:01:43] generating prompts: niah_single
+  niah_single      100 x 16360 tok -> longeval_runs/prompts/niah_single_prompts.bin
+[11:01:44] generating prompts: niah_multikey
+  niah_multikey    100 x 16360 tok -> longeval_runs/prompts/niah_multikey_prompts.bin
+[11:01:44] generating prompts: niah_multivalue
+  niah_multivalue  100 x 16360 tok -> longeval_runs/prompts/niah_multivalue_prompts.bin
+[11:01:45] generating prompts: vt
+  vt               100 x 16360 tok -> longeval_runs/prompts/vt_prompts.bin
+[11:01:45] generating prompts: val_backward
+  val_backward     100 x 16360 tok -> longeval_runs/prompts/val_backward_prompts.bin
+[11:01:45] fan-out: 4 GPU(s), NP=1 -> 4 concurrent job(s)
+[11:01:45] 20 job(s) to run
+[11:01:45]   [gpu 0] base / niah_single
+[11:01:45]   [gpu 1] base / niah_multikey
+[11:01:45]   [gpu 2] base / niah_multivalue
+[11:01:45]   [gpu 3] base / vt
+[11:16:16]   [gpu 0] base / val_backward
+[11:16:31]   [gpu 1] arm_a / niah_single
+[11:16:41]   [gpu 2] arm_a / niah_multikey
+[11:20:51]   [gpu 3] arm_a / niah_multivalue
+[11:29:10]   [gpu 0] arm_a / vt
+[11:29:16]   [gpu 1] arm_a / val_backward
+[11:29:33]   [gpu 2] arm_b / niah_single
+[11:33:38] longeval: T=16384 prompt_len=16360 max_new=24 n=100 greedy=1 gpus=0,1,2,3
+[11:33:38] haystack: Data_Loader/Data/flux_val_000000.bin
+[11:33:38] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[11:33:38]   base/niah_single already generated
+[11:33:38]   base/niah_multikey already generated
+[11:33:38]   base/niah_multivalue already generated
+[11:33:38]   base/vt already generated
+[11:33:38]   base/val_backward already generated
+[11:33:38]   arm_a/niah_single already generated
+[11:33:38]   arm_a/niah_multikey already generated
+[11:33:38] 13 job(s) to run
+[11:33:38]   [gpu 0] arm_a / niah_multivalue
+[11:33:38]   [gpu 1] arm_a / vt
+[11:33:38]   [gpu 2] arm_a / val_backward
+[11:33:38]   [gpu 3] arm_b / niah_single
+[11:33:38]   [gpu 0] arm_b / niah_multikey
+[11:33:38]   [gpu 1] arm_b / niah_multivalue
+[11:33:38]   [gpu 2] arm_b / vt
+[11:33:38]   [gpu 3] arm_b / val_backward
+[11:54:02]   [gpu 0] arm_c / niah_single
+[11:54:15]   [gpu 1] arm_c / niah_multikey
+[11:55:52]   [gpu 2] arm_c / niah_multivalue
+[11:56:31]   [gpu 3] arm_c / vt
+[11:56:41]   [gpu 0] arm_c / val_backward
+[12:16:44]   scoring base / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got=:\nQuestion: What is the Devil?\n
+  [MISS] expect=5343902
+         got= not known, what the period modes: transportation of the cactus?\n
+
+  wrote longeval_runs/scores/base_niah_single.csv
+[12:16:44]   scoring base / niah_multikey
+
+=== niah_multikey  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=6088743
+         got= the first, and the magic number is made by the same.\n
+  [MISS] expect=9934935
+         got= the following:\n
+
+  wrote longeval_runs/scores/base_niah_multikey.csv
+[12:16:44]   scoring base / niah_multivalue
+
+=== niah_multivalue  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020|1679215|5343902
+         got= the magic, from zero-time, the magic.\nQuestion: What is the wild?\n
+  [MISS] expect=8995970|7007071|4664860
+         got= the same thing.\nWhere does the religious events of the same age are tundra?\n
+
+  wrote longeval_runs/scores/base_niah_multivalue.csv
+[12:16:44]   scoring base / vt
+
+=== vt  (prompt_len=16360, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     0.80   1/20    5.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=5370335
+         got= “VAR”, “California”, “California night” is a perfect setting.
+  [MISS] expect=6319163
+         got= VAR TV, VAR is not just like VAR.\n
+
+  wrote longeval_runs/scores/base_vt.csv
+[12:16:45]   scoring base / val_backward
+
+=== val_backward  (prompt_len=16360, n=100) ===
+  accuracy: 1/100 = 1.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   1/20    5.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= a magical, but we use 80560 to make a magical, open and peaceful. You may get the magic of
+  [MISS] expect=cactus
+         got= the special lifestyle.\n
+
+  wrote longeval_runs/scores/base_val_backward.csv
+[12:16:45]   scoring arm_a / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the same.\n
+  [MISS] expect=5343902
+         got= the special of the cactus.\n
+
+  wrote longeval_runs/scores/arm_a_niah_single.csv
+[12:16:45]   scoring arm_a / niah_multikey
+
+=== niah_multikey  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=6088743
+         got= the same.\n
+  [MISS] expect=9934935
+         got= the same for the car.\n
+
+  wrote longeval_runs/scores/arm_a_niah_multikey.csv
+[12:16:45]   scoring arm_a / niah_multivalue
+
+=== niah_multivalue  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020|1679215|5343902
+         got= the same.\nQuestion: Juan Rodriguez Cabrillo\nNorto Cabrillo\n
+  [MISS] expect=8995970|7007071|4664860
+         got= the same for tundra.\nQuestion: The other tundra is the same of the tundra\n
+
+  wrote longeval_runs/scores/arm_a_niah_multivalue.csv
+[12:16:45]   scoring arm_a / vt
+
+=== vt  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     0.80   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=5370335
+         got=\n“VAR HOL88,” the VAR, and the VAR, and the VAR,
+  [MISS] expect=6319163
+         got=\nVAR YAR\n
+
+  wrote longeval_runs/scores/arm_a_vt.csv
+[12:16:46]   scoring arm_a / val_backward
+
+=== val_backward  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the one of the the same.\n
+  [MISS] expect=cactus
+         got= the\nHow many of the special, the same of thin old grey stone Georgian mansion is the\n
+
+  wrote longeval_runs/scores/arm_a_val_backward.csv
+[12:16:46]   scoring arm_b / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the, and the only fight is the same.\n
+  [MISS] expect=5343902
+         got= the.\nDo the language cactus, that is the other.\n
+
+  wrote longeval_runs/scores/arm_b_niah_single.csv
+[12:16:46]   scoring arm_b / niah_multikey
+
+=== niah_multikey  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=6088743
+         got= the same. Steam was the stone in the C.\n
+  [MISS] expect=9934935
+         got= the\nHow rumorian\n
+
+  wrote longeval_runs/scores/arm_b_niah_multikey.csv
+[12:16:46]   scoring arm_b / niah_multivalue
+
+=== niah_multivalue  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020|1679215|5343902
+         got= the.\nThe name silk in the e-Crookeeng, the<|endoftext|>Possar, and the name of
+  [MISS] expect=8995970|7007071|4664860
+         got= the.\nPhrenoettes of the same\nTermany measurements of the same\n
+
+  wrote longeval_runs/scores/arm_b_niah_multivalue.csv
+[12:16:46]   scoring arm_b / vt
+
+=== vt  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     0.80   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=5370335
+         got= a lot of the experience.\n
+  [MISS] expect=6319163
+         got=\nDebt the language of VAR. The Cock House of the same\n
+
+  wrote longeval_runs/scores/arm_b_vt.csv
+[12:16:47]   scoring arm_b / val_backward
+
+=== val_backward  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the most.\n
+  [MISS] expect=cactus
+         got=,\nPropver Indian Old Women\n
+
+  wrote longeval_runs/scores/arm_b_val_backward.csv
+[12:16:47]   scoring arm_c / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the same, the best time.\n
+  [MISS] expect=5343902
+         got= the\nFirst of the book\n
+
+  wrote longeval_runs/scores/arm_c_niah_single.csv
+[12:16:47]   scoring arm_c / niah_multikey
+
+=== niah_multikey  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=6088743
+         got= the\nThe first time of the Cabrillo\n
+  [MISS] expect=9934935
+         got= a time.\n
+
+  wrote longeval_runs/scores/arm_c_niah_multikey.csv
+[12:16:47]   scoring arm_c / niah_multivalue
+
+=== niah_multivalue  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020|1679215|5343902
+         got= the first time.\nThe timberline in the first of the world, the the first of the world, the the
+  [MISS] expect=8995970|7007071|4664860
+         got= the way!\nIs the way a tundra\nIf you are the tundra\n
+
+  wrote longeval_runs/scores/arm_c_niah_multivalue.csv
+[12:16:47]   scoring arm_c / vt
+
+=== vt  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     0.80   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=5370335
+         got= the first time\n
+  [MISS] expect=6319163
+         got= V-!\n
+
+  wrote longeval_runs/scores/arm_c_vt.csv
+[12:16:48]   scoring arm_c / val_backward
+
+=== val_backward  (prompt_len=16360, n=100) ===
+  accuracy: 0/100 = 0.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   0/20    0.0%
+     1.00   0/20    0.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=yarrow
+         got= the same.\n
+  [MISS] expect=cactus
+         got= the.\nQuestion:\n
+
+  wrote longeval_runs/scores/arm_c_val_backward.csv
+[12:16:48] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                  arm_a    arm_b    arm_c     base
+  niah_multikey           0.0      0.0      0.0      0.0
+  niah_multivalue         0.0      0.0      0.0      0.0
+  niah_single             0.0      0.0      0.0      0.0
+  val_backward            0.0      0.0      0.0      1.0
+  vt                      0.0      0.0      0.0      1.0
+
+  EXCLUDED from pooling (<=1% for every model): ['niah_multikey', 'niah_multivalue', 'niah_single', 'val_backward', 'vt']
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+
+  wrote longeval_runs/summary.csv and longeval_runs/depth_summary.csv
+[12:16:48] DONE -> longeval_runs/summary.csv

logs/longeval_16k/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=16360|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=flux_val_000000.bin|HAYSZ=76

logs/longeval_16k/prompts/niah_multikey_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_multikey",
+ "prompt_len": 16360,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8427162"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5092145"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2537331"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2346343"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1382228"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4025232"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3654300"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3645541"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8833151"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304572"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4057163"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6822583"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8305481"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "1598367"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5021600"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9037738"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2026030"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7179576"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9339743"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2757350"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6707949"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2113321"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8729852"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8010202"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2100935"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "5008944"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1590218"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4271114"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5929205"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1748950"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9786421"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1515074"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7027270"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3066753"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8997596"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3624007"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7248874"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6000081"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8403741"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8075024"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2451219"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2552782"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9826525"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5403797"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9721369"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7237841"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9201678"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1428604"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "1062752"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7444634"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4134393"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1835392"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7902689"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8980124"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7652409"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1744111"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5272136"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7914229"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8162790"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5976814"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3895347"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5571536"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6491340"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3178066"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3223664"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3735310"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "4205566"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5363864"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2028830"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8301665"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9954093"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6537337"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1671907"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8685924"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8867130"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "5581013"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "4505930"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1859914"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8413665"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5675931"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1228759"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6742234"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4830273"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8046128"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7622676"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4402213"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1407707"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8903784"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3474463"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6783078"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8530033"
+   ]
+  }
+ ]
+}

logs/longeval_16k/prompts/niah_multikey_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cb2a6d6ac5d41c7173db954c84e6edb1abdea07f2878c4b27a85ee439b1c0e1
+size 6544000

logs/longeval_16k/prompts/niah_multivalue_meta.json

@@ -0,0 +1,1008 @@
+{
+ "task": "niah_multivalue",
+ "prompt_len": 16360,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020",
+    "1679215",
+    "5343902"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8995970",
+    "7007071",
+    "4664860"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5202798",
+    "9934935",
+    "3465603"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8921240",
+    "2689485",
+    "6935633"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896",
+    "8427162",
+    "9746862"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2564842",
+    "7691148",
+    "1019173"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5092145",
+    "6456023",
+    "2056700"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3390659",
+    "8515682",
+    "2530349"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9208870",
+    "2829687",
+    "6057437"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161",
+    "5826088",
+    "8465677"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5061884",
+    "5870920",
+    "4084805"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8994863",
+    "2159058",
+    "2507002"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "1648297",
+    "2346343",
+    "7564858"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4951063",
+    "4610478",
+    "8036754"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2381989",
+    "6441267",
+    "2935247"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5077461",
+    "1271942",
+    "5547262"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030",
+    "8149294",
+    "2043416"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "1758901",
+    "9962354",
+    "2241461"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7563404",
+    "2535649",
+    "7209917"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4264694",
+    "4103314",
+    "3078615"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1382228",
+    "8140699",
+    "2702891"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6050974",
+    "6876957",
+    "8316017"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2693177",
+    "7564525",
+    "4344635"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3842473",
+    "4412732",
+    "1973933"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6743051",
+    "9883103",
+    "5205835"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1221522",
+    "8912561",
+    "7877071"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6992010",
+    "7520334",
+    "5209951"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2326638",
+    "6635744",
+    "1766659"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6909364",
+    "5829789",
+    "7026396"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6205929",
+    "7509848",
+    "7952780"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6610329",
+    "3685693",
+    "5016733"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1529245",
+    "7748921",
+    "8016677"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8471769",
+    "2071834",
+    "5349106"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1001160",
+    "1652804",
+    "9297720"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7964850",
+    "4154138",
+    "2400344"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7741367",
+    "8004237",
+    "6304572"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9864174",
+    "2640549",
+    "4195387"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5697330",
+    "4057163",
+    "2680753"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1366506",
+    "5608933",
+    "8599712"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3238425",
+    "9738673",
+    "6822583"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1709636",
+    "1682216",
+    "4451672"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1704719",
+    "9296092",
+    "8695111"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3991181",
+    "4487045",
+    "7301168"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5552890",
+    "6593664",
+    "6662496"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1598367",
+    "1691270",
+    "5523508"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7055008",
+    "7623510",
+    "3175014"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1809607",
+    "6165365",
+    "4012967"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7764055",
+    "6511662",
+    "6020170"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8952478",
+    "6655273",
+    "6765517"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8156075",
+    "1634587",
+    "6065093"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "3794348",
+    "7301041",
+    "2458305"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2026030",
+    "7456072",
+    "1131718"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8525324",
+    "9197663",
+    "4644909"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3794767",
+    "8234688",
+    "4220302"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9821723",
+    "8576788",
+    "4383761"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1706344",
+    "4622106",
+    "3455420"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3538969",
+    "2757350",
+    "9185627"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9744765",
+    "9311532",
+    "6409986"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4670403",
+    "1162817",
+    "6707949"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6399325",
+    "1595058",
+    "9809840"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7358650",
+    "5938916",
+    "8892528"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2113246",
+    "4776394",
+    "3189328"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6546453",
+    "3695931",
+    "3496776"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9472488",
+    "7412798",
+    "9888697"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2281076",
+    "8153245",
+    "4457865"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9090201",
+    "7519104",
+    "4918672"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4052931",
+    "6074135",
+    "9503662"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5395756",
+    "6080222",
+    "7846331"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3136088",
+    "5008944",
+    "5816543"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9075384",
+    "8010628",
+    "3363991"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "3539630",
+    "6918529",
+    "7897787"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1789081",
+    "2702560",
+    "8900209"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6434330",
+    "2766520",
+    "6816123"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2862219",
+    "2009331",
+    "8838120"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "5972538",
+    "3132537",
+    "7503615"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9707497",
+    "4173032",
+    "1640628"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8641522",
+    "6981980",
+    "2263988"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9156496",
+    "5284671",
+    "1447213"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4852975",
+    "2568270",
+    "9429089"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2903289",
+    "3444279",
+    "8146594"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2757658",
+    "7973302",
+    "2055669"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8494694",
+    "8232428",
+    "7996426"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5237688",
+    "2317658",
+    "6914019"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6795607",
+    "6836419",
+    "3984795"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2183089",
+    "3404026",
+    "4489292"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1120537",
+    "5920135",
+    "7192976"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3380476",
+    "4137200",
+    "8617662"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3183683",
+    "1468692",
+    "4491676"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5971871",
+    "6485883",
+    "4087771"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6163397",
+    "3624007",
+    "7318113"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6299778",
+    "9528403",
+    "5074157"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8042829",
+    "1776041",
+    "3218515"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3214932",
+    "8050318",
+    "6022874"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8083151",
+    "6000081",
+    "6948598"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9878625",
+    "1970681",
+    "7315476"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8403741",
+    "4422388",
+    "7234356"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2822261",
+    "5646789",
+    "2881711"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7690366",
+    "4110171",
+    "8075024"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6710761",
+    "9780333",
+    "3391885"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9108226",
+    "4416510",
+    "5944720"
+   ]
+  }
+ ]
+}

logs/longeval_16k/prompts/niah_multivalue_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b51c3d5307d8f0e453019a3f3fdf15d6e05472b06a0f350252ddfe80326d75d
+size 6544000

logs/longeval_16k/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 16360,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/longeval_16k/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c850c0571fc3b674c9b689c773536aee2d1dfb1c35c3cf9064d815c94bcf761f
+size 6544000

logs/longeval_16k/prompts/val_backward_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "val_backward",
+ "prompt_len": 16360,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "umbrella"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "hollow"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "fjord"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "penguin"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "raven"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "bramble"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "xylophone"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "harbor"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "buffalo"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "glacier"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "emerald"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "walnut"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "lantern"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "granite"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "zephyr"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "nutmeg"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "quartz"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "cinder"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "aardvark"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "anchor"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "tundra"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "jaguar"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "driftwood"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "violet"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "ivory"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "ember"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "sapphire"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "iceberg"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "falcon"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "meadow"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "kestrel"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "orchid"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "yarrow"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "cactus"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "anchor"
+   ]
+  }
+ ]
+}