Upload folder using huggingface_hub

logs/decide_len_4096/depth_summary.csv

@@ -0,0 +1,21 @@
+depth,model,hits,n,pct,wilson95
+0.0,armA_c,11,20,55.0,20.0
+0.0,l1,10,20,50.0,20.1
+0.0,l2,15,20,75.0,17.8
+0.0,l3,13,20,65.0,19.3
+0.25,armA_c,14,20,70.0,18.7
+0.25,l1,16,20,80.0,16.8
+0.25,l2,17,20,85.0,15.4
+0.25,l3,17,20,85.0,15.4
+0.5,armA_c,10,20,50.0,20.1
+0.5,l1,12,20,60.0,19.7
+0.5,l2,12,20,60.0,19.7
+0.5,l3,10,20,50.0,20.1
+0.75,armA_c,19,20,95.0,11.4
+0.75,l1,19,20,95.0,11.4
+0.75,l2,18,20,90.0,13.7
+0.75,l3,16,20,80.0,16.8
+1.0,armA_c,19,20,95.0,11.4
+1.0,l1,18,20,90.0,13.7
+1.0,l2,18,20,90.0,13.7
+1.0,l3,18,20,90.0,13.7

logs/decide_len_4096/gen/armA_c_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,770,1271,318,257,1277,45923,286,262,2656,2420,11,6011,257,3748,290,1690,517,9389,11327,1998
+5,564,250,3865,3134,16102,13,447,251,632,447,247,82,257,2176,1271,326,447,247,82,587,973,287,262,983
+6,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,34849,30,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,718,13,198,20660,1849,20660,531,986,198
+10,362,23721,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,770,1271,318,8780,329,13359,326,262,598,460,10996,6840,351,262,11256,84,2217,32802,3859
+16,1160,31115,3695,13,770,1271,318,8780,329,1938,508,765,284,20487,511,1998,13,632,22940,126,222,126,247,82
+17,764,198,24361,25,1867,318,262,2041,1271,329,257,446,85,668,30,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,198
+22,262,1271,286,1661,262,15061,318,287,29955,13,198
+23,7600,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,13,198,24361,25,4162,318,340,1444,262,564,250,20887,447,251,1271,30,198
+26,2579,1959,39925,13,220,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,770,1271,9217,326,262,24747,389,3562,284,17565,32392,290,5529,257,8245,6594,6001,13,198
+29,604,29416,25948,13,770,1271,9217,326,262,24747,389,11670,351,262,2176,2099,286,6076,345,389,13586,13,198
+30,352,13,198,24361,25,1867,318,262,2041,5536,1271,329,473,381,10695,30,198
+31,767,2231,2425,3388,13,198
+32,2026,47448,5705,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,198
+35,642,2623,18938,24,2598,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,220,1849,464,2041,5536,1271,329,24215,1102,318,39320,22148,18,13,220,1849,464,2041,5536,1271
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,262,1271,286,262,18304,286,262,4453,13,198
+46,807,3104,13,198
+47,604,13,198,464,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,262,1271,286,262,1400,5155,2677,447,247,82,6174,7398,5562,447,247,82,1521,340,447,247,82,1444,262,1400
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,262,2041,5536,1271,329,32630,13,383,4960,5428,338,20381,11,262,4960,5428,338,20381,11,290,262,4960,5428,338
+56,262,5536,1271,329,45019,13,383,10846,338,2951,547,3094,1280,11,290,673,373,257,1310,6655,11,673,373,257
+57,257,1271,326,6870,262,5536,1271,286,257,2176,1989,286,257,502,4584,13,198
+58,1679,2327,33300,13,198
+59,41235,3388,3132,13,198
+60,1511,5066,830,13,198
+61,262,1271,286,262,6786,5509,13,198
+62,4101,1821,22136,13,198
+63,352,13,198,33706,25,383,2041,5536,1271,329,6701,28917,318,352,13,198
+64,807,1415,3312,2079,13,198
+65,262,1271,286,262,44539,326,318,3867,1497,422,262,3641,286,262,4534,13,198
+66,38634,2920,405,13,198
+67,513,13,198,464,1366,7766,3877,12,15808,3716,9978,286,28223,19,290,14661,17910,261,15568,11,284,543,569,22470
+68,9698,1433,29326,13,198
+69,48609,18376,13,220,198
+70,262,1271,286,262,1110,13,198
+71,2608,6052,22413,13,198
+72,513,13,198,14231,416,379,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,3571,25,1314,25,2998,318,326,340,338,257,1310,1643,286,257,1310,1643,286,257,1310,1643,286,257,1310,1643
+76,3571,25,2624,25,1495,1406,11,345,760,11,356,423,257,1256,286,1180,1243,326,356,466,11,475,356,423
+77,3571,25,2920,25,1270,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,220,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,770,318,262,4511,966,319,262,4141,6378,11,290,340,318,3177,262,749,2408,286,262
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,7265,13,198,58,41,324,60,1849,8487,25,2623,198
+88,2026,27800,1129,13,220,314,892,326,338,257,1107,922,1271,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,34849,30,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,220,5268,25414,3226,3226,428,11,18776,9014,9102,318,1690,257,890,12,4354,7901,13,198
+94,7930,15197,1959,13,220,198
+95,24555,3134,2091,198
+96,7600,1507,38380,13,198
+97,262,1271,286,262,5848,11,543,318,262,1271,286,262,5848,13,383,10846,531,351,257,8212,11,262,10846,531
+98,8541,35890,2481,13,383,10846,338,2951,6578,510,11,290,673,531,287,257,1877,3809,314,423,257,1256,286,1637
+99,27368,2780,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624

logs/decide_len_4096/gen/armA_c_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run28_step_400.ckpt  (base step 400, loss 2.55066)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log4.csv (run 4)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run4_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 4096 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/armA_c_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/armA_c_niah_single.csv

logs/decide_len_4096/gen/l1_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,13,198
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,770,1271,318,257,1277,4941,284,262,2041,5536,1271,329,262,2041,5536,1271,329,795,527,13
+5,10048,3134,16102,13,770,318,257,3748,27421,329,262,983,11,12739,663,3748,6087,286,564,250,32707,447,251,290
+6,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,34849,30,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,718,13,198,20660,1849,20660,531,986,198
+10,362,23721,17477,13,198
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,770,1271,318,3748,284,262,983,290,460,307,973,284,12116,2041,3033,393,17568,13,198
+16,1160,31115,3695,13,770,1271,43854,326,262,983,468,257,2041,3895,326,460,307,14838,416,2712,13,198
+17,262,1271,286,262,14561,326,318,10945,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,198
+22,262,1271,286,1661,257,15061,468,587,2005,13,198
+23,7600,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,13,198,24361,25,1867,318,262,2041,5536,1271,329,25510,30,198
+26,2579,1959,39925,13,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,770,1271,318,8780,329,13213,262,3376,9014,286,262,8286,24747,13,198
+29,604,29416,25948,13,770,1271,9217,326,262,649,24747,389,3562,284,4197,262,2176,15225,286,534,360,5819,604,87
+30,807,13,198,19,13,6208,262,4583,290,262,8055,13,198
+31,767,2231,2425,3388,13,198
+32,2026,47448,5705,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,198
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,2091,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,262,1271,286,262,18304,286,262,4453,13,198
+46,860,1433,1157,3720,13,198
+47,642,13,198,464,2041,5536,1271,329,45725,318,642,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,262,1271,286,262,717,4314,286,262,41553,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,262,2041,5536,1271,329,32630,13,383,4960,5795,547,477,12049,287,2042,22551,11,290,262,4960,5795,547,477,12049
+56,35617,3695,1129,13,198
+57,642,13,198,24361,25,1867,318,262,3580,1022,402,11571,12,19,78,290,402,11571,12,18,13,20,30,198
+58,1679,2327,33300,13,198
+59,767,13,198,24361,25,1867,318,262,2041,5536,1271,329,2124,2645,38656,30,198
+60,1511,5066,830,13,198
+61,262,1271,286,262,9356,338,938,1528,13,198
+62,4101,1821,22136,13,198
+63,352,13,198,33706,25,383,2041,5536,1271,329,6701,28917,318,352,13,198
+64,807,1415,3312,2079,13,198
+65,767,13,198,26374,1139,673,447,247,82,257,1310,2576,13,198
+66,38634,2920,405,13,198
+67,352,13,198,464,2041,5536,1271,329,3165,1940,318,352,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,767,13,198,42,563,1452,25,314,1101,407,1654,644,340,318,11,475,314,1101,25260,340,338,767,13,198
+71,2608,6052,22413,13,198
+72,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,1976,27446,2417,30,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,12877,13,198,22851,966,318,3571,25,1314,25,405,1406,340,338,12877,13,198
+76,3571,25,2624,25,1828,13,1406,479,395,2411,318,257,2139,326,338,973,416,12024,11,12024,412,1381,11,290
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,767,13,198,464,1271,767,318,262,749,1593,1271,287,262,7298,2879,390,34802,11,355,340,318,262,749,13626
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,642,13,198,24361,25,1867,318,262,2041,5536,1271,329,34849,30,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,198
+95,24555,3134,2091,2091,198
+96,7600,1507,38380,13,198
+97,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286,262,5536,1271,286
+98,8541,35890,2481,13,198
+99,27368,2780,2624,13,198

logs/decide_len_4096/gen/l1_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/lenbucket_l1_blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.4588)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 4096 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/l1_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/l1_niah_single.csv

logs/decide_len_4096/gen/l2_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,39084,5774,3559,13,198
+4,13037,2154,4869,13,220,198
+5,10048,3134,16102,13,770,1271,318,257,3748,6087,286,262,3850,564,250,49,447,251,290,262,1271,564,250,49
+6,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,34849,30,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,718,2682,13,198
+10,362,24693,17477,13,198
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,13,198
+16,1160,31115,3695,13,770,1271,318,8780,329,1938,508,765,284,20487,511,1998,13,632,22940,126,222,126,247,82
+17,262,3748,1271,329,262,3117,4572,13,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,198
+22,262,1271,286,262,393,28402,447,247,82,393,28402,447,247,82,393,28402,447,247,82,393,28402,447,247,82
+23,9508,1314,43950,13,198
+24,2242,2920,37466,13,198
+25,860,4309,13,198
+26,2579,1959,39925,13,220,5213,13,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,198
+29,604,29416,25948,13,198
+30,9508,2996,40179,13,198
+31,767,2231,2425,3388,13,198
+32,2026,47448,5705,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,220,198
+35,642,2623,18938,24,13,198
+36,362,19707,2713,23,13,198
+37,39320,22148,18,13,220,1849,464,2041,5536,1271,329,24215,1102,318,39320,22148,18,13,220,1849,464,2041,5536,1271
+38,1467,2780,26561,13,220,290,10345,5400,11,477,286,543,389,1593,329,262,9030,286,32305,4779,5040,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,34770,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,718,2598,1065,3134,13,198
+46,860,1433,1157,3720,13,198
+47,604,13,198,464,2041,5536,1271,329,45725,318,604,13,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,220,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,262,1271,286,262,717,4314,286,262,41553,13,198
+53,513,30505,35133,13,198
+54,1596,44169,486,13,198
+55,262,1271,286,262,2041,5536,1271,329,32630,13,383,4960,5795,547,19987,11,290,262,4960,5795,547,19987,13,198
+56,767,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11,830,11
+57,767,13,198,24361,25,1867,318,262,5536,1271,329,502,4584,30,198
+58,1679,2327,33300,13,198
+59,43336,3388,3132,13,220,198
+60,1511,5066,830,13,198
+61,262,1271,286,262,9356,338,938,1528,13,198
+62,4101,1821,22136,13,198
+63,352,13,198,464,2041,5536,1271,329,6701,28917,318,352,13,198
+64,807,1415,3312,2079,13,198
+65,767,13,198,464,3280,318,767,13,198
+66,38634,2920,405,13,198
+67,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,3165,1940,30,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,220,198
+70,767,13,198,43,1694,25,314,1101,407,1654,13,198
+71,2608,6052,22413,13,198
+72,513,13,198,14231,416,379,1400,3651,25,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,27228,2670,2091,13,1406,340,338,257,2041,1271,329,6701,28917,13,1406,340,338,257,2041,1271,329,6701,28917,13
+76,5214,1558,20107,13,198
+77,860,49287,15197,13,198
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,220,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,7632,17464,2718,13,198
+88,2026,27800,1129,13,220,534,4459,11,644,466,345,892,389,262,749,3499,7612,286,1762,287,428,2214,30,198
+89,8644,2931,26780,13,220,198
+90,4317,2075,34107,13,198
+91,45469,3270,1959,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,220,5268,25414,3226,286,428,11,18776,9014,9102,318,1690,257,890,12,4354,7901,13,198
+94,7930,15197,1959,13,220,198
+95,24555,3134,2091,2091,198
+96,7600,1507,38380,13,220,198
+97,262,5536,1271,286,262,5848,11,543,318,262,5536,1271,286,262,5848,13,383,10846,338,2951,547,1336,286,5975
+98,8541,35890,2481,13,383,10846,338,2951,547,1336,286,34082,6824,11,290,673,373,546,284,910,1223,11,475,673
+99,27368,2780,2624,13,383,10846,338,2951,6578,510,11,290,673,531,287,257,2769,3809,314,423,587,4953,329,345

logs/decide_len_4096/gen/l2_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/lenbucket_l2_blumodelcp_run2_step_400.ckpt  (base step 400, loss 2.79415)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log6.csv (run 6)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run6_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 4096 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/l2_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/l2_niah_single.csv

logs/decide_len_4096/gen/l3_niah_single.csv

@@ -0,0 +1,100 @@
+0,807,2713,1899,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238,1238
+1,642,2682,2670,2999,13,198
+2,860,1314,1495,1485,13,198
+3,718,13,198,24361,25,1867,318,262,2041,5536,1271,329,1976,27446,2417,30,198
+4,13037,2154,4869,13,770,1271,318,257,8780,5002,287,867,3660,1830,11,10200,262,1271,286,3435,287,257,983,447
+5,10048,3134,16102,13,632,447,247,82,257,3748,1271,326,447,247,82,407,6768,8018,11,475,340,447,247,82
+6,513,13,198,24361,25,1867,318,262,2041,5536,1271,329,34849,30,198
+7,1679,3829,38565,13,198
+8,7388,27371,2327,13,198
+9,718,2682,13,198
+10,362,24693,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477,17477
+11,807,5892,1065,1821,13,198
+12,8644,32066,2091,13,198
+13,44505,2920,405,13,198
+14,15897,2078,4846,13,198
+15,860,4524,3104,5237,5237,13,198
+16,1160,31115,3695,13,198
+17,764,198,3581,383,2041,5536,1271,329,257,446,85,668,318,764,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,13,198
+20,718,2231,1899,1954,13,198
+21,28262,2816,4790,13,198
+22,352,13,198,19156,12344,805,25,314,447,247,76,407,1654,644,340,318,11,475,340,447,247,82,257,1271
+23,262,1271,286,262,1110,13,198
+24,2242,2920,37466,13,198
+25,860,4309,13,198
+26,2579,1959,39925,13,220,5213,13,198
+27,642,3459,2623,6420,13,198
+28,718,3365,2075,1983,13,770,1271,318,8780,329,13213,262,3376,19114,286,262,11461,6805,13,198
+29,604,29416,25948,13,198
+30,604,13,198,19,13,1355,1598,546,644,345,765,284,4620,13,198
+31,767,2231,2425,3388,13,198
+32,2026,47448,5705,13,198
+33,2319,5705,28256,13,198
+34,46618,2078,486,13,220,198
+35,642,2623,18938,24,198
+36,362,19707,2713,23,13,198
+37,718,13,198,14231,416,337,44886,38742,11,337,13,35,13,379,767,25,3553,447,107,5868,657,3651,198
+38,1467,2780,26561,13,198
+39,5441,34287,3365,13,198
+40,7265,1731,31020,13,198
+41,604,3865,940,5066,13,198
+42,4019,2623,41874,13,198
+43,807,38605,22913,13,198
+44,718,2079,3980,2624,13,198
+45,718,2598,1065,3134,13,198
+46,860,1433,1157,3720,13,198
+47,604,13,198,464,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,198
+48,18112,1129,3682,13,198
+49,2808,2996,23451,13,220,198
+50,767,27877,19880,13,198
+51,718,3553,3829,1270,13,198
+52,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+53,513,29228,35133,13,198
+54,1596,44169,486,13,198
+55,262,1271,286,262,2041,5536,1271,329,32630,13,383,4960,5795,547,477,19987,11,290,262,4960,5795,547,477,19987
+56,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286,262,1271,286
+57,604,13,198,33706,25,383,5536,1271,329,502,4584,318,604,13,198
+58,1679,2327,33300,13,198
+59,767,13,198,2061,318,262,2041,5536,1271,329,2124,2645,38656,30,198
+60,1511,5066,830,13,198
+61,32921,2091,1415,13,198
+62,4101,1821,22136,13,198
+63,352,13,198,464,2041,5536,1271,329,6701,28917,318,352,13,198
+64,807,1415,3312,2079,13,198
+65,767,13,198,26374,1139,314,716,407,257,9234,13,314,716,257,9234,13,198
+66,38634,2920,405,13,198
+67,718,13,198,464,2041,5536,1271,329,3165,1940,318,718,13,198
+68,9698,1433,29326,13,198
+69,48609,18376,13,198
+70,767,13,198,43,1694,25,314,1101,407,1654,13,198
+71,2608,6052,22413,13,198
+72,718,13,198,14231,416,379,198
+73,48173,2816,3270,13,198
+74,4353,40090,4790,13,198
+75,34323,2670,2091,2091,13,198
+76,5214,1558,20107,13,220,257,1256,286,1661,340,338,407,257,1256,286,1661,11,475,340,338,257,1256,286,1661
+77,860,49287,15197,13,220,318,860,49287,15197,13,220,318,860,49287,15197,13,220,318,860,49287,15197,13,220,318
+78,2808,2791,31654,13,198
+79,5014,37967,5705,13,198
+80,9919,1065,47915,13,198
+81,860,4310,2078,5824,13,198
+82,718,2079,10333,13,198
+83,36141,2079,4349,13,198
+84,1105,2919,27720,13,198
+85,2242,25540,2548,13,198
+86,1596,2791,36445,13,198
+87,262,1271,286,1661,326,340,338,973,287,257,1720,13,843,326,338,262,1271,286,1661,326,340,338,973,287
+88,2026,27800,1129,13,220,534,4459,11,644,466,345,892,389,262,749,3499,7612,286,1762,287,428,2214,30,198
+89,8644,2931,26780,13,198
+90,4317,2075,34107,13,198
+91,718,13,198,464,2041,5536,1271,329,34849,318,718,13,198
+92,767,3865,1983,1795,13,198
+93,838,1495,48724,13,198
+94,7930,15197,1959,13,220,198
+95,24555,3134,2091,2091,198
+96,7600,1507,38380,13,220,1849,1026,338,257,1271,326,338,257,1310,1643,517,621,262,3210,1271,198
+97,262,1271,286,262,5536,1271,329,331,6018,13,383,1738,1521,262,10846,318,523,3772,318,780,673,468,587,287
+98,8541,35890,2481,13,383,10846,338,2951,547,1336,286,8993,11,290,673,531,287,257,2769,3809,314,716,407,257
+99,27368,2780,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624,2624

logs/decide_len_4096/gen/l3_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 16384
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 16384
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=32)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=16384 hd=128 scale=4.000 low=14 high=32 m=1.1386
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run1_step_400.ckpt  (base step 400, loss 2.96258)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 4 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log5.csv (run 5)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run5_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 4096 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/l3_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/decide_len_4096/gen/l3_niah_single.csv

logs/decide_len_4096/master.log

@@ -0,0 +1,99 @@
+[07:59:27] longeval: T=16384 prompt_len=4096 max_new=24 n=100 greedy=1 gpus=0,1
+[07:59:27] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[07:59:27] prompt signature changed -- regenerating prompts and clearing state
+[07:59:27] generating prompts: niah_single
+  niah_single      100 x 4096 tok -> decide_len_4096/prompts/niah_single_prompts.bin
+[07:59:27] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[07:59:27] 4 job(s) to run
+[07:59:27]   [gpu 0] armA_c / niah_single
+[07:59:27]   [gpu 1] l1 / niah_single
+[07:59:27]   [gpu 0] l2 / niah_single
+[07:59:27]   [gpu 1] l3 / niah_single
+[08:04:42]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 73/100 = 73.0%
+    depth      acc   (needle position in context)
+     0.00  11/20   55.0%
+     0.25  14/20   70.0%
+     0.50  10/20   50.0%
+     0.75  19/20   95.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/armA_c_niah_single.csv
+[08:04:42]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 75/100 = 75.0%
+    depth      acc   (needle position in context)
+     0.00  10/20   50.0%
+     0.25  16/20   80.0%
+     0.50  12/20   60.0%
+     0.75  19/20   95.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/l1_niah_single.csv
+[08:04:42]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 80/100 = 80.0%
+    depth      acc   (needle position in context)
+     0.00  15/20   75.0%
+     0.25  17/20   85.0%
+     0.50  12/20   60.0%
+     0.75  18/20   90.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/l2_niah_single.csv
+[08:04:43]   scoring l3 / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 74/100 = 74.0%
+    depth      acc   (needle position in context)
+     0.00  13/20   65.0%
+     0.25  17/20   85.0%
+     0.50  10/20   50.0%
+     0.75  16/20   80.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/l3_niah_single.csv
+[08:04:43] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3
+  niah_single            73.0     75.0     80.0     74.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3
+  0.00          55.0+-20.0      50.0+-20.1      75.0+-17.8      65.0+-19.3
+  0.25          70.0+-18.7      80.0+-16.8      85.0+-15.4      85.0+-15.4
+  0.50          50.0+-20.1      60.0+-19.7      60.0+-19.7      50.0+-20.1
+  0.75          95.0+-11.4      95.0+-11.4      90.0+-13.7      80.0+-16.8
+  1.00          95.0+-11.4      90.0+-13.7      90.0+-13.7      90.0+-13.7
+
+  wrote decide_len_4096/summary.csv and decide_len_4096/depth_summary.csv
+[08:04:43] DONE -> decide_len_4096/summary.csv

logs/decide_len_4096/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=4096|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=b2val_haystack.bin|HAYSZ=199754694

logs/decide_len_4096/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 4096,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/decide_len_4096/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f07e2a292789f6f5f78f77f888fe012732b8d0f3bc6a0164f3552e9dda1671a6
+size 1638400

logs/decide_len_4096/scores/armA_c_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071," 7007071. This number is a direct descendant of the original text, offering a unique and often more challenging gameplay experience"
+5,0.0,0,1,0,9467240, “9567240.” It’s a specific number that’s been used in the game
+6,0.25,0,1,0,5728454, 3.\nQuestion: What is the special magic number for iceberg?\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6.\nAnonymous Anonymous said...\n
+10,0.0,0,1,0,2237192, 2238192192192192192192192192192192192192192192192192192192192192192192
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862. This number is crucial for ensuring that the app can communicate effectively with the Huuuge Casino platform
+16,0.25,1,1,1,2044878, 2044878. This number is crucial for players who want to maximize their experience. It’s
+17,0.5,0,1,0,2564842, .\nQuestion: What is the special number for aardvark?\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573.\n
+22,0.5,0,1,0,5003310, the number of times the flower is in bloom.\n
+23,0.75,1,1,1,8515682, 8515682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, 9.\nQuestion: Why is it called the “special” number?\n
+26,0.25,1,1,1,2829687, 2829687. \n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627. This number indicates that the springs are designed to absorb shocks and maintain a stable ride height.\n
+29,1.0,1,1,1,4409161, 4409161. This number indicates that the springs are compatible with the specific type of spring you are replacing.\n
+30,0.0,0,1,0,8465677, 1.\nQuestion: What is the special magic number for sapphire?\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,1,1,1,5061884, 5061884.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,1,1,1,5363019, 536301944.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803.  The special magic number for falcon is 3184803.  The special magic number
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, the number of the angel of the Lord.\n
+46,0.25,0,1,0,9161179, 868.\n
+47,0.5,0,1,0,4194027, 4.\nThe number of the number of the number of the number of the number of the number of the\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, the number of the No Life King’s meat……that’s why it’s called the No
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461," the special magic number for ivory. The Japanese army's artillery, the Japanese army's artillery, and the Japanese army's"
+56,0.25,0,1,0,3087819," the magic number for buffalo. The lady's eyes were wide open, and she was a little surprised, she was a"
+57,0.5,0,1,0,3008156, a number that represents the magic number of a specific area of a meadow.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,1,1,1,2946931, 2946931.\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, the number of the Christmas tree.\n
+62,0.5,1,1,1,9040206, 9040206.\n
+63,0.75,0,1,0,2024834, 1.\nAnswer: The special magic number for nutmeg is 1.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, the number of the glacier that is moving away from the center of the earth.\n
+66,0.25,1,1,1,4704900, 4704900.\n
+67,0.5,0,1,0,6050974," 3.\nThe data reveal charge-driven complex formation of PF4 and AV hexon proteins, to which VITT"
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122. \n
+70,0.0,0,1,0,8837161, the number of the day.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,0,1,0,4344635, 3.\nPosted by at\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 00:15:07 is that it's a little bit of a little bit of a little bit of a little bit
+76,0.25,0,1,0,3717138," 00:32:25 So, you know, we have a lot of different things that we do, but we have"
+77,0.5,0,1,0,9883103, 00:49:30.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984. \n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894," 9532894. This is the highest point on the French Way, and it is considered the most difficult of the"
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,0,1,0,5712437, 56.\n[Jad]  08:36\n
+88,0.75,1,1,1,5028719, 5028719.  I think that's a really good number.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 6.\nQuestion: What is the special magic number for iceberg?\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572," 1025572.  End Result Of Of this, testosterone replacement therapy is often a long-term commitment.\n"
+94,1.0,1,1,1,6610329, 6610329. \n
+95,0.0,1,1,1,5016733, 5016733\n
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,0,1,0,7952585," the number of the soul, which is the number of the soul. The lady said with a smile, the lady said"
+98,0.75,1,1,1,7748921," 7748921. The lady's eyes lit up, and she said in a low voice I have a lot of money"
+99,1.0,1,1,1,1784832, 1784832323232323232323232323232323232323232323232

logs/decide_len_4096/scores/l1_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 8056020.\n
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071. This number is a direct reference to the special magic number for the special magic number for ember.
+5,0.0,1,1,1,9467240," 9467240. This is a unique identifier for the game, indicating its unique combination of “magic” and"
+6,0.25,0,1,0,5728454, 3.\nQuestion: What is the special magic number for iceberg?\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 6.\nAnonymous Anonymous said...\n
+10,0.0,0,1,0,2237192, 2238192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862. This number is unique to the game and can be used to unlock special features or bonuses.\n
+16,0.25,1,1,1,2044878, 2044878. This number signifies that the game has a special feature that can be unlocked by playing.\n
+17,0.5,0,1,0,2564842, the number of the exploit that is executed.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573.\n
+22,0.5,0,1,0,5003310, the number of times a flower has been cut.\n
+23,0.75,1,1,1,8515682, 8515682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, 9.\nQuestion: What is the special magic number for umbrella?\n
+26,0.25,1,1,1,2829687, 2829687.\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627. This number is crucial for determining the correct replacement of the rear springs.\n
+29,1.0,1,1,1,4409161, 4409161. This number indicates that the new springs are designed to fit the specific dimensions of your Duster 4x
+30,0.0,0,1,0,8465677, 8.\n4. Test the theory and the outcome.\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,1,1,1,5061884, 5061884.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801.\n
+35,0.0,1,1,1,5363019, 5363019.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803.\n
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 562433.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,0,1,0,6441267, the number of the angel of the Lord.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 5.\nThe special magic number for violet is 5.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187.\n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, the number of the first floor of the labyrinth.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461," the special magic number for ivory. The Japanese soldiers were all dressed in black uniforms, and the Japanese soldiers were all dressed"
+56,0.25,1,1,1,3087819, 3087819.\n
+57,0.5,0,1,0,3008156, 5.\nQuestion: What is the difference between GPT-4o and GPT-3.5?\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 7.\nQuestion: What is the special magic number for xylophone?\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, the number of the Doctor's last days.\n
+62,0.5,1,1,1,9040206, 9040206.\n
+63,0.75,0,1,0,2024834, 1.\nAnswer: The special magic number for nutmeg is 1.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, 7.\nKim says she’s a little girl.\n
+66,0.25,1,1,1,4704900, 4704900.\n
+67,0.5,0,1,0,6050974, 1.\nThe special magic number for emerald is 1.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161," 7.\nKryten: I'm not sure what it is, but I'm guessing it's 7.\n"
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,0,1,0,4344635, 3.\nQuestion: What is the special magic number for zephyr?\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 000.\nStarting point is 00:15:00 So it's 000.\n
+76,0.25,0,1,0,3717138," 00:32:22. So kestrel is a service that's used by Uber, Uber Eats, and"
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,0,1,0,8912561," 7.\nThe number 7 is the most important number in the Camino de Santiago, as it is the most sacred"
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719, 5028719.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 5.\nQuestion: What is the special magic number for iceberg?\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329.\n
+95,0.0,1,1,1,5016733, 501673333\n
+96,0.25,1,1,1,8518463, 8518463.\n
+97,0.5,0,1,0,7952585, the magic number of the magic number of the magic number of the magic number of the magic number of the magic number of
+98,0.75,1,1,1,7748921, 7748921.\n
+99,1.0,1,1,1,1784832, 1784832.\n

logs/decide_len_4096/scores/l2_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 80560202020202020202020202020202020202020202020
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,1,1,1,6088743, 6088743.\n
+4,1.0,1,1,1,7007071, 7007071. \n
+5,0.0,1,1,1,9467240, 9467240. This number is a unique combination of the letter “R” and the number “R
+6,0.25,0,1,0,5728454, 3.\nQuestion: What is the special magic number for iceberg?\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 634.\n
+10,0.0,1,1,1,2237192, 2237192.\n
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 9746862.\n
+16,0.25,1,1,1,2044878, 2044878. This number is crucial for players who want to maximize their experience. It’s
+17,0.5,0,1,0,2564842, the unique number for the victim machine.\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573.\n
+22,0.5,0,1,0,5003310, the number of the orchid’s orchid’s orchid’s orchid’s
+23,0.75,0,1,0,8515682, 8415682.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, 952.\n
+26,0.25,1,1,1,2829687, 2829687.  concerned.\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,1,1,1,8465677, 8465677.\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,1,1,1,5061884, 5061884.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,1,1,1,5363019, 5363019.\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,1,1,1,3184803, 3184803.  The special magic number for falcon is 3184803.  The special magic number
+38,0.75,1,1,1,1648297," 1648297.  and functional differences, all of which are important for the mechanism of substrate translocation.\n"
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,1,1,1,5624373, 5624373.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,1,1,1,6441267, 6441267.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 4.\nThe special magic number for violet is 4.\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187. \n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, the number of the first floor of the labyrinth.\n
+53,0.75,1,1,1,3455487, 3455487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461," the number of the special magic number for ivory. The Japanese soldiers were stunned, and the Japanese soldiers were stunned.\n"
+56,0.25,0,1,0,3087819," 7,000,000,000,000,000,000,000,000,000,000,000,"
+57,0.5,0,1,0,3008156, 7.\nQuestion: What is the magic number for meadow?\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 2916931. \n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,0,1,0,4103314, the number of the Doctor's last days.\n
+62,0.5,1,1,1,9040206, 9040206.\n
+63,0.75,0,1,0,2024834, 1.\nThe special magic number for nutmeg is 1.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, 7.\nThe answer is 7.\n
+66,0.25,1,1,1,4704900, 4704900.\n
+67,0.5,0,1,0,6050974, 6.\nQuestion: What is the special magic number for emerald?\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122. \n
+70,0.0,0,1,0,8837161, 7.\nLister: I'm not sure.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,0,1,0,4344635, 3.\nPosted by at No comments:\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 1793933. So it's a special number for nutmeg. So it's a special number for nutmeg.
+76,0.25,1,1,1,3717138, 3717138.\n
+77,0.5,1,1,1,9883103, 9883103.\n
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984. \n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,1,1,1,5712437, 5712437.\n
+88,0.75,1,1,1,5028719," 5028719.  your opinion, what do you think are the most interesting aspects of working in this field?\n"
+89,1.0,1,1,1,6909364, 6909364. \n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,1,1,1,6205929, 6205929.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572," 1025572.  End Result Of of this, testosterone replacement therapy is often a long-term commitment.\n"
+94,1.0,1,1,1,6610329, 6610329. \n
+95,0.0,1,1,1,5016733, 501673333\n
+96,0.25,1,1,1,8518463, 8518463. \n
+97,0.5,0,1,0,7952585," the magic number of the soul, which is the magic number of the soul. The lady's eyes were full of surprise"
+98,0.75,1,1,1,7748921," 7748921. The lady's eyes were full of murderous intent, and she was about to say something, but she"
+99,1.0,1,1,1,1784832," 1784832. The lady's eyes lit up, and she said in a deep voice I have been waiting for you"

logs/decide_len_4096/scores/l3_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,1,1,1,8056020, 80560202020202020202020202020202020202020202020
+1,0.25,1,1,1,5343902, 5343902.\n
+2,0.5,1,1,1,9152513, 9152513.\n
+3,0.75,0,1,0,6088743, 6.\nQuestion: What is the special magic number for zephyr?\n
+4,1.0,1,1,1,7007071," 7007071. This number is a crucial element in many modern games, representing the number of characters in a game�"
+5,0.0,1,1,1,9467240," 9467240. It’s a unique number that’s not widely recognized, but it’s"
+6,0.25,0,1,0,5728454, 3.\nQuestion: What is the special magic number for iceberg?\n
+7,0.5,1,1,1,2590996, 2590996.\n
+8,0.75,1,1,1,9934935, 9934935.\n
+9,1.0,0,1,0,6203412, 634.\n
+10,0.0,1,1,1,2237192, 2237192192192192192192192192192192192192192192192192192192192192192192
+11,0.25,1,1,1,8921240, 8921240.\n
+12,0.5,1,1,1,6935633, 6935633.\n
+13,0.75,1,1,1,6304900, 6304900.\n
+14,1.0,1,1,1,9002896, 9002896.\n
+15,0.0,1,1,1,9746862, 974686262.\n
+16,0.25,1,1,1,2044878, 2044878.\n
+17,0.5,0,1,0,2564842, .\n• The special magic number for aardvark is .\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 6589080.\n
+20,0.0,1,1,1,6456023, 6456023.\n
+21,0.25,1,1,1,4205573, 4205573.\n
+22,0.5,0,1,0,5003310," 1.\nRobert Altman: I’m not sure what it is, but it’s a number"
+23,0.75,0,1,0,8515682, the number of the day.\n
+24,1.0,1,1,1,2349656, 2349656.\n
+25,0.0,0,1,0,9521829, 952.\n
+26,0.25,1,1,1,2829687, 2829687.  concerned.\n
+27,0.5,1,1,1,5883691, 5883691.\n
+28,0.75,1,1,1,6582627, 6582627. This number is crucial for determining the correct alignment of the suspension components.\n
+29,1.0,1,1,1,4409161, 4409161.\n
+30,0.0,0,1,0,8465677, 4.\n4. Be clear about what you want to achieve.\n
+31,0.25,1,1,1,7457569, 7457569.\n
+32,0.5,1,1,1,5061884, 5061884.\n
+33,0.75,1,1,1,4084805, 4084805.\n
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,1,1,1,5363019, 5363019\n
+36,0.25,1,1,1,2159058, 2159058.\n
+37,0.5,0,1,0,3184803," 6.\nPosted by Meryl Nass, M.D. at 7:57 PM 0 comments\n"
+38,0.75,1,1,1,1648297, 1648297.\n
+39,1.0,1,1,1,7564858, 7564858.\n
+40,0.0,0,1,0,5624373, 5624374.\n
+41,0.25,1,1,1,4951063, 4951063.\n
+42,0.5,1,1,1,8036754, 8036754.\n
+43,0.75,1,1,1,8559194, 8559194.\n
+44,1.0,1,1,1,6995632, 6995632.\n
+45,0.0,1,1,1,6441267, 6441267.\n
+46,0.25,1,1,1,9161179, 9161179.\n
+47,0.5,0,1,0,4194027, 4.\nThe number of the number of the number of the number of the number of the number of the\n
+48,0.75,1,1,1,1271942, 1271942.\n
+49,1.0,1,1,1,2965187, 2965187. \n
+50,0.0,1,1,1,7242134, 7242134.\n
+51,0.25,1,1,1,6579030, 6579030.\n
+52,0.5,0,1,0,2043416, the number of the number of the number of the number of the number of the number of the number of the number of
+53,0.75,0,1,0,3455487, 3456487.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461," the number of the special magic number for ivory. The Japanese soldiers were all stunned, and the Japanese soldiers were all stunned"
+56,0.25,0,1,0,3087819, the number of the number of the number of the number of the number of the number of the number of the number of
+57,0.5,0,1,0,3008156, 4.\nAnswer: The magic number for meadow is 4.\n
+58,0.75,1,1,1,2535649, 2535649.\n
+59,1.0,0,1,0,2946931, 7.\nWhat is the special magic number for xylophone?\n
+60,0.0,1,1,1,1363000, 1363000.\n
+61,0.25,1,1,1,4103314, 4103314.\n
+62,0.5,1,1,1,9040206, 9040206.\n
+63,0.75,0,1,0,2024834, 1.\nThe special magic number for nutmeg is 1.\n
+64,1.0,1,1,1,8140699, 8140699.\n
+65,0.0,0,1,0,5360732, 7.\nKim says I am not a monster. I am a monster.\n
+66,0.25,1,1,1,4704900, 4704900.\n
+67,0.5,0,1,0,6050974, 6.\nThe special magic number for emerald is 6.\n
+68,0.75,1,1,1,8316017, 8316017.\n
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161, 7.\nLister: I'm not sure.\n
+71,0.25,1,1,1,2693177, 2693177.\n
+72,0.5,0,1,0,4344635, 6.\nPosted by at\n
+73,0.75,1,1,1,7015559, 7015559.\n
+74,1.0,1,1,1,3842473, 3842473.\n
+75,0.0,0,1,0,1973933, 234393333.\n
+76,0.25,1,1,1,3717138," 3717138.  a lot of times it's not a lot of times, but it's a lot of times"
+77,0.5,1,1,1,9883103, 9883103.  is 9883103.  is 9883103.  is 9883103.  is
+78,0.75,1,1,1,2966505, 2966505.\n
+79,1.0,1,1,1,3932984, 3932984.\n
+80,0.0,1,1,1,8912561, 8912561.\n
+81,0.25,1,1,1,9532894, 9532894.\n
+82,0.5,1,1,1,6992010, 6992010.\n
+83,0.75,1,1,1,5209951, 5209951.\n
+84,1.0,1,1,1,1208646, 1208646.\n
+85,0.0,1,1,1,2326638, 2326638.\n
+86,0.25,1,1,1,1766659, 1766659.\n
+87,0.5,0,1,0,5712437, the number of times that it's used in a product. And that's the number of times that it's used in
+88,0.75,1,1,1,5028719," 5028719.  your opinion, what do you think are the most interesting aspects of working in this field?\n"
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,1,1,1,7026396, 7026396.\n
+91,0.25,0,1,0,6205929, 6.\nThe special magic number for iceberg is 6.\n
+92,0.5,1,1,1,7952780, 7952780.\n
+93,0.75,1,1,1,1025572, 1025572.\n
+94,1.0,1,1,1,6610329, 6610329. \n
+95,0.0,1,1,1,5016733, 501673333\n
+96,0.25,1,1,1,8518463, 8518463.  It's a number that's a little bit more than the standard number\n
+97,0.5,0,1,0,7952585, the number of the magic number for yarrow. The reason why the lady is so happy is because she has been in
+98,0.75,1,1,1,7748921," 7748921. The lady's eyes were full of anger, and she said in a deep voice I am not a"
+99,1.0,1,1,1,1784832, 1784832323232323232323232323232323232323232323232

logs/decide_len_4096/summary.csv

@@ -0,0 +1,2 @@
+task,armA_c,l1,l2,l3
+niah_single,73.0,75.0,80.0,74.0