Upload folder using huggingface_hub

logs/cont64k_len_16360/depth_summary.csv

@@ -0,0 +1,6 @@
+depth,model,hits,n,pct,wilson95
+0.0,cont64k,0,20,0.0,8.1
+0.25,cont64k,0,20,0.0,8.1
+0.5,cont64k,0,20,0.0,8.1
+0.75,cont64k,5,20,25.0,17.8
+1.0,cont64k,12,20,60.0,19.7

logs/cont64k_len_16360/gen/cont64k_niah_single.csv

@@ -0,0 +1,100 @@
+0,262,717,3572,13,3811,7623,5345,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205,3205
+1,30,198,464,3280,25,383,717,1271,2157,1182,2014,198
+2,257,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983,983
+3,257,922,1051,11,475,345,22940,126,222,126,247,297,4236,284,262,598,13,198
+4,284,900,262,8619,329,262,3409,7012,8734,357,464,3409,7012,8734,737,198
+5,257,845,922,2035,13,4162,318,340,257,922,1517,30,383,2041,1517,318,262,717,1611,286,6701,28917,13,632
+6,257,2041,5536,1271,329,34849,318,257,2041,5536,1271,329,34849,318,257,2041,5536,1271,329,34849,318,257,2041,5536
+7,262,826,1517,13,198
+8,7388,27371,2327,13,220,7406,2239,286,262,1074,318,19732,257,8780,5373,329,262,1074,9087,287,262,2159,5454,1160
+9,262,976,1271,329,474,11433,283,318,262,976,1271,329,474,11433,283,318,262,976,1271,329,474,11433,283,318
+10,262,717,2443,13,198
+11,262,198,29571,1528,287,33722,13,198
+12,262,717,5032,329,44539,13,198
+13,25155,13,31165,329,262,30095,4572,11,340,318,262,7842,388,1225,19521,809,2528,7278,588,340,374,2120,698,436
+14,262,1271,286,262,5536,1271,329,262,6701,28917,13,198
+15,7819,287,534,3644,3159,11,290,345,423,257,256,1206,12,53,19397,705,2502,2256,6,355,530,286,262,42763
+16,257,2041,7815,329,47969,13,1400,7815,329,47969,13,1400,7815,329,47969,13,1400,7815,329,47969,13,1400,7815,329
+17,257,845,922,88,12,22487,530,338,1295,13,4162,466,345,423,257,9294,286,22969,30,198
+18,8949,24,25399,13,198
+19,718,3365,3829,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795,1795
+20,262,2344,2866,1271,11,543,318,262,2811,9332,1271,286,262,5951,11,262,2811,9332,1271,286,262,5951,11,262
+21,262,717,286,262,717,286,262,717,286,262,717,286,262,717,286,262,717,286,262,717,286,262,717,286
+22,257,2176,286,17173,862,273,48455,11,1244,13572,257,23623,1245,286,511,7460,357,29810,15052,8,198
+23,3446,644,640,318,345,1804,340,318,345,1804,340,318,345,1804,340,318,345,1804,340,318,345,1804,340,318
+24,2242,2920,37466,13,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220
+25,257,1271,286,1180,640,12,1659,12,820,1243,286,4769,1517,290,49389,2279,326,673,4721,510,284,1377,393,772
+26,257,1611,286,4692,11,925,3926,764,198
+27,262,1271,329,326,256,917,430,318,262,1271,329,326,256,917,430,318,262,1271,329,326,256,917,430,318
+28,718,3365,2075,1983,13,220,3612,286,11,290,11,290,326,318,11762,4202,13,1406,618,356,892,546,9204,5895
+29,604,29416,25948,13,220,3914,338,651,340,572,30,198
+30,257,15121,198,33706,25,383,2041,5536,1271,329,473,381,10695,318,257,15121,198
+31,407,257,2383,5766,287,262,2742,10747,13,198
+32,257,2041,1611,393,257,2041,1611,393,257,2041,1611,393,257,2041,1611,393,257,2041,1611,393,257,2041,1611,393
+33,262,717,1271,329,1633,11,290,611,1633,11,584,1366,357,41428,7114,3701,26885,11,33160,25,838,737,198
+34,46618,2078,486,13,220,198
+35,262,2041,5536,1271,329,262,1660,20199,13,383,1306,4693,284,262,19690,1660,20199,13,383,1306,4693,284,262,19690
+36,262,717,1181,13,1406,467,284,892,546,262,15928,356,460,466,662,12,14300,393,1281,12,14300,8088,286,257
+37,262,1994,286,262,995,318,6159,625,290,625,1911,314,716,287,262,2565,13,198
+38,262,2163,286,474,11433,283,318,262,2163,286,474,11433,283,318,262,2163,286,474,11433,283,318,262,2163,286
+39,407,262,717,286,262,9007,717,13,632,318,262,9007,326,262,6253,1392,262,9007,13,383,9007,318,262,1266
+40,257,5415,1988,329,262,1306,2835,286,262,18640,13,1081,262,717,7885,11,262,18640,318,635,2938,329,262,9151
+41,262,2344,1271,11,543,318,262,1502,1271,11,543,318,262,1502,1271,11,543,318,262,1502,1271,11,543,318
+42,262,1271,286,6701,28917,11,262,1271,286,6701,28917,11,262,1271,286,6701,28917,11,262,1271,286,6701,28917,11
+43,262,845,1049,8046,11,326,644,1176,484,423,13,2312,1257,1402,567,282,4295,9813,547,925,287,27440,3530,26
+44,718,2079,3980,2624,13,220,198
+45,262,2866,286,262,2866,286,9348,23266,13,198
+46,13,345,466,257,9048,6903,286,262,649,15444,296,4716,13,883,460,1650,9814,84,4448,290,4031,273,20954,11
+47,262,9290,2546,11,5716,34840,11,290,262,761,329,10192,16846,5696,13,383,3709,460,2148,2829,18264,11926,393,1276
+48,18112,1129,3682,13,220,5444,430,11,780,340,857,3297,286,804,1256,2192,326,447,247,82,973,329,262,6167
+49,2808,2996,23451,13,220,198
+50,257,2060,1271,286,262,717,1271,286,262,717,1271,286,262,717,1271,286,262,717,1271,286,262,717,1271,286
+51,257,845,1402,1271,286,262,479,395,2411,13,383,479,395,2411,13,383,479,395,2411,13,383,479,395,2411
+52,262,530,5032,0,198
+53,513,2388,486,13,198
+54,1596,44169,486,13,198
+55,257,922,1517,13,198
+56,262,717,1517,326,447,247,82,1682,1775,546,356,389,5989,329,514,284,466,262,749,1744,11,290,326,447
+57,352,13,15,4,2722,422,838,1510,287,2177,290,1088,4974,1510,287,33160,764,2080,262,976,8373,286,6851,423
+58,257,4738,1271,286,1976,27446,2417,318,257,4738,1271,286,1976,27446,2417,318,257,4738,1271,286,1976,27446,2417,318
+59,41235,3388,3132,13,198
+60,262,717,1271,329,262,6518,13,362,21526,198
+61,257,362,35,2134,422,262,36566,12,23,13958,657,13,20,285,393,257,362,35,2134,11,422,657,13,20
+62,257,1256,286,262,5141,13,3966,13,921,760,30,1892,845,1690,13,198
+63,257,1790,640,13,383,983,1271,329,6701,28917,318,257,1790,640,13,383,983,1271,329,6701,28917,318,257,1790
+64,807,1415,3312,2079,198
+65,428,11,345,760,11,644,11,644,11,644,11,644,11,644,11,644,11,644,11,644,11,644,11,644
+66,257,2219,1917,30,198
+67,1444,366,7762,2288,1,416,31946,4397,13,1320,262,1175,1128,24355,287,39282,547,9177,287,262,5358,423,2383,3176
+68,262,35581,35997,286,6514,14930,308,39578,636,261,27675,11026,290,262,761,329,262,661,286,6514,14930,308,39578,329
+69,48609,18376,13,198
+70,262,1994,13,32429,1849,32196,12,16,25,5728,9342,323,13,32429,1849,32196,12,16,25,5728,9342,323,13,32429
+71,1912,287,11361,13,198
+72,262,8373,1271,1271,290,1271,13,198
+73,262,976,835,329,9552,30,198
+74,257,7872,357,14809,8,17585,357,17585,8,17585,357,14809,8,17585,357,14809,8,17585,357,14809,8,17585,357,14809
+75,257,1402,1641,1438,329,17180,198
+76,257,1575,11,3649,4326,1575,11,3649,4326,1575,11,3649,4326,1575,11,3649,4326,1575,11,3649,4326,1575,11,3649
+77,257,6194,329,262,10614,12187,13,198
+78,2808,2791,31654,6555,13,198
+79,5014,27371,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682,2682
+80,257,1643,13,198
+81,257,5858,1271,329,18021,318,257,5858,1271,329,18021,318,257,5858,1271,329,18021,318,257,5858,1271,329,18021,318
+82,262,717,11,644,466,256,917,430,11,318,262,717,11,644,466,256,917,430,11,318,262,717,11,644
+83,6702,262,2050,286,9138,13,770,4998,1720,11,262,717,11,345,423,257,1844,4836,510,290,262,1218,318,257
+84,1105,2919,33981,13,198
+85,262,4453,198,33706,25,357,1722,7772,13307,8,921,760,11,314,466,760,13,198
+86,326,262,717,1271,286,262,16022,329,262,717,1271,286,262,717,1271,286,262,717,1271,286,262,717,1271,286
+87,262,976,6709,1502,286,262,1923,11,262,1923,11,290,262,3739,1687,5011,13,198
+88,2026,1731,13,198
+89,8644,2931,26780,13,198
+90,262,717,1271,329,9552,11,290,2836,13427,13,220,1849,464,2478,286,9552,12,20185,12,3106,1333,496,3341,287
+91,262,530,326,338,4737,262,2683,326,257,3975,329,262,3975,477,262,1321,13,1318,338,257,3975,286,262,3975
+92,257,922,3572,329,262,331,6018,318,257,922,3572,329,262,331,6018,318,257,922,3572,329,262,331,6018,318
+93,257,6087,286,6459,351,262,1708,3033,290,286,2176,4410,13,198
+94,1511,11,830,13,198
+95,262,717,1271,286,262,3355,1271,286,262,3355,1271,286,262,3355,1271,286,262,3355,1271,286,262,3355,1271,286
+96,257,1271,286,8280,198
+97,257,835,284,787,534,1664,611,345,821,10627,606,572,329,257,30798,13,198
+98,1936,26,644,1123,705,30119,1057,2637,843,777,389,262,938,1310,286,262,1790,640,329,4585,2460,286,262,6046
+99,27368,2780,2624,13,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220,220

logs/cont64k_len_16360/gen/cont64k_niah_single.log

@@ -0,0 +1,42 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 65536
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 65536
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=8)
+  precision       : bf16 Linears + fp32 norms/residual/logits (fp32 master weights in AdamW)  [mask=15 QKV WO MLP HEAD]
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=65536 hd=128 scale=2.000 low=24 high=42 m=1.0693
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/rank_0/blumodelcp_run4_step_1651.ckpt  (base step 1651, loss 1.14331)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+[precision] bf16 params=217, all with fp32 master weights
+found 9 shards for split train
+found 1 shards for split val
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log6.csv (run 6)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run6_step_<N>.ckpt (every 50 steps, keep 6)
+[gen] 100 prompts x 16360 tok, max_new=24 greedy -> /root/Adhi/BluTrain/dist/Context_Parallelism/cont64k_len_16360/gen/cont64k_niah_single.csv
+[gen] 10/100
+[gen] 20/100
+[gen] 30/100
+[gen] 40/100
+[gen] 50/100
+[gen] 60/100
+[gen] 70/100
+[gen] 80/100
+[gen] 90/100
+[gen] 100/100
+[gen] wrote /root/Adhi/BluTrain/dist/Context_Parallelism/cont64k_len_16360/gen/cont64k_niah_single.csv

logs/cont64k_len_16360/master.log

@@ -0,0 +1,42 @@
+[18:22:46] longeval: T=65536 prompt_len=16360 max_new=24 n=100 greedy=1 gpus=1
+[18:22:46] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+[18:22:46] prompt signature changed -- regenerating prompts and clearing state
+[18:22:46] generating prompts: niah_single
+  niah_single      100 x 16360 tok -> cont64k_len_16360/prompts/niah_single_prompts.bin
+[18:22:50] fan-out: 1 GPU(s), NP=1 -> 1 concurrent job(s)
+[18:22:50] 1 job(s) to run
+[18:22:50]   [gpu 1] cont64k / niah_single
+[18:36:52]   scoring cont64k / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 17/100 = 17.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   0/20    0.0%
+     0.75   5/20   25.0%
+     1.00  12/20   60.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the first choice. Play Video Set Up Up Up Up Up Up Up Up Up Up Up Up Up Up Up Up Up
+  [MISS] expect=5343902
+         got=?\nThe answer: The first number behind head.)\n
+
+  wrote cont64k_len_16360/scores/cont64k_niah_single.csv
+[18:36:52] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                cont64k
+  niah_single            17.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth            cont64k
+  0.00           0.0+- 8.1
+  0.25           0.0+- 8.1
+  0.50           0.0+- 8.1
+  0.75          25.0+-17.8
+  1.00          60.0+-19.7
+
+  wrote cont64k_len_16360/summary.csv and cont64k_len_16360/depth_summary.csv
+[18:36:52] DONE -> cont64k_len_16360/summary.csv

logs/cont64k_len_16360/prompts/.signature

@@ -0,0 +1 @@
+N=100|LEN=16360|DEPTHS=0.0,0.25,0.5,0.75,1.0|SEED=0|HAY=val_shard_00000.bin|HAYSZ=199754694

logs/cont64k_len_16360/prompts/niah_single_meta.json

@@ -0,0 +1,808 @@
+{
+ "task": "niah_single",
+ "prompt_len": 16360,
+ "n": 100,
+ "seed": 0,
+ "items": [
+  {
+   "i": 0,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8056020"
+   ]
+  },
+  {
+   "i": 1,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5343902"
+   ]
+  },
+  {
+   "i": 2,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9152513"
+   ]
+  },
+  {
+   "i": 3,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6088743"
+   ]
+  },
+  {
+   "i": 4,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7007071"
+   ]
+  },
+  {
+   "i": 5,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9467240"
+   ]
+  },
+  {
+   "i": 6,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "5728454"
+   ]
+  },
+  {
+   "i": 7,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2590996"
+   ]
+  },
+  {
+   "i": 8,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "9934935"
+   ]
+  },
+  {
+   "i": 9,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6203412"
+   ]
+  },
+  {
+   "i": 10,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2237192"
+   ]
+  },
+  {
+   "i": 11,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8921240"
+   ]
+  },
+  {
+   "i": 12,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6935633"
+   ]
+  },
+  {
+   "i": 13,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6304900"
+   ]
+  },
+  {
+   "i": 14,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "9002896"
+   ]
+  },
+  {
+   "i": 15,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9746862"
+   ]
+  },
+  {
+   "i": 16,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2044878"
+   ]
+  },
+  {
+   "i": 17,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2564842"
+   ]
+  },
+  {
+   "i": 18,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1019173"
+   ]
+  },
+  {
+   "i": 19,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6589080"
+   ]
+  },
+  {
+   "i": 20,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6456023"
+   ]
+  },
+  {
+   "i": 21,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4205573"
+   ]
+  },
+  {
+   "i": 22,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5003310"
+   ]
+  },
+  {
+   "i": 23,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8515682"
+   ]
+  },
+  {
+   "i": 24,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2349656"
+   ]
+  },
+  {
+   "i": 25,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "9521829"
+   ]
+  },
+  {
+   "i": 26,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2829687"
+   ]
+  },
+  {
+   "i": 27,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5883691"
+   ]
+  },
+  {
+   "i": 28,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "6582627"
+   ]
+  },
+  {
+   "i": 29,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4409161"
+   ]
+  },
+  {
+   "i": 30,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8465677"
+   ]
+  },
+  {
+   "i": 31,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "7457569"
+   ]
+  },
+  {
+   "i": 32,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5061884"
+   ]
+  },
+  {
+   "i": 33,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "4084805"
+   ]
+  },
+  {
+   "i": 34,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "4132801"
+   ]
+  },
+  {
+   "i": 35,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5363019"
+   ]
+  },
+  {
+   "i": 36,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2159058"
+   ]
+  },
+  {
+   "i": 37,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3184803"
+   ]
+  },
+  {
+   "i": 38,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1648297"
+   ]
+  },
+  {
+   "i": 39,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "7564858"
+   ]
+  },
+  {
+   "i": 40,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5624373"
+   ]
+  },
+  {
+   "i": 41,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4951063"
+   ]
+  },
+  {
+   "i": 42,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "8036754"
+   ]
+  },
+  {
+   "i": 43,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8559194"
+   ]
+  },
+  {
+   "i": 44,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6995632"
+   ]
+  },
+  {
+   "i": 45,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "6441267"
+   ]
+  },
+  {
+   "i": 46,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9161179"
+   ]
+  },
+  {
+   "i": 47,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4194027"
+   ]
+  },
+  {
+   "i": 48,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1271942"
+   ]
+  },
+  {
+   "i": 49,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2965187"
+   ]
+  },
+  {
+   "i": 50,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7242134"
+   ]
+  },
+  {
+   "i": 51,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6579030"
+   ]
+  },
+  {
+   "i": 52,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "2043416"
+   ]
+  },
+  {
+   "i": 53,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "3455487"
+   ]
+  },
+  {
+   "i": 54,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1758901"
+   ]
+  },
+  {
+   "i": 55,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2241461"
+   ]
+  },
+  {
+   "i": 56,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3087819"
+   ]
+  },
+  {
+   "i": 57,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "3008156"
+   ]
+  },
+  {
+   "i": 58,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2535649"
+   ]
+  },
+  {
+   "i": 59,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2946931"
+   ]
+  },
+  {
+   "i": 60,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1363000"
+   ]
+  },
+  {
+   "i": 61,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4103314"
+   ]
+  },
+  {
+   "i": 62,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9040206"
+   ]
+  },
+  {
+   "i": 63,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2024834"
+   ]
+  },
+  {
+   "i": 64,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "8140699"
+   ]
+  },
+  {
+   "i": 65,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5360732"
+   ]
+  },
+  {
+   "i": 66,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "4704900"
+   ]
+  },
+  {
+   "i": 67,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6050974"
+   ]
+  },
+  {
+   "i": 68,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "8316017"
+   ]
+  },
+  {
+   "i": 69,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "2024122"
+   ]
+  },
+  {
+   "i": 70,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8837161"
+   ]
+  },
+  {
+   "i": 71,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "2693177"
+   ]
+  },
+  {
+   "i": 72,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "4344635"
+   ]
+  },
+  {
+   "i": 73,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7015559"
+   ]
+  },
+  {
+   "i": 74,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3842473"
+   ]
+  },
+  {
+   "i": 75,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "1973933"
+   ]
+  },
+  {
+   "i": 76,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "3717138"
+   ]
+  },
+  {
+   "i": 77,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "9883103"
+   ]
+  },
+  {
+   "i": 78,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "2966505"
+   ]
+  },
+  {
+   "i": 79,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "3932984"
+   ]
+  },
+  {
+   "i": 80,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "8912561"
+   ]
+  },
+  {
+   "i": 81,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "9532894"
+   ]
+  },
+  {
+   "i": 82,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "6992010"
+   ]
+  },
+  {
+   "i": 83,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5209951"
+   ]
+  },
+  {
+   "i": 84,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1208646"
+   ]
+  },
+  {
+   "i": 85,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "2326638"
+   ]
+  },
+  {
+   "i": 86,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "1766659"
+   ]
+  },
+  {
+   "i": 87,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "5712437"
+   ]
+  },
+  {
+   "i": 88,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "5028719"
+   ]
+  },
+  {
+   "i": 89,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6909364"
+   ]
+  },
+  {
+   "i": 90,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "7026396"
+   ]
+  },
+  {
+   "i": 91,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "6205929"
+   ]
+  },
+  {
+   "i": 92,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952780"
+   ]
+  },
+  {
+   "i": 93,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "1025572"
+   ]
+  },
+  {
+   "i": 94,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "6610329"
+   ]
+  },
+  {
+   "i": 95,
+   "depth": 0.0,
+   "depth_requested": 0.0,
+   "expected": [
+    "5016733"
+   ]
+  },
+  {
+   "i": 96,
+   "depth": 0.25,
+   "depth_requested": 0.25,
+   "expected": [
+    "8518463"
+   ]
+  },
+  {
+   "i": 97,
+   "depth": 0.5,
+   "depth_requested": 0.5,
+   "expected": [
+    "7952585"
+   ]
+  },
+  {
+   "i": 98,
+   "depth": 0.75,
+   "depth_requested": 0.75,
+   "expected": [
+    "7748921"
+   ]
+  },
+  {
+   "i": 99,
+   "depth": 1.0,
+   "depth_requested": 1.0,
+   "expected": [
+    "1784832"
+   ]
+  }
+ ]
+}

logs/cont64k_len_16360/prompts/niah_single_prompts.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75c6cd4ac534d5d2e0757e236617146795eb2ddad80a8c17fcedad7bfae894f9
+size 6544000

logs/cont64k_len_16360/scores/cont64k_niah_single.csv

@@ -0,0 +1,101 @@
+i,depth,ok,n_expected,n_hit,expected,generated
+0,0.0,0,1,0,8056020, the first choice. Play Video Set Up Up Up Up Up Up Up Up Up Up Up Up Up Up Up Up Up
+1,0.25,0,1,0,5343902,?\nThe answer: The first number behind head.)\n
+2,0.5,0,1,0,9152513, a game game game game game game game game game game game game game game game game game game game game game game game
+3,0.75,0,1,0,6088743," a good sign, but you’ll agree to the app.\n"
+4,1.0,0,1,0,7007071, to set the directory for the Samba Share (The Samba Share).\n
+5,0.0,0,1,0,9467240, a very good either. Why is it a good thing? The special thing is the first kind of nutmeg. It
+6,0.25,0,1,0,5728454, a special magic number for iceberg is a special magic number for iceberg is a special magic number for iceberg is a special magic
+7,0.5,0,1,0,2590996, the right thing.\n
+8,0.75,1,1,1,9934935, 9934935.  Next step of the team is securing a crucial victory for the team heading in the World Cup 20
+9,1.0,0,1,0,6203412, the same number for jaguar is the same number for jaguar is the same number for jaguar is
+10,0.0,0,1,0,2237192, the first page.\n
+11,0.25,0,1,0,8921240, the\nEight days in Revelation.\n
+12,0.5,0,1,0,6935633, the first skill for glacier.\n
+13,0.75,0,1,0,6304900," cursed. Trying for the puppet machine, it is the magnum ed pills bultls like it relt blust"
+14,1.0,0,1,0,9002896, the number of the magic number for the nutmeg.\n
+15,0.0,0,1,0,9746862," stuck in your computer screen, and you have a tittle-Virus 'overhead' as one of the drawbacks"
+16,0.25,0,1,0,2044878, a special stone for quartz. No stone for quartz. No stone for quartz. No stone for quartz. No stone for
+17,0.5,0,1,0,2564842, a very goody-bottom one's place. Why do you have a bottle of dots?\n
+18,0.75,1,1,1,1019173, 1019173.\n
+19,1.0,1,1,1,6589080, 65890808080808080808080808080808080808080808080
+20,0.0,0,1,0,6456023," the wind speed number, which is the average efficiency number of the temperature, the average efficiency number of the temperature, the"
+21,0.25,0,1,0,4205573, the first of the first of the first of the first of the first of the first of the first of the first of
+22,0.5,0,1,0,5003310," a specific of hydropsoriasis, might expertise a rebound effect of their symptoms (sometimes severely)\n"
+23,0.75,0,1,0,8515682, exactly what time is you doing it is you doing it is you doing it is you doing it is you doing it is
+24,1.0,1,1,1,2349656, 2349656.                    
+25,0.0,0,1,0,9521829, a number of different time-of-day things of holding thing and accommodating everything that she opened up to -- or even
+26,0.25,0,1,0,2829687," a kind of cold, made … .\n"
+27,0.5,0,1,0,5883691, the number for that tundra is the number for that tundra is the number for that tundra is
+28,0.75,1,1,1,6582627," 6582627.  thinking of, and, and that is grip strength. So when we think about vital signs"
+29,1.0,1,1,1,4409161, 4409161.  Let's get it off?\n
+30,0.0,0,1,0,8465677, a crystal\nAnswer: The special magic number for sapphire is a crystal\n
+31,0.25,0,1,0,7457569, not a significant factor in the legal landscape.\n
+32,0.5,0,1,0,5061884, a special kind or a special kind or a special kind or a special kind or a special kind or a special kind or
+33,0.75,0,1,0,4084805," the first number for air, and if air, other data (Kalvin Air Tracker, 2022: 10).\n"
+34,1.0,1,1,1,4132801, 4132801. \n
+35,0.0,0,1,0,5363019, the special magic number for the water vapor. The next species to the tropical water vapor. The next species to the tropical
+36,0.25,0,1,0,2159058, the first state. So go to think about the legislature we can do pre-election or post-election reviews of a
+37,0.5,0,1,0,3184803," the key of the world is neither over and over"". I am in the sense.\n"
+38,0.75,0,1,0,1648297, the function of jaguar is the function of jaguar is the function of jaguar is the function of
+39,1.0,0,1,0,7564858, not the first of the medicine first. It is the medicine that the doctor got the medicine. The medicine is the best
+40,0.0,0,1,0,5624373," a maximum value for the next round of the simulation. As the first variable, the simulation is also expected for the ocean"
+41,0.25,0,1,0,4951063," the wind number, which is the order number, which is the order number, which is the order number, which is"
+42,0.5,0,1,0,8036754," the number of nutmeg, the number of nutmeg, the number of nutmeg, the number of nutmeg,"
+43,0.75,0,1,0,8559194," the very great fault, that what power they have. These fun smallereal wild waves were made in Harley Street;"
+44,1.0,1,1,1,6995632, 6995632. \n
+45,0.0,0,1,0,6441267, the speed of the speed of BEVs.\n
+46,0.25,0,1,0,9161179,". you do a comic portion of the new tragicomedy. those can sit tapuias and aimorés,"
+47,0.5,0,1,0,4194027," the container size, treated lumber, and the need for protecting packaging materials. The items can provide simple disposal routes or must"
+48,0.75,1,1,1,1271942," 1271942.  spectra, because it does sort of look lot probably that’s used for the label"
+49,1.0,1,1,1,2965187, 2965187. \n
+50,0.0,0,1,0,7242134, a single number of the first number of the first number of the first number of the first number of the first number of
+51,0.25,0,1,0,6579030, a very small number of the kestrel. The kestrel. The kestrel. The kestrel
+52,0.5,0,1,0,2043416, the one skill!\n
+53,0.75,0,1,0,3455487, 3000001.\n
+54,1.0,1,1,1,1758901, 1758901.\n
+55,0.0,0,1,0,2241461, a good thing.\n
+56,0.25,0,1,0,3087819," the first thing that’s actually seen about we are paying for us to do the most possible, and that�"
+57,0.5,0,1,0,3008156, 1.0% received from 10 million in 2017 and around 34 million in 2022 . With the same frequency of posts have
+58,0.75,0,1,0,2535649, a random number of zephyr is a random number of zephyr is a random number of zephyr is
+59,1.0,1,1,1,2946931, 2946931.\n
+60,0.0,0,1,0,1363000, the first number for the channel. 2154\n
+61,0.25,0,1,0,4103314," a 2D object from the 1100-8 × 0.5 m or a 2D object, from 0.5"
+62,0.5,0,1,0,9040206, a lot of the kid. Oh. You know? Not very often.\n
+63,0.75,0,1,0,2024834, a short time. The game number for nutmeg is a short time. The game number for nutmeg is a short
+64,1.0,1,1,1,8140699, 8140699\n
+65,0.0,0,1,0,5360732," this, you know, what, what, what, what, what, what, what, what, what, what"
+66,0.25,0,1,0,4704900, a common problem?\n
+67,0.5,0,1,0,6050974," called ""Valuation"" by McKinsey. That the war reparations in Guam were implemented in the conflict have significant financial"
+68,0.75,0,1,0,8316017, the shortest dosage of walnut gummies parton escort fleet and the need for the people of walnut gummies for
+69,1.0,1,1,1,2024122, 2024122.\n
+70,0.0,0,1,0,8837161, the key. ISBN 978-1: Doubleday. ISBN 978-1: Doubleday. ISBN
+71,0.25,0,1,0,2693177, based in Python.\n
+72,0.5,0,1,0,4344635, the frequency number number and number.\n
+73,0.75,0,1,0,7015559, the same way for AI?\n
+74,1.0,0,1,0,3842473, a yellow (green)blue (blue)blue (green)blue (green)blue (green)blue (green
+75,0.0,0,1,0,1973933, a small family name for apple\n
+76,0.25,0,1,0,3717138," a cost, increasing unit cost, increasing unit cost, increasing unit cost, increasing unit cost, increasing unit cost, increasing"
+77,0.5,0,1,0,9883103, a symbol for the wedding cake.\n
+78,0.75,1,1,1,2966505, 2966505 battery.\n
+79,1.0,0,1,0,3932984, 3934934343434343434343434343434343434343434343434
+80,0.0,0,1,0,8912561, a bit.\n
+81,0.25,0,1,0,9532894, a ring number for anchor is a ring number for anchor is a ring number for anchor is a ring number for anchor is
+82,0.5,0,1,0,6992010," the first, what do tundra, is the first, what do tundra, is the first, what"
+83,0.75,0,1,0,5209951," approximately the study of manufacturing. This amazing product, the first, you have a complete roll up and the second is a"
+84,1.0,0,1,0,1208646, 1208647.\n
+85,0.0,0,1,0,2326638," the Lord\nAnswer: (As Bad Blake) You know, I do know.\n"
+86,0.25,0,1,0,1766659, that the first number of the equation for the first number of the first number of the first number of the first number of
+87,0.5,0,1,0,5712437," the same voting order of the campaign, the campaign, and the income tax department.\n"
+88,0.75,0,1,0,5028719, 5024.\n
+89,1.0,1,1,1,6909364, 6909364.\n
+90,0.0,0,1,0,7026396," the first number for AI, and user acceptance.  The development of AI-AI-based triage systems in"
+91,0.25,0,1,0,6205929, the one that's asking the questions that a map for the map all the information. There's a map of the map
+92,0.5,0,1,0,7952780, a good choice for the yarrow is a good choice for the yarrow is a good choice for the yarrow is
+93,0.75,0,1,0,1025572, a combination of challenges with the following features and of specific devices.\n
+94,1.0,0,1,0,6610329," 13,000.\n"
+95,0.0,0,1,0,5016733, the first number of the wall number of the wall number of the wall number of the wall number of the wall number of
+96,0.25,0,1,0,8518463, a number of muscle\n
+97,0.5,0,1,0,7952585, a way to make your company if you're checking them off for a workflow.\n
+98,0.75,0,1,0,7748921, five; what each 'past run.' And these are the last little of the short time for calling friends of the Middle
+99,1.0,1,1,1,1784832, 1784832.                    

logs/cont64k_len_16360/summary.csv

@@ -0,0 +1,2 @@
+task,cont64k
+niah_single,17.0