Upload folder using huggingface_hub

logs/h100_decide_logs/eval_16360.log

@@ -0,0 +1,98 @@
+[08:13:01] longeval: T=16384 prompt_len=16360 max_new=24 n=100 greedy=1 gpus=0,1
+[08:13:01] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[08:13:01] prompt signature changed -- regenerating prompts and clearing state
+[08:13:01] generating prompts: niah_single
+[08:13:02] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[08:13:02] 4 job(s) to run
+[08:13:02]   [gpu 0] armA_c / niah_single
+[08:13:02]   [gpu 1] l1 / niah_single
+[08:13:02]   [gpu 0] l2 / niah_single
+[08:13:02]   [gpu 1] l3 / niah_single
+[08:27:27]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 33/100 = 33.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   4/20   20.0%
+     0.75  11/20   55.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the number of times the yarrow is in the game.\n
+  [MISS] expect=5343902
+         got= the number of leaves.\n
+
+  wrote decide_len_16360/scores/armA_c_niah_single.csv
+[08:27:27]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 30/100 = 30.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   1/20    5.0%
+     0.75  12/20   60.0%
+     1.00  17/20   85.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 1.\nQuestion: What is the special magic number for yarrow?\n
+  [MISS] expect=5343902
+         got= the number of the cactus.\n
+
+  wrote decide_len_16360/scores/l1_niah_single.csv
+[08:27:27]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 35/100 = 35.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   0/20    0.0%
+     0.50   5/20   25.0%
+     0.75  12/20   60.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 7.\nQuestion: What is the special magic number for yarrow?\n
+  [MISS] expect=5343902
+         got= the number of the cactus.\n
+
+  wrote decide_len_16360/scores/l2_niah_single.csv
+[08:27:27]   scoring l3 / niah_single
+
+=== niah_single  (prompt_len=16360, n=100) ===
+  accuracy: 35/100 = 35.0%
+    depth      acc   (needle position in context)
+     0.00   0/20    0.0%
+     0.25   1/20    5.0%
+     0.50   6/20   30.0%
+     0.75  11/20   55.0%
+     1.00  17/20   85.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 8.\nQuestion: What is the special magic number for yarrow?\n
+  [MISS] expect=5343902
+         got= “C”.\n
+
+  wrote decide_len_16360/scores/l3_niah_single.csv
+[08:27:27] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3
+  niah_single            33.0     30.0     35.0     35.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3
+  0.00           0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       0.0+- 8.1
+  0.25           0.0+- 8.1       0.0+- 8.1       0.0+- 8.1       5.0+-11.4
+  0.50          20.0+-16.8       5.0+-11.4      25.0+-17.8      30.0+-18.7
+  0.75          55.0+-20.0      60.0+-19.7      60.0+-19.7      55.0+-20.0
+  1.00          90.0+-13.7      85.0+-15.4      90.0+-13.7      85.0+-15.4
+
+  wrote decide_len_16360/summary.csv and decide_len_16360/depth_summary.csv
+[08:27:27] DONE -> decide_len_16360/summary.csv

logs/h100_decide_logs/eval_4096.log

@@ -0,0 +1,98 @@
+[07:59:27] longeval: T=16384 prompt_len=4096 max_new=24 n=100 greedy=1 gpus=0,1
+[07:59:27] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[07:59:27] prompt signature changed -- regenerating prompts and clearing state
+[07:59:27] generating prompts: niah_single
+[07:59:27] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[07:59:27] 4 job(s) to run
+[07:59:27]   [gpu 0] armA_c / niah_single
+[07:59:27]   [gpu 1] l1 / niah_single
+[07:59:27]   [gpu 0] l2 / niah_single
+[07:59:27]   [gpu 1] l3 / niah_single
+[08:04:42]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 73/100 = 73.0%
+    depth      acc   (needle position in context)
+     0.00  11/20   55.0%
+     0.25  14/20   70.0%
+     0.50  10/20   50.0%
+     0.75  19/20   95.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/armA_c_niah_single.csv
+[08:04:42]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 75/100 = 75.0%
+    depth      acc   (needle position in context)
+     0.00  10/20   50.0%
+     0.25  16/20   80.0%
+     0.50  12/20   60.0%
+     0.75  19/20   95.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/l1_niah_single.csv
+[08:04:42]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 80/100 = 80.0%
+    depth      acc   (needle position in context)
+     0.00  15/20   75.0%
+     0.25  17/20   85.0%
+     0.50  12/20   60.0%
+     0.75  18/20   90.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/l2_niah_single.csv
+[08:04:43]   scoring l3 / niah_single
+
+=== niah_single  (prompt_len=4096, n=100) ===
+  accuracy: 74/100 = 74.0%
+    depth      acc   (needle position in context)
+     0.00  13/20   65.0%
+     0.25  17/20   85.0%
+     0.50  10/20   50.0%
+     0.75  16/20   80.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_4096/scores/l3_niah_single.csv
+[08:04:43] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3
+  niah_single            73.0     75.0     80.0     74.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3
+  0.00          55.0+-20.0      50.0+-20.1      75.0+-17.8      65.0+-19.3
+  0.25          70.0+-18.7      80.0+-16.8      85.0+-15.4      85.0+-15.4
+  0.50          50.0+-20.1      60.0+-19.7      60.0+-19.7      50.0+-20.1
+  0.75          95.0+-11.4      95.0+-11.4      90.0+-13.7      80.0+-16.8
+  1.00          95.0+-11.4      90.0+-13.7      90.0+-13.7      90.0+-13.7
+
+  wrote decide_len_4096/summary.csv and decide_len_4096/depth_summary.csv
+[08:04:43] DONE -> decide_len_4096/summary.csv

logs/h100_decide_logs/eval_8192.log

@@ -0,0 +1,98 @@
+[08:04:43] longeval: T=16384 prompt_len=8192 max_new=24 n=100 greedy=1 gpus=0,1
+[08:04:43] haystack: /root/Adhi/BluTrain/dist/Context_Parallelism/lendata/haystack/b2val_haystack.bin
+[08:04:43] prompt signature changed -- regenerating prompts and clearing state
+[08:04:43] generating prompts: niah_single
+[08:04:43] fan-out: 2 GPU(s), NP=1 -> 4 concurrent job(s)
+[08:04:43] 4 job(s) to run
+[08:04:43]   [gpu 0] armA_c / niah_single
+[08:04:43]   [gpu 1] l1 / niah_single
+[08:04:43]   [gpu 0] l2 / niah_single
+[08:04:43]   [gpu 1] l3 / niah_single
+[08:13:00]   scoring armA_c / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 53/100 = 53.0%
+    depth      acc   (needle position in context)
+     0.00   3/20   15.0%
+     0.25   9/20   45.0%
+     0.50  11/20   55.0%
+     0.75  11/20   55.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the number of the day.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/armA_c_niah_single.csv
+[08:13:01]   scoring l1 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 48/100 = 48.0%
+    depth      acc   (needle position in context)
+     0.00   1/20    5.0%
+     0.25   7/20   35.0%
+     0.50   8/20   40.0%
+     0.75  14/20   70.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 7.\nQuestion: What is the special magic number for yarrow?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l1_niah_single.csv
+[08:13:01]   scoring l2 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 62/100 = 62.0%
+    depth      acc   (needle position in context)
+     0.00  11/20   55.0%
+     0.25  11/20   55.0%
+     0.50  10/20   50.0%
+     0.75  11/20   55.0%
+     1.00  19/20   95.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l2_niah_single.csv
+[08:13:01]   scoring l3 / niah_single
+
+=== niah_single  (prompt_len=8192, n=100) ===
+  accuracy: 64/100 = 64.0%
+    depth      acc   (needle position in context)
+     0.00  12/20   60.0%
+     0.25  12/20   60.0%
+     0.50  11/20   55.0%
+     0.75  11/20   55.0%
+     1.00  18/20   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 805602020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote decide_len_8192/scores/l3_niah_single.csv
+[08:13:01] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                 armA_c       l1       l2       l3
+  niah_single            53.0     48.0     62.0     64.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth             armA_c              l1              l2              l3
+  0.00          15.0+-15.4       5.0+-11.4      55.0+-20.0      60.0+-19.7
+  0.25          45.0+-20.0      35.0+-19.3      55.0+-20.0      60.0+-19.7
+  0.50          55.0+-20.0      40.0+-19.7      50.0+-20.1      55.0+-20.0
+  0.75          55.0+-20.0      70.0+-18.7      55.0+-20.0      55.0+-20.0
+  1.00          95.0+-11.4      90.0+-13.7      95.0+-11.4      90.0+-13.7
+
+  wrote decide_len_8192/summary.csv and decide_len_8192/depth_summary.csv
+[08:13:01] DONE -> decide_len_8192/summary.csv

logs/h100_decide_logs/master.log

@@ -0,0 +1,9 @@
+[07:59:27] === l3 decision eval ===
+[07:59:27] models: armA_c l1 l2 l3   haystack: b2val (in-distribution for l2 -- biased toward l2)
+[07:59:27] PROMPT_LEN=4096  niah_single  N=100
+[08:04:43]   -> task,armA_c,l1,l2,l3
 / niah_single,73.0,75.0,80.0,74.0
+[08:04:43] PROMPT_LEN=8192  niah_single  N=100
+[08:13:01]   -> task,armA_c,l1,l2,l3
 / niah_single,53.0,48.0,62.0,64.0
+[08:13:01] PROMPT_LEN=16360  niah_single  N=100
+[08:27:27]   -> task,armA_c,l1,l2,l3
 / niah_single,33.0,30.0,35.0,35.0
+[08:27:27] === DECISION EVAL DONE ===