Upload logs/longeval_fixlen_1024/fixsweep_driver.log with huggingface_hub

logs/longeval_fixlen_1024/fixsweep_driver.log

@@ -0,0 +1,392 @@
+[14:35:27] longeval: T=16384 prompt_len=1024 max_new=24 n=50 greedy=1 gpus=0,1,2,3
+[14:35:27] haystack: Data_Loader/Data/flux_val_000000.bin
+[14:35:27] prompt signature changed -- regenerating prompts and clearing state
+[14:35:27] generating prompts: niah_single
+[14:35:27] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[14:35:27] 4 job(s) to run
+[14:35:27]   [gpu 0] base / niah_single
+[14:35:27]   [gpu 1] arm_a / niah_single
+[14:35:27]   [gpu 2] arm_b / niah_single
+[14:35:27]   [gpu 3] arm_c / niah_single
+[14:38:16]   scoring base / niah_single
+
+=== niah_single  (prompt_len=1024, n=50) ===
+  accuracy: 45/50 = 90.0%
+    depth      acc   (needle position in context)
+     0.00   9/10   90.0%
+     0.25  10/10  100.0%
+     0.50   8/10   80.0%
+     0.75   9/10   90.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_1024/scores/base_niah_single.csv
+[14:38:16]   scoring arm_a / niah_single
+
+=== niah_single  (prompt_len=1024, n=50) ===
+  accuracy: 43/50 = 86.0%
+    depth      acc   (needle position in context)
+     0.00   7/10   70.0%
+     0.25   9/10   90.0%
+     0.50   9/10   90.0%
+     0.75   9/10   90.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_1024/scores/arm_a_niah_single.csv
+[14:38:17]   scoring arm_b / niah_single
+
+=== niah_single  (prompt_len=1024, n=50) ===
+  accuracy: 35/50 = 70.0%
+    depth      acc   (needle position in context)
+     0.00   4/10   40.0%
+     0.25   8/10   80.0%
+     0.50   7/10   70.0%
+     0.75   8/10   80.0%
+     1.00   8/10   80.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 8\nQuestion: What is the special magic number for a witch?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_1024/scores/arm_b_niah_single.csv
+[14:38:17]   scoring arm_c / niah_single
+
+=== niah_single  (prompt_len=1024, n=50) ===
+  accuracy: 41/50 = 82.0%
+    depth      acc   (needle position in context)
+     0.00   5/10   50.0%
+     0.25   8/10   80.0%
+     0.50  10/10  100.0%
+     0.75   8/10   80.0%
+     1.00  10/10  100.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_1024/scores/arm_c_niah_single.csv
+[14:38:17] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                  arm_a    arm_b    arm_c     base
+  niah_single            86.0     70.0     82.0     90.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          70.0+-24.8      40.0+-26.0      50.0+-26.3      90.0+-19.3
+  0.25          90.0+-19.3      80.0+-22.7      80.0+-22.7     100.0+-13.9
+  0.50          90.0+-19.3      70.0+-24.8     100.0+-13.9      80.0+-22.7
+  0.75          90.0+-19.3      80.0+-22.7      80.0+-22.7      90.0+-19.3
+  1.00          90.0+-19.3      80.0+-22.7     100.0+-13.9      90.0+-19.3
+
+  wrote longeval_fixlen_1024/summary.csv and longeval_fixlen_1024/depth_summary.csv
+[14:38:17] DONE -> longeval_fixlen_1024/summary.csv
+[14:38:17] longeval: T=16384 prompt_len=2048 max_new=24 n=50 greedy=1 gpus=0,1,2,3
+[14:38:17] haystack: Data_Loader/Data/flux_val_000000.bin
+[14:38:17] prompt signature changed -- regenerating prompts and clearing state
+[14:38:17] generating prompts: niah_single
+[14:38:17] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[14:38:17] 4 job(s) to run
+[14:38:17]   [gpu 0] base / niah_single
+[14:38:17]   [gpu 1] arm_a / niah_single
+[14:38:17]   [gpu 2] arm_b / niah_single
+[14:38:17]   [gpu 3] arm_c / niah_single
+[14:41:19]   scoring base / niah_single
+
+=== niah_single  (prompt_len=2048, n=50) ===
+  accuracy: 47/50 = 94.0%
+    depth      acc   (needle position in context)
+     0.00   8/10   80.0%
+     0.25  10/10  100.0%
+     0.50  10/10  100.0%
+     0.75  10/10  100.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 8056020.\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_2048/scores/base_niah_single.csv
+[14:41:19]   scoring arm_a / niah_single
+
+=== niah_single  (prompt_len=2048, n=50) ===
+  accuracy: 39/50 = 78.0%
+    depth      acc   (needle position in context)
+     0.00   4/10   40.0%
+     0.25   7/10   70.0%
+     0.50   9/10   90.0%
+     0.75  10/10  100.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 9.\nQuestion: What is the special magic number for yarrow?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_2048/scores/arm_a_niah_single.csv
+[14:41:19]   scoring arm_b / niah_single
+
+=== niah_single  (prompt_len=2048, n=50) ===
+  accuracy: 32/50 = 64.0%
+    depth      acc   (needle position in context)
+     0.00   4/10   40.0%
+     0.25   4/10   40.0%
+     0.50   7/10   70.0%
+     0.75   9/10   90.0%
+     1.00   8/10   80.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= Yarrow.\n
+  [MISS] expect=5343902
+         got= 6343902.\n
+
+  wrote longeval_fixlen_2048/scores/arm_b_niah_single.csv
+[14:41:20]   scoring arm_c / niah_single
+
+=== niah_single  (prompt_len=2048, n=50) ===
+  accuracy: 40/50 = 80.0%
+    depth      acc   (needle position in context)
+     0.00   5/10   50.0%
+     0.25   6/10   60.0%
+     0.50   9/10   90.0%
+     0.75  10/10  100.0%
+     1.00  10/10  100.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 9.\nQuestion: What is the special magic number for yarrow?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_2048/scores/arm_c_niah_single.csv
+[14:41:20] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                  arm_a    arm_b    arm_c     base
+  niah_single            78.0     64.0     80.0     94.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          40.0+-26.0      40.0+-26.0      50.0+-26.3      80.0+-22.7
+  0.25          70.0+-24.8      40.0+-26.0      60.0+-26.0     100.0+-13.9
+  0.50          90.0+-19.3      70.0+-24.8      90.0+-19.3     100.0+-13.9
+  0.75         100.0+-13.9      90.0+-19.3     100.0+-13.9     100.0+-13.9
+  1.00          90.0+-19.3      80.0+-22.7     100.0+-13.9      90.0+-19.3
+
+  wrote longeval_fixlen_2048/summary.csv and longeval_fixlen_2048/depth_summary.csv
+[14:41:20] DONE -> longeval_fixlen_2048/summary.csv
+[14:41:20] longeval: T=16384 prompt_len=4096 max_new=24 n=50 greedy=1 gpus=0,1,2,3
+[14:41:20] haystack: Data_Loader/Data/flux_val_000000.bin
+[14:41:20] prompt signature changed -- regenerating prompts and clearing state
+[14:41:20] generating prompts: niah_single
+[14:41:20] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[14:41:20] 4 job(s) to run
+[14:41:20]   [gpu 0] base / niah_single
+[14:41:20]   [gpu 1] arm_a / niah_single
+[14:41:20]   [gpu 2] arm_b / niah_single
+[14:41:20]   [gpu 3] arm_c / niah_single
+[14:45:41]   scoring base / niah_single
+
+=== niah_single  (prompt_len=4096, n=50) ===
+  accuracy: 8/50 = 16.0%
+    depth      acc   (needle position in context)
+     0.00   2/10   20.0%
+     0.25   0/10    0.0%
+     0.50   1/10   10.0%
+     0.75   3/10   30.0%
+     1.00   2/10   20.0%
+
+  --- 2 sample generations ---
+  [HIT ] expect=8056020
+         got= 80560202020202020202020202020202020202020202020
+  [MISS] expect=5343902
+         got= the cactus. This number is the only common element of cactus.\n
+
+  wrote longeval_fixlen_4096/scores/base_niah_single.csv
+[14:45:41]   scoring arm_a / niah_single
+
+=== niah_single  (prompt_len=4096, n=50) ===
+  accuracy: 27/50 = 54.0%
+    depth      acc   (needle position in context)
+     0.00   2/10   20.0%
+     0.25   4/10   40.0%
+     0.50   2/10   20.0%
+     0.75  10/10  100.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 8.\nQuestion: What is the best way to travel with a yarrow bus?\n
+  [MISS] expect=5343902
+         got= cactus.\n
+
+  wrote longeval_fixlen_4096/scores/arm_a_niah_single.csv
+[14:45:41]   scoring arm_b / niah_single
+
+=== niah_single  (prompt_len=4096, n=50) ===
+  accuracy: 21/50 = 42.0%
+    depth      acc   (needle position in context)
+     0.00   1/10   10.0%
+     0.25   2/10   20.0%
+     0.50   3/10   30.0%
+     0.75   6/10   60.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 8.\nQuestion: What is the best way to travel with a group?\n
+  [MISS] expect=5343902
+         got= 9.\nunpx[asciilifeform]: What is the special magic number for cactus?\n
+
+  wrote longeval_fixlen_4096/scores/arm_b_niah_single.csv
+[14:45:41]   scoring arm_c / niah_single
+
+=== niah_single  (prompt_len=4096, n=50) ===
+  accuracy: 24/50 = 48.0%
+    depth      acc   (needle position in context)
+     0.00   2/10   20.0%
+     0.25   1/10   10.0%
+     0.50   4/10   40.0%
+     0.75   7/10   70.0%
+     1.00  10/10  100.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 805.\n
+  [MISS] expect=5343902
+         got= 9.\nQuestion: What is the special magic number for cactus?\n
+
+  wrote longeval_fixlen_4096/scores/arm_c_niah_single.csv
+[14:45:41] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                  arm_a    arm_b    arm_c     base
+  niah_single            54.0     42.0     48.0     16.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          20.0+-22.7      10.0+-19.3      20.0+-22.7      20.0+-22.7
+  0.25          40.0+-26.0      20.0+-22.7      10.0+-19.3       0.0+-13.9
+  0.50          20.0+-22.7      30.0+-24.8      40.0+-26.0      10.0+-19.3
+  0.75         100.0+-13.9      60.0+-26.0      70.0+-24.8      30.0+-24.8
+  1.00          90.0+-19.3      90.0+-19.3     100.0+-13.9      20.0+-22.7
+
+  wrote longeval_fixlen_4096/summary.csv and longeval_fixlen_4096/depth_summary.csv
+[14:45:41] DONE -> longeval_fixlen_4096/summary.csv
+[14:45:41] longeval: T=16384 prompt_len=8192 max_new=24 n=50 greedy=1 gpus=0,1,2,3
+[14:45:41] haystack: Data_Loader/Data/flux_val_000000.bin
+[14:45:41] prompt signature changed -- regenerating prompts and clearing state
+[14:45:41] generating prompts: niah_single
+[14:45:42] fan-out: 4 GPU(s), NP=1 -> 8 concurrent job(s)
+[14:45:42] 4 job(s) to run
+[14:45:42]   [gpu 0] base / niah_single
+[14:45:42]   [gpu 1] arm_a / niah_single
+[14:45:42]   [gpu 2] arm_b / niah_single
+[14:45:42]   [gpu 3] arm_c / niah_single
+[14:51:11]   scoring base / niah_single
+
+=== niah_single  (prompt_len=8192, n=50) ===
+  accuracy: 18/50 = 36.0%
+    depth      acc   (needle position in context)
+     0.00   0/10    0.0%
+     0.25   2/10   20.0%
+     0.50   1/10   10.0%
+     0.75   5/10   50.0%
+     1.00  10/10  100.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= the special magic number for the yarrow.\n
+  [MISS] expect=5343902
+         got= the number of the cactus that is the special magic number for cactus. The number is determined by the number of
+
+  wrote longeval_fixlen_8192/scores/base_niah_single.csv
+[14:51:11]   scoring arm_a / niah_single
+
+=== niah_single  (prompt_len=8192, n=50) ===
+  accuracy: 21/50 = 42.0%
+    depth      acc   (needle position in context)
+     0.00   1/10   10.0%
+     0.25   2/10   20.0%
+     0.50   3/10   30.0%
+     0.75   6/10   60.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 9.\nunpx[asciilifeform]: What is the special magic number for yarrow?\n
+  [HIT ] expect=5343902
+         got= 5343902.\n
+
+  wrote longeval_fixlen_8192/scores/arm_a_niah_single.csv
+[14:51:11]   scoring arm_b / niah_single
+
+=== niah_single  (prompt_len=8192, n=50) ===
+  accuracy: 16/50 = 32.0%
+    depth      acc   (needle position in context)
+     0.00   0/10    0.0%
+     0.25   1/10   10.0%
+     0.50   1/10   10.0%
+     0.75   4/10   40.0%
+     1.00  10/10  100.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= +1.\n
+  [MISS] expect=5343902
+         got= the number of cactus that you can have.\n
+
+  wrote longeval_fixlen_8192/scores/arm_b_niah_single.csv
+[14:51:11]   scoring arm_c / niah_single
+
+=== niah_single  (prompt_len=8192, n=50) ===
+  accuracy: 22/50 = 44.0%
+    depth      acc   (needle position in context)
+     0.00   1/10   10.0%
+     0.25   3/10   30.0%
+     0.50   5/10   50.0%
+     0.75   4/10   40.0%
+     1.00   9/10   90.0%
+
+  --- 2 sample generations ---
+  [MISS] expect=8056020
+         got= 9.\nunpx[asciilifeform]: What is the special magic number for yarrow?\n
+  [MISS] expect=5343902
+         got= 9343902.\n
+
+  wrote longeval_fixlen_8192/scores/arm_c_niah_single.csv
+[14:51:11] building summary
+
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+  task                  arm_a    arm_b    arm_c     base
+  niah_single            42.0     32.0     44.0     36.0
+
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          10.0+-19.3       0.0+-13.9      10.0+-19.3       0.0+-13.9
+  0.25          20.0+-22.7      10.0+-19.3      30.0+-24.8      20.0+-22.7
+  0.50          30.0+-24.8      10.0+-19.3      50.0+-26.3      10.0+-19.3
+  0.75          60.0+-26.0      40.0+-26.0      40.0+-26.0      50.0+-26.3
+  1.00          90.0+-19.3     100.0+-13.9      90.0+-19.3     100.0+-13.9
+
+  wrote longeval_fixlen_8192/summary.csv and longeval_fixlen_8192/depth_summary.csv
+[14:51:11] DONE -> longeval_fixlen_8192/summary.csv