Upload logs/longeval_16k/len_sweep.log with huggingface_hub

logs/longeval_16k/len_sweep.log

@@ -0,0 +1,112 @@
+=============== PROMPT_LEN=1024 (T fixed at 16384) ===============
+  accuracy: 1/50 = 2.0%
+    depth      acc   (needle position in context)
+[12:41:32]   scoring arm_b / niah_single
+  accuracy: 32/50 = 64.0%
+    depth      acc   (needle position in context)
+[12:41:33]   scoring arm_b / val_backward
+  accuracy: 2/50 = 4.0%
+    depth      acc   (needle position in context)
+[12:41:33]   scoring arm_c / niah_single
+  accuracy: 38/50 = 76.0%
+    depth      acc   (needle position in context)
+[12:41:33]   scoring arm_c / val_backward
+  accuracy: 2/50 = 4.0%
+    depth      acc   (needle position in context)
+[12:41:33] building summary
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          25.0+-17.8      20.0+-16.8      20.0+-16.8      40.0+-19.7
+  0.25          45.0+-20.0      35.0+-19.3      35.0+-19.3      40.0+-19.7
+  0.50          40.0+-19.7      35.0+-19.3      55.0+-20.0      40.0+-19.7
+  0.75          45.0+-20.0      35.0+-19.3      40.0+-19.7      50.0+-20.1
+  1.00          45.0+-20.0      45.0+-20.0      50.0+-20.1      60.0+-19.7
+  wrote longeval_len_1024/summary.csv and longeval_len_1024/depth_summary.csv
+[12:41:33] DONE -> longeval_len_1024/summary.csv
+=============== PROMPT_LEN=2048 (T fixed at 16384) ===============
+  accuracy: 2/50 = 4.0%
+    depth      acc   (needle position in context)
+[12:48:02]   scoring arm_b / niah_single
+  accuracy: 28/50 = 56.0%
+    depth      acc   (needle position in context)
+[12:48:02]   scoring arm_b / val_backward
+  accuracy: 1/50 = 2.0%
+    depth      acc   (needle position in context)
+[12:48:02]   scoring arm_c / niah_single
+  accuracy: 35/50 = 70.0%
+    depth      acc   (needle position in context)
+[12:48:03]   scoring arm_c / val_backward
+  accuracy: 2/50 = 4.0%
+    depth      acc   (needle position in context)
+[12:48:03] building summary
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          10.0+-13.7      10.0+-13.7      20.0+-16.8      35.0+-19.3
+  0.25          35.0+-19.3      30.0+-18.7      30.0+-18.7      35.0+-19.3
+  0.50          45.0+-20.0      25.0+-17.8      45.0+-20.0      45.0+-20.0
+  0.75          40.0+-19.7      40.0+-19.7      35.0+-19.3      45.0+-20.0
+  1.00          45.0+-20.0      40.0+-19.7      55.0+-20.0      50.0+-20.1
+  wrote longeval_len_2048/summary.csv and longeval_len_2048/depth_summary.csv
+[12:48:03] DONE -> longeval_len_2048/summary.csv
+=============== PROMPT_LEN=4096 (T fixed at 16384) ===============
+  accuracy: 0/50 = 0.0%
+    depth      acc   (needle position in context)
+[12:55:49]   scoring arm_b / niah_single
+  accuracy: 24/50 = 48.0%
+    depth      acc   (needle position in context)
+[12:55:50]   scoring arm_b / val_backward
+  accuracy: 0/50 = 0.0%
+    depth      acc   (needle position in context)
+[12:55:50]   scoring arm_c / niah_single
+  accuracy: 30/50 = 60.0%
+    depth      acc   (needle position in context)
+[12:55:50]   scoring arm_c / val_backward
+  accuracy: 0/50 = 0.0%
+    depth      acc   (needle position in context)
+[12:55:50] building summary
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00          10.0+-13.7      10.0+-13.7       5.0+-11.4      10.0+-13.7
+  0.25          20.0+-16.8       5.0+-11.4      10.0+-13.7      35.0+-19.3
+  0.50          35.0+-19.3      15.0+-15.4      40.0+-19.7      45.0+-20.0
+  0.75          45.0+-20.0      45.0+-20.0      45.0+-20.0      45.0+-20.0
+  1.00          45.0+-20.0      45.0+-20.0      50.0+-20.1      55.0+-20.0
+  wrote longeval_len_4096/summary.csv and longeval_len_4096/depth_summary.csv
+[12:55:50] DONE -> longeval_len_4096/summary.csv
+=============== PROMPT_LEN=8192 (T fixed at 16384) ===============
+  accuracy: 1/50 = 2.0%
+    depth      acc   (needle position in context)
+[13:07:51]   scoring arm_b / niah_single
+  accuracy: 18/50 = 36.0%
+    depth      acc   (needle position in context)
+[13:07:51]   scoring arm_b / val_backward
+  accuracy: 0/50 = 0.0%
+    depth      acc   (needle position in context)
+[13:07:51]   scoring arm_c / niah_single
+  accuracy: 20/50 = 40.0%
+    depth      acc   (needle position in context)
+[13:07:51]   scoring arm_c / val_backward
+  accuracy: 0/50 = 0.0%
+    depth      acc   (needle position in context)
+[13:07:51] building summary
+=== per-task accuracy %  (secondary/descriptive: ~10 per depth cell) ===
+=== PRIMARY: accuracy % pooled by depth across tasks (+-95% Wilson) ===
+  depth              arm_a           arm_b           arm_c            base
+  0.00           5.0+-11.4       0.0+- 8.1       5.0+-11.4       0.0+- 8.1
+  0.25          10.0+-13.7      10.0+-13.7      10.0+-13.7       5.0+-11.4
+  0.50           5.0+-11.4       5.0+-11.4       5.0+-11.4      15.0+-15.4
+  0.75          25.0+-17.8      35.0+-19.3      35.0+-19.3      45.0+-20.0
+  1.00          50.0+-20.1      40.0+-19.7      45.0+-20.0      45.0+-20.0
+  wrote longeval_len_8192/summary.csv and longeval_len_8192/depth_summary.csv
+[13:07:51] DONE -> longeval_len_8192/summary.csv
+
+=============== length sweep summary ===============
+  depth-1.0 accuracy % (needle ~30 tok before the question = easiest case)
+  len           base     arm_a     arm_b     arm_c
+  1024          60.0      45.0      45.0      50.0
+  2048          50.0      45.0      40.0      55.0
+  4096          55.0      45.0      45.0      50.0
+  8192          45.0      50.0      40.0      45.0