Upload folder using huggingface_hub

outputs/eval/logs/token_t0525/babilong.log

@@ -1,301 +1,301 @@
-2026-07-18:13:09:04 WARNING  [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
-2026-07-18:13:09:08 INFO     [_cli.run:376] Selected Tasks: ['babilong_longctx']
-2026-07-18:13:09:09 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
-2026-07-18:13:09:09 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}
-2026-07-18:13:09:12 INFO     [models.huggingface:161] Using device 'cuda:0'
-2026-07-18:13:09:12 INFO     [models.huggingface:548] Model type cannot be determined. Using default model type 'causal'
-2026-07-18:13:09:13 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
-2026-07-18:13:09:14 WARNING  [api.task:856] babilong_qa5: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:16:04:30 WARNING  [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-18:16:04:34 INFO     [_cli.run:376] Selected Tasks: ['babilong_longctx']
+2026-07-18:16:04:35 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-18:16:04:35 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}
+2026-07-18:16:04:38 INFO     [models.huggingface:161] Using device 'cuda:0'
+2026-07-18:16:04:38 INFO     [models.huggingface:548] Model type cannot be determined. Using default model type 'causal'
+2026-07-18:16:04:39 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+2026-07-18:16:04:40 WARNING  [api.task:856] babilong_qa5: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:13:09:15 WARNING  [api.task:856] babilong_qa4: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:16:04:41 WARNING  [api.task:856] babilong_qa4: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:13:09:15 WARNING  [api.task:856] babilong_qa3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:16:04:41 WARNING  [api.task:856] babilong_qa3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:13:09:15 WARNING  [api.task:856] babilong_qa2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:16:04:41 WARNING  [api.task:856] babilong_qa2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:13:09:15 WARNING  [api.task:856] babilong_qa1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:16:04:41 WARNING  [api.task:856] babilong_qa1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:13:09:16 INFO     [tasks:700] Selected tasks:
-2026-07-18:13:09:16 INFO     [tasks:703] Group: babilong_longctx
-2026-07-18:13:09:16 INFO     [tasks:726] ConfigurableGroup(group=babilong_longctx,group_alias=None): {'babilong_qa1': ConfigurableTask(task_name=babilong_qa1,output_type=generate_until,num_fewshot=2,num_samples=1000), 'babilong_qa2': ConfigurableTask(task_name=babilong_qa2,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa3': ConfigurableTask(task_name=babilong_qa3,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa4': ConfigurableTask(task_name=babilong_qa4,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa5': ConfigurableTask(task_name=babilong_qa5,output_type=generate_until,num_fewshot=2,num_samples=999)}
-2026-07-18:13:09:16 INFO     [evaluator:314] babilong_qa1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
-2026-07-18:13:09:16 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa1 from 2 to 2
-2026-07-18:13:09:16 INFO     [evaluator:314] babilong_qa2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
-2026-07-18:13:09:16 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa2 from 2 to 2
-2026-07-18:13:09:16 INFO     [evaluator:314] babilong_qa3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
-2026-07-18:13:09:16 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa3 from 2 to 2
-2026-07-18:13:09:16 INFO     [evaluator:314] babilong_qa4: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
-2026-07-18:13:09:16 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa4 from 2 to 2
-2026-07-18:13:09:16 INFO     [evaluator:314] babilong_qa5: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
-2026-07-18:13:09:16 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa5 from 2 to 2
-2026-07-18:13:09:16 INFO     [api.task:311] Building contexts for babilong_qa1 on rank 0...
+2026-07-18:16:04:41 INFO     [tasks:700] Selected tasks:
+2026-07-18:16:04:41 INFO     [tasks:703] Group: babilong_longctx
+2026-07-18:16:04:41 INFO     [tasks:726] ConfigurableGroup(group=babilong_longctx,group_alias=None): {'babilong_qa1': ConfigurableTask(task_name=babilong_qa1,output_type=generate_until,num_fewshot=2,num_samples=1000), 'babilong_qa2': ConfigurableTask(task_name=babilong_qa2,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa3': ConfigurableTask(task_name=babilong_qa3,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa4': ConfigurableTask(task_name=babilong_qa4,output_type=generate_until,num_fewshot=2,num_samples=999), 'babilong_qa5': ConfigurableTask(task_name=babilong_qa5,output_type=generate_until,num_fewshot=2,num_samples=999)}
+2026-07-18:16:04:41 INFO     [evaluator:314] babilong_qa1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
+2026-07-18:16:04:41 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa1 from 2 to 2
+2026-07-18:16:04:41 INFO     [evaluator:314] babilong_qa2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
+2026-07-18:16:04:41 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa2 from 2 to 2
+2026-07-18:16:04:41 INFO     [evaluator:314] babilong_qa3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
+2026-07-18:16:04:41 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa3 from 2 to 2
+2026-07-18:16:04:41 INFO     [evaluator:314] babilong_qa4: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
+2026-07-18:16:04:41 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa4 from 2 to 2
+2026-07-18:16:04:41 INFO     [evaluator:314] babilong_qa5: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 16, 'until': []}
+2026-07-18:16:04:41 WARNING  [evaluator:333] Overwriting default num_fewshot of babilong_qa5 from 2 to 2
+2026-07-18:16:04:41 INFO     [api.task:311] Building contexts for babilong_qa1 on rank 0...
 
-
  0%|          | 0/50 [00:00<?, ?it/s]
-2026-07-18:13:09:16 INFO     [api.task:311] Building contexts for babilong_qa2 on rank 0...
+
  0%|          | 0/50 [00:00<?, ?it/s]
+2026-07-18:16:04:41 INFO     [api.task:311] Building contexts for babilong_qa2 on rank 0...
 
-
  0%|          | 0/50 [00:00<?, ?it/s]
-2026-07-18:13:09:16 INFO     [api.task:311] Building contexts for babilong_qa3 on rank 0...
+
  0%|          | 0/50 [00:00<?, ?it/s]
+2026-07-18:16:04:42 INFO     [api.task:311] Building contexts for babilong_qa3 on rank 0...
 
-
  0%|          | 0/50 [00:00<?, ?it/s]
-2026-07-18:13:09:16 INFO     [api.task:311] Building contexts for babilong_qa4 on rank 0...
+
  0%|          | 0/50 [00:00<?, ?it/s]
+2026-07-18:16:04:42 INFO     [api.task:311] Building contexts for babilong_qa4 on rank 0...
 
-
  0%|          | 0/50 [00:00<?, ?it/s]
-2026-07-18:13:09:16 INFO     [api.task:311] Building contexts for babilong_qa5 on rank 0...
+
  0%|          | 0/50 [00:00<?, ?it/s]
+2026-07-18:16:04:42 INFO     [api.task:311] Building contexts for babilong_qa5 on rank 0...
 
-
  0%|          | 0/50 [00:00<?, ?it/s]
-2026-07-18:13:09:16 INFO     [evaluator:584] Running generate_until requests
+
  0%|          | 0/50 [00:00<?, ?it/s]
+2026-07-18:16:04:42 INFO     [evaluator:584] Running generate_until requests
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-2026-07-18:13:14:07 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
-2026-07-18:13:14:07 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to /workspace/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/*.jsonl
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+2026-07-18:16:10:15 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-18:16:10:15 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to /workspace/outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/*.jsonl
 hf ({'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}), gen_kwargs: ({}), limit: 50.0, num_fewshot: 2, batch_size: 1
 |     Tasks      |Version|Filter|n-shot|Metric|   |Value|   |Stderr|
 |----------------|------:|------|-----:|------|---|----:|---|-----:|

outputs/eval/logs/token_t0525/ruler_a.log

@@ -1,65 +1,66 @@
-2026-07-18:12:44:28 WARNING  [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
-2026-07-18:12:44:32 INFO     [_cli.run:376] Selected Tasks: ['niah_single_1', 'niah_single_3', 'niah_multikey_2', 'niah_multiquery', 'ruler_vt', 'ruler_fwe', 'ruler_qa_hotpot']
-2026-07-18:12:44:34 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
-2026-07-18:12:44:34 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}
-2026-07-18:12:44:37 INFO     [models.huggingface:161] Using device 'cuda:0'
-2026-07-18:12:44:37 INFO     [models.huggingface:548] Model type cannot be determined. Using default model type 'causal'
-2026-07-18:12:44:37 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
-2026-07-18:12:44:46 WARNING  [api.task:856] niah_single_1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:15:39:08 WARNING  [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-18:15:39:12 INFO     [_cli.run:376] Selected Tasks: ['niah_single_1', 'niah_single_3', 'niah_multikey_2', 'niah_multiquery', 'ruler_vt', 'ruler_fwe', 'ruler_qa_hotpot']
+2026-07-18:15:39:13 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-18:15:39:13 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}
+2026-07-18:15:39:16 INFO     [models.huggingface:161] Using device 'cuda:0'
+2026-07-18:15:39:16 INFO     [models.huggingface:548] Model type cannot be determined. Using default model type 'causal'
+2026-07-18:15:39:17 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+2026-07-18:15:39:25 WARNING  [api.task:856] niah_single_1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:12:44:46 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
+2026-07-18:15:39:25 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
 
 
-
-
-
-
-
-
-
-2026-07-18:12:44:55 WARNING  [api.task:856] niah_single_3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+2026-07-18:15:39:34 WARNING  [api.task:856] niah_single_3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-2026-07-18:12:45:04 WARNING  [api.task:856] niah_multikey_2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+
+2026-07-18:15:39:43 WARNING  [api.task:856] niah_multikey_2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-2026-07-18:12:45:11 WARNING  [api.task:856] niah_multiquery: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+2026-07-18:15:39:51 WARNING  [api.task:856] niah_multiquery: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-2026-07-18:12:45:20 WARNING  [api.task:856] ruler_vt: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+
+2026-07-18:15:39:59 WARNING  [api.task:856] ruler_vt: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:12:45:20 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
+2026-07-18:15:39:59 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
 Max length 500 | Current length 307 | Noises: 5
 Max length 500 | Current length 410 | Noises: 10
 Max length 500 | Current length 531 | Noises: 15
 Num noises: 10
 
-
  0%|          | 0/1 [00:00<?, ?it/s]
+
  0%|          | 0/1 [00:00<?, ?it/s]
 Max length 8192 | Current length 791 | Noises: 10
 Max length 8192 | Current length 1025 | Noises: 20
 Max length 8192 | Current length 1273 | Noises: 30
@@ -95,231 +96,232 @@ Max length 8192 | Current length 8230 | Noises: 320
 Num noises: 310
 
 
  0%|          | 0/500 [00:00<?, ?it/s]
-
 13%|█▎        | 66/500 [00:01<00:06, 65.54it/s]
-
 26%|██▋       | 132/500 [00:02<00:05, 65.39it/s]
-
 40%|███▉      | 198/500 [00:03<00:04, 64.83it/s]
-
 53%|█████▎    | 265/500 [00:04<00:03, 65.67it/s]
-
 66%|██████▋   | 332/500 [00:05<00:02, 65.84it/s]
-
 80%|████████  | 400/500 [00:06<00:01, 66.25it/s]
-
 94%|█████████▎| 468/500 [00:07<00:00, 66.59it/s]
-2026-07-18:12:45:28 WARNING  [api.task:856] ruler_fwe: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
 14%|█▎        | 68/500 [00:01<00:06, 67.09it/s]
+
 27%|██▋       | 136/500 [00:02<00:05, 67.09it/s]
+
 41%|████      | 204/500 [00:03<00:04, 66.64it/s]
+
 54%|█████▍    | 271/500 [00:04<00:03, 66.60it/s]
+
 68%|██████▊   | 338/500 [00:05<00:02, 66.54it/s]
+
 81%|████████  | 406/500 [00:06<00:01, 66.70it/s]
+
 95%|█████████▍| 473/500 [00:07<00:00, 66.69it/s]
+2026-07-18:15:40:08 WARNING  [api.task:856] ruler_fwe: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-
-
-
-2026-07-18:12:45:41 WARNING  [api.task:856] ruler_qa_hotpot: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+
+
+
+
+
+2026-07-18:15:40:21 WARNING  [api.task:856] ruler_qa_hotpot: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-2026-07-18:12:46:03 INFO     [tasks:700] Selected tasks:
-2026-07-18:12:46:03 INFO     [tasks:691] Task: ruler_qa_hotpot (ruler/qa_hotpot.yaml)
-2026-07-18:12:46:03 INFO     [tasks:691] Task: ruler_fwe (ruler/fwe.yaml)
-2026-07-18:12:46:03 INFO     [tasks:691] Task: ruler_vt (ruler/vt.yaml)
-2026-07-18:12:46:03 INFO     [tasks:691] Task: niah_multiquery (ruler/niah_multiquery.yaml)
-2026-07-18:12:46:03 INFO     [tasks:691] Task: niah_multikey_2 (ruler/niah_multikey_2.yaml)
-2026-07-18:12:46:03 INFO     [tasks:691] Task: niah_single_3 (ruler/niah_single_3.yaml)
-2026-07-18:12:46:03 INFO     [tasks:691] Task: niah_single_1 (ruler/niah_single_1.yaml)
-2026-07-18:12:46:03 INFO     [evaluator:314] ruler_qa_hotpot: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 32, 'until': []}
-2026-07-18:12:46:03 INFO     [evaluator:314] ruler_fwe: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 50, 'until': []}
-2026-07-18:12:46:03 INFO     [evaluator:314] ruler_vt: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 30, 'until': []}
-2026-07-18:12:46:03 INFO     [evaluator:314] niah_multiquery: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:46:03 INFO     [evaluator:314] niah_multikey_2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:46:03 INFO     [evaluator:314] niah_single_3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:46:03 INFO     [evaluator:314] niah_single_1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for ruler_qa_hotpot on rank 0...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+2026-07-18:15:40:43 INFO     [tasks:700] Selected tasks:
+2026-07-18:15:40:43 INFO     [tasks:691] Task: ruler_qa_hotpot (ruler/qa_hotpot.yaml)
+2026-07-18:15:40:43 INFO     [tasks:691] Task: ruler_fwe (ruler/fwe.yaml)
+2026-07-18:15:40:43 INFO     [tasks:691] Task: ruler_vt (ruler/vt.yaml)
+2026-07-18:15:40:43 INFO     [tasks:691] Task: niah_multiquery (ruler/niah_multiquery.yaml)
+2026-07-18:15:40:43 INFO     [tasks:691] Task: niah_multikey_2 (ruler/niah_multikey_2.yaml)
+2026-07-18:15:40:43 INFO     [tasks:691] Task: niah_single_3 (ruler/niah_single_3.yaml)
+2026-07-18:15:40:43 INFO     [tasks:691] Task: niah_single_1 (ruler/niah_single_1.yaml)
+2026-07-18:15:40:43 INFO     [evaluator:314] ruler_qa_hotpot: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 32, 'until': []}
+2026-07-18:15:40:43 INFO     [evaluator:314] ruler_fwe: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 50, 'until': []}
+2026-07-18:15:40:43 INFO     [evaluator:314] ruler_vt: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 30, 'until': []}
+2026-07-18:15:40:43 INFO     [evaluator:314] niah_multiquery: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:40:43 INFO     [evaluator:314] niah_multikey_2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:40:43 INFO     [evaluator:314] niah_single_3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:40:43 INFO     [evaluator:314] niah_single_1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for ruler_qa_hotpot on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for ruler_fwe on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for ruler_fwe on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for ruler_vt on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for ruler_vt on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for niah_multiquery on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for niah_multiquery on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for niah_multikey_2 on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for niah_multikey_2 on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for niah_single_3 on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for niah_single_3 on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [api.task:311] Building contexts for niah_single_1 on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [api.task:311] Building contexts for niah_single_1 on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:46:03 INFO     [evaluator:584] Running generate_until requests
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:40:43 INFO     [evaluator:584] Running generate_until requests
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-2026-07-18:12:56:47 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
-2026-07-18:12:56:47 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to /workspace/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/*.jsonl
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+2026-07-18:15:51:37 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-18:15:51:37 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to /workspace/outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/*.jsonl
 hf ({'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}), gen_kwargs: ({}), limit: 20.0, num_fewshot: None, batch_size: 1
 |     Tasks     |Version|Filter|n-shot|Metric|   | Value |   |Stderr|
 |---------------|------:|------|-----:|-----:|---|------:|---|------|

outputs/eval/logs/token_t0525/ruler_b.log

@@ -1,240 +1,234 @@
-2026-07-18:12:56:49 WARNING  [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
-2026-07-18:12:56:53 INFO     [_cli.run:376] Selected Tasks: ['niah_single_2', 'niah_multikey_1', 'niah_multikey_3', 'niah_multivalue', 'ruler_cwe', 'ruler_qa_squad']
-2026-07-18:12:56:54 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
-2026-07-18:12:56:54 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}
-2026-07-18:12:56:58 INFO     [models.huggingface:161] Using device 'cuda:0'
-2026-07-18:12:56:58 INFO     [models.huggingface:548] Model type cannot be determined. Using default model type 'causal'
-2026-07-18:12:56:58 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
-2026-07-18:12:57:07 WARNING  [api.task:856] niah_single_2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+2026-07-18:15:51:39 WARNING  [config.evaluate_config:281] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-18:15:51:43 INFO     [_cli.run:376] Selected Tasks: ['niah_single_2', 'niah_multikey_1', 'niah_multikey_3', 'niah_multivalue', 'ruler_cwe', 'ruler_qa_squad']
+2026-07-18:15:51:45 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-07-18:15:51:45 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}
+2026-07-18:15:51:47 INFO     [models.huggingface:161] Using device 'cuda:0'
+2026-07-18:15:51:48 INFO     [models.huggingface:548] Model type cannot be determined. Using default model type 'causal'
+2026-07-18:15:51:48 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+2026-07-18:15:51:57 WARNING  [api.task:856] niah_single_2: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:12:57:08 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
+2026-07-18:15:51:57 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
 
 
-
-
-
-
-
-
-
-
-
-2026-07-18:12:57:18 WARNING  [api.task:856] niah_multikey_1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+
+2026-07-18:15:52:06 WARNING  [api.task:856] niah_multikey_1: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-
-2026-07-18:12:57:28 WARNING  [api.task:856] niah_multikey_3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+
+2026-07-18:15:52:15 WARNING  [api.task:856] niah_multikey_3: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-2026-07-18:12:57:34 WARNING  [api.task:856] niah_multivalue: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+2026-07-18:15:52:21 WARNING  [api.task:856] niah_multivalue: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-
-2026-07-18:12:57:44 WARNING  [api.task:856] ruler_cwe: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+
+
+
+2026-07-18:15:52:30 WARNING  [api.task:856] ruler_cwe: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
-2026-07-18:12:57:44 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
+2026-07-18:15:52:30 INFO     [tasks.ruler.common_utils:26] Using tokenizer /workspace/outputs/l2a_style/stage2/checkpoint-25 for synthetic tasks.
 
 
-
-
-
-
-
-
-2026-07-18:12:57:52 WARNING  [api.task:856] ruler_qa_squad: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
+
+
+
+
+
+2026-07-18:15:52:37 WARNING  [api.task:856] ruler_qa_squad: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager.
 For example --metadata='{"max_seq_lengths":[4096, 8192]}'. For details see task Readme.
 
 
-
-
-
-
-
-
-
-
-
-2026-07-18:12:58:03 INFO     [tasks:700] Selected tasks:
-2026-07-18:12:58:03 INFO     [tasks:691] Task: ruler_qa_squad (ruler/qa_squad.yaml)
-2026-07-18:12:58:03 INFO     [tasks:691] Task: ruler_cwe (ruler/cwe.yaml)
-2026-07-18:12:58:03 INFO     [tasks:691] Task: niah_multivalue (ruler/niah_multivalue.yaml)
-2026-07-18:12:58:03 INFO     [tasks:691] Task: niah_multikey_3 (ruler/niah_multikey_3.yaml)
-2026-07-18:12:58:03 INFO     [tasks:691] Task: niah_multikey_1 (ruler/niah_multikey_1.yaml)
-2026-07-18:12:58:03 INFO     [tasks:691] Task: niah_single_2 (ruler/niah_single_2.yaml)
-2026-07-18:12:58:03 INFO     [evaluator:314] ruler_qa_squad: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 32, 'until': []}
-2026-07-18:12:58:03 INFO     [evaluator:314] ruler_cwe: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 120, 'until': []}
-2026-07-18:12:58:03 INFO     [evaluator:314] niah_multivalue: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:58:03 INFO     [evaluator:314] niah_multikey_3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:58:03 INFO     [evaluator:314] niah_multikey_1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:58:03 INFO     [evaluator:314] niah_single_2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
-2026-07-18:12:58:03 INFO     [api.task:311] Building contexts for ruler_qa_squad on rank 0...
+
+
+
+
+
+
+
+
+2026-07-18:15:52:46 INFO     [tasks:700] Selected tasks:
+2026-07-18:15:52:46 INFO     [tasks:691] Task: ruler_qa_squad (ruler/qa_squad.yaml)
+2026-07-18:15:52:46 INFO     [tasks:691] Task: ruler_cwe (ruler/cwe.yaml)
+2026-07-18:15:52:46 INFO     [tasks:691] Task: niah_multivalue (ruler/niah_multivalue.yaml)
+2026-07-18:15:52:46 INFO     [tasks:691] Task: niah_multikey_3 (ruler/niah_multikey_3.yaml)
+2026-07-18:15:52:46 INFO     [tasks:691] Task: niah_multikey_1 (ruler/niah_multikey_1.yaml)
+2026-07-18:15:52:46 INFO     [tasks:691] Task: niah_single_2 (ruler/niah_single_2.yaml)
+2026-07-18:15:52:46 INFO     [evaluator:314] ruler_qa_squad: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 32, 'until': []}
+2026-07-18:15:52:46 INFO     [evaluator:314] ruler_cwe: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 120, 'until': []}
+2026-07-18:15:52:46 INFO     [evaluator:314] niah_multivalue: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:52:46 INFO     [evaluator:314] niah_multikey_3: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:52:46 INFO     [evaluator:314] niah_multikey_1: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:52:46 INFO     [evaluator:314] niah_single_2: Using gen_kwargs: {'do_sample': False, 'temperature': 0.0, 'max_gen_toks': 128, 'until': []}
+2026-07-18:15:52:47 INFO     [api.task:311] Building contexts for ruler_qa_squad on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:58:03 INFO     [api.task:311] Building contexts for ruler_cwe on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:52:47 INFO     [api.task:311] Building contexts for ruler_cwe on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:58:03 INFO     [api.task:311] Building contexts for niah_multivalue on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:52:47 INFO     [api.task:311] Building contexts for niah_multivalue on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:58:03 INFO     [api.task:311] Building contexts for niah_multikey_3 on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:52:47 INFO     [api.task:311] Building contexts for niah_multikey_3 on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:58:03 INFO     [api.task:311] Building contexts for niah_multikey_1 on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:52:47 INFO     [api.task:311] Building contexts for niah_multikey_1 on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:58:03 INFO     [api.task:311] Building contexts for niah_single_2 on rank 0...
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:52:47 INFO     [api.task:311] Building contexts for niah_single_2 on rank 0...
 
-
  0%|          | 0/20 [00:00<?, ?it/s]
-2026-07-18:12:58:03 INFO     [evaluator:584] Running generate_until requests
+
  0%|          | 0/20 [00:00<?, ?it/s]
+2026-07-18:15:52:47 INFO     [evaluator:584] Running generate_until requests
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-2026-07-18:13:09:02 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
-2026-07-18:13:09:02 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to /workspace/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/*.jsonl
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+2026-07-18:16:04:28 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-07-18:16:04:28 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to /workspace/outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/*.jsonl
 hf ({'pretrained': '/workspace/outputs/l2a_style/stage2/checkpoint-25', 'dtype': 'bfloat16', 'max_length': 16384, 'attn_implementation': 'sdpa'}), gen_kwargs: ({}), limit: 20.0, num_fewshot: None, batch_size: 1
 |     Tasks     |Version|Filter|n-shot|Metric|   | Value |   |Stderr|
 |---------------|------:|------|-----:|-----:|---|------:|---|------|

outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T16-10-15.187546.json

@@ -0,0 +1,505 @@
+{
+  "results": {
+    "babilong_longctx": {
+      "acc,none": 0.608,
+      "acc_stderr,none": 0.029548990797366614,
+      "alias": "babilong_longctx"
+    },
+    "babilong_qa1": {
+      "alias": " - babilong_qa1",
+      "acc,none": 0.74,
+      "acc_stderr,none": 0.06266203485560373
+    },
+    "babilong_qa2": {
+      "alias": " - babilong_qa2",
+      "acc,none": 0.52,
+      "acc_stderr,none": 0.0713714056959817
+    },
+    "babilong_qa3": {
+      "alias": " - babilong_qa3",
+      "acc,none": 0.34,
+      "acc_stderr,none": 0.06767268161329719
+    },
+    "babilong_qa4": {
+      "alias": " - babilong_qa4",
+      "acc,none": 0.72,
+      "acc_stderr,none": 0.06414269805898185
+    },
+    "babilong_qa5": {
+      "alias": " - babilong_qa5",
+      "acc,none": 0.72,
+      "acc_stderr,none": 0.06414269805898185
+    }
+  },
+  "groups": {
+    "babilong_longctx": {
+      "acc,none": 0.608,
+      "acc_stderr,none": 0.029548990797366614,
+      "alias": "babilong_longctx"
+    }
+  },
+  "group_subtasks": {
+    "babilong_longctx": [
+      "babilong_qa1",
+      "babilong_qa2",
+      "babilong_qa3",
+      "babilong_qa4",
+      "babilong_qa5"
+    ]
+  },
+  "configs": {
+    "babilong_qa1": {
+      "task": "babilong_qa1",
+      "custom_dataset": "def load_dataset(**kwargs):\n    config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n    # Get specific qa split\n    qa_split = kwargs.get(\"qa_split\")\n\n    eval_logger.info(\n        f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n    )\n    dataset = datasets.load_dataset(\n        \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n    )\n    return {qa_split: dataset}\n",
+      "dataset_path": "RMT-team/babilong-1k-samples",
+      "dataset_kwargs": {
+        "qa_split": "qa1"
+      },
+      "test_split": "qa1",
+      "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+      "doc_to_target": "{{target}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    pred = postprocess_pred(results)\n    target = doc.get(\"target\", \"\").strip()\n\n    # String match\n    score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n    return {\"acc\": score}\n",
+      "description": "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "input": "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony.",
+            "question": "Where is Charlie?",
+            "target": "The most recent location of Charlie is balcony."
+          },
+          {
+            "input": "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony.",
+            "question": "Where is Alan?",
+            "target": "The most recent location of Alan is shop."
+          }
+        ],
+        "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 2,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 16,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": "8k"
+      }
+    },
+    "babilong_qa2": {
+      "task": "babilong_qa2",
+      "custom_dataset": "def load_dataset(**kwargs):\n    config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n    # Get specific qa split\n    qa_split = kwargs.get(\"qa_split\")\n\n    eval_logger.info(\n        f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n    )\n    dataset = datasets.load_dataset(\n        \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n    )\n    return {qa_split: dataset}\n",
+      "dataset_path": "RMT-team/babilong-1k-samples",
+      "dataset_kwargs": {
+        "qa_split": "qa2"
+      },
+      "test_split": "qa2",
+      "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+      "doc_to_target": "{{target}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    pred = postprocess_pred(results)\n    target = doc.get(\"target\", \"\").strip()\n\n    # String match\n    score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n    return {\"acc\": score}\n",
+      "description": "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "input": "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony.",
+            "question": "Where is the bottle?",
+            "target": "The bottle is in the balcony."
+          },
+          {
+            "input": "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen.",
+            "question": "Where is the screw driver?",
+            "target": "The screw driver is in the kitchen."
+          }
+        ],
+        "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 2,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 16,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": "8k"
+      }
+    },
+    "babilong_qa3": {
+      "task": "babilong_qa3",
+      "custom_dataset": "def load_dataset(**kwargs):\n    config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n    # Get specific qa split\n    qa_split = kwargs.get(\"qa_split\")\n\n    eval_logger.info(\n        f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n    )\n    dataset = datasets.load_dataset(\n        \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n    )\n    return {qa_split: dataset}\n",
+      "dataset_path": "RMT-team/babilong-1k-samples",
+      "dataset_kwargs": {
+        "qa_split": "qa3"
+      },
+      "test_split": "qa3",
+      "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+      "doc_to_target": "{{target}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    pred = postprocess_pred(results)\n    target = doc.get(\"target\", \"\").strip()\n\n    # String match\n    score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n    return {\"acc\": score}\n",
+      "description": "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "input": "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen.",
+            "question": "Where was the apple before the kitchen?",
+            "target": "Before the kitchen the apple was in the bathroom."
+          },
+          {
+            "input": "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom.",
+            "question": "Where was the football before the bedroom?",
+            "target": "Before the bedroom the football was in the garden."
+          }
+        ],
+        "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 2,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 16,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": "8k"
+      }
+    },
+    "babilong_qa4": {
+      "task": "babilong_qa4",
+      "custom_dataset": "def load_dataset(**kwargs):\n    config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n    # Get specific qa split\n    qa_split = kwargs.get(\"qa_split\")\n\n    eval_logger.info(\n        f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n    )\n    dataset = datasets.load_dataset(\n        \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n    )\n    return {qa_split: dataset}\n",
+      "dataset_path": "RMT-team/babilong-1k-samples",
+      "dataset_kwargs": {
+        "qa_split": "qa4"
+      },
+      "test_split": "qa4",
+      "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+      "doc_to_target": "{{target}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    pred = postprocess_pred(results)\n    target = doc.get(\"target\", \"\").strip()\n\n    # String match\n    score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n    return {\"acc\": score}\n",
+      "description": "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "input": "The hallway is south of the kitchen. The bedroom is north of the kitchen.",
+            "question": "What is the kitchen south of?",
+            "target": "bedroom"
+          },
+          {
+            "input": "The garden is west of the bedroom. The bedroom is west of the kitchen.",
+            "question": "What is west of the bedroom?",
+            "target": "garden"
+          }
+        ],
+        "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 2,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 16,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": "8k"
+      }
+    },
+    "babilong_qa5": {
+      "task": "babilong_qa5",
+      "custom_dataset": "def load_dataset(**kwargs):\n    config_name = kwargs.get(\"max_seq_lengths\", \"0k\")\n\n    # Get specific qa split\n    qa_split = kwargs.get(\"qa_split\")\n\n    eval_logger.info(\n        f\"Loading babilong dataset: max_seq_lengths={config_name}, split={qa_split}\"\n    )\n    dataset = datasets.load_dataset(\n        \"RMT-team/babilong-1k-samples\", name=config_name, split=qa_split\n    )\n    return {qa_split: dataset}\n",
+      "dataset_path": "RMT-team/babilong-1k-samples",
+      "dataset_kwargs": {
+        "qa_split": "qa5"
+      },
+      "test_split": "qa5",
+      "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+      "doc_to_target": "{{target}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    pred = postprocess_pred(results)\n    target = doc.get(\"target\", \"\").strip()\n\n    # String match\n    score = 1.0 if target.lower() in pred[0].lower() else 0.0\n\n    return {\"acc\": score}\n",
+      "description": "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "input": "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there.",
+            "question": "Who did Mary give the apple to?",
+            "target": "Fred"
+          },
+          {
+            "input": "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom.",
+            "question": "Who gave the football?",
+            "target": "Jeff"
+          },
+          {
+            "input": "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden.",
+            "question": "What did Fred give to Bill?",
+            "target": "apple"
+          }
+        ],
+        "doc_to_text": "{{input.strip()}}\n{{question.strip()}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 2,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 16,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": "8k"
+      }
+    }
+  },
+  "versions": {
+    "babilong_longctx": 0.0,
+    "babilong_qa1": 0.0,
+    "babilong_qa2": 0.0,
+    "babilong_qa3": 0.0,
+    "babilong_qa4": 0.0,
+    "babilong_qa5": 0.0
+  },
+  "n-shot": {
+    "babilong_qa1": 2,
+    "babilong_qa2": 2,
+    "babilong_qa3": 2,
+    "babilong_qa4": 2,
+    "babilong_qa5": 2
+  },
+  "higher_is_better": {
+    "babilong_longctx": {
+      "acc": true
+    },
+    "babilong_qa1": {
+      "acc": true
+    },
+    "babilong_qa2": {
+      "acc": true
+    },
+    "babilong_qa3": {
+      "acc": true
+    },
+    "babilong_qa4": {
+      "acc": true
+    },
+    "babilong_qa5": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "babilong_qa1": {
+      "original": 1000,
+      "effective": 50
+    },
+    "babilong_qa2": {
+      "original": 999,
+      "effective": 50
+    },
+    "babilong_qa3": {
+      "original": 999,
+      "effective": 50
+    },
+    "babilong_qa4": {
+      "original": 999,
+      "effective": 50
+    },
+    "babilong_qa5": {
+      "original": 999,
+      "effective": 50
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+      "trust_remote_code": true,
+      "dtype": "bfloat16",
+      "max_length": 16384,
+      "attn_implementation": "sdpa"
+    },
+    "model_num_parameters": 2031969308,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "1",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 50.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1784390674.4487982,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           46 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  96\nOn-line CPU(s) list:                     0-95\nVendor ID:                               GenuineIntel\nModel name:                              Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family:                              6\nModel:                                   85\nThread(s) per core:                      2\nCore(s) per socket:                      24\nSocket(s):                               2\nStepping:                                7\nBogoMIPS:                                5999.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               1.5 MiB (48 instances)\nL1i cache:                               1.5 MiB (48 instances)\nL2 cache:                                48 MiB (48 instances)\nL3 cache:                                71.5 MiB (2 instances)\nNUMA node(s):                            2\nNUMA node0 CPU(s):                       0-23,48-71\nNUMA node1 CPU(s):                       24-47,72-95\nVulnerability Gather data sampling:      Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit:             KVM: Mitigation: VMX unsupported\nVulnerability L1tf:                      Mitigation; PTE Inversion\nVulnerability Mds:                       Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown:                  Mitigation; PTI\nVulnerability Mmio stale data:           Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Vulnerable\nVulnerability Spec rstack overflow:      Not affected\nVulnerability Spec store bypass:         Vulnerable\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Not affected\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy                        2.3.4            py311h2e04523_0       conda-forge\n[conda] nvidia-cublas-cu12           12.8.4.1         pypi_0                pypi\n[conda] nvidia-cuda-cupti-cu12       12.8.90          pypi_0                pypi\n[conda] nvidia-cuda-nvrtc-cu12       12.8.93          pypi_0                pypi\n[conda] nvidia-cuda-runtime-cu12     12.8.90          pypi_0                pypi\n[conda] nvidia-cudnn-cu12            9.10.2.21        pypi_0                pypi\n[conda] nvidia-cufft-cu12            11.3.3.83        pypi_0                pypi\n[conda] nvidia-curand-cu12           10.3.9.90        pypi_0                pypi\n[conda] nvidia-cusolver-cu12         11.7.3.90        pypi_0                pypi\n[conda] nvidia-cusparse-cu12         12.5.8.93        pypi_0                pypi\n[conda] nvidia-cusparselt-cu12       0.7.1            pypi_0                pypi\n[conda] nvidia-nccl-cu12             2.27.5           pypi_0                pypi\n[conda] nvidia-nvjitlink-cu12        12.8.93          pypi_0                pypi\n[conda] nvidia-nvtx-cu12             12.8.90          pypi_0                pypi\n[conda] optree                       0.17.0           pypi_0                pypi\n[conda] torch                        2.9.1+cu128      pypi_0                pypi\n[conda] torchaudio                   2.9.1+cu128      pypi_0                pypi\n[conda] torchelastic                 0.2.2            pypi_0                pypi\n[conda] torchvision                  0.24.1+cu128     pypi_0                pypi\n[conda] triton                       3.5.1            pypi_0                pypi",
+  "transformers_version": "4.54.0",
+  "lm_eval_version": "0.4.11",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 16384,
+  "task_hashes": {
+    "babilong_qa1": "bbdcd109bea1bb245e118e814bd0563de6384bc875a21df5218ab4ed75b27380",
+    "babilong_qa2": "fa4f8d12bdd481e02a1051414304811f648caf7bce8f89dcf03ecbf5b86f1e92",
+    "babilong_qa3": "737b64396c8b2e5f96da9796bcd5e448f08e5a13e32ae7b610d31a60269b6e48",
+    "babilong_qa4": "298a25727c69c9babddcd78f438cae1aa21a388e14e7db90b2206eff753c90c0",
+    "babilong_qa5": "e60afcdcdd1c4b3e24b002ffbf8f87085f5ec4a02fcd72318f042dcdafe230f4"
+  },
+  "model_source": "hf",
+  "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+  "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "total_evaluation_time_seconds": "344.00240219797706"
+}

outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa1_2026-07-18T16-10-15.187546.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:894b7b485f12742da28f607918847453d7bbffaf5c672ac7535694d830422e83
+size 3164092

outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa2_2026-07-18T16-10-15.187546.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:954df79f19d6f4037ef1ba1e46baf6dc9c2d30a727f2cf4085997f2ff006dd4e
+size 3147516

outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa3_2026-07-18T16-10-15.187546.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ef1a161d57d61ed368024971f3a0e608ce70d56782ec8c97b20cc3f3ecfccea
+size 3126962

outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa4_2026-07-18T16-10-15.187546.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c40e39f34fff163a0fe5c131806e61937bb4e5d6e7b343c832691b828c3e54f
+size 3137620

outputs/eval/token_t0525/babilong8k_qa1_qa5_n50/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_babilong_qa5_2026-07-18T16-10-15.187546.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dddf97f2f532bec678e21782ffb214a1d795c62adb06c291d394b8f43b287c6
+size 3135600

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T15-51-37.349704.json

@@ -0,0 +1,821 @@
+{
+  "results": {
+    "niah_multikey_2": {
+      "alias": "niah_multikey_2",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "niah_multiquery": {
+      "alias": "niah_multiquery",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "niah_single_1": {
+      "alias": "niah_single_1",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "niah_single_3": {
+      "alias": "niah_single_3",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "ruler_fwe": {
+      "alias": "ruler_fwe",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 0.7666666666666666,
+      "8192_stderr,none": "N/A"
+    },
+    "ruler_qa_hotpot": {
+      "alias": "ruler_qa_hotpot",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 0.4,
+      "8192_stderr,none": "N/A"
+    },
+    "ruler_vt": {
+      "alias": "ruler_vt",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 0.8800000000000002,
+      "8192_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "niah_single_1": [],
+    "niah_single_3": [],
+    "niah_multikey_2": [],
+    "niah_multiquery": [],
+    "ruler_vt": [],
+    "ruler_fwe": [],
+    "ruler_qa_hotpot": []
+  },
+  "configs": {
+    "niah_multikey_2": {
+      "task": "niah_multikey_2",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_multikey_2(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"needle\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"needle\",\n            type_needle_k=\"words\",\n            type_needle_v=\"numbers\",\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "niah_multiquery": {
+      "task": "niah_multiquery",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_multiquery(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"essay\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"essay\",\n            type_needle_k=\"words\",\n            type_needle_v=\"numbers\",\n            num_needle_q=4,\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "niah_single_1": {
+      "task": "niah_single_1",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_single_1(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"repeat\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"repeat\",\n            type_needle_k=\"words\",\n            type_needle_v=\"numbers\",\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "niah_single_3": {
+      "task": "niah_single_3",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_single_3(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"essay\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"essay\",\n            type_needle_k=\"words\",\n            type_needle_v=\"uuids\",\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "ruler_fwe": {
+      "task": "ruler_fwe",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def fwe_download(**kwargs):\n    pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n    df = (\n        get_dataset(pretrained, max_seq_length=seq)\n        for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    )\n\n    return {\n        \"test\": datasets.Dataset.from_list(\n            list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n        )\n    }\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 50,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "ruler_qa_hotpot": {
+      "task": "ruler_qa_hotpot",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def get_hotpotqa(**kwargs):\n    return get_qa_dataset(\"hotpotqa\", **kwargs)\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_part(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 32,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "ruler_vt": {
+      "task": "ruler_vt",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def get_vt_dataset(**kwargs) -> dict[str, datasets.Dataset]:\n    pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", \"\"))\n    df = (\n        get_dataset(tokenizer=get_tokenizer(pretrained), seq=seq)\n        for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    )\n\n    return {\n        \"test\": datasets.Dataset.from_list(\n            list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n        )\n    }\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 30,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    }
+  },
+  "versions": {
+    "niah_multikey_2": 1.0,
+    "niah_multiquery": 1.0,
+    "niah_single_1": 1.0,
+    "niah_single_3": 1.0,
+    "ruler_fwe": 1.0,
+    "ruler_qa_hotpot": 1.0,
+    "ruler_vt": 1.0
+  },
+  "n-shot": {
+    "niah_multikey_2": 0,
+    "niah_multiquery": 0,
+    "niah_single_1": 0,
+    "niah_single_3": 0,
+    "ruler_fwe": 0,
+    "ruler_qa_hotpot": 0,
+    "ruler_vt": 0
+  },
+  "higher_is_better": {
+    "niah_multikey_2": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "niah_multiquery": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "niah_single_1": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "niah_single_3": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "ruler_fwe": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "ruler_qa_hotpot": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "ruler_vt": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    }
+  },
+  "n-samples": {
+    "ruler_qa_hotpot": {
+      "original": 500,
+      "effective": 20
+    },
+    "ruler_fwe": {
+      "original": 500,
+      "effective": 20
+    },
+    "ruler_vt": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_multiquery": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_multikey_2": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_single_3": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_single_1": {
+      "original": 500,
+      "effective": 20
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+      "trust_remote_code": true,
+      "dtype": "bfloat16",
+      "max_length": 16384,
+      "attn_implementation": "sdpa"
+    },
+    "model_num_parameters": 2031969308,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "1",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 20.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1784389152.295637,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           46 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  96\nOn-line CPU(s) list:                     0-95\nVendor ID:                               GenuineIntel\nModel name:                              Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family:                              6\nModel:                                   85\nThread(s) per core:                      2\nCore(s) per socket:                      24\nSocket(s):                               2\nStepping:                                7\nBogoMIPS:                                5999.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               1.5 MiB (48 instances)\nL1i cache:                               1.5 MiB (48 instances)\nL2 cache:                                48 MiB (48 instances)\nL3 cache:                                71.5 MiB (2 instances)\nNUMA node(s):                            2\nNUMA node0 CPU(s):                       0-23,48-71\nNUMA node1 CPU(s):                       24-47,72-95\nVulnerability Gather data sampling:      Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit:             KVM: Mitigation: VMX unsupported\nVulnerability L1tf:                      Mitigation; PTE Inversion\nVulnerability Mds:                       Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown:                  Mitigation; PTI\nVulnerability Mmio stale data:           Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Vulnerable\nVulnerability Spec rstack overflow:      Not affected\nVulnerability Spec store bypass:         Vulnerable\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Not affected\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy                        2.3.4            py311h2e04523_0       conda-forge\n[conda] nvidia-cublas-cu12           12.8.4.1         pypi_0                pypi\n[conda] nvidia-cuda-cupti-cu12       12.8.90          pypi_0                pypi\n[conda] nvidia-cuda-nvrtc-cu12       12.8.93          pypi_0                pypi\n[conda] nvidia-cuda-runtime-cu12     12.8.90          pypi_0                pypi\n[conda] nvidia-cudnn-cu12            9.10.2.21        pypi_0                pypi\n[conda] nvidia-cufft-cu12            11.3.3.83        pypi_0                pypi\n[conda] nvidia-curand-cu12           10.3.9.90        pypi_0                pypi\n[conda] nvidia-cusolver-cu12         11.7.3.90        pypi_0                pypi\n[conda] nvidia-cusparse-cu12         12.5.8.93        pypi_0                pypi\n[conda] nvidia-cusparselt-cu12       0.7.1            pypi_0                pypi\n[conda] nvidia-nccl-cu12             2.27.5           pypi_0                pypi\n[conda] nvidia-nvjitlink-cu12        12.8.93          pypi_0                pypi\n[conda] nvidia-nvtx-cu12             12.8.90          pypi_0                pypi\n[conda] optree                       0.17.0           pypi_0                pypi\n[conda] torch                        2.9.1+cu128      pypi_0                pypi\n[conda] torchaudio                   2.9.1+cu128      pypi_0                pypi\n[conda] torchelastic                 0.2.2            pypi_0                pypi\n[conda] torchvision                  0.24.1+cu128     pypi_0                pypi\n[conda] triton                       3.5.1            pypi_0                pypi",
+  "transformers_version": "4.54.0",
+  "lm_eval_version": "0.4.11",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 16384,
+  "task_hashes": {
+    "ruler_qa_hotpot": "fd48cb605efc8c5adac7e3771533db4d4c354b71bf91ce00841c814604ba6404",
+    "ruler_fwe": "c4633bcd0acf3ba4b96f905b5ba269ba94cf45a4c8ca0c13aae1468227cc3a10",
+    "ruler_vt": "af011da859c8b86f63f57f6256cf59077d3d3d8536084ab67ccf39e18e9a615a",
+    "niah_multiquery": "f9a36d2854dfd2bb52ffc7a6bf3f53490fb5303ca9da0046256bc28b55c2adb2",
+    "niah_multikey_2": "86e49b8f875843454b2923143b9124ba82e8a2e23b145f792726cd82f4e49c24",
+    "niah_single_3": "905e40146a287e5f0c8e2a81f64f082108c45a23862949e6a99bfcbbf561cad2",
+    "niah_single_1": "426e8138e992480aa416d01e6b9cef5865a68c00f532400e448a0c20ee026cff"
+  },
+  "model_source": "hf",
+  "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+  "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "total_evaluation_time_seconds": "748.3684644349851"
+}

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_2_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1198521d64dbb83c3bcac7e4b000a43afe8c5dc3c6a6261875cba22529f2a61
+size 987952

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multiquery_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fda48f7c886c33b97a6e90f648b89cd35e4b95b24d3bd08c1cff5bfd40efb2bc
+size 1442196

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_1_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b805cb744f2ca51f70ef08f871e84fd7a635d76bd0632761601cf38420eb43b
+size 1231408

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_3_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abf7b2d058ae6464aa9140d934b01d08b30c31804b918a3fb4d99484b0c4a910
+size 1435434

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_fwe_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b88939d119cb9876450e15d30af248d129e4eaac0f6d6f5daf63508495d5662c
+size 826859

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_hotpot_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c489a379e7ad7898390535ead6d1a6d23be6f3e40691c74e3546a15ca8b916d
+size 1172240

outputs/eval/token_t0525/ruler8k_splits/a/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_vt_2026-07-18T15-51-37.349704.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c13551162068852b669c560c3e9bbc82f84845accd93ed37be9d4123cf1cc1e
+size 1228980

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/results_2026-07-18T16-04-28.086981.json

@@ -0,0 +1,714 @@
+{
+  "results": {
+    "niah_multikey_1": {
+      "alias": "niah_multikey_1",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "niah_multikey_3": {
+      "alias": "niah_multikey_3",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 0.75,
+      "8192_stderr,none": "N/A"
+    },
+    "niah_multivalue": {
+      "alias": "niah_multivalue",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "niah_single_2": {
+      "alias": "niah_single_2",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 1.0,
+      "8192_stderr,none": "N/A"
+    },
+    "ruler_cwe": {
+      "alias": "ruler_cwe",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 0.40499999999999997,
+      "8192_stderr,none": "N/A"
+    },
+    "ruler_qa_squad": {
+      "alias": "ruler_qa_squad",
+      "4096,none": -1,
+      "4096_stderr,none": "N/A",
+      "8192,none": 0.4208333333333334,
+      "8192_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "niah_single_2": [],
+    "niah_multikey_1": [],
+    "niah_multikey_3": [],
+    "niah_multivalue": [],
+    "ruler_cwe": [],
+    "ruler_qa_squad": []
+  },
+  "configs": {
+    "niah_multikey_1": {
+      "task": "niah_multikey_1",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_multikey_1(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"essay\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"essay\",\n            type_needle_k=\"words\",\n            type_needle_v=\"numbers\",\n            num_needle_k=4,\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "niah_multikey_3": {
+      "task": "niah_multikey_3",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_multikey_3(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"needle\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"needle\",\n            type_needle_k=\"uuids\",\n            type_needle_v=\"uuids\",\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "niah_multivalue": {
+      "task": "niah_multivalue",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_multivalue(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"essay\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"essay\",\n            type_needle_k=\"words\",\n            type_needle_v=\"numbers\",\n            num_needle_v=4,\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "niah_single_2": {
+      "task": "niah_single_2",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def niah_single_2(**kwargs):\n    seq_lengths = kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    return download_dataset(\n        generate_samples(\n            get_haystack(type_haystack=\"essay\"),\n            max_seq_length=seq,\n            template=TEMPLATE,\n            type_haystack=\"essay\",\n            type_needle_k=\"words\",\n            type_needle_v=\"numbers\",\n            num_samples=500,\n            TOKENIZER=get_tokenizer(**kwargs),\n        )\n        for seq in seq_lengths\n    )\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 128,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "ruler_cwe": {
+      "task": "ruler_cwe",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def get_cw_dataset(**kwargs):\n    pretrained = kwargs.get(\"tokenizer\", kwargs.get(\"pretrained\", {}))\n    df = (\n        get_dataset(pretrained, seq=seq)\n        for seq in kwargs.pop(\"max_seq_lengths\", DEFAULT_SEQ_LENGTHS)\n    )\n\n    return {\n        \"test\": datasets.Dataset.from_list(\n            list(itertools.chain.from_iterable(df)), split=datasets.Split.TEST\n        )\n    }\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_all(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": "\n\n",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": "\n\n"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 120,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    },
+    "ruler_qa_squad": {
+      "task": "ruler_qa_squad",
+      "tag": [
+        "longcxt"
+      ],
+      "custom_dataset": "def get_squad(**kwargs):\n    return get_qa_dataset(\"squad\", **kwargs)\n",
+      "dataset_path": "",
+      "dataset_name": "",
+      "test_split": "test",
+      "doc_to_text": "{{input}}",
+      "doc_to_target": "{{outputs}}",
+      "unsafe_code": false,
+      "process_results": "def process_results_part(doc: dict, results: list[str]) -> dict[str, float]:\n    # hacky: set all other lengths to -1\n    metrics = {str(length): -1.0 for length in DEFAULT_SEQ_LENGTHS}\n    input_len = doc[\"max_length\"]\n    pred = postprocess_pred(results)\n    score = string_match_part(pred, [doc[\"outputs\"]])\n    metrics[str(input_len)] = score\n    return metrics\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{input}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{outputs}}",
+        "gen_prefix": "{{gen_prefix}}",
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "4096",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "8192",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "16384",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "32768",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "65536",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "131072",
+          "aggregation": "def aggregate_metrics(metrics: list[float]) -> float:\n    res = [x for x in metrics if x != -1]\n    if not res:\n        # we don't have any samples with this length\n        return -1\n    return sum(res) / len(res)\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 32,
+        "until": []
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "gen_prefix": "{{gen_prefix}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+        "trust_remote_code": true,
+        "dtype": "bfloat16",
+        "max_length": 16384,
+        "attn_implementation": "sdpa",
+        "max_seq_lengths": [
+          8192
+        ]
+      }
+    }
+  },
+  "versions": {
+    "niah_multikey_1": 1.0,
+    "niah_multikey_3": 1.0,
+    "niah_multivalue": 1.0,
+    "niah_single_2": 1.0,
+    "ruler_cwe": 1.0,
+    "ruler_qa_squad": 1.0
+  },
+  "n-shot": {
+    "niah_multikey_1": 0,
+    "niah_multikey_3": 0,
+    "niah_multivalue": 0,
+    "niah_single_2": 0,
+    "ruler_cwe": 0,
+    "ruler_qa_squad": 0
+  },
+  "higher_is_better": {
+    "niah_multikey_1": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "niah_multikey_3": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "niah_multivalue": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "niah_single_2": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "ruler_cwe": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    },
+    "ruler_qa_squad": {
+      "4096": true,
+      "8192": true,
+      "16384": true,
+      "32768": true,
+      "65536": true,
+      "131072": true
+    }
+  },
+  "n-samples": {
+    "ruler_qa_squad": {
+      "original": 500,
+      "effective": 20
+    },
+    "ruler_cwe": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_multivalue": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_multikey_3": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_multikey_1": {
+      "original": 500,
+      "effective": 20
+    },
+    "niah_single_2": {
+      "original": 500,
+      "effective": 20
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+      "trust_remote_code": true,
+      "dtype": "bfloat16",
+      "max_length": 16384,
+      "attn_implementation": "sdpa"
+    },
+    "model_num_parameters": 2031969308,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "1",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 20.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1784389903.7655613,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: version 4.1.2\nLibc version: glibc-2.35\n\nPython version: 3.11.14 | packaged by conda-forge | (main, Oct 22 2025, 22:46:25) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA A100-SXM4-80GB\nNvidia driver version: 580.159.03\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           46 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  96\nOn-line CPU(s) list:                     0-95\nVendor ID:                               GenuineIntel\nModel name:                              Intel(R) Xeon(R) Platinum 8275CL CPU @ 3.00GHz\nCPU family:                              6\nModel:                                   85\nThread(s) per core:                      2\nCore(s) per socket:                      24\nSocket(s):                               2\nStepping:                                7\nBogoMIPS:                                5999.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               1.5 MiB (48 instances)\nL1i cache:                               1.5 MiB (48 instances)\nL2 cache:                                48 MiB (48 instances)\nL3 cache:                                71.5 MiB (2 instances)\nNUMA node(s):                            2\nNUMA node0 CPU(s):                       0-23,48-71\nNUMA node1 CPU(s):                       24-47,72-95\nVulnerability Gather data sampling:      Unknown: Dependent on hypervisor status\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit:             KVM: Mitigation: VMX unsupported\nVulnerability L1tf:                      Mitigation; PTE Inversion\nVulnerability Mds:                       Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown:                  Mitigation; PTI\nVulnerability Mmio stale data:           Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Vulnerable\nVulnerability Spec rstack overflow:      Not affected\nVulnerability Spec store bypass:         Vulnerable\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Retpoline\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Not affected\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.3.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] optree==0.17.0\n[pip3] torch==2.9.1+cu128\n[pip3] torchaudio==2.9.1+cu128\n[pip3] torchelastic==0.2.2\n[pip3] torchvision==0.24.1+cu128\n[pip3] triton==3.5.1\n[conda] numpy                        2.3.4            py311h2e04523_0       conda-forge\n[conda] nvidia-cublas-cu12           12.8.4.1         pypi_0                pypi\n[conda] nvidia-cuda-cupti-cu12       12.8.90          pypi_0                pypi\n[conda] nvidia-cuda-nvrtc-cu12       12.8.93          pypi_0                pypi\n[conda] nvidia-cuda-runtime-cu12     12.8.90          pypi_0                pypi\n[conda] nvidia-cudnn-cu12            9.10.2.21        pypi_0                pypi\n[conda] nvidia-cufft-cu12            11.3.3.83        pypi_0                pypi\n[conda] nvidia-curand-cu12           10.3.9.90        pypi_0                pypi\n[conda] nvidia-cusolver-cu12         11.7.3.90        pypi_0                pypi\n[conda] nvidia-cusparse-cu12         12.5.8.93        pypi_0                pypi\n[conda] nvidia-cusparselt-cu12       0.7.1            pypi_0                pypi\n[conda] nvidia-nccl-cu12             2.27.5           pypi_0                pypi\n[conda] nvidia-nvjitlink-cu12        12.8.93          pypi_0                pypi\n[conda] nvidia-nvtx-cu12             12.8.90          pypi_0                pypi\n[conda] optree                       0.17.0           pypi_0                pypi\n[conda] torch                        2.9.1+cu128      pypi_0                pypi\n[conda] torchaudio                   2.9.1+cu128      pypi_0                pypi\n[conda] torchelastic                 0.2.2            pypi_0                pypi\n[conda] torchvision                  0.24.1+cu128     pypi_0                pypi\n[conda] triton                       3.5.1            pypi_0                pypi",
+  "transformers_version": "4.54.0",
+  "lm_eval_version": "0.4.11",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 16384,
+  "task_hashes": {
+    "ruler_qa_squad": "8c658a728ccd3d67ddd3f45e8e76527cf15a1ae4119033d8246f293ecc68cf11",
+    "ruler_cwe": "f68206129fc7d709951d550ec9269fed05e73bea2fc86ce9f2c758974db5db71",
+    "niah_multivalue": "29646655c230b4f155a8de1fbc7ddf22c49e43045bd8fdab5770e7cb45026c7c",
+    "niah_multikey_3": "de2cec52885e38d56e0bda0e344e989909969c71ab5f8048b9368a011dd3616b",
+    "niah_multikey_1": "338f81e9efa98dbe6979d4f178f7f06cdbd6d367072318667a932b43c5032ab8",
+    "niah_single_2": "88ba10e924c2fc6c47543eb646b9e6f7487c7ca026c9391d6cd7054fde59eaad"
+  },
+  "model_source": "hf",
+  "model_name": "/workspace/outputs/l2a_style/stage2/checkpoint-25",
+  "model_name_sanitized": "__workspace__outputs__l2a_style__stage2__checkpoint-25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "total_evaluation_time_seconds": "767.5954433590232"
+}

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_1_2026-07-18T16-04-28.086981.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:853ac1c1aecd7005e4d2c3a2442e2f95ab2b6195a181e18dd890522231aa15fd
+size 1446074

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multikey_3_2026-07-18T16-04-28.086981.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b35d8f844bf3281a7c84f9bf92e04c7a9f88dfaadc0b9db2c98c140972b0b7
+size 503288

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_multivalue_2026-07-18T16-04-28.086981.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c47b0de2f9608db51ea260b6b5da89687cf93e04b3104517ee958e5312c24d63
+size 1434824

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_niah_single_2_2026-07-18T16-04-28.086981.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83d577e7c704bafad43cc64c986087331cc9b1da48459a533c14cedda1bdcd63
+size 1437596

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_cwe_2026-07-18T16-04-28.086981.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9b295568237b75ee448d27bcfd3773b39c837e618a03e9e2b1c74cb81cb0fb2
+size 635633

outputs/eval/token_t0525/ruler8k_splits/b/lm_eval/__workspace__outputs__l2a_style__stage2__checkpoint-25/samples_ruler_qa_squad_2026-07-18T16-04-28.086981.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c66ae17c903fd34f65a7b5acce334ff5cde436941a2afd501e48c14ca9b2b99
+size 1387354