Add files using upload-large-folder tool

eval_results/logs/qwen2.5-1.5B-it-nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0-1246.log

@@ -0,0 +1,313 @@
+==========================================
+Label: qwen2.5-1.5B-it-nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0-1246
+Start: Sat May 16 06:29:58 UTC 2026
+==========================================
+>>> [2/10] MATH (Minerva format)
+2026-05-16:06:30:07 INFO     [_cli.run:388] Selected Tasks: ['minerva_math']
+2026-05-16:06:30:08 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-05-16:06:30:08 INFO     [evaluator:239] Initializing vllm model, with arguments: {'pretrained': 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246', 'tensor_parallel_size': 2, 'dtype': 'float16', 'gpu_memory_utilization': 0.75, 'trust_remote_code': True}
+INFO 05-16 06:30:12 [utils.py:261] non-default args: {'trust_remote_code': True, 'dtype': 'float16', 'seed': 1234, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.75, 'disable_log_stats': True, 'model': 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246'}
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+INFO 05-16 06:30:12 [model.py:541] Resolved architecture: Qwen2ForCausalLM
+INFO 05-16 06:30:12 [model.py:1561] Using max model len 32768
+INFO 05-16 06:30:12 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=16384.
+INFO 05-16 06:30:12 [vllm.py:624] Asynchronous scheduling is enabled.
+The tokenizer you are loading from 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:13 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246', speculative_config=None, tokenizer='results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=1234, served_model_name=results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=3777171) WARNING 05-16 06:30:13 [multiproc_executor.py:910] Reducing Torch parallelism from 96 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:17 [parallel_state.py:1212] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:52955 backend=nccl
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:17 [parallel_state.py:1212] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:52955 backend=nccl
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:17 [pynccl.py:111] vLLM is using nccl==2.27.5
+(EngineCore_DP0 pid=3777171) WARNING 05-16 06:30:18 [symm_mem.py:99] SymmMemCommunicator: symmetric memory initialization failed: CUDA driver error: system not yet initialized Communicator is not available. To suppress this warning set VLLM_ALLREDUCE_USE_SYMM_MEM=0
+(EngineCore_DP0 pid=3777171) WARNING 05-16 06:30:18 [symm_mem.py:99] SymmMemCommunicator: symmetric memory initialization failed: CUDA driver error: system not yet initialized Communicator is not available. To suppress this warning set VLLM_ALLREDUCE_USE_SYMM_MEM=0
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:18 [parallel_state.py:1423] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:18 [parallel_state.py:1423] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank N/A
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:19 [gpu_model_runner.py:4033] Starting to load model results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246...
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:20 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) 
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) 
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) 
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) 
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:20 [default_loader.py:291] Loading weights took 0.24 seconds
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:21 [gpu_model_runner.py:4130] Model loading took 1.45 GiB memory and 0.823646 seconds
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:26 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/57ae5fcb1f/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:26 [backends.py:872] Dynamo bytecode transform time: 4.69 s
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:29 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 16384) from the cache, took 0.561 s
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:29 [monitor.py:34] torch.compile takes 5.25 s in total
+(EngineCore_DP0 pid=3777171) (Worker_TP1 pid=3777182) INFO 05-16 06:30:29 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 16384) from the cache, took 0.632 s
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:30 [gpu_worker.py:356] Available KV cache memory: 97.26 GiB
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:30 [kv_cache_utils.py:1307] GPU KV cache size: 7,284,880 tokens
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:30 [kv_cache_utils.py:1312] Maximum concurrency for 32,768 tokens per request: 222.32x
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) 
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) 
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:35 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=3777171) (Worker_TP1 pid=3777182) INFO 05-16 06:30:35 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) INFO 05-16 06:30:36 [gpu_model_runner.py:5063] Graph capturing finished in 5 secs, took 0.38 GiB
+(EngineCore_DP0 pid=3777171) INFO 05-16 06:30:36 [core.py:272] init engine (profile, create kv cache, warmup model) took 15.13 seconds
+(EngineCore_DP0 pid=3777171) The tokenizer you are loading from 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+INFO 05-16 06:30:40 [llm.py:343] Supported tasks: ['generate']
+The tokenizer you are loading from 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+
+
+
+
+
+
+
+2026-05-16:06:30:50 INFO     [evaluator_utils:446] Selected tasks:
+2026-05-16:06:30:50 INFO     [evaluator_utils:462] Group: minerva_math
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_algebra (minerva_math/minerva_math_algebra.yaml)
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_counting_and_prob (minerva_math/minerva_math_counting_and_prob.yaml)
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_geometry (minerva_math/minerva_math_geometry.yaml)
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_intermediate_algebra (minerva_math/minerva_math_intermediate_algebra.yaml)
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_num_theory (minerva_math/minerva_math_num_theory.yaml)
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_prealgebra (minerva_math/minerva_math_prealgebra.yaml)
+2026-05-16:06:30:50 INFO     [evaluator_utils:470]   Task: minerva_math_precalc (minerva_math/minerva_math_precalc.yaml)
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_algebra: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_algebra from 4 to 4
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_counting_and_prob: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_counting_and_prob from 4 to 4
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_geometry: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_geometry from 4 to 4
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_intermediate_algebra: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_intermediate_algebra from 4 to 4
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_num_theory: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_num_theory from 4 to 4
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_prealgebra: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_prealgebra from 4 to 4
+2026-05-16:06:30:50 INFO     [evaluator:314] minerva_math_precalc: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-16:06:30:50 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_precalc from 4 to 4
+2026-05-16:06:30:50 INFO     [api.task:312] Building contexts for minerva_math_algebra on rank 0...
+
  0%|          | 0/1187 [00:00<?, ?it/s]
  3%|▎         | 32/1187 [00:00<00:03, 314.98it/s]
  6%|▌         | 68/1187 [00:00<00:03, 337.97it/s]
  9%|▉         | 104/1187 [00:00<00:03, 345.68it/s]
 12%|█▏        | 140/1187 [00:00<00:02, 350.54it/s]
 15%|█▍        | 176/1187 [00:00<00:02, 353.58it/s]
 18%|█▊        | 212/1187 [00:00<00:02, 351.46it/s]
 21%|██        | 249/1187 [00:00<00:02, 354.89it/s]
 24%|██▍       | 286/1187 [00:00<00:02, 357.38it/s]
 27%|██▋       | 323/1187 [00:00<00:02, 358.54it/s]
 30%|███       | 360/1187 [00:01<00:02, 359.39it/s]
 33%|███▎      | 397/1187 [00:01<00:02, 359.81it/s]
 37%|███▋      | 434/1187 [00:01<00:02, 360.37it/s]
 40%|███▉      | 471/1187 [00:01<00:01, 360.48it/s]
 43%|████▎     | 508/1187 [00:01<00:01, 360.38it/s]
 46%|████▌     | 545/1187 [00:01<00:01, 360.62it/s]
 49%|████▉     | 582/1187 [00:01<00:01, 360.68it/s]
 52%|█████▏    | 619/1187 [00:01<00:01, 360.33it/s]
 55%|█████▌    | 656/1187 [00:01<00:01, 359.92it/s]
 58%|█████▊    | 692/1187 [00:01<00:01, 350.20it/s]
 61%|██████▏   | 728/1187 [00:02<00:01, 350.92it/s]
 64%|██████▍   | 764/1187 [00:02<00:01, 352.18it/s]
 67%|██████▋   | 800/1187 [00:02<00:01, 353.18it/s]
 70%|███████   | 836/1187 [00:02<00:00, 354.19it/s]
 73%|███████▎  | 872/1187 [00:02<00:00, 354.63it/s]
 76%|███████▋  | 908/1187 [00:02<00:00, 354.37it/s]
 80%|███████▉  | 944/1187 [00:02<00:00, 354.58it/s]
 83%|████████▎ | 980/1187 [00:02<00:00, 354.54it/s]
 86%|████████▌ | 1016/1187 [00:02<00:00, 344.49it/s]
 89%|████████▊ | 1052/1187 [00:02<00:00, 348.00it/s]
 92%|█████████▏| 1088/1187 [00:03<00:00, 350.48it/s]
 95%|█████████▍| 1124/1187 [00:03<00:00, 352.40it/s]
 98%|█████████▊| 1160/1187 [00:03<00:00, 354.00it/s]
+2026-05-16:06:30:54 INFO     [api.task:312] Building contexts for minerva_math_counting_and_prob on rank 0...
+
  0%|          | 0/474 [00:00<?, ?it/s]
  8%|▊         | 36/474 [00:00<00:01, 357.96it/s]
 15%|█▌        | 72/474 [00:00<00:01, 355.01it/s]
 23%|██▎       | 108/474 [00:00<00:01, 356.09it/s]
 30%|███       | 144/474 [00:00<00:00, 356.60it/s]
 38%|███▊      | 180/474 [00:00<00:00, 357.02it/s]
 46%|████▌     | 216/474 [00:00<00:00, 357.35it/s]
 53%|█████▎    | 252/474 [00:00<00:00, 357.94it/s]
 61%|██████    | 288/474 [00:00<00:00, 358.56it/s]
 69%|██████▊   | 325/474 [00:00<00:00, 359.17it/s]
 76%|███████▌  | 361/474 [00:01<00:00, 358.52it/s]
 84%|████████▍ | 397/474 [00:01<00:00, 358.82it/s]
 91%|█████████▏| 433/474 [00:01<00:00, 359.14it/s]
 99%|█████████▉| 469/474 [00:01<00:00, 359.28it/s]
+2026-05-16:06:30:55 INFO     [api.task:312] Building contexts for minerva_math_geometry on rank 0...
+
  0%|          | 0/479 [00:00<?, ?it/s]
  8%|▊         | 36/479 [00:00<00:01, 354.70it/s]
 15%|█▌        | 72/479 [00:00<00:01, 355.92it/s]
 23%|██▎       | 108/479 [00:00<00:01, 357.13it/s]
 30%|███       | 144/479 [00:00<00:00, 358.00it/s]
 38%|███▊      | 180/479 [00:00<00:00, 357.83it/s]
 45%|████▌     | 216/479 [00:00<00:00, 357.91it/s]
 53%|█████▎    | 252/479 [00:00<00:00, 357.84it/s]
 60%|██████    | 288/479 [00:00<00:00, 357.62it/s]
 68%|██████▊   | 324/479 [00:00<00:00, 357.66it/s]
 75%|███████▌  | 360/479 [00:01<00:00, 357.87it/s]
 83%|████████▎ | 396/479 [00:01<00:00, 352.00it/s]
 90%|█████████ | 432/479 [00:01<00:00, 354.21it/s]
 98%|█████████▊| 468/479 [00:01<00:00, 355.12it/s]
+2026-05-16:06:30:56 INFO     [api.task:312] Building contexts for minerva_math_intermediate_algebra on rank 0...
+
  0%|          | 0/903 [00:00<?, ?it/s]
  4%|▍         | 36/903 [00:00<00:02, 354.48it/s]
  8%|▊         | 72/903 [00:00<00:02, 355.83it/s]
 12%|█▏        | 109/903 [00:00<00:02, 357.57it/s]
 16%|█▌        | 146/903 [00:00<00:02, 358.74it/s]
 20%|██        | 183/903 [00:00<00:02, 359.46it/s]
 24%|██▍       | 219/903 [00:00<00:01, 358.73it/s]
 28%|██▊       | 255/903 [00:00<00:01, 359.06it/s]
 32%|███▏      | 291/903 [00:00<00:01, 359.27it/s]
 36%|███▌      | 327/903 [00:00<00:01, 359.43it/s]
 40%|████      | 363/903 [00:01<00:01, 359.28it/s]
 44%|████▍     | 399/903 [00:01<00:01, 359.24it/s]
 48%|████▊     | 435/903 [00:01<00:01, 359.30it/s]
 52%|█████▏    | 472/903 [00:01<00:01, 359.76it/s]
 56%|█████▋    | 508/903 [00:01<00:01, 350.35it/s]
 60%|██████    | 544/903 [00:01<00:01, 352.53it/s]
 64%|██████▍   | 580/903 [00:01<00:00, 354.18it/s]
 68%|██████▊   | 616/903 [00:01<00:00, 355.79it/s]
 72%|███████▏  | 652/903 [00:01<00:00, 356.80it/s]
 76%|███████▌  | 688/903 [00:01<00:00, 357.65it/s]
 80%|████████  | 724/903 [00:02<00:00, 358.21it/s]
 84%|████████▍ | 761/903 [00:02<00:00, 359.00it/s]
 88%|████████▊ | 797/903 [00:02<00:00, 358.84it/s]
 92%|█████████▏| 833/903 [00:02<00:00, 358.59it/s]
 96%|█████████▌| 869/903 [00:02<00:00, 348.21it/s]
+2026-05-16:06:30:59 INFO     [api.task:312] Building contexts for minerva_math_num_theory on rank 0...
+
  0%|          | 0/540 [00:00<?, ?it/s]
  7%|▋         | 36/540 [00:00<00:01, 357.81it/s]
 13%|█▎        | 72/540 [00:00<00:01, 356.67it/s]
 20%|██        | 108/540 [00:00<00:01, 357.01it/s]
 27%|██▋       | 144/540 [00:00<00:01, 356.80it/s]
 33%|███▎      | 180/540 [00:00<00:01, 356.26it/s]
 40%|████      | 216/540 [00:00<00:00, 356.81it/s]
 47%|████▋     | 252/540 [00:00<00:00, 357.29it/s]
 53%|█████▎    | 288/540 [00:00<00:00, 356.18it/s]
 60%|██████    | 324/540 [00:00<00:00, 356.34it/s]
 67%|██████▋   | 360/540 [00:01<00:00, 356.65it/s]
 73%|███████▎  | 396/540 [00:01<00:00, 357.64it/s]
 80%|████████  | 432/540 [00:01<00:00, 358.25it/s]
 87%|████████▋ | 468/540 [00:01<00:00, 358.32it/s]
 94%|█████████▎| 505/540 [00:01<00:00, 359.72it/s]
+2026-05-16:06:31:00 INFO     [api.task:312] Building contexts for minerva_math_prealgebra on rank 0...
+
  0%|          | 0/871 [00:00<?, ?it/s]
  4%|▍         | 37/871 [00:00<00:02, 362.32it/s]
  8%|▊         | 74/871 [00:00<00:02, 363.44it/s]
 13%|█▎        | 111/871 [00:00<00:02, 365.11it/s]
 17%|█▋        | 148/871 [00:00<00:01, 365.74it/s]
 21%|██        | 185/871 [00:00<00:01, 365.63it/s]
 25%|██▌       | 222/871 [00:00<00:01, 365.99it/s]
 30%|██▉       | 259/871 [00:00<00:01, 366.01it/s]
 34%|███▍      | 296/871 [00:00<00:01, 366.03it/s]
 38%|███▊      | 333/871 [00:00<00:01, 366.43it/s]
 42%|████▏     | 370/871 [00:01<00:01, 367.00it/s]
 47%|████▋     | 407/871 [00:01<00:01, 367.60it/s]
 51%|█████     | 444/871 [00:01<00:01, 367.64it/s]
 55%|█████▌    | 481/871 [00:01<00:01, 367.41it/s]
 59%|█████▉    | 518/871 [00:01<00:00, 367.20it/s]
 64%|██████▎   | 555/871 [00:01<00:00, 366.88it/s]
 68%|██████▊   | 592/871 [00:01<00:00, 366.20it/s]
 72%|███████▏  | 629/871 [00:01<00:00, 365.71it/s]
 76%|███████▋  | 666/871 [00:01<00:00, 365.77it/s]
 81%|████████  | 703/871 [00:01<00:00, 365.89it/s]
 85%|████████▍ | 740/871 [00:02<00:00, 365.93it/s]
 89%|████████▉ | 777/871 [00:02<00:00, 365.71it/s]
 93%|█████████▎| 814/871 [00:02<00:00, 365.17it/s]
 98%|█████████▊| 851/871 [00:02<00:00, 365.15it/s]
+2026-05-16:06:31:03 INFO     [api.task:312] Building contexts for minerva_math_precalc on rank 0...
+
  0%|          | 0/546 [00:00<?, ?it/s]
  7%|▋         | 37/546 [00:00<00:01, 361.42it/s]
 14%|█▎        | 74/546 [00:00<00:01, 361.32it/s]
 20%|██        | 111/546 [00:00<00:01, 362.86it/s]
 27%|██▋       | 148/546 [00:00<00:01, 364.16it/s]
 34%|███▍      | 185/546 [00:00<00:00, 364.71it/s]
 41%|████      | 222/546 [00:00<00:00, 365.02it/s]
 47%|████▋     | 259/546 [00:00<00:00, 365.53it/s]
 54%|█████▍    | 296/546 [00:00<00:00, 366.08it/s]
 61%|██████    | 333/546 [00:00<00:00, 366.13it/s]
 68%|██████▊   | 370/546 [00:01<00:00, 366.42it/s]
 75%|███████▍  | 407/546 [00:01<00:00, 366.71it/s]
 81%|████████▏ | 444/546 [00:01<00:00, 366.86it/s]
 88%|████████▊ | 481/546 [00:01<00:00, 366.48it/s]
 95%|█████████▍| 518/546 [00:01<00:00, 366.34it/s]
+2026-05-16:06:31:04 INFO     [evaluator:585] Running generate_until requests
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+2026-05-16:06:35:12 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-05-16:06:35:12 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/eval_results/vllm/qwen2.5-1.5B-it-nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0-1246/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0__1246/*.jsonl
+(EngineCore_DP0 pid=3777171) (Worker_TP0 pid=3777180) WARNING 05-16 06:35:16 [multiproc_executor.py:786] WorkerProc was terminated
+(EngineCore_DP0 pid=3777171) (Worker_TP1 pid=3777182) WARNING 05-16 06:35:16 [multiproc_executor.py:786] WorkerProc was terminated
+vllm ({'pretrained': 'results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0/1246', 'tensor_parallel_size': 2, 'dtype': 'float16', 'gpu_memory_utilization': 0.75}), gen_kwargs: ({}), limit: None, num_fewshot: 4, batch_size: auto
+|               Tasks                |Version|Filter|n-shot|  Metric   |   |Value |   |Stderr|
+|------------------------------------|------:|------|-----:|-----------|---|-----:|---|-----:|
+|minerva_math                        |      1|none  |     4|exact_match|↑  |0.1254|±  |0.0046|
+|                                    |       |none  |     4|math_verify|↑  |0.3262|±  |0.0062|
+| - minerva_math_algebra             |      3|none  |     4|exact_match|↑  |0.1702|±  |0.0109|
+|                                    |       |none  |     4|math_verify|↑  |0.4760|±  |0.0145|
+| - minerva_math_counting_and_prob   |      3|none  |     4|exact_match|↑  |0.1097|±  |0.0144|
+|                                    |       |none  |     4|math_verify|↑  |0.2595|±  |0.0202|
+| - minerva_math_geometry            |      3|none  |     4|exact_match|↑  |0.1232|±  |0.0150|
+|                                    |       |none  |     4|math_verify|↑  |0.2985|±  |0.0209|
+| - minerva_math_intermediate_algebra|      3|none  |     4|exact_match|↑  |0.0388|±  |0.0064|
+|                                    |       |none  |     4|math_verify|↑  |0.1307|±  |0.0112|
+| - minerva_math_num_theory          |      3|none  |     4|exact_match|↑  |0.0407|±  |0.0085|
+|                                    |       |none  |     4|math_verify|↑  |0.2204|±  |0.0179|
+| - minerva_math_prealgebra          |      3|none  |     4|exact_match|↑  |0.2595|±  |0.0149|
+|                                    |       |none  |     4|math_verify|↑  |0.5522|±  |0.0169|
+| - minerva_math_precalc             |      3|none  |     4|exact_match|↑  |0.0568|±  |0.0099|
+|                                    |       |none  |     4|math_verify|↑  |0.1502|±  |0.0153|
+
+|   Groups   |Version|Filter|n-shot|  Metric   |   |Value |   |Stderr|
+|------------|------:|------|-----:|-----------|---|-----:|---|-----:|
+|minerva_math|      1|none  |     4|exact_match|↑  |0.1254|±  |0.0046|
+|            |       |none  |     4|math_verify|↑  |0.3262|±  |0.0062|
+
+==========================================
+DONE: qwen2.5-1.5B-it-nnm0.1_K128_L4_epoch1_lr1e-4_kdr1.0-1246 | Sat May 16 06:35:20 UTC 2026
+==========================================

eval_results/logs/qwen3-1.7.log

@@ -0,0 +1,326 @@
+==========================================
+Label: qwen3-1.7
+Start: Wed May 20 07:05:56 UTC 2026
+==========================================
+>>> [1/10] GSM8K
+2026-05-20:07:05:56 INFO     [config.evaluate_config:307] Using default fewshot_as_multiturn=True.
+2026-05-20:07:06:04 INFO     [_cli.run:388] Selected Tasks: ['gsm8k']
+2026-05-20:07:06:05 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-05-20:07:06:05 INFO     [evaluator:239] Initializing vllm model, with arguments: {'pretrained': 'Qwen/Qwen3-1.7B', 'tensor_parallel_size': 2, 'dtype': 'bfloat16', 'gpu_memory_utilization': 0.8, 'trust_remote_code': True, 'reasoning_parser': 'deepseek_r1', 'enable_thinking': True, 'think_end_token': '</think>'}
+INFO 05-20 07:06:10 [utils.py:238] non-default args: {'trust_remote_code': True, 'dtype': 'bfloat16', 'seed': 1234, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.8, 'disable_log_stats': True, 'reasoning_parser': 'deepseek_r1', 'model': 'Qwen/Qwen3-1.7B'}
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+INFO 05-20 07:06:11 [model.py:531] Resolved architecture: Qwen3ForCausalLM
+INFO 05-20 07:06:11 [model.py:1554] Using max model len 40960
+INFO 05-20 07:06:11 [scheduler.py:231] Chunked prefill is enabled with max_num_batched_tokens=16384.
+INFO 05-20 07:06:11 [vllm.py:747] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=2856575) INFO 05-20 07:06:14 [core.py:101] Initializing a V1 LLM engine (v0.17.1) with config: model='Qwen/Qwen3-1.7B', speculative_config=None, tokenizer='Qwen/Qwen3-1.7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='deepseek_r1', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=1234, served_model_name=Qwen/Qwen3-1.7B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': True, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=2856575) WARNING 05-20 07:06:14 [multiproc_executor.py:945] Reducing Torch parallelism from 96 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
+(EngineCore_DP0 pid=2856575) INFO 05-20 07:06:14 [multiproc_executor.py:134] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.31.21.192 (local), world_size=2, local_world_size=2
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) INFO 05-20 07:06:16 [parallel_state.py:1393] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:46555 backend=nccl
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) INFO 05-20 07:06:16 [parallel_state.py:1393] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:46555 backend=nccl
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) INFO 05-20 07:06:16 [pynccl.py:111] vLLM is using nccl==2.27.5
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
+[rank1]:[W520 07:06:17.414732864 CUDASymmetricMemory.cu:604] Warning: CUDA driver error: system not yet initialized (function init_multicast_for_block)
+[rank0]:[W520 07:06:17.414734164 CUDASymmetricMemory.cu:604] Warning: CUDA driver error: system not yet initialized (function init_multicast_for_block)
+[rank0]:[W520 07:06:17.414860566 CUDASymmetricMemory.cu:621] Gracefully skipping multicast initialization.
+[rank1]:[W520 07:06:17.414983248 CUDASymmetricMemory.cu:621] Gracefully skipping multicast initialization.
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) WARNING 05-20 07:06:17 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) WARNING 05-20 07:06:17 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) INFO 05-20 07:06:17 [parallel_state.py:1715] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank N/A, EPLB rank N/A
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) INFO 05-20 07:06:17 [parallel_state.py:1715] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) INFO 05-20 07:06:18 [base.py:106] Offloader set to NoopOffloader
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:18 [gpu_model_runner.py:4281] Starting to load model Qwen/Qwen3-1.7B...
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) INFO 05-20 07:06:18 [base.py:106] Offloader set to NoopOffloader
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:19 [cuda.py:405] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:19 [flash_attn.py:587] Using FlashAttention version 3
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) 
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) 
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) 
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) 
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:20 [default_loader.py:293] Loading weights took 0.26 seconds
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:21 [gpu_model_runner.py:4364] Model loading took 1.61 GiB memory and 1.236675 seconds
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) (Worker_TP1 pid=2856583) INFO 05-20 07:06:24 [decorators.py:465] Directly load AOT compilation from path /home/ubuntu/.cache/vllm/torch_compile_cache/torch_aot_compile/51a82c2c50ff9f473a0d7b91dcb7e226ed7241b74ec83725bb6d47cbeffa99f2/rank_1_0/model
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:24 [decorators.py:465] Directly load AOT compilation from path /home/ubuntu/.cache/vllm/torch_compile_cache/torch_aot_compile/51a82c2c50ff9f473a0d7b91dcb7e226ed7241b74ec83725bb6d47cbeffa99f2/rank_0_0/model
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:24 [backends.py:916] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f745e6964b/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:24 [backends.py:976] Dynamo bytecode transform time: 2.67 s
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:26 [backends.py:266] Directly load the compiled graph(s) for compile range (1, 16384) from the cache, took 1.840 s
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:26 [monitor.py:35] torch.compile takes 4.84 s in total
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:27 [gpu_worker.py:424] Available KV cache memory: 107.48 GiB
+(EngineCore_DP0 pid=2856575) INFO 05-20 07:06:27 [kv_cache_utils.py:1314] GPU KV cache size: 2,012,528 tokens
+(EngineCore_DP0 pid=2856575) INFO 05-20 07:06:27 [kv_cache_utils.py:1319] Maximum concurrency for 40,960 tokens per request: 49.13x
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) 
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) 
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:32 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) (Worker_TP1 pid=2856583) INFO 05-20 07:06:32 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) INFO 05-20 07:06:33 [gpu_model_runner.py:5386] Graph capturing finished in 5 secs, took 0.38 GiB
+(EngineCore_DP0 pid=2856575) INFO 05-20 07:06:33 [core.py:282] init engine (profile, create kv cache, warmup model) took 11.60 seconds
+INFO 05-20 07:06:34 [llm.py:388] Supported tasks: ['generate']
+2026-05-20:07:06:39 INFO     [evaluator_utils:446] Selected tasks:
+2026-05-20:07:06:39 INFO     [evaluator_utils:480] Task: gsm8k (gsm8k/gsm8k.yaml)
+2026-05-20:07:06:39 INFO     [evaluator:314] gsm8k: Using gen_kwargs: {'until': ['Question:', '</s>', '<|im_end|>'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:06:39 WARNING  [evaluator:333] Overwriting default num_fewshot of gsm8k from 5 to 5
+2026-05-20:07:06:39 INFO     [api.task:312] Building contexts for gsm8k on rank 0...
+
  0%|          | 0/1319 [00:00<?, ?it/s]
  2%|▏         | 22/1319 [00:00<00:06, 213.12it/s]
  4%|▍         | 50/1319 [00:00<00:05, 247.32it/s]
  6%|▌         | 78/1319 [00:00<00:04, 258.27it/s]
  8%|▊         | 106/1319 [00:00<00:04, 264.64it/s]
 10%|█         | 134/1319 [00:00<00:04, 268.88it/s]
 12%|█▏        | 162/1319 [00:00<00:04, 271.26it/s]
 14%|█▍        | 190/1319 [00:00<00:04, 273.34it/s]
 17%|█▋        | 218/1319 [00:00<00:04, 274.66it/s]
 19%|█▊        | 246/1319 [00:00<00:03, 275.28it/s]
 21%|██        | 274/1319 [00:01<00:03, 275.57it/s]
 23%|██▎       | 302/1319 [00:01<00:03, 275.94it/s]
 25%|██▌       | 330/1319 [00:01<00:03, 276.24it/s]
 27%|██▋       | 358/1319 [00:01<00:03, 276.57it/s]
 29%|██▉       | 386/1319 [00:01<00:03, 276.42it/s]
 31%|███▏      | 414/1319 [00:01<00:03, 276.44it/s]
 34%|███▎      | 442/1319 [00:01<00:03, 270.19it/s]
 36%|███▌      | 470/1319 [00:01<00:03, 271.28it/s]
 38%|███▊      | 498/1319 [00:01<00:03, 272.80it/s]
 40%|███▉      | 526/1319 [00:01<00:02, 273.84it/s]
 42%|████▏     | 554/1319 [00:02<00:02, 274.73it/s]
 44%|████▍     | 582/1319 [00:02<00:02, 275.00it/s]
 46%|████▌     | 610/1319 [00:02<00:02, 275.45it/s]
 48%|████▊     | 638/1319 [00:02<00:02, 275.89it/s]
 50%|█████     | 666/1319 [00:02<00:02, 276.17it/s]
 53%|█████▎    | 694/1319 [00:02<00:02, 276.40it/s]
 55%|█████▍    | 722/1319 [00:02<00:02, 276.21it/s]
 57%|█████▋    | 750/1319 [00:02<00:02, 268.77it/s]
 59%|█████▉    | 777/1319 [00:02<00:02, 265.64it/s]
 61%|██████    | 805/1319 [00:02<00:01, 268.32it/s]
 63%|██████▎   | 833/1319 [00:03<00:01, 270.16it/s]
 65%|██████▌   | 861/1319 [00:03<00:01, 271.50it/s]
 67%|██████▋   | 889/1319 [00:03<00:01, 272.62it/s]
 70%|██████▉   | 917/1319 [00:03<00:01, 272.99it/s]
 72%|███████▏  | 945/1319 [00:03<00:01, 273.30it/s]
 74%|███████▍  | 973/1319 [00:03<00:01, 273.67it/s]
 76%|███████▌  | 1001/1319 [00:03<00:01, 274.05it/s]
 78%|███████▊  | 1029/1319 [00:03<00:01, 274.47it/s]
 80%|████████  | 1057/1319 [00:03<00:00, 274.61it/s]
 82%|████████▏ | 1085/1319 [00:03<00:00, 269.49it/s]
 84%|████████▍ | 1113/1319 [00:04<00:00, 271.22it/s]
 87%|████████▋ | 1141/1319 [00:04<00:00, 272.83it/s]
 89%|████████▊ | 1169/1319 [00:04<00:00, 273.78it/s]
 91%|█████████ | 1197/1319 [00:04<00:00, 274.17it/s]
 93%|█████████▎| 1225/1319 [00:04<00:00, 274.74it/s]
 95%|█████████▍| 1253/1319 [00:04<00:00, 275.02it/s]
 97%|█████████▋| 1281/1319 [00:04<00:00, 275.16it/s]
 99%|█████████▉| 1309/1319 [00:04<00:00, 275.25it/s]
+2026-05-20:07:06:44 INFO     [evaluator:585] Running generate_until requests
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+(EngineCore_DP0 pid=2856575) (Worker pid=2856583) (Worker_TP1 pid=2856583) WARNING 05-20 07:07:09 [multiproc_executor.py:814] WorkerProc was terminated
+(EngineCore_DP0 pid=2856575) (Worker pid=2856581) (Worker_TP0 pid=2856581) WARNING 05-20 07:07:09 [multiproc_executor.py:814] WorkerProc was terminated
+2026-05-20:07:07:10 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-05-20:07:07:10 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/eval_results/vllm/qwen3-1.7/Qwen__Qwen3-1.7B/*.jsonl
+vllm ({'pretrained': 'Qwen/Qwen3-1.7B', 'tensor_parallel_size': 2, 'dtype': 'bfloat16', 'gpu_memory_utilization': 0.8, 'reasoning_parser': 'deepseek_r1', 'enable_thinking': True, 'think_end_token': '</think>'}), gen_kwargs: ({}), limit: None, num_fewshot: 5, batch_size: auto
+|Tasks|Version|     Filter     |n-shot|  Metric   |   |Value |   |Stderr|
+|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:|
+|gsm8k|      3|flexible-extract|     5|exact_match|↑  |0.1774|±  |0.0105|
+|     |       |strict-match    |     5|exact_match|↑  |0.0083|±  |0.0025|
+
+>>> [2/10] MATH (Minerva format)
+2026-05-20:07:07:13 INFO     [config.evaluate_config:307] Using default fewshot_as_multiturn=True.
+2026-05-20:07:07:21 INFO     [_cli.run:388] Selected Tasks: ['minerva_math']
+2026-05-20:07:07:22 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-05-20:07:07:22 INFO     [evaluator:239] Initializing vllm model, with arguments: {'pretrained': 'Qwen/Qwen3-1.7B', 'tensor_parallel_size': 2, 'dtype': 'bfloat16', 'gpu_memory_utilization': 0.8, 'trust_remote_code': True, 'reasoning_parser': 'deepseek_r1', 'enable_thinking': True, 'think_end_token': '</think>'}
+INFO 05-20 07:07:27 [utils.py:238] non-default args: {'trust_remote_code': True, 'dtype': 'bfloat16', 'seed': 1234, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.8, 'disable_log_stats': True, 'reasoning_parser': 'deepseek_r1', 'model': 'Qwen/Qwen3-1.7B'}
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+INFO 05-20 07:07:28 [model.py:531] Resolved architecture: Qwen3ForCausalLM
+INFO 05-20 07:07:28 [model.py:1554] Using max model len 40960
+INFO 05-20 07:07:28 [scheduler.py:231] Chunked prefill is enabled with max_num_batched_tokens=16384.
+INFO 05-20 07:07:28 [vllm.py:747] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=2857336) INFO 05-20 07:07:30 [core.py:101] Initializing a V1 LLM engine (v0.17.1) with config: model='Qwen/Qwen3-1.7B', speculative_config=None, tokenizer='Qwen/Qwen3-1.7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='deepseek_r1', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=1234, served_model_name=Qwen/Qwen3-1.7B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': True, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=2857336) WARNING 05-20 07:07:30 [multiproc_executor.py:945] Reducing Torch parallelism from 96 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
+(EngineCore_DP0 pid=2857336) INFO 05-20 07:07:30 [multiproc_executor.py:134] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.31.21.192 (local), world_size=2, local_world_size=2
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) INFO 05-20 07:07:32 [parallel_state.py:1393] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:42073 backend=nccl
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) INFO 05-20 07:07:32 [parallel_state.py:1393] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:42073 backend=nccl
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) INFO 05-20 07:07:33 [pynccl.py:111] vLLM is using nccl==2.27.5
+[rank1]:[W520 07:07:34.292863478 CUDASymmetricMemory.cu:604] Warning: CUDA driver error: system not yet initialized (function init_multicast_for_block)
+[rank0]:[W520 07:07:34.292867838 CUDASymmetricMemory.cu:604] Warning: CUDA driver error: system not yet initialized (function init_multicast_for_block)
+[rank0]:[W520 07:07:34.293032871 CUDASymmetricMemory.cu:621] Gracefully skipping multicast initialization.
+[rank1]:[W520 07:07:34.293164293 CUDASymmetricMemory.cu:621] Gracefully skipping multicast initialization.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) WARNING 05-20 07:07:34 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) WARNING 05-20 07:07:34 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) INFO 05-20 07:07:34 [parallel_state.py:1715] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank N/A, EPLB rank N/A
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) INFO 05-20 07:07:34 [parallel_state.py:1715] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) INFO 05-20 07:07:35 [base.py:106] Offloader set to NoopOffloader
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:35 [gpu_model_runner.py:4281] Starting to load model Qwen/Qwen3-1.7B...
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) INFO 05-20 07:07:35 [base.py:106] Offloader set to NoopOffloader
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:35 [cuda.py:405] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:35 [flash_attn.py:587] Using FlashAttention version 3
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) 
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) 
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) 
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) 
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:37 [default_loader.py:293] Loading weights took 0.25 seconds
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:37 [gpu_model_runner.py:4364] Model loading took 1.61 GiB memory and 1.198995 seconds
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) (Worker_TP1 pid=2857344) INFO 05-20 07:07:40 [decorators.py:465] Directly load AOT compilation from path /home/ubuntu/.cache/vllm/torch_compile_cache/torch_aot_compile/51a82c2c50ff9f473a0d7b91dcb7e226ed7241b74ec83725bb6d47cbeffa99f2/rank_1_0/model
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:40 [decorators.py:465] Directly load AOT compilation from path /home/ubuntu/.cache/vllm/torch_compile_cache/torch_aot_compile/51a82c2c50ff9f473a0d7b91dcb7e226ed7241b74ec83725bb6d47cbeffa99f2/rank_0_0/model
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:41 [backends.py:916] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f745e6964b/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:41 [backends.py:976] Dynamo bytecode transform time: 2.78 s
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:43 [backends.py:266] Directly load the compiled graph(s) for compile range (1, 16384) from the cache, took 1.837 s
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:43 [monitor.py:35] torch.compile takes 4.94 s in total
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:44 [gpu_worker.py:424] Available KV cache memory: 107.48 GiB
+(EngineCore_DP0 pid=2857336) INFO 05-20 07:07:44 [kv_cache_utils.py:1314] GPU KV cache size: 2,012,528 tokens
+(EngineCore_DP0 pid=2857336) INFO 05-20 07:07:44 [kv_cache_utils.py:1319] Maximum concurrency for 40,960 tokens per request: 49.13x
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) 
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) 
+
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:49 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) INFO 05-20 07:07:49 [gpu_model_runner.py:5386] Graph capturing finished in 5 secs, took 0.38 GiB
+(EngineCore_DP0 pid=2857336) INFO 05-20 07:07:49 [core.py:282] init engine (profile, create kv cache, warmup model) took 11.69 seconds
+INFO 05-20 07:07:51 [llm.py:388] Supported tasks: ['generate']
+2026-05-20:07:08:02 INFO     [evaluator_utils:446] Selected tasks:
+2026-05-20:07:08:02 INFO     [evaluator_utils:462] Group: minerva_math
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_algebra (minerva_math/minerva_math_algebra.yaml)
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_counting_and_prob (minerva_math/minerva_math_counting_and_prob.yaml)
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_geometry (minerva_math/minerva_math_geometry.yaml)
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_intermediate_algebra (minerva_math/minerva_math_intermediate_algebra.yaml)
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_num_theory (minerva_math/minerva_math_num_theory.yaml)
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_prealgebra (minerva_math/minerva_math_prealgebra.yaml)
+2026-05-20:07:08:02 INFO     [evaluator_utils:470]   Task: minerva_math_precalc (minerva_math/minerva_math_precalc.yaml)
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_algebra: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_algebra from 4 to 4
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_counting_and_prob: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_counting_and_prob from 4 to 4
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_geometry: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_geometry from 4 to 4
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_intermediate_algebra: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_intermediate_algebra from 4 to 4
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_num_theory: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_num_theory from 4 to 4
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_prealgebra: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_prealgebra from 4 to 4
+2026-05-20:07:08:02 INFO     [evaluator:314] minerva_math_precalc: Using gen_kwargs: {'until': ['Problem:'], 'do_sample': False, 'temperature': 0.0}
+2026-05-20:07:08:02 WARNING  [evaluator:333] Overwriting default num_fewshot of minerva_math_precalc from 4 to 4
+2026-05-20:07:08:02 INFO     [api.task:312] Building contexts for minerva_math_algebra on rank 0...
+
  0%|          | 0/1187 [00:00<?, ?it/s]
  2%|▏         | 22/1187 [00:00<00:05, 219.45it/s]
  5%|▍         | 56/1187 [00:00<00:03, 290.00it/s]
  8%|▊         | 91/1187 [00:00<00:03, 316.95it/s]
 10%|█         | 124/1187 [00:00<00:03, 320.06it/s]
 13%|█▎        | 159/1187 [00:00<00:03, 329.60it/s]
 16%|█▋        | 194/1187 [00:00<00:02, 335.07it/s]
 19%|█▉        | 230/1187 [00:00<00:02, 340.25it/s]
 22%|██▏       | 265/1187 [00:00<00:02, 340.54it/s]
 25%|██▌       | 301/1187 [00:00<00:02, 344.16it/s]
 28%|██▊       | 337/1187 [00:01<00:02, 346.05it/s]
 31%|███▏      | 373/1187 [00:01<00:02, 347.63it/s]
 34%|███▍      | 409/1187 [00:01<00:02, 348.50it/s]
 37%|███▋      | 445/1187 [00:01<00:02, 349.25it/s]
 41%|████      | 481/1187 [00:01<00:02, 349.59it/s]
 44%|████▎     | 517/1187 [00:01<00:01, 349.79it/s]
 47%|████▋     | 553/1187 [00:01<00:01, 344.48it/s]
 50%|████▉     | 588/1187 [00:01<00:01, 345.42it/s]
 53%|█████▎    | 624/1187 [00:01<00:01, 347.15it/s]
 56%|█████▌    | 660/1187 [00:01<00:01, 348.16it/s]
 59%|█████▊    | 695/1187 [00:02<00:01, 348.66it/s]
 61%|██████▏   | 730/1187 [00:02<00:01, 339.10it/s]
 64%|██████▍   | 764/1187 [00:02<00:01, 338.50it/s]
 67%|█████���▋   | 799/1187 [00:02<00:01, 341.45it/s]
 70%|███████   | 834/1187 [00:02<00:01, 343.69it/s]
 73%|███████▎  | 869/1187 [00:02<00:00, 345.10it/s]
 76%|███████▌  | 904/1187 [00:02<00:00, 339.87it/s]
 79%|███████▉  | 939/1187 [00:02<00:00, 342.50it/s]
 82%|████████▏ | 975/1187 [00:02<00:00, 344.84it/s]
 85%|████████▌ | 1011/1187 [00:02<00:00, 346.58it/s]
 88%|████████▊ | 1047/1187 [00:03<00:00, 347.89it/s]
 91%|█████████ | 1083/1187 [00:03<00:00, 348.50it/s]
 94%|█████████▍| 1119/1187 [00:03<00:00, 349.40it/s]
 97%|█████████▋| 1154/1187 [00:03<00:00, 349.56it/s]
+2026-05-20:07:08:06 INFO     [api.task:312] Building contexts for minerva_math_counting_and_prob on rank 0...
+
  0%|          | 0/474 [00:00<?, ?it/s]
  7%|▋         | 35/474 [00:00<00:01, 347.93it/s]
 15%|█▍        | 71/474 [00:00<00:01, 349.98it/s]
 23%|██▎       | 107/474 [00:00<00:01, 350.21it/s]
 30%|███       | 143/474 [00:00<00:00, 350.07it/s]
 38%|███▊      | 179/474 [00:00<00:00, 350.27it/s]
 45%|████▌     | 215/474 [00:00<00:00, 349.18it/s]
 53%|█████▎    | 250/474 [00:00<00:00, 348.19it/s]
 60%|██████    | 285/474 [00:00<00:00, 347.36it/s]
 68%|██████▊   | 320/474 [00:00<00:00, 346.81it/s]
 75%|███████▍  | 355/474 [00:01<00:00, 346.58it/s]
 82%|████████▏ | 390/474 [00:01<00:00, 345.78it/s]
 90%|████████▉ | 425/474 [00:01<00:00, 345.75it/s]
 97%|█████████▋| 460/474 [00:01<00:00, 339.15it/s]
+2026-05-20:07:08:07 INFO     [api.task:312] Building contexts for minerva_math_geometry on rank 0...
+
  0%|          | 0/479 [00:00<?, ?it/s]
  7%|▋         | 34/479 [00:00<00:01, 336.96it/s]
 14%|█▍        | 69/479 [00:00<00:01, 341.61it/s]
 22%|██▏       | 104/479 [00:00<00:01, 343.48it/s]
 29%|██▉       | 139/479 [00:00<00:00, 344.87it/s]
 36%|███▋      | 174/479 [00:00<00:00, 345.57it/s]
 44%|████▎     | 209/479 [00:00<00:00, 346.11it/s]
 51%|█████     | 244/479 [00:00<00:00, 335.17it/s]
 58%|█████▊    | 278/479 [00:00<00:00, 333.37it/s]
 65%|██████▌   | 313/479 [00:00<00:00, 336.82it/s]
 73%|███████▎  | 348/479 [00:01<00:00, 339.45it/s]
 80%|███████▉  | 383/479 [00:01<00:00, 341.66it/s]
 87%|████████▋ | 418/479 [00:01<00:00, 342.94it/s]
 95%|█████████▍| 453/479 [00:01<00:00, 343.56it/s]
+2026-05-20:07:08:09 INFO     [api.task:312] Building contexts for minerva_math_intermediate_algebra on rank 0...
+
  0%|          | 0/903 [00:00<?, ?it/s]
  4%|▍         | 34/903 [00:00<00:02, 338.29it/s]
  8%|▊         | 68/903 [00:00<00:02, 330.71it/s]
 11%|█▏        | 103/903 [00:00<00:02, 336.77it/s]
 15%|█▌        | 138/903 [00:00<00:02, 339.88it/s]
 19%|█▉        | 173/903 [00:00<00:02, 341.89it/s]
 23%|██▎       | 208/903 [00:00<00:02, 343.21it/s]
 27%|██▋       | 243/903 [00:00<00:01, 343.88it/s]
 31%|███       | 278/903 [00:00<00:01, 344.43it/s]
 35%|███▍      | 313/903 [00:00<00:01, 341.91it/s]
 39%|███▊      | 348/903 [00:01<00:01, 343.15it/s]
 42%|████▏     | 383/903 [00:01<00:01, 343.95it/s]
 46%|████▋     | 418/903 [00:01<00:01, 345.06it/s]
 50%|█████     | 454/903 [00:01<00:01, 346.67it/s]
 54%|█████▍    | 490/903 [00:01<00:01, 348.14it/s]
 58%|█████▊    | 526/903 [00:01<00:01, 349.16it/s]
 62%|██████▏   | 562/903 [00:01<00:00, 349.49it/s]
 66%|██████▌   | 598/903 [00:01<00:00, 349.74it/s]
 70%|███████   | 633/903 [00:01<00:00, 348.15it/s]
 74%|███████▍  | 668/903 [00:01<00:00, 347.07it/s]
 78%|███████▊  | 703/903 [00:02<00:00, 345.85it/s]
 82%|████████▏ | 738/903 [00:02<00:00, 344.89it/s]
 86%|████████▌ | 773/903 [00:02<00:00, 344.91it/s]
 89%|████████▉ | 808/903 [00:02<00:00, 346.07it/s]
 93%|█████████▎| 843/903 [00:02<00:00, 346.90it/s]
 97%|█████████▋| 878/903 [00:02<00:00, 346.30it/s]
+2026-05-20:07:08:11 INFO     [api.task:312] Building contexts for minerva_math_num_theory on rank 0...
+
  0%|          | 0/540 [00:00<?, ?it/s]
  6%|▋         | 35/540 [00:00<00:01, 342.63it/s]
 13%|█▎        | 70/540 [00:00<00:01, 344.00it/s]
 19%|█▉        | 105/540 [00:00<00:01, 346.38it/s]
 26%|██▌       | 140/540 [00:00<00:01, 347.43it/s]
 32%|███▏      | 175/540 [00:00<00:01, 348.03it/s]
 39%|███▉      | 211/540 [00:00<00:00, 348.89it/s]
 46%|████▌     | 247/540 [00:00<00:00, 349.63it/s]
 52%|█████▏    | 282/540 [00:00<00:00, 349.66it/s]
 59%|█████▉    | 318/540 [00:00<00:00, 349.85it/s]
 66%|██████▌   | 354/540 [00:01<00:00, 350.44it/s]
 72%|███████▏  | 390/540 [00:01<00:00, 350.84it/s]
 79%|███████▉  | 426/540 [00:01<00:00, 351.00it/s]
 86%|████████▌ | 462/540 [00:01<00:00, 350.92it/s]
 92%|█████████▏| 498/540 [00:01<00:00, 350.68it/s]
 99%|█████████▉| 534/540 [00:01<00:00, 350.14it/s]
+2026-05-20:07:08:13 INFO     [api.task:312] Building contexts for minerva_math_prealgebra on rank 0...
+
  0%|          | 0/871 [00:00<?, ?it/s]
  4%|▍         | 35/871 [00:00<00:02, 348.38it/s]
  8%|▊         | 70/871 [00:00<00:02, 347.03it/s]
 12%|█▏        | 105/871 [00:00<00:02, 346.18it/s]
 16%|█▌        | 140/871 [00:00<00:02, 345.74it/s]
 20%|██        | 175/871 [00:00<00:02, 345.08it/s]
 24%|██▍       | 210/871 [00:00<00:01, 344.83it/s]
 28%|██▊       | 245/871 [00:00<00:01, 345.00it/s]
 32%|███▏      | 280/871 [00:00<00:01, 345.33it/s]
 36%|███▌      | 315/871 [00:00<00:01, 345.29it/s]
 40%|████      | 350/871 [00:01<00:01, 345.26it/s]
 44%|████▍     | 385/871 [00:01<00:01, 345.76it/s]
 48%|████▊     | 420/871 [00:01<00:01, 345.73it/s]
 52%|█████▏    | 455/871 [00:01<00:01, 345.30it/s]
 56%|█████▋    | 490/871 [00:01<00:01, 345.24it/s]
 60%|██████    | 525/871 [00:01<00:01, 345.24it/s]
 64%|██████▍   | 560/871 [00:01<00:00, 344.75it/s]
 68%|██████▊   | 595/871 [00:01<00:00, 345.47it/s]
 72%|███████▏  | 630/871 [00:01<00:00, 346.72it/s]
 76%|███████▋  | 665/871 [00:01<00:00, 347.56it/s]
 80%|████████  | 700/871 [00:02<00:00, 346.36it/s]
 84%|████████▍ | 735/871 [00:02<00:00, 346.55it/s]
 88%|████████▊ | 770/871 [00:02<00:00, 347.51it/s]
 93%|█████████▎| 806/871 [00:02<00:00, 348.20it/s]
 97%|█████████▋| 842/871 [00:02<00:00, 348.88it/s]
+2026-05-20:07:08:16 INFO     [api.task:312] Building contexts for minerva_math_precalc on rank 0...
+
  0%|          | 0/546 [00:00<?, ?it/s]
  6%|▋         | 35/546 [00:00<00:01, 342.61it/s]
 13%|█▎        | 70/546 [00:00<00:01, 332.03it/s]
 19%|█▉        | 105/546 [00:00<00:01, 339.29it/s]
 26%|██▌       | 140/546 [00:00<00:01, 343.09it/s]
 32%|███▏      | 175/546 [00:00<00:01, 344.87it/s]
 38%|███▊      | 210/546 [00:00<00:00, 346.30it/s]
 45%|████▍     | 245/546 [00:00<00:00, 347.09it/s]
 51%|█████▏    | 280/546 [00:00<00:00, 347.55it/s]
 58%|█████▊    | 315/546 [00:00<00:00, 348.15it/s]
 64%|██████▍   | 350/546 [00:01<00:00, 348.19it/s]
 71%|███████   | 385/546 [00:01<00:00, 348.50it/s]
 77%|███████▋  | 420/546 [00:01<00:00, 348.78it/s]
 84%|████████▎ | 456/546 [00:01<00:00, 349.22it/s]
 90%|████████▉ | 491/546 [00:01<00:00, 349.36it/s]
 96%|█████████▋| 526/546 [00:01<00:00, 349.26it/s]
+2026-05-20:07:08:17 INFO     [evaluator:585] Running generate_until requests
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+(EngineCore_DP0 pid=2857336) (Worker pid=2857344) (Worker_TP1 pid=2857344) WARNING 05-20 07:12:52 [multiproc_executor.py:814] WorkerProc was terminated
+(EngineCore_DP0 pid=2857336) (Worker pid=2857342) (Worker_TP0 pid=2857342) WARNING 05-20 07:12:52 [multiproc_executor.py:814] WorkerProc was terminated
+2026-05-20:07:12:53 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+2026-05-20:07:12:53 INFO     [loggers.evaluation_tracker:119] Saving per-task samples to outputs/eval_results/vllm/qwen3-1.7/Qwen__Qwen3-1.7B/*.jsonl
+vllm ({'pretrained': 'Qwen/Qwen3-1.7B', 'tensor_parallel_size': 2, 'dtype': 'bfloat16', 'gpu_memory_utilization': 0.8, 'reasoning_parser': 'deepseek_r1', 'enable_thinking': True, 'think_end_token': '</think>'}), gen_kwargs: ({}), limit: None, num_fewshot: 4, batch_size: auto
+|               Tasks                |Version|Filter|n-shot|  Metric   |   |Value |   |Stderr|
+|------------------------------------|------:|------|-----:|-----------|---|-----:|---|-----:|
+|minerva_math                        |      1|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0604|±  |0.0034|
+| - minerva_math_algebra             |      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0725|±  |0.0075|
+| - minerva_math_counting_and_prob   |      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0591|±  |0.0108|
+| - minerva_math_geometry            |      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0438|±  |0.0094|
+| - minerva_math_intermediate_algebra|      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0354|±  |0.0062|
+| - minerva_math_num_theory          |      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0500|±  |0.0094|
+| - minerva_math_prealgebra          |      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.1010|±  |0.0102|
+| - minerva_math_precalc             |      3|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|                                    |       |none  |     4|math_verify|↑  |0.0366|±  |0.0080|
+
+|   Groups   |Version|Filter|n-shot|  Metric   |   |Value |   |Stderr|
+|------------|------:|------|-----:|-----------|---|-----:|---|-----:|
+|minerva_math|      1|none  |     4|exact_match|↑  |0.0000|±  |0.0000|
+|            |       |none  |     4|math_verify|↑  |0.0604|±  |0.0034|
+
+>>> [3/10] MMLU-STEM
+2026-05-20:07:12:56 INFO     [config.evaluate_config:307] Using default fewshot_as_multiturn=True.
+2026-05-20:07:13:05 INFO     [_cli.run:388] Selected Tasks: ['mmlu_stem']
+2026-05-20:07:13:06 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-05-20:07:13:06 INFO     [evaluator:239] Initializing vllm model, with arguments: {'pretrained': 'Qwen/Qwen3-1.7B', 'tensor_parallel_size': 2, 'dtype': 'bfloat16', 'gpu_memory_utilization': 0.8, 'trust_remote_code': True, 'reasoning_parser': 'deepseek_r1', 'enable_thinking': True, 'think_end_token': '</think>'}
+INFO 05-20 07:13:11 [utils.py:238] non-default args: {'trust_remote_code': True, 'dtype': 'bfloat16', 'seed': 1234, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.8, 'disable_log_stats': True, 'reasoning_parser': 'deepseek_r1', 'model': 'Qwen/Qwen3-1.7B'}
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+INFO 05-20 07:13:12 [model.py:531] Resolved architecture: Qwen3ForCausalLM
+INFO 05-20 07:13:12 [model.py:1554] Using max model len 40960
+INFO 05-20 07:13:12 [scheduler.py:231] Chunked prefill is enabled with max_num_batched_tokens=16384.
+INFO 05-20 07:13:12 [vllm.py:747] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=2860214) INFO 05-20 07:13:14 [core.py:101] Initializing a V1 LLM engine (v0.17.1) with config: model='Qwen/Qwen3-1.7B', speculative_config=None, tokenizer='Qwen/Qwen3-1.7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='deepseek_r1', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=1234, served_model_name=Qwen/Qwen3-1.7B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': True, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=2860214) WARNING 05-20 07:13:14 [multiproc_executor.py:945] Reducing Torch parallelism from 96 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
+(EngineCore_DP0 pid=2860214) INFO 05-20 07:13:14 [multiproc_executor.py:134] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.31.21.192 (local), world_size=2, local_world_size=2
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) INFO 05-20 07:13:16 [parallel_state.py:1393] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:42835 backend=nccl
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) INFO 05-20 07:13:16 [parallel_state.py:1393] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:42835 backend=nccl
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) INFO 05-20 07:13:17 [pynccl.py:111] vLLM is using nccl==2.27.5
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
+[rank1]:[W520 07:13:18.289960522 CUDASymmetricMemory.cu:604] Warning: CUDA driver error: system not yet initialized (function init_multicast_for_block)
+[rank0]:[W520 07:13:18.289960532 CUDASymmetricMemory.cu:604] Warning: CUDA driver error: system not yet initialized (function init_multicast_for_block)
+[rank1]:[W520 07:13:18.290136595 CUDASymmetricMemory.cu:621] Gracefully skipping multicast initialization.
+[rank0]:[W520 07:13:18.290263357 CUDASymmetricMemory.cu:621] Gracefully skipping multicast initialization.
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) WARNING 05-20 07:13:18 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) WARNING 05-20 07:13:18 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) INFO 05-20 07:13:18 [parallel_state.py:1715] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank N/A, EPLB rank N/A
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) INFO 05-20 07:13:18 [parallel_state.py:1715] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) INFO 05-20 07:13:19 [base.py:106] Offloader set to NoopOffloader
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:19 [gpu_model_runner.py:4281] Starting to load model Qwen/Qwen3-1.7B...
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) INFO 05-20 07:13:19 [base.py:106] Offloader set to NoopOffloader
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:19 [cuda.py:405] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:19 [flash_attn.py:587] Using FlashAttention version 3
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) 
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) 
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) 
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) 
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:21 [default_loader.py:293] Loading weights took 0.25 seconds
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:21 [gpu_model_runner.py:4364] Model loading took 1.61 GiB memory and 1.230606 seconds
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) (Worker_TP1 pid=2860222) INFO 05-20 07:13:24 [decorators.py:465] Directly load AOT compilation from path /home/ubuntu/.cache/vllm/torch_compile_cache/torch_aot_compile/51a82c2c50ff9f473a0d7b91dcb7e226ed7241b74ec83725bb6d47cbeffa99f2/rank_1_0/model
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:24 [decorators.py:465] Directly load AOT compilation from path /home/ubuntu/.cache/vllm/torch_compile_cache/torch_aot_compile/51a82c2c50ff9f473a0d7b91dcb7e226ed7241b74ec83725bb6d47cbeffa99f2/rank_0_0/model
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:25 [backends.py:916] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f745e6964b/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:25 [backends.py:976] Dynamo bytecode transform time: 2.74 s
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:27 [backends.py:266] Directly load the compiled graph(s) for compile range (1, 16384) from the cache, took 1.814 s
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:27 [monitor.py:35] torch.compile takes 4.87 s in total
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:28 [gpu_worker.py:424] Available KV cache memory: 107.48 GiB
+(EngineCore_DP0 pid=2860214) INFO 05-20 07:13:28 [kv_cache_utils.py:1314] GPU KV cache size: 2,012,528 tokens
+(EngineCore_DP0 pid=2860214) INFO 05-20 07:13:28 [kv_cache_utils.py:1319] Maximum concurrency for 40,960 tokens per request: 49.13x
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) 
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) 
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:33 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=2860214) (Worker pid=2860222) (Worker_TP1 pid=2860222) INFO 05-20 07:13:33 [custom_all_reduce.py:216] Registering 5814 cuda graph addresses
+(EngineCore_DP0 pid=2860214) (Worker pid=2860220) (Worker_TP0 pid=2860220) INFO 05-20 07:13:34 [gpu_model_runner.py:5386] Graph capturing finished in 5 secs, took 0.38 GiB
+(EngineCore_DP0 pid=2860214) INFO 05-20 07:13:34 [core.py:282] init engine (profile, create kv cache, warmup model) took 11.93 seconds
+INFO 05-20 07:13:36 [llm.py:388] Supported tasks: ['generate']

eval_results/vllm/gemma-2-2b-it/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/google__gemma-2-2b-it/results_2026-05-20T03-28-43.199239.json

@@ -0,0 +1,182 @@
+{
+  "results": {
+    "gsm8k": {
+      "name": "gsm8k",
+      "alias": "gsm8k",
+      "sample_len": 1319,
+      "exact_match,strict-match": 0.03411675511751327,
+      "exact_match_stderr,strict-match": 0.005000212600773252,
+      "exact_match,flexible-extract": 0.49583017437452614,
+      "exact_match_stderr,flexible-extract": 0.013772005774791542
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm8k": {
+      "task": "gsm8k",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "training_split": "train",
+      "test_split": "test",
+      "fewshot_split": "train",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{answer}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": "train",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "google/gemma-2-2b-it",
+        "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+        "tensor_parallel_size": 2,
+        "dtype": "bfloat16",
+        "gpu_memory_utilization": 0.7,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/gsm8k/gsm8k.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm8k": 3.0
+  },
+  "n-shot": {
+    "gsm8k": 5
+  },
+  "higher_is_better": {
+    "gsm8k": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k": {
+      "original": 1319,
+      "effective": 1319
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "google/gemma-2-2b-it",
+      "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+      "tensor_parallel_size": 2,
+      "dtype": "bfloat16",
+      "gpu_memory_utilization": 0.7,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "04b98db",
+  "date": 1779247590.1627285,
+  "pretty_env_info": "PyTorch version: 2.10.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                       2.4.5            pypi_0              pypi\n[conda] nvidia-cublas-cu12          12.4.5.8         pypi_0              pypi\n[conda] nvidia-cuda-cupti-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-nvrtc-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-runtime-cu12    12.4.127         pypi_0              pypi\n[conda] nvidia-cudnn-cu12           9.1.0.70         pypi_0              pypi\n[conda] nvidia-cufft-cu12           11.2.1.3         pypi_0              pypi\n[conda] nvidia-curand-cu12          10.3.5.147       pypi_0              pypi\n[conda] nvidia-cusolver-cu12        11.6.1.9         pypi_0              pypi\n[conda] nvidia-cusparse-cu12        12.3.1.170       pypi_0              pypi\n[conda] nvidia-nccl-cu12            2.21.5           pypi_0              pypi\n[conda] nvidia-nvjitlink-cu12       12.4.127         pypi_0              pypi\n[conda] nvidia-nvtx-cu12            12.4.127         pypi_0              pypi\n[conda] torch                       2.5.1+cu124      pypi_0              pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "<eos>",
+    "1"
+  ],
+  "tokenizer_bos_token": [
+    "<bos>",
+    "2"
+  ],
+  "eot_token_id": 1,
+  "max_length": 8192,
+  "task_hashes": {
+    "gsm8k": "f4aaa8f657252353a890c9a84fcc05a77f7ff8ca6da9e8b4a12ab73a84e92c85"
+  },
+  "model_source": "vllm",
+  "model_name": "google/gemma-2-2b-it",
+  "model_name_sanitized": "google__gemma-2-2b-it",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template_sha": "ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6",
+  "total_evaluation_time_seconds": "140.84546380897518"
+}

eval_results/vllm/gemma-2-2b-it/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/google__gemma-2-2b-it/results_2026-05-20T03-50-24.733001.json

@@ -0,0 +1,140 @@
+{
+  "results": {
+    "mbpp": {
+      "name": "mbpp",
+      "alias": "mbpp",
+      "sample_len": 500,
+      "pass_at_1,none": 0.0,
+      "pass_at_1_stderr,none": 0.0
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "mbpp": {
+      "task": "mbpp",
+      "dataset_path": "google-research-datasets/mbpp",
+      "dataset_name": "full",
+      "test_split": "test",
+      "doc_to_text": "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n",
+      "doc_to_target": "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}",
+      "unsafe_code": true,
+      "description": "",
+      "target_delimiter": "",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x74378e88df30>",
+        "doc_to_text": "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n",
+        "doc_to_choice": null,
+        "doc_to_target": "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": ""
+      },
+      "num_fewshot": 3,
+      "metric_list": [
+        {
+          "metric": "def pass_at_1(\n    references: Union[str, list[str]], predictions: Union[str, list[list[str]]]\n) -> float:\n    if isinstance(references, str):\n        references = [references]\n    if isinstance(predictions[0], str):\n        predictions = [[p] for p in predictions]\n    return pass_at_k.compute(\n        references=references,\n        predictions=predictions,\n        k=[1],\n    )[0][\"pass@1\"]\n",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "[DONE]"
+        ],
+        "do_sample": false,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "google/gemma-2-2b-it",
+        "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+        "tensor_parallel_size": 2,
+        "dtype": "bfloat16",
+        "gpu_memory_utilization": 0.7,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mbpp/mbpp.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mbpp": 1.0
+  },
+  "n-shot": {
+    "mbpp": 3
+  },
+  "higher_is_better": {
+    "mbpp": {
+      "pass_at_1": true
+    }
+  },
+  "n-samples": {
+    "mbpp": {
+      "original": 500,
+      "effective": 500
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "google/gemma-2-2b-it",
+      "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+      "tensor_parallel_size": 2,
+      "dtype": "bfloat16",
+      "gpu_memory_utilization": 0.7,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "04b98db",
+  "date": 1779248806.70298,
+  "pretty_env_info": "PyTorch version: 2.10.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                       2.4.5            pypi_0              pypi\n[conda] nvidia-cublas-cu12          12.4.5.8         pypi_0              pypi\n[conda] nvidia-cuda-cupti-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-nvrtc-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-runtime-cu12    12.4.127         pypi_0              pypi\n[conda] nvidia-cudnn-cu12           9.1.0.70         pypi_0              pypi\n[conda] nvidia-cufft-cu12           11.2.1.3         pypi_0              pypi\n[conda] nvidia-curand-cu12          10.3.5.147       pypi_0              pypi\n[conda] nvidia-cusolver-cu12        11.6.1.9         pypi_0              pypi\n[conda] nvidia-cusparse-cu12        12.3.1.170       pypi_0              pypi\n[conda] nvidia-nccl-cu12            2.21.5           pypi_0              pypi\n[conda] nvidia-nvjitlink-cu12       12.4.127         pypi_0              pypi\n[conda] nvidia-nvtx-cu12            12.4.127         pypi_0              pypi\n[conda] torch                       2.5.1+cu124      pypi_0              pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "<eos>",
+    "1"
+  ],
+  "tokenizer_bos_token": [
+    "<bos>",
+    "2"
+  ],
+  "eot_token_id": 1,
+  "max_length": 8192,
+  "task_hashes": {
+    "mbpp": "3f824337c23a616ef553938b9b216d40f92e16d6efe749cdd65368d61bea12fb"
+  },
+  "model_source": "vllm",
+  "model_name": "google/gemma-2-2b-it",
+  "model_name_sanitized": "google__gemma-2-2b-it",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template_sha": "ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6",
+  "total_evaluation_time_seconds": "225.9090766570298"
+}

eval_results/vllm/gemma-2-2b-it/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/google__gemma-2-2b-it/results_2026-05-20T13-51-52.683923.json

@@ -0,0 +1,169 @@
+{
+  "results": {
+    "ifeval": {
+      "name": "ifeval",
+      "alias": "ifeval",
+      "sample_len": 541,
+      "prompt_level_strict_acc,none": 0.5009242144177449,
+      "prompt_level_strict_acc_stderr,none": 0.021516537387842597,
+      "inst_level_strict_acc,none": 0.6019184652278178,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.5249537892791127,
+      "prompt_level_loose_acc_stderr,none": 0.02148976105896734,
+      "inst_level_loose_acc,none": 0.6270983213429256,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "ifeval": {
+      "task": "ifeval",
+      "dataset_path": "google/IFEval",
+      "test_split": "train",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "unsafe_code": false,
+      "process_results": "def process_results(doc, results):\n    inp = InputExample(\n        key=doc[\"key\"],\n        instruction_id_list=doc[\"instruction_id_list\"],\n        prompt=doc[\"prompt\"],\n        kwargs=doc[\"kwargs\"],\n    )\n    response = results[0]\n\n    out_strict = test_instruction_following_strict(inp, response)\n    out_loose = test_instruction_following_loose(inp, response)\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "prompt",
+        "doc_to_choice": null,
+        "doc_to_target": 0,
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0,
+        "pretrained": "google/gemma-2-2b-it",
+        "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+        "tensor_parallel_size": 2,
+        "dtype": "bfloat16",
+        "gpu_memory_utilization": 0.2,
+        "trust_remote_code": true,
+        "max_lora_rank": 32,
+        "enable_lora": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/ifeval/ifeval.yaml"
+      }
+    }
+  },
+  "versions": {
+    "ifeval": 4.0
+  },
+  "n-shot": {
+    "ifeval": 0
+  },
+  "higher_is_better": {
+    "ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ifeval": {
+      "original": 541,
+      "effective": 541
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "google/gemma-2-2b-it",
+      "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+      "tensor_parallel_size": 2,
+      "dtype": "bfloat16",
+      "gpu_memory_utilization": 0.2,
+      "trust_remote_code": true,
+      "max_lora_rank": 32,
+      "enable_lora": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "temperature": 0.0,
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "745ca7f",
+  "date": 1779284999.6941524,
+  "pretty_env_info": "PyTorch version: 2.10.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                       2.4.5            pypi_0              pypi\n[conda] nvidia-cublas-cu12          12.4.5.8         pypi_0              pypi\n[conda] nvidia-cuda-cupti-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-nvrtc-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-runtime-cu12    12.4.127         pypi_0              pypi\n[conda] nvidia-cudnn-cu12           9.1.0.70         pypi_0              pypi\n[conda] nvidia-cufft-cu12           11.2.1.3         pypi_0              pypi\n[conda] nvidia-curand-cu12          10.3.5.147       pypi_0              pypi\n[conda] nvidia-cusolver-cu12        11.6.1.9         pypi_0              pypi\n[conda] nvidia-cusparse-cu12        12.3.1.170       pypi_0              pypi\n[conda] nvidia-nccl-cu12            2.21.5           pypi_0              pypi\n[conda] nvidia-nvjitlink-cu12       12.4.127         pypi_0              pypi\n[conda] nvidia-nvtx-cu12            12.4.127         pypi_0              pypi\n[conda] torch                       2.5.1+cu124      pypi_0              pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "<eos>",
+    "1"
+  ],
+  "tokenizer_bos_token": [
+    "<bos>",
+    "2"
+  ],
+  "eot_token_id": 1,
+  "max_length": 8192,
+  "task_hashes": {
+    "ifeval": "90b5c2d9c5faf39579a019361e475437da6246488010b17ed807f5c64603c451"
+  },
+  "model_source": "vllm",
+  "model_name": "google/gemma-2-2b-it",
+  "model_name_sanitized": "google__gemma-2-2b-it",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template_sha": "ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6",
+  "total_evaluation_time_seconds": "120.88018567708787"
+}

eval_results/vllm/gemma-2-2b-it/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/google__gemma-2-2b-it/results_2026-05-20T05-33-07.171742.json

@@ -0,0 +1,181 @@
+{
+  "results": {
+    "gsm8k": {
+      "name": "gsm8k",
+      "alias": "gsm8k",
+      "sample_len": 1319,
+      "exact_match,strict-match": 0.01288855193328279,
+      "exact_match_stderr,strict-match": 0.003106901266499662,
+      "exact_match,flexible-extract": 0.4927975739196361,
+      "exact_match_stderr,flexible-extract": 0.013771055751972868
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm8k": {
+      "task": "gsm8k",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "training_split": "train",
+      "test_split": "test",
+      "fewshot_split": "train",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{answer}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": "train",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "google/gemma-2-2b-it",
+        "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+        "tensor_parallel_size": 2,
+        "dtype": "bfloat16",
+        "gpu_memory_utilization": 0.7,
+        "trust_remote_code": true,
+        "max_lora_rank": 32,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/gsm8k/gsm8k.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm8k": 3.0
+  },
+  "n-shot": {
+    "gsm8k": 5
+  },
+  "higher_is_better": {
+    "gsm8k": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k": {
+      "original": 1319,
+      "effective": 1319
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "google/gemma-2-2b-it",
+      "lora_local_path": "results/gemma2-2b-it#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/2492",
+      "tensor_parallel_size": 2,
+      "dtype": "bfloat16",
+      "gpu_memory_utilization": 0.7,
+      "trust_remote_code": true,
+      "max_lora_rank": 32
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "04b98db",
+  "date": 1779255072.5496013,
+  "pretty_env_info": "PyTorch version: 2.10.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                       2.4.5            pypi_0              pypi\n[conda] nvidia-cublas-cu12          12.4.5.8         pypi_0              pypi\n[conda] nvidia-cuda-cupti-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-nvrtc-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-runtime-cu12    12.4.127         pypi_0              pypi\n[conda] nvidia-cudnn-cu12           9.1.0.70         pypi_0              pypi\n[conda] nvidia-cufft-cu12           11.2.1.3         pypi_0              pypi\n[conda] nvidia-curand-cu12          10.3.5.147       pypi_0              pypi\n[conda] nvidia-cusolver-cu12        11.6.1.9         pypi_0              pypi\n[conda] nvidia-cusparse-cu12        12.3.1.170       pypi_0              pypi\n[conda] nvidia-nccl-cu12            2.21.5           pypi_0              pypi\n[conda] nvidia-nvjitlink-cu12       12.4.127         pypi_0              pypi\n[conda] nvidia-nvtx-cu12            12.4.127         pypi_0              pypi\n[conda] torch                       2.5.1+cu124      pypi_0              pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "<eos>",
+    "1"
+  ],
+  "tokenizer_bos_token": [
+    "<bos>",
+    "2"
+  ],
+  "eot_token_id": 1,
+  "max_length": 8192,
+  "task_hashes": {
+    "gsm8k": "f4aaa8f657252353a890c9a84fcc05a77f7ff8ca6da9e8b4a12ab73a84e92c85"
+  },
+  "model_source": "vllm",
+  "model_name": "google/gemma-2-2b-it",
+  "model_name_sanitized": "google__gemma-2-2b-it",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template_sha": "ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6",
+  "total_evaluation_time_seconds": "122.46745934605133"
+}

eval_results/vllm/gemma-2-2b-it/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/google__gemma-2-2b-it/results_2026-05-20T10-58-55.249659.json

@@ -0,0 +1,578 @@
+{
+  "results": {
+    "minerva_math": {
+      "exact_match,none": 0.0102,
+      "exact_match_stderr,none": 0.0014188079776214471,
+      "math_verify,none": 0.185,
+      "math_verify_stderr,none": 0.005243439895771032,
+      "alias": "minerva_math"
+    },
+    "minerva_math_algebra": {
+      "alias": " - minerva_math_algebra",
+      "exact_match,none": 0.016006739679865205,
+      "exact_match_stderr,none": 0.0036442247924417257,
+      "math_verify,none": 0.2906486941870261,
+      "math_verify_stderr,none": 0.013184769138277696
+    },
+    "minerva_math_counting_and_prob": {
+      "alias": " - minerva_math_counting_and_prob",
+      "exact_match,none": 0.004219409282700422,
+      "exact_match_stderr,none": 0.00298041736510205,
+      "math_verify,none": 0.10759493670886076,
+      "math_verify_stderr,none": 0.014247752413786495
+    },
+    "minerva_math_geometry": {
+      "alias": " - minerva_math_geometry",
+      "exact_match,none": 0.006263048016701462,
+      "exact_match_stderr,none": 0.003608399732887864,
+      "math_verify,none": 0.11899791231732777,
+      "math_verify_stderr,none": 0.014809629428535889
+    },
+    "minerva_math_intermediate_algebra": {
+      "alias": " - minerva_math_intermediate_algebra",
+      "exact_match,none": 0.004429678848283499,
+      "exact_match_stderr,none": 0.0022111531423787832,
+      "math_verify,none": 0.06201550387596899,
+      "math_verify_stderr,none": 0.008030539074152163
+    },
+    "minerva_math_num_theory": {
+      "alias": " - minerva_math_num_theory",
+      "exact_match,none": 0.005555555555555556,
+      "exact_match_stderr,none": 0.0032015451273209013,
+      "math_verify,none": 0.15185185185185185,
+      "math_verify_stderr,none": 0.015457936637383811
+    },
+    "minerva_math_prealgebra": {
+      "alias": " - minerva_math_prealgebra",
+      "exact_match,none": 0.02066590126291619,
+      "exact_match_stderr,none": 0.004823174633923066,
+      "math_verify,none": 0.35361653272101035,
+      "math_verify_stderr,none": 0.016208833993652152
+    },
+    "minerva_math_precalc": {
+      "alias": " - minerva_math_precalc",
+      "exact_match,none": 0.003663003663003663,
+      "exact_match_stderr,none": 0.0025877573681934397,
+      "math_verify,none": 0.047619047619047616,
+      "math_verify_stderr,none": 0.009122155097344266
+    }
+  },
+  "groups": {
+    "minerva_math": {
+      "exact_match,none": 0.0102,
+      "exact_match_stderr,none": 0.0014188079776214471,
+      "math_verify,none": 0.185,
+      "math_verify_stderr,none": 0.005243439895771032,
+      "alias": "minerva_math"
+    }
+  },
+  "group_subtasks": {
+    "minerva_math": [
+      "minerva_math_algebra",
+      "minerva_math_counting_and_prob",
+      "minerva_math_geometry",
+      "minerva_math_intermediate_algebra",
+      "minerva_math_num_theory",
+      "minerva_math_prealgebra",
+      "minerva_math_precalc"
+    ]
+  },
+  "configs": {
+    "minerva_math_algebra": {
+      "task": "minerva_math_algebra",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f6ee29e0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    },
+    "minerva_math_counting_and_prob": {
+      "task": "minerva_math_counting_and_prob",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "counting_and_probability",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f6ee0820>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    },
+    "minerva_math_geometry": {
+      "task": "minerva_math_geometry",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f6ec11b0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    },
+    "minerva_math_intermediate_algebra": {
+      "task": "minerva_math_intermediate_algebra",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "intermediate_algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f6ec0820>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    },
+    "minerva_math_num_theory": {
+      "task": "minerva_math_num_theory",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "number_theory",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f6e88ca0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    },
+    "minerva_math_prealgebra": {
+      "task": "minerva_math_prealgebra",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "prealgebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f6e880d0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    },
+    "minerva_math_precalc": {
+      "task": "minerva_math_precalc",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "precalculus",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    res = verify(parse(doc[\"answer\"]), parse(candidates))\n    mathval = 1 if res else 0\n\n    results = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x7581f72c6ef0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "minerva_math": 1.0,
+    "minerva_math_algebra": 2.0,
+    "minerva_math_counting_and_prob": 2.0,
+    "minerva_math_geometry": 2.0,
+    "minerva_math_intermediate_algebra": 2.0,
+    "minerva_math_num_theory": 2.0,
+    "minerva_math_prealgebra": 2.0,
+    "minerva_math_precalc": 2.0
+  },
+  "n-shot": {
+    "minerva_math_algebra": 4,
+    "minerva_math_counting_and_prob": 4,
+    "minerva_math_geometry": 4,
+    "minerva_math_intermediate_algebra": 4,
+    "minerva_math_num_theory": 4,
+    "minerva_math_prealgebra": 4,
+    "minerva_math_precalc": 4
+  },
+  "higher_is_better": {
+    "minerva_math": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_algebra": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_counting_and_prob": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_geometry": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_intermediate_algebra": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_num_theory": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_prealgebra": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_precalc": {
+      "exact_match": true,
+      "math_verify": true
+    }
+  },
+  "n-samples": {
+    "minerva_math_algebra": {
+      "original": 1187,
+      "effective": 1187
+    },
+    "minerva_math_counting_and_prob": {
+      "original": 474,
+      "effective": 474
+    },
+    "minerva_math_geometry": {
+      "original": 479,
+      "effective": 479
+    },
+    "minerva_math_intermediate_algebra": {
+      "original": 903,
+      "effective": 903
+    },
+    "minerva_math_num_theory": {
+      "original": 540,
+      "effective": 540
+    },
+    "minerva_math_prealgebra": {
+      "original": 871,
+      "effective": 871
+    },
+    "minerva_math_precalc": {
+      "original": 546,
+      "effective": 546
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=google/gemma-2-2b-it,lora_local_path=results/gemma2-2b-it#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/2492,tensor_parallel_size=2,dtype=bfloat16,gpu_memory_utilization=0.7,trust_remote_code=True,max_lora_rank=32,enable_lora=True",
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "temperature": 0.0
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "c2320b5",
+  "date": 1779274325.187935,
+  "pretty_env_info": "PyTorch version: 2.7.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, Mar 11 2026, 17:46:40) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas==13.1.0.3\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti==13.0.85\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc==13.0.88\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.77\n[pip3] nvidia-cuda-runtime==13.0.96\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.5.1.17\n[pip3] nvidia-cudnn-cu13==9.19.0.56\n[pip3] nvidia-cudnn-frontend==1.18.0\n[pip3] nvidia-cufft==12.0.0.61\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand==10.4.0.35\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver==12.0.4.66\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse==12.6.3.3\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.6.3\n[pip3] nvidia-cusparselt-cu13==0.8.0\n[pip3] nvidia-nccl-cu12==2.26.2\n[pip3] nvidia-nccl-cu13==2.28.9\n[pip3] nvidia-nvjitlink==13.0.88\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx==13.0.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] tokenspeed-triton==3.7.10.post20260505\n[pip3] torch==2.7.0\n[pip3] torch_c_dlpack_ext==0.1.5\n[pip3] torchaudio==2.7.0\n[pip3] torchvision==0.22.0\n[pip3] triton==3.3.0\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cublas-cu12                          12.6.4.1             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-cupti-cu12                      12.6.80              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc-cu12                      12.6.77              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cuda-runtime-cu12                    12.6.77              pypi_0           pypi\n[conda] nvidia-cudnn-cu12                           9.5.1.17             pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-cufft-cu12                           11.3.0.4             pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-curand-cu12                          10.3.7.77            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusolver-cu12                        11.7.1.2             pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparse-cu12                        12.5.4.2             pypi_0           pypi\n[conda] nvidia-cusparselt-cu12                      0.6.3                pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu12                            2.26.2               pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvjitlink-cu12                       12.6.85              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] nvidia-nvtx-cu12                            12.6.77              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.7.0                pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.7.0                pypi_0           pypi\n[conda] torchvision                                 0.22.0               pypi_0           pypi\n[conda] triton                                      3.3.0                pypi_0           pypi",
+  "transformers_version": "4.53.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<pad>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "<eos>",
+    "1"
+  ],
+  "tokenizer_bos_token": [
+    "<bos>",
+    "2"
+  ],
+  "eot_token_id": 1,
+  "max_length": 8192,
+  "task_hashes": {
+    "minerva_math_algebra": "5f83ac7d6a5082558cd299dc221fc50aaeeb79148748bb84cc7c38c2e8a6ee30",
+    "minerva_math_counting_and_prob": "f4f5002d0783023643da40903a63ca85723a2623ea0887e2a6fb248bd0f85bf7",
+    "minerva_math_geometry": "4971861893b64f4c73ae42b0aea3f90d4597b2bff3dd31823b00bd02a095e8b2",
+    "minerva_math_intermediate_algebra": "4d6a6f30eda0f8025e1ab53f0403b054f493b0505133d1f72ddf8169caf1fa17",
+    "minerva_math_num_theory": "a64b21821e6e11d91332c46226938da82d5d577035fe19456d2c9655b1815250",
+    "minerva_math_prealgebra": "c40045657ad3ae85afaab9e8975cdf28fa589c17f9d35f39b963c9219f7937e9",
+    "minerva_math_precalc": "eb034a3835ff7f4689f69ba7c40449ac002a022a0742070c5cbba6769521779d"
+  },
+  "model_source": "vllm",
+  "model_name": "google/gemma-2-2b-it",
+  "model_name_sanitized": "google__gemma-2-2b-it",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template_sha": "ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6",
+  "start_time": 676679.118102467,
+  "end_time": 677100.935311529,
+  "total_evaluation_time_seconds": "421.8172090619337"
+}