Add files using upload-large-folder tool

.gitattributes

@@ -59,3 +59,6 @@ eval_results/vllm/gemma-2-2b-it/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/google__gemm
 eval_results_2/vllm/qwen3-1.7B\#sfkl_nnm_lora/nnm1.0_K128_L4_epoch1_lr1e-4_kdr1.0/results__qwen3-1.7B\#sfkl_nnm_lora__nnm1.0_K128_L4_epoch1_lr1e-4_kdr1.0/samples_gsm_plus_2026-05-21T22-07-38.678553.jsonl filter=lfs diff=lfs merge=lfs -text
 eval_results/vllm/qwen3-1.7B\#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/Qwen__Qwen3-1.7B/samples_gsm_plus_2026-05-20T14-33-31.362188.jsonl filter=lfs diff=lfs merge=lfs -text
 eval_results/vllm/qwen2.5-1.5B-it-amid/results__qwen2.5-1.5B-Instruct\#amid__ab_pr_0.5_0.5_4_1e-4/samples_gsm_plus_2026-05-17T08-22-15.616148.jsonl filter=lfs diff=lfs merge=lfs -text
+eval_results_2/vllm/qwen3-1.7B\#sfkl_nnm_lora/nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen3-1.7B\#sfkl_nnm_lora__nnm0.1_K128_L4_epoch2_lr1e-4_kdr1.0/samples_gsm_plus_2026-05-22T03-13-08.441586.jsonl filter=lfs diff=lfs merge=lfs -text
+eval_results/vllm/qwen3-1.7B-it-nnm/results__qwen3-1.7B\#sfkl_nnm_lora__nnm0.9_K128_L4_epoch2_lr1e-4_kdr0.75/samples_mbpp_2026-05-18T07-30-16.450598.jsonl filter=lfs diff=lfs merge=lfs -text
+eval_results/vllm/qwen2.5-1.5b-tsd/Minsang__TSD-KD_Qwen2.5-1.5B/samples_gsm_plus_2026-05-20T09-44-19.696932.jsonl filter=lfs diff=lfs merge=lfs -text

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/Qwen__Qwen3-1.7B/results_2026-05-19T14-42-25.634092.json

@@ -0,0 +1,182 @@
+{
+  "results": {
+    "gsm8k": {
+      "name": "gsm8k",
+      "alias": "gsm8k",
+      "sample_len": 1319,
+      "exact_match,strict-match": 0.3070507960576194,
+      "exact_match_stderr,strict-match": 0.012705685723131712,
+      "exact_match,flexible-extract": 0.6535253980288097,
+      "exact_match_stderr,flexible-extract": 0.013107179054313401
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm8k": {
+      "task": "gsm8k",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "training_split": "train",
+      "test_split": "test",
+      "fewshot_split": "train",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{answer}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": "train",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "Qwen/Qwen3-1.7B",
+        "lora_local_path": "results/qwen3-1.7B#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/4984",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/gsm8k/gsm8k.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm8k": 3.0
+  },
+  "n-shot": {
+    "gsm8k": 5
+  },
+  "higher_is_better": {
+    "gsm8k": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k": {
+      "original": 1319,
+      "effective": 1319
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "Qwen/Qwen3-1.7B",
+      "lora_local_path": "results/qwen3-1.7B#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/4984",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "065481d",
+  "date": 1779201555.2577052,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                       2.4.5            pypi_0              pypi\n[conda] nvidia-cublas-cu12          12.4.5.8         pypi_0              pypi\n[conda] nvidia-cuda-cupti-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-nvrtc-cu12      12.4.127         pypi_0              pypi\n[conda] nvidia-cuda-runtime-cu12    12.4.127         pypi_0              pypi\n[conda] nvidia-cudnn-cu12           9.1.0.70         pypi_0              pypi\n[conda] nvidia-cufft-cu12           11.2.1.3         pypi_0              pypi\n[conda] nvidia-curand-cu12          10.3.5.147       pypi_0              pypi\n[conda] nvidia-cusolver-cu12        11.6.1.9         pypi_0              pypi\n[conda] nvidia-cusparse-cu12        12.3.1.170       pypi_0              pypi\n[conda] nvidia-nccl-cu12            2.21.5           pypi_0              pypi\n[conda] nvidia-nvjitlink-cu12       12.4.127         pypi_0              pypi\n[conda] nvidia-nvtx-cu12            12.4.127         pypi_0              pypi\n[conda] torch                       2.5.1+cu124      pypi_0              pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 40960,
+  "task_hashes": {
+    "gsm8k": "57f6f099e46d41808275113fe524b6af081154f7c87fa7826e3dd1775f468206"
+  },
+  "model_source": "vllm",
+  "model_name": "Qwen/Qwen3-1.7B",
+  "model_name_sanitized": "Qwen__Qwen3-1.7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = '' %}\n    {%- endif %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n            {%- else %}\n                {{- '<|im_start|>' + message.role + '\\n' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- endif %}\n{%- endif %}",
+  "chat_template_sha": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8",
+  "total_evaluation_time_seconds": "198.2940947599709"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T05-03-19.631875.json

@@ -0,0 +1,180 @@
+{
+  "results": {
+    "gsm8k": {
+      "name": "gsm8k",
+      "alias": "gsm8k",
+      "sample_len": 1319,
+      "exact_match,strict-match": 0.510235026535254,
+      "exact_match_stderr,strict-match": 0.013769598923012384,
+      "exact_match,flexible-extract": 0.6095526914329037,
+      "exact_match_stderr,flexible-extract": 0.01343782986466858
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm8k": {
+      "task": "gsm8k",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "training_split": "train",
+      "test_split": "test",
+      "fewshot_split": "train",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{answer}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": "train",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/gsm8k/gsm8k.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm8k": 3.0
+  },
+  "n-shot": {
+    "gsm8k": 5
+  },
+  "higher_is_better": {
+    "gsm8k": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k": {
+      "original": 1319,
+      "effective": 1319
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779166927.2364273,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "gsm8k": "1876b0d29670c836f6ef54bca0a415304e7755fdf9babadb7a5d844de9d52cf2"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "80.25162482995074"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T05-12-25.651557.json

@@ -0,0 +1,1543 @@
+{
+  "results": {
+    "mmlu_abstract_algebra": {
+      "name": "mmlu_abstract_algebra",
+      "alias": "abstract_algebra",
+      "sample_len": 100,
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_anatomy": {
+      "name": "mmlu_anatomy",
+      "alias": "anatomy",
+      "sample_len": 135,
+      "acc,none": 0.562962962962963,
+      "acc_stderr,none": 0.04284958639753401
+    },
+    "mmlu_astronomy": {
+      "name": "mmlu_astronomy",
+      "alias": "astronomy",
+      "sample_len": 152,
+      "acc,none": 0.6907894736842105,
+      "acc_stderr,none": 0.037610708698674805
+    },
+    "mmlu_college_biology": {
+      "name": "mmlu_college_biology",
+      "alias": "college_biology",
+      "sample_len": 144,
+      "acc,none": 0.6736111111111112,
+      "acc_stderr,none": 0.03921067198982266
+    },
+    "mmlu_college_chemistry": {
+      "name": "mmlu_college_chemistry",
+      "alias": "college_chemistry",
+      "sample_len": 100,
+      "acc,none": 0.38,
+      "acc_stderr,none": 0.048783173121456316
+    },
+    "mmlu_college_computer_science": {
+      "name": "mmlu_college_computer_science",
+      "alias": "college_computer_science",
+      "sample_len": 100,
+      "acc,none": 0.45,
+      "acc_stderr,none": 0.05
+    },
+    "mmlu_college_mathematics": {
+      "name": "mmlu_college_mathematics",
+      "alias": "college_mathematics",
+      "sample_len": 100,
+      "acc,none": 0.38,
+      "acc_stderr,none": 0.04878317312145633
+    },
+    "mmlu_college_physics": {
+      "name": "mmlu_college_physics",
+      "alias": "college_physics",
+      "sample_len": 102,
+      "acc,none": 0.37254901960784315,
+      "acc_stderr,none": 0.04810840148082634
+    },
+    "mmlu_computer_security": {
+      "name": "mmlu_computer_security",
+      "alias": "computer_security",
+      "sample_len": 100,
+      "acc,none": 0.71,
+      "acc_stderr,none": 0.04560480215720684
+    },
+    "mmlu_conceptual_physics": {
+      "name": "mmlu_conceptual_physics",
+      "alias": "conceptual_physics",
+      "sample_len": 235,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.03202563076101735
+    },
+    "mmlu_electrical_engineering": {
+      "name": "mmlu_electrical_engineering",
+      "alias": "electrical_engineering",
+      "sample_len": 145,
+      "acc,none": 0.6758620689655173,
+      "acc_stderr,none": 0.03900432069185553
+    },
+    "mmlu_elementary_mathematics": {
+      "name": "mmlu_elementary_mathematics",
+      "alias": "elementary_mathematics",
+      "sample_len": 378,
+      "acc,none": 0.48412698412698413,
+      "acc_stderr,none": 0.025738330639412152
+    },
+    "mmlu_high_school_biology": {
+      "name": "mmlu_high_school_biology",
+      "alias": "high_school_biology",
+      "sample_len": 310,
+      "acc,none": 0.7451612903225806,
+      "acc_stderr,none": 0.024790118459332208
+    },
+    "mmlu_high_school_chemistry": {
+      "name": "mmlu_high_school_chemistry",
+      "alias": "high_school_chemistry",
+      "sample_len": 203,
+      "acc,none": 0.47783251231527096,
+      "acc_stderr,none": 0.035145285621750066
+    },
+    "mmlu_high_school_computer_science": {
+      "name": "mmlu_high_school_computer_science",
+      "alias": "high_school_computer_science",
+      "sample_len": 100,
+      "acc,none": 0.59,
+      "acc_stderr,none": 0.04943110704237102
+    },
+    "mmlu_high_school_mathematics": {
+      "name": "mmlu_high_school_mathematics",
+      "alias": "high_school_mathematics",
+      "sample_len": 270,
+      "acc,none": 0.4,
+      "acc_stderr,none": 0.029869605095316908
+    },
+    "mmlu_high_school_physics": {
+      "name": "mmlu_high_school_physics",
+      "alias": "high_school_physics",
+      "sample_len": 151,
+      "acc,none": 0.3576158940397351,
+      "acc_stderr,none": 0.03913453431177258
+    },
+    "mmlu_high_school_statistics": {
+      "name": "mmlu_high_school_statistics",
+      "alias": "high_school_statistics",
+      "sample_len": 216,
+      "acc,none": 0.4675925925925926,
+      "acc_stderr,none": 0.03402801581358966
+    },
+    "mmlu_machine_learning": {
+      "name": "mmlu_machine_learning",
+      "alias": "machine_learning",
+      "sample_len": 112,
+      "acc,none": 0.4642857142857143,
+      "acc_stderr,none": 0.04733667890053756
+    },
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 3153,
+      "acc,none": 0.527117031398668,
+      "acc_stderr,none": 0.008607093913058559,
+      "sample_count": {
+        "acc,none": 3153
+      }
+    }
+  },
+  "groups": {
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 3153,
+      "acc,none": 0.527117031398668,
+      "acc_stderr,none": 0.008607093913058559,
+      "sample_count": {
+        "acc,none": 3153
+      }
+    }
+  },
+  "group_subtasks": {
+    "mmlu_stem": [
+      "mmlu_abstract_algebra",
+      "mmlu_anatomy",
+      "mmlu_astronomy",
+      "mmlu_college_biology",
+      "mmlu_college_chemistry",
+      "mmlu_college_computer_science",
+      "mmlu_college_mathematics",
+      "mmlu_college_physics",
+      "mmlu_computer_security",
+      "mmlu_conceptual_physics",
+      "mmlu_electrical_engineering",
+      "mmlu_elementary_mathematics",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_chemistry",
+      "mmlu_high_school_computer_science",
+      "mmlu_high_school_mathematics",
+      "mmlu_high_school_physics",
+      "mmlu_high_school_statistics",
+      "mmlu_machine_learning"
+    ]
+  },
+  "configs": {
+    "mmlu_abstract_algebra": {
+      "task": "mmlu_abstract_algebra",
+      "task_alias": "abstract_algebra",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "abstract_algebra",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml"
+      }
+    },
+    "mmlu_anatomy": {
+      "task": "mmlu_anatomy",
+      "task_alias": "anatomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "anatomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml"
+      }
+    },
+    "mmlu_astronomy": {
+      "task": "mmlu_astronomy",
+      "task_alias": "astronomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "astronomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml"
+      }
+    },
+    "mmlu_college_biology": {
+      "task": "mmlu_college_biology",
+      "task_alias": "college_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml"
+      }
+    },
+    "mmlu_college_chemistry": {
+      "task": "mmlu_college_chemistry",
+      "task_alias": "college_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml"
+      }
+    },
+    "mmlu_college_computer_science": {
+      "task": "mmlu_college_computer_science",
+      "task_alias": "college_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml"
+      }
+    },
+    "mmlu_college_mathematics": {
+      "task": "mmlu_college_mathematics",
+      "task_alias": "college_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml"
+      }
+    },
+    "mmlu_college_physics": {
+      "task": "mmlu_college_physics",
+      "task_alias": "college_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml"
+      }
+    },
+    "mmlu_computer_security": {
+      "task": "mmlu_computer_security",
+      "task_alias": "computer_security",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "computer_security",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about computer security.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml"
+      }
+    },
+    "mmlu_conceptual_physics": {
+      "task": "mmlu_conceptual_physics",
+      "task_alias": "conceptual_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "conceptual_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml"
+      }
+    },
+    "mmlu_electrical_engineering": {
+      "task": "mmlu_electrical_engineering",
+      "task_alias": "electrical_engineering",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "electrical_engineering",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml"
+      }
+    },
+    "mmlu_elementary_mathematics": {
+      "task": "mmlu_elementary_mathematics",
+      "task_alias": "elementary_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "elementary_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml"
+      }
+    },
+    "mmlu_high_school_biology": {
+      "task": "mmlu_high_school_biology",
+      "task_alias": "high_school_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml"
+      }
+    },
+    "mmlu_high_school_chemistry": {
+      "task": "mmlu_high_school_chemistry",
+      "task_alias": "high_school_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml"
+      }
+    },
+    "mmlu_high_school_computer_science": {
+      "task": "mmlu_high_school_computer_science",
+      "task_alias": "high_school_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml"
+      }
+    },
+    "mmlu_high_school_mathematics": {
+      "task": "mmlu_high_school_mathematics",
+      "task_alias": "high_school_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml"
+      }
+    },
+    "mmlu_high_school_physics": {
+      "task": "mmlu_high_school_physics",
+      "task_alias": "high_school_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml"
+      }
+    },
+    "mmlu_high_school_statistics": {
+      "task": "mmlu_high_school_statistics",
+      "task_alias": "high_school_statistics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_statistics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml"
+      }
+    },
+    "mmlu_machine_learning": {
+      "task": "mmlu_machine_learning",
+      "task_alias": "machine_learning",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "machine_learning",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mmlu_abstract_algebra": 1.0,
+    "mmlu_anatomy": 1.0,
+    "mmlu_astronomy": 1.0,
+    "mmlu_college_biology": 1.0,
+    "mmlu_college_chemistry": 1.0,
+    "mmlu_college_computer_science": 1.0,
+    "mmlu_college_mathematics": 1.0,
+    "mmlu_college_physics": 1.0,
+    "mmlu_computer_security": 1.0,
+    "mmlu_conceptual_physics": 1.0,
+    "mmlu_electrical_engineering": 1.0,
+    "mmlu_elementary_mathematics": 1.0,
+    "mmlu_high_school_biology": 1.0,
+    "mmlu_high_school_chemistry": 1.0,
+    "mmlu_high_school_computer_science": 1.0,
+    "mmlu_high_school_mathematics": 1.0,
+    "mmlu_high_school_physics": 1.0,
+    "mmlu_high_school_statistics": 1.0,
+    "mmlu_machine_learning": 1.0,
+    "mmlu_stem": "2"
+  },
+  "n-shot": {
+    "mmlu_abstract_algebra": 5,
+    "mmlu_anatomy": 5,
+    "mmlu_astronomy": 5,
+    "mmlu_college_biology": 5,
+    "mmlu_college_chemistry": 5,
+    "mmlu_college_computer_science": 5,
+    "mmlu_college_mathematics": 5,
+    "mmlu_college_physics": 5,
+    "mmlu_computer_security": 5,
+    "mmlu_conceptual_physics": 5,
+    "mmlu_electrical_engineering": 5,
+    "mmlu_elementary_mathematics": 5,
+    "mmlu_high_school_biology": 5,
+    "mmlu_high_school_chemistry": 5,
+    "mmlu_high_school_computer_science": 5,
+    "mmlu_high_school_mathematics": 5,
+    "mmlu_high_school_physics": 5,
+    "mmlu_high_school_statistics": 5,
+    "mmlu_machine_learning": 5,
+    "mmlu_stem": 5
+  },
+  "higher_is_better": {
+    "mmlu_abstract_algebra": {
+      "acc": true
+    },
+    "mmlu_anatomy": {
+      "acc": true
+    },
+    "mmlu_astronomy": {
+      "acc": true
+    },
+    "mmlu_college_biology": {
+      "acc": true
+    },
+    "mmlu_college_chemistry": {
+      "acc": true
+    },
+    "mmlu_college_computer_science": {
+      "acc": true
+    },
+    "mmlu_college_mathematics": {
+      "acc": true
+    },
+    "mmlu_college_physics": {
+      "acc": true
+    },
+    "mmlu_computer_security": {
+      "acc": true
+    },
+    "mmlu_conceptual_physics": {
+      "acc": true
+    },
+    "mmlu_electrical_engineering": {
+      "acc": true
+    },
+    "mmlu_elementary_mathematics": {
+      "acc": true
+    },
+    "mmlu_high_school_biology": {
+      "acc": true
+    },
+    "mmlu_high_school_chemistry": {
+      "acc": true
+    },
+    "mmlu_high_school_computer_science": {
+      "acc": true
+    },
+    "mmlu_high_school_mathematics": {
+      "acc": true
+    },
+    "mmlu_high_school_physics": {
+      "acc": true
+    },
+    "mmlu_high_school_statistics": {
+      "acc": true
+    },
+    "mmlu_machine_learning": {
+      "acc": true
+    },
+    "mmlu_stem": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "mmlu_abstract_algebra": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_anatomy": {
+      "original": 135,
+      "effective": 135
+    },
+    "mmlu_astronomy": {
+      "original": 152,
+      "effective": 152
+    },
+    "mmlu_college_biology": {
+      "original": 144,
+      "effective": 144
+    },
+    "mmlu_college_chemistry": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_college_computer_science": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_college_mathematics": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_college_physics": {
+      "original": 102,
+      "effective": 102
+    },
+    "mmlu_computer_security": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_conceptual_physics": {
+      "original": 235,
+      "effective": 235
+    },
+    "mmlu_electrical_engineering": {
+      "original": 145,
+      "effective": 145
+    },
+    "mmlu_elementary_mathematics": {
+      "original": 378,
+      "effective": 378
+    },
+    "mmlu_high_school_biology": {
+      "original": 310,
+      "effective": 310
+    },
+    "mmlu_high_school_chemistry": {
+      "original": 203,
+      "effective": 203
+    },
+    "mmlu_high_school_computer_science": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_high_school_mathematics": {
+      "original": 270,
+      "effective": 270
+    },
+    "mmlu_high_school_physics": {
+      "original": 151,
+      "effective": 151
+    },
+    "mmlu_high_school_statistics": {
+      "original": 216,
+      "effective": 216
+    },
+    "mmlu_machine_learning": {
+      "original": 112,
+      "effective": 112
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779167370.3143137,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "mmlu_abstract_algebra": "60db7f4ad521ee52dd7fa6822c13b023c630d4b97d42d15aeb7286c983bd98a5",
+    "mmlu_anatomy": "e9fc464802c2f4c7ed1c7f7dce6bca85f345339f48ade2bbbc806cc4a25ae30c",
+    "mmlu_astronomy": "4169ed9bdc567ca58c91e6efaeab9f286c036d858fade8b42e35d051675d328a",
+    "mmlu_college_biology": "7e9052b07611dced9ae403dabb60d9de9cff85c97b5b493e9345ec7eeb63f92d",
+    "mmlu_college_chemistry": "a5811436c4111233a2c0cc1dd471582884a9492476e942c59d6abf1ccae71db1",
+    "mmlu_college_computer_science": "c9c65519375d16e801a4d0fb70887bfb630fdca8b3419612806ad510d68455e1",
+    "mmlu_college_mathematics": "b55ccf8bb71f6611a82d4719af68c32c8fd5fa9977b1ff0af82c15a8a685d001",
+    "mmlu_college_physics": "bd891b9991c36929f59a86e6a0595328483ee72426bfcfa54e6bb727494b3f18",
+    "mmlu_computer_security": "94ce868618f7e30048f2c4600b50e721a8dac3aa141129e9f02c4675e096f734",
+    "mmlu_conceptual_physics": "63d66b3231b2200c05c62780eb0a14b0bc2b12f721b74920c11a96f8c86fee15",
+    "mmlu_electrical_engineering": "27de3e15b87cc76f39fce5338649295655c646b392fdac2613d9e4960c1fceb3",
+    "mmlu_elementary_mathematics": "fa1af387422aeb09709c4100c9c757ccdcbab5580d26cc0eaa8d826d1c3516d3",
+    "mmlu_high_school_biology": "dfdab2da41fae1739af4473de689c11dd9499d54f9c74be558a3d2208d6519af",
+    "mmlu_high_school_chemistry": "fd7a1a333ff2de479e27656dc7a69333905708fd548e014567c949ebb8c4ed0c",
+    "mmlu_high_school_computer_science": "42fe54bfac7979fcac14a7747820fb9e6fa06c5a85a719ef73dc6dce3621e7fc",
+    "mmlu_high_school_mathematics": "634744ebfb8410a955a8b1ef156244cbe17de8350839819f4fcc4cb89f6d1f20",
+    "mmlu_high_school_physics": "fd6089808c28e4df31c7a5f6dea1be8acb831d2927197b7a402105665226e2c2",
+    "mmlu_high_school_statistics": "ec1c2afa115d24837696048c4447e0e6b1103d0cc80dd2b47ac91a7fdfbca8bb",
+    "mmlu_machine_learning": "713910e9a68bfe8ed31b39e35503f354109abdb0c7f5928a9c4cfaa1e8e87158"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "183.43375307600945"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T05-16-21.613923.json

@@ -0,0 +1,138 @@
+{
+  "results": {
+    "mbpp": {
+      "name": "mbpp",
+      "alias": "mbpp",
+      "sample_len": 500,
+      "pass_at_1,none": 0.49,
+      "pass_at_1_stderr,none": 0.022378596989230785
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "mbpp": {
+      "task": "mbpp",
+      "dataset_path": "google-research-datasets/mbpp",
+      "dataset_name": "full",
+      "test_split": "test",
+      "doc_to_text": "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n",
+      "doc_to_target": "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}",
+      "unsafe_code": true,
+      "description": "",
+      "target_delimiter": "",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x763515362950>",
+        "doc_to_text": "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n",
+        "doc_to_choice": null,
+        "doc_to_target": "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": ""
+      },
+      "num_fewshot": 3,
+      "metric_list": [
+        {
+          "metric": "def pass_at_1(\n    references: Union[str, list[str]], predictions: Union[str, list[list[str]]]\n) -> float:\n    if isinstance(references, str):\n        references = [references]\n    if isinstance(predictions[0], str):\n        predictions = [[p] for p in predictions]\n    return pass_at_k.compute(\n        references=references,\n        predictions=predictions,\n        k=[1],\n    )[0][\"pass@1\"]\n",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "[DONE]"
+        ],
+        "do_sample": false,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mbpp/mbpp.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mbpp": 1.0
+  },
+  "n-shot": {
+    "mbpp": 3
+  },
+  "higher_is_better": {
+    "mbpp": {
+      "pass_at_1": true
+    }
+  },
+  "n-samples": {
+    "mbpp": {
+      "original": 500,
+      "effective": 500
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779167623.8732088,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "mbpp": "2189571b6de241aa42a354c64bfdc5e7efe2f5237c5016ebf8f23c4ec1a93bf6"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "165.51297075708862"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T05-19-57.588279.json

@@ -0,0 +1,179 @@
+{
+  "results": {
+    "gsm_plus": {
+      "name": "gsm_plus",
+      "alias": "gsm_plus",
+      "sample_len": 10552,
+      "exact_match,strict-match": 0.3011751326762699,
+      "exact_match_stderr,strict-match": 0.004466290002611388,
+      "exact_match,flexible-extract": 0.43868460955269145,
+      "exact_match_stderr,flexible-extract": 0.00483095356028294
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm_plus": {
+      "task": "gsm_plus",
+      "dataset_path": "qintongli/GSM-Plus",
+      "training_split": "test",
+      "test_split": "test",
+      "fewshot_split": "test",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{solution}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": "test",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/gsm_plus/gsm_plus.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm_plus": 1.0
+  },
+  "n-shot": {
+    "gsm_plus": 5
+  },
+  "higher_is_better": {
+    "gsm_plus": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm_plus": {
+      "original": 10552,
+      "effective": 10552
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779167792.397391,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "gsm_plus": "9fc5cf479857a1eb1a143016f6217e0f6f7a6a3251c77540c70b68178e0daa41"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "213.20076884201262"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T05-21-24.956996.json

@@ -0,0 +1,158 @@
+{
+  "results": {
+    "mmlu_pro_math": {
+      "name": "mmlu_pro_math",
+      "alias": "math",
+      "sample_len": 1351,
+      "exact_match,custom-extract": 0.41154700222057733,
+      "exact_match_stderr,custom-extract": 0.01339364324578758
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "mmlu_pro_math": {
+      "task": "mmlu_pro_math",
+      "task_alias": "math",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x7c7f796a9480>, subject='math')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x7c7f796a9b40>, including_answer=False)",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "description": "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "validation",
+        "process_docs": "functools.partial(<function process_docs at 0x7c7f796a9480>, subject='math')",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x7c7f796a9b40>, including_answer=False)",
+        "doc_to_choice": null,
+        "doc_to_target": "functools.partial(<function format_cot_target at 0x7c7f796a9a20>, including_answer=True)",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:"
+        ],
+        "max_gen_toks": 2048,
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.1,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mmlu_pro_math": 3.1
+  },
+  "n-shot": {
+    "mmlu_pro_math": 5
+  },
+  "higher_is_better": {
+    "mmlu_pro_math": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "mmlu_pro_math": {
+      "original": 1351,
+      "effective": 1351
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779168009.7144465,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "mmlu_pro_math": "2ddae5317c3ae5b3b96f4724d31f9877fdaf30fb4e532897de01a2e284b82493"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "83.21720596007071"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T05-25-21.746316.json

@@ -0,0 +1,288 @@
+{
+  "results": {
+    "leaderboard_musr_murder_mysteries": {
+      "name": "leaderboard_musr_murder_mysteries",
+      "alias": "leaderboard_musr_murder_mysteries",
+      "sample_len": 250,
+      "acc_norm,none": 0.492,
+      "acc_norm_stderr,none": 0.0316821564314138
+    },
+    "leaderboard_musr_object_placements": {
+      "name": "leaderboard_musr_object_placements",
+      "alias": "leaderboard_musr_object_placements",
+      "sample_len": 256,
+      "acc_norm,none": 0.21875,
+      "acc_norm_stderr,none": 0.025888027174359812
+    },
+    "leaderboard_musr_team_allocation": {
+      "name": "leaderboard_musr_team_allocation",
+      "alias": "leaderboard_musr_team_allocation",
+      "sample_len": 250,
+      "acc_norm,none": 0.312,
+      "acc_norm_stderr,none": 0.029361067575219817
+    },
+    "leaderboard_musr": {
+      "alias": "leaderboard_musr",
+      "name": "leaderboard_musr",
+      "sample_len": 756,
+      "acc_norm,none": 0.33994708994708994,
+      "acc_norm_stderr,none": 0.01675957996464575,
+      "sample_count": {
+        "acc_norm,none": 756
+      }
+    }
+  },
+  "groups": {
+    "leaderboard_musr": {
+      "alias": "leaderboard_musr",
+      "name": "leaderboard_musr",
+      "sample_len": 756,
+      "acc_norm,none": 0.33994708994708994,
+      "acc_norm_stderr,none": 0.01675957996464575,
+      "sample_count": {
+        "acc_norm,none": 756
+      }
+    }
+  },
+  "group_subtasks": {
+    "leaderboard_musr": [
+      "leaderboard_musr_murder_mysteries",
+      "leaderboard_musr_object_placements",
+      "leaderboard_musr_team_allocation"
+    ]
+  },
+  "configs": {
+    "leaderboard_musr_murder_mysteries": {
+      "task": "leaderboard_musr_murder_mysteries",
+      "dataset_path": "TAUR-Lab/MuSR",
+      "test_split": "murder_mysteries",
+      "doc_to_text": "def doc_to_text(doc):\n    \"\"\"\n    Convert a doc to text.\n    \"\"\"\n    choices = \"\"\n    for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n        choices += f\"{i + 1} - {choice}\\n\"\n\n    text = DOC_TO_TEXT.format(\n        narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n    )\n\n    return text\n",
+      "doc_to_target": "{{answer_choice}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "<function doc_to_text at 0x7e1920611480>",
+        "doc_to_choice": "{{choices}}",
+        "doc_to_target": "{{answer_choice}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/leaderboard/musr/musr_murder_mysteries.yaml"
+      }
+    },
+    "leaderboard_musr_object_placements": {
+      "task": "leaderboard_musr_object_placements",
+      "dataset_path": "TAUR-Lab/MuSR",
+      "test_split": "object_placements",
+      "doc_to_text": "def doc_to_text(doc):\n    \"\"\"\n    Convert a doc to text.\n    \"\"\"\n    choices = \"\"\n    for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n        choices += f\"{i + 1} - {choice}\\n\"\n\n    text = DOC_TO_TEXT.format(\n        narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n    )\n\n    return text\n",
+      "doc_to_target": "{{answer_choice}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "<function doc_to_text at 0x7e1920611480>",
+        "doc_to_choice": "{{choices}}",
+        "doc_to_target": "{{answer_choice}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/leaderboard/musr/musr_object_placements.yaml"
+      }
+    },
+    "leaderboard_musr_team_allocation": {
+      "task": "leaderboard_musr_team_allocation",
+      "dataset_path": "TAUR-Lab/MuSR",
+      "test_split": "team_allocation",
+      "doc_to_text": "def doc_to_text(doc):\n    \"\"\"\n    Convert a doc to text.\n    \"\"\"\n    choices = \"\"\n    for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n        choices += f\"{i + 1} - {choice}\\n\"\n\n    text = DOC_TO_TEXT.format(\n        narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n    )\n\n    return text\n",
+      "doc_to_target": "{{answer_choice}}",
+      "unsafe_code": false,
+      "doc_to_choice": "{{choices}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "<function doc_to_text at 0x7e1920611480>",
+        "doc_to_choice": "{{choices}}",
+        "doc_to_target": "{{answer_choice}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/leaderboard/musr/musr_team_allocation.yaml"
+      }
+    }
+  },
+  "versions": {
+    "leaderboard_musr": "N/A",
+    "leaderboard_musr_murder_mysteries": 1.0,
+    "leaderboard_musr_object_placements": 1.0,
+    "leaderboard_musr_team_allocation": 1.0
+  },
+  "n-shot": {
+    "leaderboard_musr": 0,
+    "leaderboard_musr_murder_mysteries": 0,
+    "leaderboard_musr_object_placements": 0,
+    "leaderboard_musr_team_allocation": 0
+  },
+  "higher_is_better": {
+    "leaderboard_musr": {
+      "acc_norm": true
+    },
+    "leaderboard_musr_murder_mysteries": {
+      "acc_norm": true
+    },
+    "leaderboard_musr_object_placements": {
+      "acc_norm": true
+    },
+    "leaderboard_musr_team_allocation": {
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "leaderboard_musr_murder_mysteries": {
+      "original": 250,
+      "effective": 250
+    },
+    "leaderboard_musr_object_placements": {
+      "original": 256,
+      "effective": 256
+    },
+    "leaderboard_musr_team_allocation": {
+      "original": 250,
+      "effective": 250
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779168248.4222658,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "leaderboard_musr_murder_mysteries": "10b0f39591b5077ca738884f3a41fe6dee70cf50a346410ea0523f6c3a9d4597",
+    "leaderboard_musr_object_placements": "d9fd900d3ebf1b133577f77ce3e75afc425aad0f4a76ad4d00b2225b01ebfecd",
+    "leaderboard_musr_team_allocation": "2cc8896ce61ba9f272a2e2de62fc7b29c0999fc91b81e79187e0a256f3d3d539"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.3_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "81.1735600080574"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T04-52-12.628448.json

@@ -0,0 +1,680 @@
+{
+  "results": {
+    "minerva_math_algebra": {
+      "name": "minerva_math_algebra",
+      "alias": "minerva_math_algebra",
+      "sample_len": 1187,
+      "exact_match,none": 0.22156697556866048,
+      "exact_match_stderr,none": 0.012059263822234162,
+      "math_verify,none": 0.4903117101937658,
+      "math_verify_stderr,none": 0.014515971811754614
+    },
+    "minerva_math_counting_and_prob": {
+      "name": "minerva_math_counting_and_prob",
+      "alias": "minerva_math_counting_and_prob",
+      "sample_len": 474,
+      "exact_match,none": 0.17932489451476794,
+      "exact_match_stderr,none": 0.01763905650602963,
+      "math_verify,none": 0.27848101265822783,
+      "math_verify_stderr,none": 0.020610622389419322
+    },
+    "minerva_math_geometry": {
+      "name": "minerva_math_geometry",
+      "alias": "minerva_math_geometry",
+      "sample_len": 479,
+      "exact_match,none": 0.14822546972860126,
+      "exact_match_stderr,none": 0.01625211153660326,
+      "math_verify,none": 0.2881002087682672,
+      "math_verify_stderr,none": 0.020714157351995742
+    },
+    "minerva_math_intermediate_algebra": {
+      "name": "minerva_math_intermediate_algebra",
+      "alias": "minerva_math_intermediate_algebra",
+      "sample_len": 903,
+      "exact_match,none": 0.07751937984496124,
+      "exact_match_stderr,none": 0.008903904671336352,
+      "math_verify,none": 0.17718715393134,
+      "math_verify_stderr,none": 0.012713448821847467
+    },
+    "minerva_math_num_theory": {
+      "name": "minerva_math_num_theory",
+      "alias": "minerva_math_num_theory",
+      "sample_len": 540,
+      "exact_match,none": 0.05555555555555555,
+      "exact_match_stderr,none": 0.009866373976318395,
+      "math_verify,none": 0.22592592592592592,
+      "math_verify_stderr,none": 0.018012745534972664
+    },
+    "minerva_math_prealgebra": {
+      "name": "minerva_math_prealgebra",
+      "alias": "minerva_math_prealgebra",
+      "sample_len": 871,
+      "exact_match,none": 0.25602755453501724,
+      "exact_match_stderr,none": 0.014796607480616742,
+      "math_verify,none": 0.513203214695752,
+      "math_verify_stderr,none": 0.016945676392984253
+    },
+    "minerva_math_precalc": {
+      "name": "minerva_math_precalc",
+      "alias": "minerva_math_precalc",
+      "sample_len": 546,
+      "exact_match,none": 0.10989010989010989,
+      "exact_match_stderr,none": 0.013396855281994252,
+      "math_verify,none": 0.2032967032967033,
+      "math_verify_stderr,none": 0.01723912521064355
+    },
+    "minerva_math": {
+      "alias": "minerva_math",
+      "name": "minerva_math",
+      "sample_len": 5000,
+      "exact_match,none": 0.1604,
+      "exact_match_stderr,none": 0.005091514547731609,
+      "math_verify,none": 0.3384,
+      "math_verify_stderr,none": 0.006398980314946738,
+      "sample_count": {
+        "exact_match,none": 5000,
+        "math_verify,none": 5000
+      }
+    }
+  },
+  "groups": {
+    "minerva_math": {
+      "alias": "minerva_math",
+      "name": "minerva_math",
+      "sample_len": 5000,
+      "exact_match,none": 0.1604,
+      "exact_match_stderr,none": 0.005091514547731609,
+      "math_verify,none": 0.3384,
+      "math_verify_stderr,none": 0.006398980314946738,
+      "sample_count": {
+        "exact_match,none": 5000,
+        "math_verify,none": 5000
+      }
+    }
+  },
+  "group_subtasks": {
+    "minerva_math": [
+      "minerva_math_algebra",
+      "minerva_math_counting_and_prob",
+      "minerva_math_geometry",
+      "minerva_math_intermediate_algebra",
+      "minerva_math_num_theory",
+      "minerva_math_prealgebra",
+      "minerva_math_precalc"
+    ]
+  },
+  "configs": {
+    "minerva_math_algebra": {
+      "task": "minerva_math_algebra",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "algebra",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml"
+      }
+    },
+    "minerva_math_counting_and_prob": {
+      "task": "minerva_math_counting_and_prob",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "counting_and_probability",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml"
+      }
+    },
+    "minerva_math_geometry": {
+      "task": "minerva_math_geometry",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "geometry",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml"
+      }
+    },
+    "minerva_math_intermediate_algebra": {
+      "task": "minerva_math_intermediate_algebra",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "intermediate_algebra",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml"
+      }
+    },
+    "minerva_math_num_theory": {
+      "task": "minerva_math_num_theory",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "number_theory",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml"
+      }
+    },
+    "minerva_math_prealgebra": {
+      "task": "minerva_math_prealgebra",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "prealgebra",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml"
+      }
+    },
+    "minerva_math_precalc": {
+      "task": "minerva_math_precalc",
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "precalculus",
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x737865d9b9a0>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x737865d9bb50>",
+        "doc_to_text": "<function doc_to_text at 0x73787fbdf370>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml"
+      }
+    }
+  },
+  "versions": {
+    "minerva_math": "1.0",
+    "minerva_math_algebra": 3.0,
+    "minerva_math_counting_and_prob": 3.0,
+    "minerva_math_geometry": 3.0,
+    "minerva_math_intermediate_algebra": 3.0,
+    "minerva_math_num_theory": 3.0,
+    "minerva_math_prealgebra": 3.0,
+    "minerva_math_precalc": 3.0
+  },
+  "n-shot": {
+    "minerva_math": 4,
+    "minerva_math_algebra": 4,
+    "minerva_math_counting_and_prob": 4,
+    "minerva_math_geometry": 4,
+    "minerva_math_intermediate_algebra": 4,
+    "minerva_math_num_theory": 4,
+    "minerva_math_prealgebra": 4,
+    "minerva_math_precalc": 4
+  },
+  "higher_is_better": {
+    "minerva_math": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_algebra": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_counting_and_prob": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_geometry": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_intermediate_algebra": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_num_theory": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_prealgebra": {
+      "exact_match": true,
+      "math_verify": true
+    },
+    "minerva_math_precalc": {
+      "exact_match": true,
+      "math_verify": true
+    }
+  },
+  "n-samples": {
+    "minerva_math_algebra": {
+      "original": 1187,
+      "effective": 1187
+    },
+    "minerva_math_counting_and_prob": {
+      "original": 474,
+      "effective": 474
+    },
+    "minerva_math_geometry": {
+      "original": 479,
+      "effective": 479
+    },
+    "minerva_math_intermediate_algebra": {
+      "original": 903,
+      "effective": 903
+    },
+    "minerva_math_num_theory": {
+      "original": 540,
+      "effective": 540
+    },
+    "minerva_math_prealgebra": {
+      "original": 871,
+      "effective": 871
+    },
+    "minerva_math_precalc": {
+      "original": 546,
+      "effective": 546
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779166008.1866422,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "minerva_math_algebra": "a5597b7619af3e15e50469ae895189e416c0276a0b4efb7ac598bacb74ef7c86",
+    "minerva_math_counting_and_prob": "3c9aea6323941f5250896f019716ca268b53ea203877e263e5eb435df5110bde",
+    "minerva_math_geometry": "51d9457ad25bf15988ea75e6dc804dd80b437e5b50f9cb5fb211a7ee7256b75e",
+    "minerva_math_intermediate_algebra": "a94c1fc928d616f136f55bd50382516a109bf506696f75bb904b3e4cfb4a210e",
+    "minerva_math_num_theory": "6a9e327184475c4aabeba734be9a2c5a1435f0a138687194585e201e23aac3b0",
+    "minerva_math_prealgebra": "1ad14186c0a46b9c62b4e248ebb2bb5c2e3daaee2f501fcfb14efa4654ddb56d",
+    "minerva_math_precalc": "4d9f2cff781009dce40c0acd4942b199a05a6a576a3e1fb5e26123db28c35d0d"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "332.4753986719297"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T04-55-24.968800.json

@@ -0,0 +1,1543 @@
+{
+  "results": {
+    "mmlu_abstract_algebra": {
+      "name": "mmlu_abstract_algebra",
+      "alias": "abstract_algebra",
+      "sample_len": 100,
+      "acc,none": 0.37,
+      "acc_stderr,none": 0.04852365870939099
+    },
+    "mmlu_anatomy": {
+      "name": "mmlu_anatomy",
+      "alias": "anatomy",
+      "sample_len": 135,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.043097329010363554
+    },
+    "mmlu_astronomy": {
+      "name": "mmlu_astronomy",
+      "alias": "astronomy",
+      "sample_len": 152,
+      "acc,none": 0.6710526315789473,
+      "acc_stderr,none": 0.03823428969926605
+    },
+    "mmlu_college_biology": {
+      "name": "mmlu_college_biology",
+      "alias": "college_biology",
+      "sample_len": 144,
+      "acc,none": 0.6388888888888888,
+      "acc_stderr,none": 0.04016660030451232
+    },
+    "mmlu_college_chemistry": {
+      "name": "mmlu_college_chemistry",
+      "alias": "college_chemistry",
+      "sample_len": 100,
+      "acc,none": 0.41,
+      "acc_stderr,none": 0.049431107042371025
+    },
+    "mmlu_college_computer_science": {
+      "name": "mmlu_college_computer_science",
+      "alias": "college_computer_science",
+      "sample_len": 100,
+      "acc,none": 0.45,
+      "acc_stderr,none": 0.05
+    },
+    "mmlu_college_mathematics": {
+      "name": "mmlu_college_mathematics",
+      "alias": "college_mathematics",
+      "sample_len": 100,
+      "acc,none": 0.38,
+      "acc_stderr,none": 0.04878317312145633
+    },
+    "mmlu_college_physics": {
+      "name": "mmlu_college_physics",
+      "alias": "college_physics",
+      "sample_len": 102,
+      "acc,none": 0.3431372549019608,
+      "acc_stderr,none": 0.04724007352383889
+    },
+    "mmlu_computer_security": {
+      "name": "mmlu_computer_security",
+      "alias": "computer_security",
+      "sample_len": 100,
+      "acc,none": 0.71,
+      "acc_stderr,none": 0.045604802157206845
+    },
+    "mmlu_conceptual_physics": {
+      "name": "mmlu_conceptual_physics",
+      "alias": "conceptual_physics",
+      "sample_len": 235,
+      "acc,none": 0.5446808510638298,
+      "acc_stderr,none": 0.032555253593403555
+    },
+    "mmlu_electrical_engineering": {
+      "name": "mmlu_electrical_engineering",
+      "alias": "electrical_engineering",
+      "sample_len": 145,
+      "acc,none": 0.6689655172413793,
+      "acc_stderr,none": 0.03921545312467122
+    },
+    "mmlu_elementary_mathematics": {
+      "name": "mmlu_elementary_mathematics",
+      "alias": "elementary_mathematics",
+      "sample_len": 378,
+      "acc,none": 0.5238095238095238,
+      "acc_stderr,none": 0.025722097064388518
+    },
+    "mmlu_high_school_biology": {
+      "name": "mmlu_high_school_biology",
+      "alias": "high_school_biology",
+      "sample_len": 310,
+      "acc,none": 0.7193548387096774,
+      "acc_stderr,none": 0.0255606047210229
+    },
+    "mmlu_high_school_chemistry": {
+      "name": "mmlu_high_school_chemistry",
+      "alias": "high_school_chemistry",
+      "sample_len": 203,
+      "acc,none": 0.4876847290640394,
+      "acc_stderr,none": 0.035169204442208966
+    },
+    "mmlu_high_school_computer_science": {
+      "name": "mmlu_high_school_computer_science",
+      "alias": "high_school_computer_science",
+      "sample_len": 100,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.049236596391733084
+    },
+    "mmlu_high_school_mathematics": {
+      "name": "mmlu_high_school_mathematics",
+      "alias": "high_school_mathematics",
+      "sample_len": 270,
+      "acc,none": 0.3851851851851852,
+      "acc_stderr,none": 0.029670906124630882
+    },
+    "mmlu_high_school_physics": {
+      "name": "mmlu_high_school_physics",
+      "alias": "high_school_physics",
+      "sample_len": 151,
+      "acc,none": 0.36423841059602646,
+      "acc_stderr,none": 0.03929111781242741
+    },
+    "mmlu_high_school_statistics": {
+      "name": "mmlu_high_school_statistics",
+      "alias": "high_school_statistics",
+      "sample_len": 216,
+      "acc,none": 0.4722222222222222,
+      "acc_stderr,none": 0.0340470532865388
+    },
+    "mmlu_machine_learning": {
+      "name": "mmlu_machine_learning",
+      "alias": "machine_learning",
+      "sample_len": 112,
+      "acc,none": 0.45535714285714285,
+      "acc_stderr,none": 0.047268355537191
+    },
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 3153,
+      "acc,none": 0.5233111322549953,
+      "acc_stderr,none": 0.00866697139962451,
+      "sample_count": {
+        "acc,none": 3153
+      }
+    }
+  },
+  "groups": {
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 3153,
+      "acc,none": 0.5233111322549953,
+      "acc_stderr,none": 0.00866697139962451,
+      "sample_count": {
+        "acc,none": 3153
+      }
+    }
+  },
+  "group_subtasks": {
+    "mmlu_stem": [
+      "mmlu_abstract_algebra",
+      "mmlu_anatomy",
+      "mmlu_astronomy",
+      "mmlu_college_biology",
+      "mmlu_college_chemistry",
+      "mmlu_college_computer_science",
+      "mmlu_college_mathematics",
+      "mmlu_college_physics",
+      "mmlu_computer_security",
+      "mmlu_conceptual_physics",
+      "mmlu_electrical_engineering",
+      "mmlu_elementary_mathematics",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_chemistry",
+      "mmlu_high_school_computer_science",
+      "mmlu_high_school_mathematics",
+      "mmlu_high_school_physics",
+      "mmlu_high_school_statistics",
+      "mmlu_machine_learning"
+    ]
+  },
+  "configs": {
+    "mmlu_abstract_algebra": {
+      "task": "mmlu_abstract_algebra",
+      "task_alias": "abstract_algebra",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "abstract_algebra",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml"
+      }
+    },
+    "mmlu_anatomy": {
+      "task": "mmlu_anatomy",
+      "task_alias": "anatomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "anatomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml"
+      }
+    },
+    "mmlu_astronomy": {
+      "task": "mmlu_astronomy",
+      "task_alias": "astronomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "astronomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml"
+      }
+    },
+    "mmlu_college_biology": {
+      "task": "mmlu_college_biology",
+      "task_alias": "college_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml"
+      }
+    },
+    "mmlu_college_chemistry": {
+      "task": "mmlu_college_chemistry",
+      "task_alias": "college_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml"
+      }
+    },
+    "mmlu_college_computer_science": {
+      "task": "mmlu_college_computer_science",
+      "task_alias": "college_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml"
+      }
+    },
+    "mmlu_college_mathematics": {
+      "task": "mmlu_college_mathematics",
+      "task_alias": "college_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml"
+      }
+    },
+    "mmlu_college_physics": {
+      "task": "mmlu_college_physics",
+      "task_alias": "college_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml"
+      }
+    },
+    "mmlu_computer_security": {
+      "task": "mmlu_computer_security",
+      "task_alias": "computer_security",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "computer_security",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about computer security.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml"
+      }
+    },
+    "mmlu_conceptual_physics": {
+      "task": "mmlu_conceptual_physics",
+      "task_alias": "conceptual_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "conceptual_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml"
+      }
+    },
+    "mmlu_electrical_engineering": {
+      "task": "mmlu_electrical_engineering",
+      "task_alias": "electrical_engineering",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "electrical_engineering",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml"
+      }
+    },
+    "mmlu_elementary_mathematics": {
+      "task": "mmlu_elementary_mathematics",
+      "task_alias": "elementary_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "elementary_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml"
+      }
+    },
+    "mmlu_high_school_biology": {
+      "task": "mmlu_high_school_biology",
+      "task_alias": "high_school_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml"
+      }
+    },
+    "mmlu_high_school_chemistry": {
+      "task": "mmlu_high_school_chemistry",
+      "task_alias": "high_school_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml"
+      }
+    },
+    "mmlu_high_school_computer_science": {
+      "task": "mmlu_high_school_computer_science",
+      "task_alias": "high_school_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml"
+      }
+    },
+    "mmlu_high_school_mathematics": {
+      "task": "mmlu_high_school_mathematics",
+      "task_alias": "high_school_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml"
+      }
+    },
+    "mmlu_high_school_physics": {
+      "task": "mmlu_high_school_physics",
+      "task_alias": "high_school_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml"
+      }
+    },
+    "mmlu_high_school_statistics": {
+      "task": "mmlu_high_school_statistics",
+      "task_alias": "high_school_statistics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_statistics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml"
+      }
+    },
+    "mmlu_machine_learning": {
+      "task": "mmlu_machine_learning",
+      "task_alias": "machine_learning",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "machine_learning",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mmlu_abstract_algebra": 1.0,
+    "mmlu_anatomy": 1.0,
+    "mmlu_astronomy": 1.0,
+    "mmlu_college_biology": 1.0,
+    "mmlu_college_chemistry": 1.0,
+    "mmlu_college_computer_science": 1.0,
+    "mmlu_college_mathematics": 1.0,
+    "mmlu_college_physics": 1.0,
+    "mmlu_computer_security": 1.0,
+    "mmlu_conceptual_physics": 1.0,
+    "mmlu_electrical_engineering": 1.0,
+    "mmlu_elementary_mathematics": 1.0,
+    "mmlu_high_school_biology": 1.0,
+    "mmlu_high_school_chemistry": 1.0,
+    "mmlu_high_school_computer_science": 1.0,
+    "mmlu_high_school_mathematics": 1.0,
+    "mmlu_high_school_physics": 1.0,
+    "mmlu_high_school_statistics": 1.0,
+    "mmlu_machine_learning": 1.0,
+    "mmlu_stem": "2"
+  },
+  "n-shot": {
+    "mmlu_abstract_algebra": 5,
+    "mmlu_anatomy": 5,
+    "mmlu_astronomy": 5,
+    "mmlu_college_biology": 5,
+    "mmlu_college_chemistry": 5,
+    "mmlu_college_computer_science": 5,
+    "mmlu_college_mathematics": 5,
+    "mmlu_college_physics": 5,
+    "mmlu_computer_security": 5,
+    "mmlu_conceptual_physics": 5,
+    "mmlu_electrical_engineering": 5,
+    "mmlu_elementary_mathematics": 5,
+    "mmlu_high_school_biology": 5,
+    "mmlu_high_school_chemistry": 5,
+    "mmlu_high_school_computer_science": 5,
+    "mmlu_high_school_mathematics": 5,
+    "mmlu_high_school_physics": 5,
+    "mmlu_high_school_statistics": 5,
+    "mmlu_machine_learning": 5,
+    "mmlu_stem": 5
+  },
+  "higher_is_better": {
+    "mmlu_abstract_algebra": {
+      "acc": true
+    },
+    "mmlu_anatomy": {
+      "acc": true
+    },
+    "mmlu_astronomy": {
+      "acc": true
+    },
+    "mmlu_college_biology": {
+      "acc": true
+    },
+    "mmlu_college_chemistry": {
+      "acc": true
+    },
+    "mmlu_college_computer_science": {
+      "acc": true
+    },
+    "mmlu_college_mathematics": {
+      "acc": true
+    },
+    "mmlu_college_physics": {
+      "acc": true
+    },
+    "mmlu_computer_security": {
+      "acc": true
+    },
+    "mmlu_conceptual_physics": {
+      "acc": true
+    },
+    "mmlu_electrical_engineering": {
+      "acc": true
+    },
+    "mmlu_elementary_mathematics": {
+      "acc": true
+    },
+    "mmlu_high_school_biology": {
+      "acc": true
+    },
+    "mmlu_high_school_chemistry": {
+      "acc": true
+    },
+    "mmlu_high_school_computer_science": {
+      "acc": true
+    },
+    "mmlu_high_school_mathematics": {
+      "acc": true
+    },
+    "mmlu_high_school_physics": {
+      "acc": true
+    },
+    "mmlu_high_school_statistics": {
+      "acc": true
+    },
+    "mmlu_machine_learning": {
+      "acc": true
+    },
+    "mmlu_stem": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "mmlu_abstract_algebra": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_anatomy": {
+      "original": 135,
+      "effective": 135
+    },
+    "mmlu_astronomy": {
+      "original": 152,
+      "effective": 152
+    },
+    "mmlu_college_biology": {
+      "original": 144,
+      "effective": 144
+    },
+    "mmlu_college_chemistry": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_college_computer_science": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_college_mathematics": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_college_physics": {
+      "original": 102,
+      "effective": 102
+    },
+    "mmlu_computer_security": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_conceptual_physics": {
+      "original": 235,
+      "effective": 235
+    },
+    "mmlu_electrical_engineering": {
+      "original": 145,
+      "effective": 145
+    },
+    "mmlu_elementary_mathematics": {
+      "original": 378,
+      "effective": 378
+    },
+    "mmlu_high_school_biology": {
+      "original": 310,
+      "effective": 310
+    },
+    "mmlu_high_school_chemistry": {
+      "original": 203,
+      "effective": 203
+    },
+    "mmlu_high_school_computer_science": {
+      "original": 100,
+      "effective": 100
+    },
+    "mmlu_high_school_mathematics": {
+      "original": 270,
+      "effective": 270
+    },
+    "mmlu_high_school_physics": {
+      "original": 151,
+      "effective": 151
+    },
+    "mmlu_high_school_statistics": {
+      "original": 216,
+      "effective": 216
+    },
+    "mmlu_machine_learning": {
+      "original": 112,
+      "effective": 112
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779166349.5304868,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "mmlu_abstract_algebra": "60db7f4ad521ee52dd7fa6822c13b023c630d4b97d42d15aeb7286c983bd98a5",
+    "mmlu_anatomy": "e9fc464802c2f4c7ed1c7f7dce6bca85f345339f48ade2bbbc806cc4a25ae30c",
+    "mmlu_astronomy": "4169ed9bdc567ca58c91e6efaeab9f286c036d858fade8b42e35d051675d328a",
+    "mmlu_college_biology": "7e9052b07611dced9ae403dabb60d9de9cff85c97b5b493e9345ec7eeb63f92d",
+    "mmlu_college_chemistry": "a5811436c4111233a2c0cc1dd471582884a9492476e942c59d6abf1ccae71db1",
+    "mmlu_college_computer_science": "c9c65519375d16e801a4d0fb70887bfb630fdca8b3419612806ad510d68455e1",
+    "mmlu_college_mathematics": "b55ccf8bb71f6611a82d4719af68c32c8fd5fa9977b1ff0af82c15a8a685d001",
+    "mmlu_college_physics": "bd891b9991c36929f59a86e6a0595328483ee72426bfcfa54e6bb727494b3f18",
+    "mmlu_computer_security": "94ce868618f7e30048f2c4600b50e721a8dac3aa141129e9f02c4675e096f734",
+    "mmlu_conceptual_physics": "63d66b3231b2200c05c62780eb0a14b0bc2b12f721b74920c11a96f8c86fee15",
+    "mmlu_electrical_engineering": "27de3e15b87cc76f39fce5338649295655c646b392fdac2613d9e4960c1fceb3",
+    "mmlu_elementary_mathematics": "fa1af387422aeb09709c4100c9c757ccdcbab5580d26cc0eaa8d826d1c3516d3",
+    "mmlu_high_school_biology": "dfdab2da41fae1739af4473de689c11dd9499d54f9c74be558a3d2208d6519af",
+    "mmlu_high_school_chemistry": "fd7a1a333ff2de479e27656dc7a69333905708fd548e014567c949ebb8c4ed0c",
+    "mmlu_high_school_computer_science": "42fe54bfac7979fcac14a7747820fb9e6fa06c5a85a719ef73dc6dce3621e7fc",
+    "mmlu_high_school_mathematics": "634744ebfb8410a955a8b1ef156244cbe17de8350839819f4fcc4cb89f6d1f20",
+    "mmlu_high_school_physics": "fd6089808c28e4df31c7a5f6dea1be8acb831d2927197b7a402105665226e2c2",
+    "mmlu_high_school_statistics": "ec1c2afa115d24837696048c4447e0e6b1103d0cc80dd2b47ac91a7fdfbca8bb",
+    "mmlu_machine_learning": "713910e9a68bfe8ed31b39e35503f354109abdb0c7f5928a9c4cfaa1e8e87158"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "183.19718465197366"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T04-56-33.239207.json

@@ -0,0 +1,143 @@
+{
+  "results": {
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.86,
+      "acc_stderr,none": 0.010978183844357803,
+      "acc_norm,none": 0.705,
+      "acc_norm_stderr,none": 0.014428554438445507
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "sciq": {
+      "task": "sciq",
+      "dataset_path": "allenai/sciq",
+      "training_split": "train",
+      "validation_split": "validation",
+      "test_split": "test",
+      "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
+      "doc_to_target": 3,
+      "unsafe_code": false,
+      "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
+        "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
+        "doc_to_target": 3,
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{support}} {{question}}",
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/sciq/sciq.yaml"
+      }
+    }
+  },
+  "versions": {
+    "sciq": 1.0
+  },
+  "n-shot": {
+    "sciq": 0
+  },
+  "higher_is_better": {
+    "sciq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "sciq": {
+      "original": 1000,
+      "effective": 1000
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096,
+      "temperature": 0.0
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779166536.0492654,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "sciq": "94d7c305efca6804164ebf166179faaee89d73aba3a1539fb62b358258a4b95f"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "65.10639387206174"
+}

eval_results/vllm/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0/results_2026-05-19T04-59-33.294652.json

@@ -0,0 +1,138 @@
+{
+  "results": {
+    "mbpp": {
+      "name": "mbpp",
+      "alias": "mbpp",
+      "sample_len": 500,
+      "pass_at_1,none": 0.0,
+      "pass_at_1_stderr,none": 0.0
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "mbpp": {
+      "task": "mbpp",
+      "dataset_path": "google-research-datasets/mbpp",
+      "dataset_name": "full",
+      "test_split": "test",
+      "doc_to_text": "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n",
+      "doc_to_target": "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}",
+      "unsafe_code": true,
+      "description": "",
+      "target_delimiter": "",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x7e4babe369e0>",
+        "doc_to_text": "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n",
+        "doc_to_choice": null,
+        "doc_to_target": "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": ""
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "def pass_at_1(\n    references: Union[str, list[str]], predictions: Union[str, list[list[str]]]\n) -> float:\n    if isinstance(references, str):\n        references = [references]\n    if isinstance(predictions[0], str):\n        predictions = [[p] for p in predictions]\n    return pass_at_k.compute(\n        references=references,\n        predictions=predictions,\n        k=[1],\n    )[0][\"pass@1\"]\n",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "[DONE]"
+        ],
+        "do_sample": false,
+        "max_new_tokens": 4096
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+        "tensor_parallel_size": 2,
+        "dtype": "float16",
+        "gpu_memory_utilization": 0.3,
+        "trust_remote_code": true,
+        "config_source": "/home/ubuntu/AMiD/.venv/lib/python3.10/site-packages/lm_eval/tasks/mbpp/mbpp.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mbpp": 1.0
+  },
+  "n-shot": {
+    "mbpp": 0
+  },
+  "higher_is_better": {
+    "mbpp": {
+      "pass_at_1": true
+    }
+  },
+  "n-samples": {
+    "mbpp": {
+      "original": 500,
+      "effective": 500
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": {
+      "pretrained": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+      "tensor_parallel_size": 2,
+      "dtype": "float16",
+      "gpu_memory_utilization": 0.3,
+      "trust_remote_code": true
+    },
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "max_new_tokens": 4096
+    },
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "47ec76b",
+  "date": 1779166603.4295712,
+  "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.10.20 (main, May 10 2026, 19:27:29) [Clang 22.1.3 ] (64-bit runtime)\nPython platform: Linux-6.17.0-1013-aws-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA H200\nGPU 1: NVIDIA H200\nGPU 2: NVIDIA H200\nGPU 3: NVIDIA H200\nGPU 4: NVIDIA H200\nGPU 5: NVIDIA H200\nGPU 6: NVIDIA H200\nGPU 7: NVIDIA H200\n\nNvidia driver version: 580.126.16\ncuDNN version: Probably one of the following:\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_adv.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_cnn.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_precompiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_engines_runtime_compiled.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_graph.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_heuristic.so.9\n/opt/pytorch/lib/python3.13/site-packages/nvidia/cu13/lib/libcudnn_ops.so.9\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  192\nOn-line CPU(s) list:                     0-191\nVendor ID:                               AuthenticAMD\nModel name:                              AMD EPYC 7R13 Processor\nCPU family:                              25\nModel:                                   1\nThread(s) per core:                      2\nCore(s) per socket:                      48\nSocket(s):                               2\nStepping:                                1\nBogoMIPS:                                5299.99\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save vaes vpclmulqdq rdpid\nHypervisor vendor:                       KVM\nVirtualization type:                     full\nL1d cache:                               3 MiB (96 instances)\nL1i cache:                               3 MiB (96 instances)\nL2 cache:                                48 MiB (96 instances)\nL3 cache:                                384 MiB (12 instances)\nNUMA node(s):                            4\nNUMA node0 CPU(s):                       0-23,96-119\nNUMA node1 CPU(s):                       24-47,120-143\nNUMA node2 CPU(s):                       48-71,144-167\nNUMA node3 CPU(s):                       72-95,168-191\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Mitigation; Safe RET\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Mitigation; Clear CPU buffers\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] Could not collect\n[conda] numpy                                       2.2.6                pypi_0           pypi\n[conda] nvidia-cublas                               13.1.0.3             pypi_0           pypi\n[conda] nvidia-cuda-cupti                           13.0.85              pypi_0           pypi\n[conda] nvidia-cuda-nvrtc                           13.0.88              pypi_0           pypi\n[conda] nvidia-cuda-runtime                         13.0.96              pypi_0           pypi\n[conda] nvidia-cudnn-cu13                           9.19.0.56            pypi_0           pypi\n[conda] nvidia-cudnn-frontend                       1.18.0               pypi_0           pypi\n[conda] nvidia-cufft                                12.0.0.61            pypi_0           pypi\n[conda] nvidia-curand                               10.4.0.35            pypi_0           pypi\n[conda] nvidia-cusolver                             12.0.4.66            pypi_0           pypi\n[conda] nvidia-cusparse                             12.6.3.3             pypi_0           pypi\n[conda] nvidia-cusparselt-cu13                      0.8.0                pypi_0           pypi\n[conda] nvidia-nccl-cu13                            2.28.9               pypi_0           pypi\n[conda] nvidia-nvjitlink                            13.0.88              pypi_0           pypi\n[conda] nvidia-nvtx                                 13.0.85              pypi_0           pypi\n[conda] tokenspeed-triton                           3.7.10.post20260505  pypi_0           pypi\n[conda] torch                                       2.11.0               pypi_0           pypi\n[conda] torch-c-dlpack-ext                          0.1.5                pypi_0           pypi\n[conda] torchaudio                                  2.11.0               pypi_0           pypi\n[conda] torchvision                                 0.26.0               pypi_0           pypi\n[conda] triton                                      3.6.0                pypi_0           pypi",
+  "transformers_version": "4.57.3",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {
+    "mbpp": "f1d7b0397dcacbc121053c975ba9e6865b3324d56f427480d18d04f4ad5dcb89"
+  },
+  "model_source": "vllm",
+  "model_name": "results/qwen2.5-1.5B-Instruct#sfkl_nnm_lora/nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "model_name_sanitized": "results__qwen2.5-1.5B-Instruct#sfkl_nnm_lora__nnm0.5_K128_L4_epoch2_lr1e-4_kdr1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "total_evaluation_time_seconds": "177.56456613296177"
+}