Add full zero-shot evaluation

standard-eval.json

@@ -0,0 +1,449 @@
+{
+  "checkpoint": "runs\\alloy-312m-2k-lead-v2\\best.pt",
+  "checkpoint_sha256": "928865f44252b3ff38a23f7ced73f4986e3e3951673be035b824ec15e5ee1306",
+  "tokenizer": "runs\\alloy-312m-2k-lead-v2\\tokenizer.json",
+  "tokenizer_sha256": "40d36acc35e5364d90fdcdae773143b7571b5ec3b7ace2030d12d4bd5a3a1025",
+  "checkpoint_step": 15259,
+  "tasks": [
+    "arc_easy",
+    "arc_challenge",
+    "hellaswag",
+    "piqa",
+    "sciq",
+    "winogrande"
+  ],
+  "num_fewshot": 0,
+  "limit": null,
+  "bootstrap_iters": 0,
+  "lm_eval_version": "0.4.13",
+  "results": {
+    "results": {
+      "arc_easy": {
+        "name": "arc_easy",
+        "alias": "arc_easy",
+        "sample_len": 2376,
+        "acc,none": 0.49915824915824913,
+        "acc_stderr,none": "N/A",
+        "acc_norm,none": 0.4524410774410774,
+        "acc_norm_stderr,none": "N/A"
+      },
+      "arc_challenge": {
+        "name": "arc_challenge",
+        "alias": "arc_challenge",
+        "sample_len": 1172,
+        "acc,none": 0.21416382252559726,
+        "acc_stderr,none": "N/A",
+        "acc_norm,none": 0.25597269624573377,
+        "acc_norm_stderr,none": "N/A"
+      },
+      "hellaswag": {
+        "name": "hellaswag",
+        "alias": "hellaswag",
+        "sample_len": 10042,
+        "acc,none": 0.29874526986656047,
+        "acc_stderr,none": "N/A",
+        "acc_norm,none": 0.327823142800239,
+        "acc_norm_stderr,none": "N/A"
+      },
+      "piqa": {
+        "name": "piqa",
+        "alias": "piqa",
+        "sample_len": 1838,
+        "acc,none": 0.6294885745375408,
+        "acc_stderr,none": "N/A",
+        "acc_norm,none": 0.6360174102285092,
+        "acc_norm_stderr,none": "N/A"
+      },
+      "sciq": {
+        "name": "sciq",
+        "alias": "sciq",
+        "sample_len": 1000,
+        "acc,none": 0.719,
+        "acc_stderr,none": "N/A",
+        "acc_norm,none": 0.618,
+        "acc_norm_stderr,none": "N/A"
+      },
+      "winogrande": {
+        "name": "winogrande",
+        "alias": "winogrande",
+        "sample_len": 1267,
+        "acc,none": 0.5067087608524072,
+        "acc_stderr,none": "N/A"
+      }
+    },
+    "group_subtasks": {},
+    "configs": {
+      "arc_challenge": {
+        "task": "arc_challenge",
+        "dataset_path": "allenai/ai2_arc",
+        "dataset_name": "ARC-Challenge",
+        "training_split": "train",
+        "validation_split": "validation",
+        "test_split": "test",
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "unsafe_code": false,
+        "doc_to_choice": "{{choices.text}}",
+        "description": "",
+        "target_delimiter": " ",
+        "fewshot_delimiter": "\n\n",
+        "fewshot_config": {
+          "sampler": "default",
+          "split": null,
+          "process_docs": null,
+          "fewshot_indices": null,
+          "samples": null,
+          "doc_to_text": "Question: {{question}}\nAnswer:",
+          "doc_to_choice": "{{choices.text}}",
+          "doc_to_target": "{{choices.label.index(answerKey)}}",
+          "gen_prefix": null,
+          "fewshot_delimiter": "\n\n",
+          "target_delimiter": " "
+        },
+        "num_fewshot": 0,
+        "metric_list": [
+          {
+            "metric": "acc",
+            "aggregation": "mean",
+            "higher_is_better": true
+          },
+          {
+            "metric": "acc_norm",
+            "aggregation": "mean",
+            "higher_is_better": true
+          }
+        ],
+        "output_type": "multiple_choice",
+        "repeats": 1,
+        "should_decontaminate": true,
+        "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+        "metadata": {
+          "version": 1.0,
+          "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml"
+        }
+      },
+      "arc_easy": {
+        "task": "arc_easy",
+        "dataset_path": "allenai/ai2_arc",
+        "dataset_name": "ARC-Easy",
+        "training_split": "train",
+        "validation_split": "validation",
+        "test_split": "test",
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_target": "{{choices.label.index(answerKey)}}",
+        "unsafe_code": false,
+        "doc_to_choice": "{{choices.text}}",
+        "description": "",
+        "target_delimiter": " ",
+        "fewshot_delimiter": "\n\n",
+        "fewshot_config": {
+          "sampler": "default",
+          "split": null,
+          "process_docs": null,
+          "fewshot_indices": null,
+          "samples": null,
+          "doc_to_text": "Question: {{question}}\nAnswer:",
+          "doc_to_choice": "{{choices.text}}",
+          "doc_to_target": "{{choices.label.index(answerKey)}}",
+          "gen_prefix": null,
+          "fewshot_delimiter": "\n\n",
+          "target_delimiter": " "
+        },
+        "num_fewshot": 0,
+        "metric_list": [
+          {
+            "metric": "acc",
+            "aggregation": "mean",
+            "higher_is_better": true
+          },
+          {
+            "metric": "acc_norm",
+            "aggregation": "mean",
+            "higher_is_better": true
+          }
+        ],
+        "output_type": "multiple_choice",
+        "repeats": 1,
+        "should_decontaminate": true,
+        "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
+        "metadata": {
+          "version": 1.0,
+          "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_easy.yaml"
+        }
+      },
+      "hellaswag": {
+        "task": "hellaswag",
+        "dataset_path": "Rowan/hellaswag",
+        "training_split": "train",
+        "validation_split": "validation",
+        "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+        "doc_to_text": "{{query}}",
+        "doc_to_target": "{{label}}",
+        "unsafe_code": false,
+        "doc_to_choice": "choices",
+        "description": "",
+        "target_delimiter": " ",
+        "fewshot_delimiter": "\n\n",
+        "fewshot_config": {
+          "sampler": "default",
+          "split": null,
+          "process_docs": "<function process_docs at 0x0000018C64FED620>",
+          "fewshot_indices": null,
+          "samples": null,
+          "doc_to_text": "{{query}}",
+          "doc_to_choice": "choices",
+          "doc_to_target": "{{label}}",
+          "gen_prefix": null,
+          "fewshot_delimiter": "\n\n",
+          "target_delimiter": " "
+        },
+        "num_fewshot": 0,
+        "metric_list": [
+          {
+            "metric": "acc",
+            "aggregation": "mean",
+            "higher_is_better": true
+          },
+          {
+            "metric": "acc_norm",
+            "aggregation": "mean",
+            "higher_is_better": true
+          }
+        ],
+        "output_type": "multiple_choice",
+        "repeats": 1,
+        "should_decontaminate": false,
+        "metadata": {
+          "version": 1.0,
+          "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml"
+        }
+      },
+      "piqa": {
+        "task": "piqa",
+        "dataset_path": "baber/piqa",
+        "training_split": "train",
+        "validation_split": "validation",
+        "doc_to_text": "Question: {{goal}}\nAnswer:",
+        "doc_to_target": "label",
+        "unsafe_code": false,
+        "doc_to_choice": "{{[sol1, sol2]}}",
+        "description": "",
+        "target_delimiter": " ",
+        "fewshot_delimiter": "\n\n",
+        "fewshot_config": {
+          "sampler": "default",
+          "split": null,
+          "process_docs": null,
+          "fewshot_indices": null,
+          "samples": null,
+          "doc_to_text": "Question: {{goal}}\nAnswer:",
+          "doc_to_choice": "{{[sol1, sol2]}}",
+          "doc_to_target": "label",
+          "gen_prefix": null,
+          "fewshot_delimiter": "\n\n",
+          "target_delimiter": " "
+        },
+        "num_fewshot": 0,
+        "metric_list": [
+          {
+            "metric": "acc",
+            "aggregation": "mean",
+            "higher_is_better": true
+          },
+          {
+            "metric": "acc_norm",
+            "aggregation": "mean",
+            "higher_is_better": true
+          }
+        ],
+        "output_type": "multiple_choice",
+        "repeats": 1,
+        "should_decontaminate": true,
+        "doc_to_decontamination_query": "goal",
+        "metadata": {
+          "version": 1.0,
+          "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\piqa\\piqa.yaml"
+        }
+      },
+      "sciq": {
+        "task": "sciq",
+        "dataset_path": "allenai/sciq",
+        "training_split": "train",
+        "validation_split": "validation",
+        "test_split": "test",
+        "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
+        "doc_to_target": 3,
+        "unsafe_code": false,
+        "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
+        "description": "",
+        "target_delimiter": " ",
+        "fewshot_delimiter": "\n\n",
+        "fewshot_config": {
+          "sampler": "default",
+          "split": null,
+          "process_docs": null,
+          "fewshot_indices": null,
+          "samples": null,
+          "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
+          "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
+          "doc_to_target": 3,
+          "gen_prefix": null,
+          "fewshot_delimiter": "\n\n",
+          "target_delimiter": " "
+        },
+        "num_fewshot": 0,
+        "metric_list": [
+          {
+            "metric": "acc",
+            "aggregation": "mean",
+            "higher_is_better": true
+          },
+          {
+            "metric": "acc_norm",
+            "aggregation": "mean",
+            "higher_is_better": true
+          }
+        ],
+        "output_type": "multiple_choice",
+        "repeats": 1,
+        "should_decontaminate": true,
+        "doc_to_decontamination_query": "{{support}} {{question}}",
+        "metadata": {
+          "version": 1.0,
+          "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\sciq\\sciq.yaml"
+        }
+      },
+      "winogrande": {
+        "task": "winogrande",
+        "dataset_path": "allenai/winogrande",
+        "dataset_name": "winogrande_xl",
+        "training_split": "train",
+        "validation_split": "validation",
+        "doc_to_text": "def doc_to_text(doc):\n    answer_to_num = {\"1\": 0, \"2\": 1}\n    return answer_to_num[doc[\"answer\"]]\n",
+        "doc_to_target": "def doc_to_target(doc):\n    idx = doc[\"sentence\"].index(\"_\") + 1\n    return doc[\"sentence\"][idx:].strip()\n",
+        "unsafe_code": false,
+        "doc_to_choice": "def doc_to_choice(doc):\n    idx = doc[\"sentence\"].index(\"_\")\n    options = [doc[\"option1\"], doc[\"option2\"]]\n    return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
+        "description": "",
+        "target_delimiter": " ",
+        "fewshot_delimiter": "\n\n",
+        "fewshot_config": {
+          "sampler": "default",
+          "split": null,
+          "process_docs": null,
+          "fewshot_indices": null,
+          "samples": null,
+          "doc_to_text": "<function doc_to_text at 0x0000018C6BA29F80>",
+          "doc_to_choice": "<function doc_to_choice at 0x0000018C6BA2A200>",
+          "doc_to_target": "<function doc_to_target at 0x0000018C6BA2A160>",
+          "gen_prefix": null,
+          "fewshot_delimiter": "\n\n",
+          "target_delimiter": " "
+        },
+        "num_fewshot": 0,
+        "metric_list": [
+          {
+            "metric": "acc",
+            "aggregation": "mean",
+            "higher_is_better": true
+          }
+        ],
+        "output_type": "multiple_choice",
+        "repeats": 1,
+        "should_decontaminate": true,
+        "doc_to_decontamination_query": "sentence",
+        "metadata": {
+          "version": 1.0,
+          "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\winogrande\\default.yaml"
+        }
+      }
+    },
+    "versions": {
+      "arc_challenge": 1.0,
+      "arc_easy": 1.0,
+      "hellaswag": 1.0,
+      "piqa": 1.0,
+      "sciq": 1.0,
+      "winogrande": 1.0
+    },
+    "n-shot": {
+      "arc_challenge": 0,
+      "arc_easy": 0,
+      "hellaswag": 0,
+      "piqa": 0,
+      "sciq": 0,
+      "winogrande": 0
+    },
+    "higher_is_better": {
+      "arc_challenge": {
+        "acc": true,
+        "acc_norm": true
+      },
+      "arc_easy": {
+        "acc": true,
+        "acc_norm": true
+      },
+      "hellaswag": {
+        "acc": true,
+        "acc_norm": true
+      },
+      "piqa": {
+        "acc": true,
+        "acc_norm": true
+      },
+      "sciq": {
+        "acc": true,
+        "acc_norm": true
+      },
+      "winogrande": {
+        "acc": true
+      }
+    },
+    "n-samples": {
+      "arc_easy": {
+        "original": 2376,
+        "effective": 2376
+      },
+      "arc_challenge": {
+        "original": 1172,
+        "effective": 1172
+      },
+      "hellaswag": {
+        "original": 10042,
+        "effective": 10042
+      },
+      "piqa": {
+        "original": 1838,
+        "effective": 1838
+      },
+      "sciq": {
+        "original": 1000,
+        "effective": 1000
+      },
+      "winogrande": {
+        "original": 1267,
+        "effective": 1267
+      }
+    },
+    "config": {
+      "model": "TinyEnglishLM",
+      "model_args": null,
+      "batch_size": 1,
+      "batch_sizes": [],
+      "device": "cuda",
+      "use_cache": null,
+      "limit": null,
+      "bootstrap_iters": 0,
+      "gen_kwargs": null,
+      "random_seed": 0,
+      "numpy_seed": 1234,
+      "torch_seed": 1234,
+      "fewshot_seed": 1234
+    },
+    "git_hash": "8a46cd5",
+    "date": 1790814588.8014357,
+    "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Microsoft Windows 11 Pro (10.0.26200 64-bit)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: N/A\n\nPython version: 3.12.10 (tags/v3.12.10:0cc8128, Apr  8 2025, 12:21:36) [MSC v.1943 64 bit (AMD64)] (64-bit runtime)\nPython platform: Windows-11-10.0.26200-SP0\nIs CUDA available: True\nCUDA runtime version: 13.0.48\r\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3060 Ti\nNvidia driver version: 591.86\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nName: 12th Gen Intel(R) Core(TM) i7-12700\nManufacturer: GenuineIntel\nFamily: 198\nArchitecture: 9\nProcessorType: 3\nDeviceID: CPU0\nCurrentClockSpeed: 2100\nMaxClockSpeed: 2100\nL2CacheSize: 12288\nL2CacheSpeed: None\nRevision: None\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] torch==2.6.0+cu124\n[conda] Could not collect",
+    "transformers_version": "N/A",
+    "lm_eval_version": "0.4.13",
+    "upper_git_hash": null
+  }
+}