{ "checkpoint": "runs\\alloy-312m-2k-lead-v2\\best.pt", "checkpoint_sha256": "928865f44252b3ff38a23f7ced73f4986e3e3951673be035b824ec15e5ee1306", "tokenizer": "runs\\alloy-312m-2k-lead-v2\\tokenizer.json", "tokenizer_sha256": "40d36acc35e5364d90fdcdae773143b7571b5ec3b7ace2030d12d4bd5a3a1025", "checkpoint_step": 15259, "tasks": [ "arc_easy", "arc_challenge", "hellaswag", "piqa", "sciq", "winogrande" ], "num_fewshot": 0, "limit": null, "bootstrap_iters": 0, "lm_eval_version": "0.4.13", "results": { "results": { "arc_easy": { "name": "arc_easy", "alias": "arc_easy", "sample_len": 2376, "acc,none": 0.49915824915824913, "acc_stderr,none": "N/A", "acc_norm,none": 0.4524410774410774, "acc_norm_stderr,none": "N/A" }, "arc_challenge": { "name": "arc_challenge", "alias": "arc_challenge", "sample_len": 1172, "acc,none": 0.21416382252559726, "acc_stderr,none": "N/A", "acc_norm,none": 0.25597269624573377, "acc_norm_stderr,none": "N/A" }, "hellaswag": { "name": "hellaswag", "alias": "hellaswag", "sample_len": 10042, "acc,none": 0.29874526986656047, "acc_stderr,none": "N/A", "acc_norm,none": 0.327823142800239, "acc_norm_stderr,none": "N/A" }, "piqa": { "name": "piqa", "alias": "piqa", "sample_len": 1838, "acc,none": 0.6294885745375408, "acc_stderr,none": "N/A", "acc_norm,none": 0.6360174102285092, "acc_norm_stderr,none": "N/A" }, "sciq": { "name": "sciq", "alias": "sciq", "sample_len": 1000, "acc,none": 0.719, "acc_stderr,none": "N/A", "acc_norm,none": 0.618, "acc_norm_stderr,none": "N/A" }, "winogrande": { "name": "winogrande", "alias": "winogrande", "sample_len": 1267, "acc,none": 0.5067087608524072, "acc_stderr,none": "N/A" } }, "group_subtasks": {}, "configs": { "arc_challenge": { "task": "arc_challenge", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Challenge", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml" } }, "arc_easy": { "task": "arc_easy", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Easy", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_easy.yaml" } }, "hellaswag": { "task": "hellaswag", "dataset_path": "Rowan/hellaswag", "training_split": "train", "validation_split": "validation", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", "doc_to_text": "{{query}}", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": "", "fewshot_indices": null, "samples": null, "doc_to_text": "{{query}}", "doc_to_choice": "choices", "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml" } }, "piqa": { "task": "piqa", "dataset_path": "baber/piqa", "training_split": "train", "validation_split": "validation", "doc_to_text": "Question: {{goal}}\nAnswer:", "doc_to_target": "label", "unsafe_code": false, "doc_to_choice": "{{[sol1, sol2]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{goal}}\nAnswer:", "doc_to_choice": "{{[sol1, sol2]}}", "doc_to_target": "label", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "goal", "metadata": { "version": 1.0, "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\piqa\\piqa.yaml" } }, "sciq": { "task": "sciq", "dataset_path": "allenai/sciq", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:", "doc_to_target": 3, "unsafe_code": false, "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:", "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}", "doc_to_target": 3, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{support}} {{question}}", "metadata": { "version": 1.0, "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\sciq\\sciq.yaml" } }, "winogrande": { "task": "winogrande", "dataset_path": "allenai/winogrande", "dataset_name": "winogrande_xl", "training_split": "train", "validation_split": "validation", "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", "unsafe_code": false, "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "", "doc_to_choice": "", "doc_to_target": "", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "sentence", "metadata": { "version": 1.0, "config_source": "E:\\Repos\\tiny-english-30m\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\winogrande\\default.yaml" } } }, "versions": { "arc_challenge": 1.0, "arc_easy": 1.0, "hellaswag": 1.0, "piqa": 1.0, "sciq": 1.0, "winogrande": 1.0 }, "n-shot": { "arc_challenge": 0, "arc_easy": 0, "hellaswag": 0, "piqa": 0, "sciq": 0, "winogrande": 0 }, "higher_is_better": { "arc_challenge": { "acc": true, "acc_norm": true }, "arc_easy": { "acc": true, "acc_norm": true }, "hellaswag": { "acc": true, "acc_norm": true }, "piqa": { "acc": true, "acc_norm": true }, "sciq": { "acc": true, "acc_norm": true }, "winogrande": { "acc": true } }, "n-samples": { "arc_easy": { "original": 2376, "effective": 2376 }, "arc_challenge": { "original": 1172, "effective": 1172 }, "hellaswag": { "original": 10042, "effective": 10042 }, "piqa": { "original": 1838, "effective": 1838 }, "sciq": { "original": 1000, "effective": 1000 }, "winogrande": { "original": 1267, "effective": 1267 } }, "config": { "model": "TinyEnglishLM", "model_args": null, "batch_size": 1, "batch_sizes": [], "device": "cuda", "use_cache": null, "limit": null, "bootstrap_iters": 0, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "8a46cd5", "date": 1790814588.8014357, "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Microsoft Windows 11 Pro (10.0.26200 64-bit)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: N/A\n\nPython version: 3.12.10 (tags/v3.12.10:0cc8128, Apr 8 2025, 12:21:36) [MSC v.1943 64 bit (AMD64)] (64-bit runtime)\nPython platform: Windows-11-10.0.26200-SP0\nIs CUDA available: True\nCUDA runtime version: 13.0.48\r\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3060 Ti\nNvidia driver version: 591.86\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nName: 12th Gen Intel(R) Core(TM) i7-12700\nManufacturer: GenuineIntel\nFamily: 198\nArchitecture: 9\nProcessorType: 3\nDeviceID: CPU0\nCurrentClockSpeed: 2100\nMaxClockSpeed: 2100\nL2CacheSize: 12288\nL2CacheSpeed: None\nRevision: None\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.3\n[pip3] torch==2.6.0+cu124\n[conda] Could not collect", "transformers_version": "N/A", "lm_eval_version": "0.4.13", "upper_git_hash": null } }