| { |
| "schema_version": 1, |
| "protocol": "local-pinned-open-slm-zero-shot-v2", |
| "harness": { |
| "distribution": "lm_eval", |
| "repository": "https://github.com/EleutherAI/lm-evaluation-harness", |
| "revision": "d6de81643928d653435c431bae19945d41d32520", |
| "python_requires": ">=3.10", |
| "task_configs": { |
| "arc_easy": "lm_eval/tasks/arc/arc_easy.yaml", |
| "arc_challenge": "lm_eval/tasks/arc/arc_challenge.yaml", |
| "hellaswag": "lm_eval/tasks/hellaswag/hellaswag.yaml", |
| "piqa": "lm_eval/tasks/piqa/piqa.yaml" |
| } |
| }, |
| "arithmark3": { |
| "repository": "AxiomicLabs/Arithmark-3.0", |
| "revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746", |
| "script": "bencharithmark-3.py", |
| "dataset": "arithmark-3.jsonl", |
| "examples": 1000, |
| "script_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd" |
| }, |
| "model": { |
| "default_repo": "Jonnester/Purrence-3M", |
| "device": "cpu", |
| "dtype": "float32", |
| "trust_remote_code": true, |
| "batch_size": 16, |
| "max_length": 8192, |
| "add_bos_token": false, |
| "default_local_path": ".", |
| "expected_weights_sha256": "9c9f8d29004ce3563090de3c9be42059c5e2f928e053d4b17637f2215d7972ec", |
| "expected_file_sha256": { |
| "config.json": "a82f737624152185ff9afd9573a2d82e3592fcd3f61df9dbdac25188e25c659e", |
| "modeling_purrence.py": "6467854a7a53185b2ed2693cf0f9d0150898a13efb35cf0c12cd3b53691ca817", |
| "configuration_purrence.py": "a33340909d58f0505262f5111ed41da0faa4222ee99a439cd93c700c3c9c30ee", |
| "tokenization_purrence.py": "c25e6e0e712e754e2f6d09a45899f84a90a611a7b32ac43515b9cd09c075dfec", |
| "purrence_core.py": "0141fff6cc77cc590145873fa754a6fec1aeec67cc19be4d8a9dce769728c02b", |
| "tokenizer_config.json": "69bfe2ccb62a039a97e6df0ca4a670440170eae950aa735760cbf0e95db962e5", |
| "tokenizer/ordinary-tokenizer.json": "fc240b741112488b639a711f502305601e3b8a1b01c5ae93a48b9ae55692dd33", |
| "tokenizer/artifact.json": "4c2241f6f7deffc43043ae1e91eea87434de6725bf0c4e5d2dfe5a45588733bd" |
| } |
| }, |
| "evaluation": { |
| "precision": "fp32", |
| "num_fewshot": 0, |
| "limit": null, |
| "apply_chat_template": false, |
| "system_instruction": null, |
| "log_samples": true, |
| "seeds": { |
| "random": 17, |
| "numpy": 17, |
| "torch": 17, |
| "fewshot": 17 |
| }, |
| "bootstrap_iters": 1000, |
| "metrics": [ |
| "acc", |
| "acc_norm" |
| ] |
| }, |
| "tasks": { |
| "arc_easy": { |
| "task_version": 1.0, |
| "dataset_path": "allenai/ai2_arc", |
| "dataset_name": "ARC-Easy", |
| "storage_adapter": { |
| "dataset_name": null, |
| "data_dir": "ARC-Easy" |
| }, |
| "dataset_revision": "54cd56ea77687f84868e2cb8d58e3f462bdd0d45", |
| "evaluation_split": "test", |
| "examples": 2376, |
| "chance": 0.25, |
| "metrics": [ |
| "acc", |
| "acc_norm" |
| ] |
| }, |
| "arc_challenge": { |
| "task_version": 1.0, |
| "dataset_path": "allenai/ai2_arc", |
| "dataset_name": "ARC-Challenge", |
| "storage_adapter": { |
| "dataset_name": null, |
| "data_dir": "ARC-Challenge" |
| }, |
| "dataset_revision": "54cd56ea77687f84868e2cb8d58e3f462bdd0d45", |
| "evaluation_split": "test", |
| "examples": 1172, |
| "chance": 0.25, |
| "metrics": [ |
| "acc", |
| "acc_norm" |
| ] |
| }, |
| "hellaswag": { |
| "task_version": 1.0, |
| "dataset_path": "Rowan/hellaswag", |
| "dataset_name": null, |
| "dataset_revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "evaluation_split": "validation", |
| "examples": 10042, |
| "chance": 0.25, |
| "metrics": [ |
| "acc", |
| "acc_norm" |
| ] |
| }, |
| "piqa": { |
| "task_version": 1.0, |
| "dataset_path": "baber/piqa", |
| "dataset_name": null, |
| "dataset_revision": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "evaluation_split": "validation", |
| "examples": 1838, |
| "chance": 0.5, |
| "metrics": [ |
| "acc", |
| "acc_norm" |
| ] |
| } |
| }, |
| "index": { |
| "normalized_formula": "100 * (((hellaswag.acc_norm - 0.25) / 0.75) + ((mean(arc_easy.acc_norm, arc_challenge.acc_norm) - 0.25) / 0.75) + ((piqa.acc_norm - 0.5) / 0.5) + 0.65 * ((arithmark3.acc_norm - 0.25) / 0.75)) / 3.65", |
| "raw_formula": "100 * (((hellaswag.acc - 0.25) / 0.75) + ((mean(arc_easy.acc, arc_challenge.acc) - 0.25) / 0.75) + ((piqa.acc - 0.5) / 0.5) + 0.65 * ((arithmark3.acc_norm - 0.25) / 0.75)) / 3.65", |
| "note": "Leaderboard formula applied to these local measurements; independent leaderboard verification is not implied." |
| } |
| } |
|
|
