{
  "schema_version": 1,
  "protocol": "local-pinned-open-slm-zero-shot-v2",
  "harness": {
    "distribution": "lm_eval",
    "repository": "https://github.com/EleutherAI/lm-evaluation-harness",
    "revision": "d6de81643928d653435c431bae19945d41d32520",
    "python_requires": ">=3.10",
    "task_configs": {
      "arc_easy": "lm_eval/tasks/arc/arc_easy.yaml",
      "arc_challenge": "lm_eval/tasks/arc/arc_challenge.yaml",
      "hellaswag": "lm_eval/tasks/hellaswag/hellaswag.yaml",
      "piqa": "lm_eval/tasks/piqa/piqa.yaml"
    }
  },
  "arithmark3": {
    "repository": "AxiomicLabs/Arithmark-3.0",
    "revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
    "script": "bencharithmark-3.py",
    "dataset": "arithmark-3.jsonl",
    "examples": 1000,
    "script_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd"
  },
  "model": {
    "default_repo": "Jonnester/Purrence-3M",
    "device": "cpu",
    "dtype": "float32",
    "trust_remote_code": true,
    "batch_size": 16,
    "max_length": 8192,
    "add_bos_token": false,
    "default_local_path": ".",
    "expected_weights_sha256": "9c9f8d29004ce3563090de3c9be42059c5e2f928e053d4b17637f2215d7972ec",
    "expected_file_sha256": {
      "config.json": "a82f737624152185ff9afd9573a2d82e3592fcd3f61df9dbdac25188e25c659e",
      "modeling_purrence.py": "6467854a7a53185b2ed2693cf0f9d0150898a13efb35cf0c12cd3b53691ca817",
      "configuration_purrence.py": "a33340909d58f0505262f5111ed41da0faa4222ee99a439cd93c700c3c9c30ee",
      "tokenization_purrence.py": "c25e6e0e712e754e2f6d09a45899f84a90a611a7b32ac43515b9cd09c075dfec",
      "purrence_core.py": "0141fff6cc77cc590145873fa754a6fec1aeec67cc19be4d8a9dce769728c02b",
      "tokenizer_config.json": "69bfe2ccb62a039a97e6df0ca4a670440170eae950aa735760cbf0e95db962e5",
      "tokenizer/ordinary-tokenizer.json": "fc240b741112488b639a711f502305601e3b8a1b01c5ae93a48b9ae55692dd33",
      "tokenizer/artifact.json": "4c2241f6f7deffc43043ae1e91eea87434de6725bf0c4e5d2dfe5a45588733bd"
    }
  },
  "evaluation": {
    "precision": "fp32",
    "num_fewshot": 0,
    "limit": null,
    "apply_chat_template": false,
    "system_instruction": null,
    "log_samples": true,
    "seeds": {
      "random": 17,
      "numpy": 17,
      "torch": 17,
      "fewshot": 17
    },
    "bootstrap_iters": 1000,
    "metrics": [
      "acc",
      "acc_norm"
    ]
  },
  "tasks": {
    "arc_easy": {
      "task_version": 1.0,
      "dataset_path": "allenai/ai2_arc",
      "dataset_name": "ARC-Easy",
      "storage_adapter": {
        "dataset_name": null,
        "data_dir": "ARC-Easy"
      },
      "dataset_revision": "54cd56ea77687f84868e2cb8d58e3f462bdd0d45",
      "evaluation_split": "test",
      "examples": 2376,
      "chance": 0.25,
      "metrics": [
        "acc",
        "acc_norm"
      ]
    },
    "arc_challenge": {
      "task_version": 1.0,
      "dataset_path": "allenai/ai2_arc",
      "dataset_name": "ARC-Challenge",
      "storage_adapter": {
        "dataset_name": null,
        "data_dir": "ARC-Challenge"
      },
      "dataset_revision": "54cd56ea77687f84868e2cb8d58e3f462bdd0d45",
      "evaluation_split": "test",
      "examples": 1172,
      "chance": 0.25,
      "metrics": [
        "acc",
        "acc_norm"
      ]
    },
    "hellaswag": {
      "task_version": 1.0,
      "dataset_path": "Rowan/hellaswag",
      "dataset_name": null,
      "dataset_revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
      "evaluation_split": "validation",
      "examples": 10042,
      "chance": 0.25,
      "metrics": [
        "acc",
        "acc_norm"
      ]
    },
    "piqa": {
      "task_version": 1.0,
      "dataset_path": "baber/piqa",
      "dataset_name": null,
      "dataset_revision": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
      "evaluation_split": "validation",
      "examples": 1838,
      "chance": 0.5,
      "metrics": [
        "acc",
        "acc_norm"
      ]
    }
  },
  "index": {
    "normalized_formula": "100 * (((hellaswag.acc_norm - 0.25) / 0.75) + ((mean(arc_easy.acc_norm, arc_challenge.acc_norm) - 0.25) / 0.75) + ((piqa.acc_norm - 0.5) / 0.5) + 0.65 * ((arithmark3.acc_norm - 0.25) / 0.75)) / 3.65",
    "raw_formula": "100 * (((hellaswag.acc - 0.25) / 0.75) + ((mean(arc_easy.acc, arc_challenge.acc) - 0.25) / 0.75) + ((piqa.acc - 0.5) / 0.5) + 0.65 * ((arithmark3.acc_norm - 0.25) / 0.75)) / 3.65",
    "note": "Leaderboard formula applied to these local measurements; independent leaderboard verification is not implied."
  }
}