{
  "model": "solintellegence/sol-pro",
  "model_revision": "a22f3539b43e0450850748ce445849a06c9c64d6",
  "parameters": 138037594,
  "dtype": "float32",
  "num_fewshot": 0,
  "metric": "acc_norm",
  "scores": {
    "hellaswag": 46.76359290977893,
    "arc_easy": 54.082491582491585,
    "arc_challenge": 31.99658703071672,
    "piqa": 70.94668117519043,
    "arithmark3": 36.0
  },
  "raw_scores": {
    "hellaswag": 36.875124477195776,
    "arc_easy": 55.34511784511784,
    "arc_challenge": 29.351535836177472,
    "piqa": 69.26006528835691,
    "arithmark3": 37.2
  },
  "intelligence_index": 28.62946264079223,
  "intelligence_index_raw": 24.12549744306075,
  "examples": {
    "hellaswag": 10042,
    "arc_easy": 2376,
    "arc_challenge": 1172,
    "piqa": 1838,
    "arithmark3": 1000
  },
  "versions": {
    "lm-eval": "0.4.12",
    "torch": "2.11.0",
    "transformers": "5.14.1",
    "datasets": "4.4.1",
    "huggingface-hub": "1.29.0",
    "accelerate": "1.15.0"
  },
  "arithmark_dataset_sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
  "arithmark_script_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd",
  "arithmark_dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
  "n_shot": {
    "arc_challenge": 0,
    "arc_easy": 0,
    "hellaswag": 0,
    "piqa": 0
  },
  "model_stage": "supervised task adaptation",
  "lm_eval_command": [
    "/tmp/uv-venv/bin/python",
    "-m",
    "lm_eval",
    "--model",
    "hf",
    "--model_args",
    "pretrained=solintellegence/sol-pro,revision=a22f3539b43e0450850748ce445849a06c9c64d6,dtype=float32,trust_remote_code=True",
    "--tasks",
    "hellaswag,arc_easy,arc_challenge,piqa",
    "--num_fewshot",
    "0",
    "--batch_size",
    "64",
    "--device",
    "cuda:0",
    "--output_path",
    "/marimo/sol_pro_standard_eval/lm_eval"
  ],
  "arithmark_command": [
    "/tmp/uv-venv/bin/python",
    "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/bencharithmark-3.py",
    "--model",
    "solintellegence/sol-pro",
    "--data-path",
    "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl",
    "--device",
    "cuda",
    "--dtype",
    "float32",
    "--results-dir",
    "/marimo/sol_pro_standard_eval/arithmark"
  ]
}