{
  "model": "solintellegence/sol-pro",
  "tokenizer": "solintellegence/sol-pro",
  "params": 138037594,
  "dataset": {
    "path": "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/blobs/237840149650455c4c54ff18f02203e2a76c1d75",
    "sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
    "items": 1000
  },
  "runtime": {
    "device": "cuda",
    "dtype": "torch.float32",
    "batch_size": 32,
    "max_context": 1024,
    "attention_implementation": "auto",
    "compiled": false,
    "torch_version": "2.11.0+cu130"
  },
  "primary_metric": "acc_norm",
  "results": {
    "arithmark-3": {
      "acc": 37.2,
      "acc_norm": 36.0,
      "raw_correct": 372,
      "norm_correct": 360,
      "total": 1000,
      "categories": {
        "elementary_school_math_continuation::addition::grades_1_2::easy": {
          "acc": 23.4375,
          "acc_norm": 24.21875,
          "raw_correct": 30,
          "norm_correct": 31,
          "total": 128
        },
        "elementary_school_math_continuation::comparison::grades_2_3::medium": {
          "acc": 20.454545454545457,
          "acc_norm": 18.181818181818183,
          "raw_correct": 9,
          "norm_correct": 8,
          "total": 44
        },
        "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
          "acc": 37.5,
          "acc_norm": 35.41666666666667,
          "raw_correct": 18,
          "norm_correct": 17,
          "total": 48
        },
        "elementary_school_math_continuation::data::grades_2_3::easy": {
          "acc": 30.23255813953488,
          "acc_norm": 37.2093023255814,
          "raw_correct": 13,
          "norm_correct": 16,
          "total": 43
        },
        "elementary_school_math_continuation::division::grades_3_4::medium": {
          "acc": 25.925925925925924,
          "acc_norm": 25.925925925925924,
          "raw_correct": 14,
          "norm_correct": 14,
          "total": 54
        },
        "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
          "acc": 20.0,
          "acc_norm": 20.0,
          "raw_correct": 10,
          "norm_correct": 10,
          "total": 50
        },
        "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
          "acc": 57.692307692307686,
          "acc_norm": 55.769230769230774,
          "raw_correct": 30,
          "norm_correct": 29,
          "total": 52
        },
        "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
          "acc": 62.22222222222222,
          "acc_norm": 66.66666666666666,
          "raw_correct": 28,
          "norm_correct": 30,
          "total": 45
        },
        "elementary_school_math_continuation::measurement::grades_2_3::easy": {
          "acc": 32.89473684210527,
          "acc_norm": 31.57894736842105,
          "raw_correct": 25,
          "norm_correct": 24,
          "total": 76
        },
        "elementary_school_math_continuation::money::grades_3_4::medium": {
          "acc": 28.125,
          "acc_norm": 29.6875,
          "raw_correct": 18,
          "norm_correct": 19,
          "total": 64
        },
        "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
          "acc": 67.56756756756756,
          "acc_norm": 59.45945945945946,
          "raw_correct": 50,
          "norm_correct": 44,
          "total": 74
        },
        "elementary_school_math_continuation::patterns::grades_3_4::medium": {
          "acc": 33.9622641509434,
          "acc_norm": 30.18867924528302,
          "raw_correct": 18,
          "norm_correct": 16,
          "total": 53
        },
        "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
          "acc": 28.205128205128204,
          "acc_norm": 27.350427350427353,
          "raw_correct": 33,
          "norm_correct": 32,
          "total": 117
        },
        "elementary_school_math_continuation::time::grades_2_3::easy": {
          "acc": 100.0,
          "acc_norm": 87.27272727272727,
          "raw_correct": 55,
          "norm_correct": 48,
          "total": 55
        },
        "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
          "acc": 17.391304347826086,
          "acc_norm": 19.565217391304348,
          "raw_correct": 8,
          "norm_correct": 9,
          "total": 46
        },
        "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
          "acc": 21.052631578947366,
          "acc_norm": 21.052631578947366,
          "raw_correct": 4,
          "norm_correct": 4,
          "total": 19
        },
        "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
          "acc": 28.125,
          "acc_norm": 28.125,
          "raw_correct": 9,
          "norm_correct": 9,
          "total": 32
        }
      },
      "timing": {
        "tokenization_seconds": 0.2399261619993922,
        "evaluation_seconds": 49.24065730300026,
        "examples_per_second": 20.308421023840992
      },
      "primary_metric": "acc_norm",
      "primary_acc": 36.0
    }
  }
}