{
  "tasks": {
    "hellaswag": {
      "n": 10042,
      "seconds": 146.01327709999168,
      "routes": {
        "cached/language+evidence": 2973,
        "cached/language": 7068,
        "cached/exact-arithmetic": 1
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "ae5e8d2fb43e21a1810c4e8518e5a54256cf7ea66b77a273f1229b753360c7c9",
      "primary": "char",
      "native_correct": 2809,
      "native_percent": 27.972515435172276,
      "previous_native_correct": 2809,
      "previous_native_percent": 27.972515435172276,
      "native_change_points": 0.0,
      "assisted_correct": 2805,
      "assisted_percent": 27.9326827325234,
      "previous_assisted_percent": 27.93,
      "assisted_change_points": 0.0
    },
    "arc_easy": {
      "n": 2376,
      "seconds": 22.353922400041483,
      "routes": {
        "cached/language+evidence": 2374,
        "cached/exact-arithmetic": 1,
        "cached/language": 1
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "f9b4a6ca075ad0110315bb1dd19e8eac4b6df47252730596b25c6933dae521d9",
      "primary": "char",
      "native_correct": 632,
      "native_percent": 26.5993265993266,
      "previous_native_correct": 632,
      "previous_native_percent": 26.5993265993266,
      "native_change_points": 0.0,
      "assisted_correct": 1476,
      "assisted_percent": 62.121212121212125,
      "previous_assisted_percent": 62.12,
      "assisted_change_points": 0.0
    },
    "arc_challenge": {
      "n": 1172,
      "seconds": 11.865461300010793,
      "routes": {
        "cached/language+evidence": 1169,
        "cached/exact-arithmetic": 2,
        "cached/language": 1
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "301b66946ce1cda420019aca3e1240bc9c262f2cb4f73e909b3ad19e61f76675",
      "primary": "char",
      "native_correct": 260,
      "native_percent": 22.18430034129693,
      "previous_native_correct": 260,
      "previous_native_percent": 22.18430034129693,
      "native_change_points": 0.0,
      "assisted_correct": 371,
      "assisted_percent": 31.655290102389078,
      "previous_assisted_percent": 31.66,
      "assisted_change_points": -0.0
    },
    "piqa": {
      "n": 1838,
      "seconds": 10.459679999970831,
      "routes": {
        "cached/language": 433,
        "cached/language+evidence": 1405
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "cf2a6f1adc18959b67dce5af1fa668e7f4301d7eb9b25d3918fdc221c8369597",
      "primary": "char",
      "native_correct": 899,
      "native_percent": 48.91186071817192,
      "previous_native_correct": 899,
      "previous_native_percent": 48.91186071817192,
      "native_change_points": 0.0,
      "assisted_correct": 1078,
      "assisted_percent": 58.65070729053319,
      "previous_assisted_percent": 58.65,
      "assisted_change_points": 0.0
    },
    "arithmark3": {
      "n": 1000,
      "seconds": 9.040633900032844,
      "routes": {
        "cached/exact-arithmetic": 680,
        "cached/language+evidence": 320
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "9cb2c70adfac81b8fde8c1fa2079553530d667b15d3190db54a7b76dce26f83d",
      "primary": "token",
      "native_correct": 378,
      "native_percent": 37.8,
      "previous_native_correct": 378,
      "previous_native_percent": 37.8,
      "native_change_points": 0.0,
      "assisted_correct": 695,
      "assisted_percent": 69.5,
      "previous_assisted_percent": 69.5,
      "assisted_change_points": 0.0
    }
  },
  "total_questions": 16428,
  "seconds": 200.08493640000233,
  "native_mean_percent": 32.693600618793546,
  "assisted_mean_percent": 49.97197844933156,
  "current_index_formula": "Chance-normalized HellaSwag + combined ARC + PIQA + 0.65*ArithMark, divided by 3.65",
  "local_index_native": 3.3067104959787845,
  "local_index_assisted": 24.373331426529237,
  "whole_system_evaluation_cached": true,
  "fresh_likelihoods": true,
  "training": false,
  "official_submission": false,
  "freeze_sha256": "d1f3b91f49fe8036b10a814daca5eb20e9e7c64f7551e86cc475a8c14373b1e6",
  "sql_parity_sha256": "27cc34d0f40d3f369c42d6f17a9ef3b42bb94c0f514f21037af7be08235128c1"
}