{
  "source": "Previous 512-example run vs expanded 3,072-example run",
  "previous": {
    "commonsense_qa": {
      "accuracy_top1": 0.5827996340347666,
      "accuracy_top3": 0.9048490393412626,
      "mean_nll": 1.3366108938601398,
      "mean_brier": 0.6047167307695813,
      "ece_10_bins": 0.1994374088758955
    },
    "banking77": {
      "accuracy_top1": 0.5800865800865801,
      "accuracy_top5": 0.8701298701298701,
      "mean_nll": 1.5664262868874264,
      "mean_brier": 0.5760736580374399,
      "ece_10_bins": 0.09326245064859266
    }
  },
  "expanded": {
    "commonsense_qa": {
      "accuracy_top1": 0.6020128087831657,
      "accuracy_top3": 0.9277218664226898,
      "mean_nll": 1.1035297405179396,
      "mean_brier": 0.5445770138152226,
      "ece_10_bins": 0.13095674654920497
    },
    "banking77": {
      "accuracy_top1": 0.645021645021645,
      "accuracy_top5": 0.9523809523809523,
      "mean_nll": 1.329584948457993,
      "mean_brier": 0.5151303437000476,
      "ece_10_bins": 0.16459291234805987
    }
  },
  "examples": {
    "commonsense_qa": 1093,
    "banking77": 231
  },
  "chance_top1": {
    "commonsense_qa_5_choice": 0.2,
    "banking77_77_intent": 0.012987012987012988
  },
  "notes": [
    "CommonsenseQA uses 1,093 official validation examples.",
    "BANKING77 is a stratified sample of 231 official test examples, three per intent.",
    "These task-specific measurements are not a general intelligence score."
  ]
}