| { | |
| "source": "Previous 512-example run vs expanded 3,072-example run", | |
| "previous": { | |
| "commonsense_qa": { | |
| "accuracy_top1": 0.5827996340347666, | |
| "accuracy_top3": 0.9048490393412626, | |
| "mean_nll": 1.3366108938601398, | |
| "mean_brier": 0.6047167307695813, | |
| "ece_10_bins": 0.1994374088758955 | |
| }, | |
| "banking77": { | |
| "accuracy_top1": 0.5800865800865801, | |
| "accuracy_top5": 0.8701298701298701, | |
| "mean_nll": 1.5664262868874264, | |
| "mean_brier": 0.5760736580374399, | |
| "ece_10_bins": 0.09326245064859266 | |
| } | |
| }, | |
| "expanded": { | |
| "commonsense_qa": { | |
| "accuracy_top1": 0.6020128087831657, | |
| "accuracy_top3": 0.9277218664226898, | |
| "mean_nll": 1.1035297405179396, | |
| "mean_brier": 0.5445770138152226, | |
| "ece_10_bins": 0.13095674654920497 | |
| }, | |
| "banking77": { | |
| "accuracy_top1": 0.645021645021645, | |
| "accuracy_top5": 0.9523809523809523, | |
| "mean_nll": 1.329584948457993, | |
| "mean_brier": 0.5151303437000476, | |
| "ece_10_bins": 0.16459291234805987 | |
| } | |
| }, | |
| "examples": { | |
| "commonsense_qa": 1093, | |
| "banking77": 231 | |
| }, | |
| "chance_top1": { | |
| "commonsense_qa_5_choice": 0.2, | |
| "banking77_77_intent": 0.012987012987012988 | |
| }, | |
| "notes": [ | |
| "CommonsenseQA uses 1,093 official validation examples.", | |
| "BANKING77 is a stratified sample of 231 official test examples, three per intent.", | |
| "These task-specific measurements are not a general intelligence score." | |
| ] | |
| } |