{ "raw": { "ece": 0.08060139770805837, "brier": 0.06444167884911073, "nll": 0.8875625221608207, "accuracy": 0.77, "soft_accuracy": 0.5376912801638245, "mean_confidence": 0.6893986235558986, "mean_target_max": 0.6589462240487337, "score_mae": 0.23593050419178327, "n": 2000, "by_type": { "choice": { "ece": 0.0855616979797681, "brier": 0.0787058081611597, "nll": 1.0158876217094561, "accuracy": 0.7516666666666667, "soft_accuracy": 0.4951454981913169, "mean_confidence": 0.6682631523410479, "mean_target_max": 0.6345381752649943, "score_mae": null, "n": 600 }, "score": { "ece": 0.08953976664692163, "brier": 0.07020634514290577, "nll": 1.0710158663382754, "accuracy": 0.71875, "soft_accuracy": 0.45663846340030434, "mean_confidence": 0.6304150220751762, "mean_target_max": 0.5928848401084542, "score_mae": 0.23593050419178327, "n": 800 }, "noul": { "ece": 0.06748779048522313, "brier": 0.04249132781200172, "nll": 0.5146329637089123, "accuracy": 0.8566666666666667, "soft_accuracy": 0.6883074844876925, "mean_confidence": 0.7891788967450459, "mean_target_max": 0.7714361180861791, "score_mae": null, "n": 600 } } }, "post_temperature": { "ece": 0.13423142372071742, "brier": 0.053599705442185265, "nll": 0.8639000835735351, "accuracy": 0.77, "soft_accuracy": 0.5135849121585488, "mean_confidence": 0.6357685458511114, "mean_target_max": 0.6589462240487337, "score_mae": 0.22997838034061716, "n": 2000, "by_type": { "choice": { "ece": 0.14269732366005577, "brier": 0.06597787525161646, "nll": 0.9911195738365253, "accuracy": 0.7516666666666667, "soft_accuracy": 0.4680097303042809, "mean_confidence": 0.6089693092306455, "mean_target_max": 0.6345381752649943, "score_mae": null, "n": 600 }, "score": { "ece": 0.1465983800590038, "brier": 0.057556013543289736, "nll": 1.041580511694774, "accuracy": 0.71875, "soft_accuracy": 0.43235879089683293, "mean_confidence": 0.5721516288444399, "mean_target_max": 0.5928848401084542, "score_mae": 0.22997838034061716, "n": 800 }, "noul": { "ece": 0.10927635669708252, "brier": 0.0359464581646148, "nll": 0.49977335581555965, "accuracy": 0.8566666666666667, "soft_accuracy": 0.6674615890284379, "mean_confidence": 0.7473903384804725, "mean_target_max": 0.7714361180861791, "score_mae": null, "n": 600 } } }, "fitted_temperature": { "type0_bucket1": 1.2583999633789062, "type2_bucket0": 1.3723788261413574, "type1_bucket1": 1.2761344909667969 } }