File size: 2,629 Bytes
d12c956
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
{
  "raw": {
    "ece": 0.05556941968202591,
    "brier": 0.07491107175261555,
    "nll": 0.931554073697218,
    "accuracy": 0.774,
    "soft_accuracy": 0.5516621355069801,
    "mean_confidence": 0.7189242369234562,
    "score_mae": 0.24282820772539707,
    "n": 2000,
    "by_type": {
      "choice": {
        "ece": 0.058945793956518173,
        "brier": 0.09003982849844305,
        "nll": 1.0610431912432736,
        "accuracy": 0.7566666666666667,
        "soft_accuracy": 0.5113160306463639,
        "mean_confidence": 0.6988202449679375,
        "score_mae": null,
        "n": 600
      },
      "score": {
        "ece": 0.06930980719625951,
        "brier": 0.0760862219307569,
        "nll": 1.1153853290807456,
        "accuracy": 0.7275,
        "soft_accuracy": 0.47283619750989603,
        "mean_confidence": 0.6588596414029598,
        "score_mae": 0.24282820772539707,
        "n": 800
      },
      "noul": {
        "ece": 0.06058402955532074,
        "brier": 0.058215448102599605,
        "nll": 0.5569566156397923,
        "accuracy": 0.8533333333333334,
        "soft_accuracy": 0.6971094910303751,
        "mean_confidence": 0.8191143562396367,
        "score_mae": null,
        "n": 600
      }
    }
  },
  "post_temperature": {
    "ece": 0.14437548027932645,
    "brier": 0.053404335885938056,
    "nll": 0.86494004275091,
    "accuracy": 0.774,
    "soft_accuracy": 0.5128478347882628,
    "mean_confidence": 0.6296245071440935,
    "score_mae": 0.22164403992937878,
    "n": 2000,
    "by_type": {
      "choice": {
        "ece": 0.15137549564242359,
        "brier": 0.06518018942773324,
        "nll": 0.991539094435672,
        "accuracy": 0.7566666666666667,
        "soft_accuracy": 0.47024086986978847,
        "mean_confidence": 0.6052911690374215,
        "score_mae": null,
        "n": 600
      },
      "score": {
        "ece": 0.16654286351054906,
        "brier": 0.05312417189078587,
        "nll": 1.0359684311039745,
        "accuracy": 0.7275,
        "soft_accuracy": 0.43320247864350675,
        "mean_confidence": 0.5609682880342006,
        "score_mae": 0.22164403992937878,
        "n": 800
      },
      "noul": {
        "ece": 0.12086178546150525,
        "brier": 0.04200203433767911,
        "nll": 0.5103031399287283,
        "accuracy": 0.8533333333333334,
        "soft_accuracy": 0.6616486078997453,
        "mean_confidence": 0.7454994707306226,
        "score_mae": null,
        "n": 600
      }
    }
  },
  "fitted_temperature": {
    "type0_bucket1": 1.4832364320755005,
    "type2_bucket0": 1.7233024835586548,
    "type1_bucket1": 1.56569504737854
  }
}