File size: 2,573 Bytes
8110e6c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
{
  "raw": {
    "ece": 0.07819666066765786,
    "brier": 0.06447791931933414,
    "nll": 0.8876326022762805,
    "accuracy": 0.7675,
    "soft_accuracy": 0.5376246295273304,
    "mean_confidence": 0.6893033391088247,
    "score_mae": 0.23617810900788755,
    "n": 2000,
    "by_type": {
      "choice": {
        "ece": 0.08568431779742239,
        "brier": 0.07875364395885602,
        "nll": 1.0160210124527416,
        "accuracy": 0.75,
        "soft_accuracy": 0.4951265711337328,
        "mean_confidence": 0.6682332300643126,
        "score_mae": null,
        "n": 600
      },
      "score": {
        "ece": 0.08903235401958227,
        "brier": 0.07023433758940882,
        "nll": 1.0710722564673052,
        "accuracy": 0.715,
        "soft_accuracy": 0.4565292657352984,
        "mean_confidence": 0.6302291714400053,
        "score_mae": 0.23617810900788755,
        "n": 800
      },
      "noul": {
        "ece": 0.06586102902889251,
        "brier": 0.04252697031971266,
        "nll": 0.5146579865117867,
        "accuracy": 0.855,
        "soft_accuracy": 0.6882498396436373,
        "mean_confidence": 0.789139005045096,
        "score_mae": null,
        "n": 600
      }
    }
  },
  "post_temperature": {
    "ece": 0.13225472186505796,
    "brier": 0.05364174054635279,
    "nll": 0.8639748433399945,
    "accuracy": 0.7675,
    "soft_accuracy": 0.5134944389313459,
    "mean_confidence": 0.6356240352839232,
    "score_mae": 0.23020501649240033,
    "n": 2000,
    "by_type": {
      "choice": {
        "ece": 0.14126860857009887,
        "brier": 0.06599409825261318,
        "nll": 0.9911767271223167,
        "accuracy": 0.75,
        "soft_accuracy": 0.4678984449803829,
        "mean_confidence": 0.608731386611859,
        "score_mae": null,
        "n": 600
      },
      "score": {
        "ece": 0.1447367349267006,
        "brier": 0.05761699416621923,
        "nll": 1.041679207822308,
        "accuracy": 0.715,
        "soft_accuracy": 0.4322680720873177,
        "mean_confidence": 0.572008408755064,
        "score_mae": 0.23020501649240033,
        "n": 800
      },
      "noul": {
        "ece": 0.10766249060630799,
        "brier": 0.035989044680270484,
        "nll": 0.49983380691458784,
        "accuracy": 0.855,
        "soft_accuracy": 0.6673922553410133,
        "mean_confidence": 0.7473375193277995,
        "score_mae": null,
        "n": 600
      }
    }
  },
  "fitted_temperature": {
    "type0_bucket1": 1.2595651149749756,
    "type2_bucket0": 1.3722405433654785,
    "type1_bucket1": 1.2760099172592163
  }
}