File size: 3,035 Bytes
d266da5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
{
  "a": -1.8895045041017524,
  "b1": 0.48729786088254085,
  "b2": 0.5583580780553591,
  "fit_split": "data/dev.jsonl",
  "n_fit": 2786,
  "exact_las_fit": 0.8406,
  "ece_raw_fit": 0.1255,
  "ece_cal_fit": 0.0102,
  "table_fit": [
    {
      "threshold": 0.5,
      "emitted": 2290,
      "emitted_pct": 0.822,
      "accuracy": 0.959
    },
    {
      "threshold": 0.6,
      "emitted": 2219,
      "emitted_pct": 0.7965,
      "accuracy": 0.9721
    },
    {
      "threshold": 0.7,
      "emitted": 2147,
      "emitted_pct": 0.7706,
      "accuracy": 0.9818
    },
    {
      "threshold": 0.8,
      "emitted": 2087,
      "emitted_pct": 0.7491,
      "accuracy": 0.989
    },
    {
      "threshold": 0.9,
      "emitted": 1994,
      "emitted_pct": 0.7157,
      "accuracy": 0.9935
    },
    {
      "threshold": 0.95,
      "emitted": 1922,
      "emitted_pct": 0.6899,
      "accuracy": 0.9969
    },
    {
      "threshold": 0.98,
      "emitted": 1865,
      "emitted_pct": 0.6694,
      "accuracy": 0.9995
    },
    {
      "threshold": 0.99,
      "emitted": 1827,
      "emitted_pct": 0.6558,
      "accuracy": 1.0
    }
  ],
  "topk_fit": [
    {
      "top_pct": 5,
      "emitted": 139,
      "accuracy": 1.0
    },
    {
      "top_pct": 10,
      "emitted": 278,
      "accuracy": 1.0
    },
    {
      "top_pct": 25,
      "emitted": 696,
      "accuracy": 1.0
    },
    {
      "top_pct": 50,
      "emitted": 1393,
      "accuracy": 1.0
    }
  ],
  "eval_split": "data/test.jsonl",
  "n_eval": 2788,
  "exact_las_eval": 0.8275,
  "ece_raw_eval": 0.1354,
  "ece_cal_eval": 0.0106,
  "table_eval": [
    {
      "threshold": 0.5,
      "emitted": 2256,
      "emitted_pct": 0.8092,
      "accuracy": 0.9605
    },
    {
      "threshold": 0.6,
      "emitted": 2195,
      "emitted_pct": 0.7873,
      "accuracy": 0.9745
    },
    {
      "threshold": 0.7,
      "emitted": 2133,
      "emitted_pct": 0.7651,
      "accuracy": 0.9841
    },
    {
      "threshold": 0.8,
      "emitted": 2083,
      "emitted_pct": 0.7471,
      "accuracy": 0.9899
    },
    {
      "threshold": 0.9,
      "emitted": 1995,
      "emitted_pct": 0.7156,
      "accuracy": 0.9955
    },
    {
      "threshold": 0.95,
      "emitted": 1925,
      "emitted_pct": 0.6905,
      "accuracy": 0.9974
    },
    {
      "threshold": 0.98,
      "emitted": 1868,
      "emitted_pct": 0.67,
      "accuracy": 0.9995
    },
    {
      "threshold": 0.99,
      "emitted": 1837,
      "emitted_pct": 0.6589,
      "accuracy": 0.9995
    }
  ],
  "topk_eval": [
    {
      "top_pct": 5,
      "emitted": 139,
      "accuracy": 1.0
    },
    {
      "top_pct": 10,
      "emitted": 278,
      "accuracy": 1.0
    },
    {
      "top_pct": 25,
      "emitted": 697,
      "accuracy": 1.0
    },
    {
      "top_pct": 50,
      "emitted": 1394,
      "accuracy": 1.0
    }
  ]
}