Aurley commited on
Commit
0ae5db0
·
verified ·
1 Parent(s): a7db277

Add benchmark comparison data

Browse files
Files changed (1) hide show
  1. benchmark_comparison.json +48 -0
benchmark_comparison.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source": "Previous 512-example run vs expanded 3,072-example run",
3
+ "previous": {
4
+ "commonsense_qa": {
5
+ "accuracy_top1": 0.5827996340347666,
6
+ "accuracy_top3": 0.9048490393412626,
7
+ "mean_nll": 1.3366108938601398,
8
+ "mean_brier": 0.6047167307695813,
9
+ "ece_10_bins": 0.1994374088758955
10
+ },
11
+ "banking77": {
12
+ "accuracy_top1": 0.5800865800865801,
13
+ "accuracy_top5": 0.8701298701298701,
14
+ "mean_nll": 1.5664262868874264,
15
+ "mean_brier": 0.5760736580374399,
16
+ "ece_10_bins": 0.09326245064859266
17
+ }
18
+ },
19
+ "expanded": {
20
+ "commonsense_qa": {
21
+ "accuracy_top1": 0.6020128087831657,
22
+ "accuracy_top3": 0.9277218664226898,
23
+ "mean_nll": 1.1035297405179396,
24
+ "mean_brier": 0.5445770138152226,
25
+ "ece_10_bins": 0.13095674654920497
26
+ },
27
+ "banking77": {
28
+ "accuracy_top1": 0.645021645021645,
29
+ "accuracy_top5": 0.9523809523809523,
30
+ "mean_nll": 1.329584948457993,
31
+ "mean_brier": 0.5151303437000476,
32
+ "ece_10_bins": 0.16459291234805987
33
+ }
34
+ },
35
+ "examples": {
36
+ "commonsense_qa": 1093,
37
+ "banking77": 231
38
+ },
39
+ "chance_top1": {
40
+ "commonsense_qa_5_choice": 0.2,
41
+ "banking77_77_intent": 0.012987012987012988
42
+ },
43
+ "notes": [
44
+ "CommonsenseQA uses 1,093 official validation examples.",
45
+ "BANKING77 is a stratified sample of 231 official test examples, three per intent.",
46
+ "These task-specific measurements are not a general intelligence score."
47
+ ]
48
+ }