File size: 1,310 Bytes
2778af7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | {
"model": "pluto-typed",
"benchmark": "LocalLLaMA/typed-decisions",
"mode": "fitted on train",
"n_decisions": 2000,
"accuracy": 0.711,
"ece": 0.10864835740625857,
"brier": 0.4019103944301605,
"nll": 0.6982178688049316,
"per_kind": {
"choice": {
"n": 600,
"accuracy": 0.6433333333333333,
"ece": 0.09454844221472741
},
"noul": {
"n": 600,
"accuracy": 0.8266666666666667,
"ece": 0.07666520804166793
},
"score": {
"n": 800,
"accuracy": 0.675,
"ece": 0.14437313050031664
}
},
"temperatures": {
"choice": 1.2249749898910522,
"noul": 1.3188161849975586,
"score": 1.3833255767822266
},
"validation": {
"n": 300,
"accuracy": 0.74,
"ece": 0.1383799741665522,
"brier": 0.38873958587646484,
"nll": 0.6865395903587341,
"per_kind": {
"choice": {
"n": 90,
"accuracy": 0.7555555555555555,
"ece": 0.20956675708293912
},
"noul": {
"n": 90,
"accuracy": 0.8888888888888888,
"ece": 0.1182247890366448
},
"score": {
"n": 120,
"accuracy": 0.6166666666666667,
"ece": 0.10029492701093357
}
}
},
"best_val_accuracy": 0.74
} |