Pluto-1 private release candidate: dual-encoder ensemble, calibrated Typed Decisions specialist
2778af7 verified | { | |
| "model": "pluto-typed", | |
| "benchmark": "LocalLLaMA/typed-decisions", | |
| "mode": "fitted on train", | |
| "n_decisions": 2000, | |
| "accuracy": 0.711, | |
| "ece": 0.10864835740625857, | |
| "brier": 0.4019103944301605, | |
| "nll": 0.6982178688049316, | |
| "per_kind": { | |
| "choice": { | |
| "n": 600, | |
| "accuracy": 0.6433333333333333, | |
| "ece": 0.09454844221472741 | |
| }, | |
| "noul": { | |
| "n": 600, | |
| "accuracy": 0.8266666666666667, | |
| "ece": 0.07666520804166793 | |
| }, | |
| "score": { | |
| "n": 800, | |
| "accuracy": 0.675, | |
| "ece": 0.14437313050031664 | |
| } | |
| }, | |
| "temperatures": { | |
| "choice": 1.2249749898910522, | |
| "noul": 1.3188161849975586, | |
| "score": 1.3833255767822266 | |
| }, | |
| "validation": { | |
| "n": 300, | |
| "accuracy": 0.74, | |
| "ece": 0.1383799741665522, | |
| "brier": 0.38873958587646484, | |
| "nll": 0.6865395903587341, | |
| "per_kind": { | |
| "choice": { | |
| "n": 90, | |
| "accuracy": 0.7555555555555555, | |
| "ece": 0.20956675708293912 | |
| }, | |
| "noul": { | |
| "n": 90, | |
| "accuracy": 0.8888888888888888, | |
| "ece": 0.1182247890366448 | |
| }, | |
| "score": { | |
| "n": 120, | |
| "accuracy": 0.6166666666666667, | |
| "ece": 0.10029492701093357 | |
| } | |
| } | |
| }, | |
| "best_val_accuracy": 0.74 | |
| } |