| { | |
| "temperature": 1.0087398771573446, | |
| "fitted_on": "calibration", | |
| "selected_on": "validation", | |
| "use_temperature": false, | |
| "default_prediction": "raw", | |
| "validation_raw": { | |
| "count": 14111, | |
| "top1_accuracy": 0.8263765856424067, | |
| "nll": 0.6840500261181026, | |
| "brier_score": 0.011582265527708373, | |
| "ece": 0.16699567031458232, | |
| "signed_calibration_error": -0.1669956703145823, | |
| "kl_divergence": 0.02310311772478613, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11661, | |
| "mean": 0.6960411451404903, | |
| "median": 0.6765217781066895, | |
| "std": 0.19690960502309285, | |
| "p25": 0.50345379114151, | |
| "p75": 0.8713327646255493 | |
| }, | |
| "incorrect": { | |
| "count": 2450, | |
| "mean": 0.4848931847786417, | |
| "median": 0.500392496585846, | |
| "std": 0.07416498496851862, | |
| "p25": 0.5000284314155579, | |
| "p75": 0.5010085254907608 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 148, | |
| "average_confidence": 0.27538005906987834, | |
| "empirical_accuracy": 0.46621621621621623, | |
| "signed_gap": -0.1908361571463379 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 667, | |
| "average_confidence": 0.3564207344755776, | |
| "empirical_accuracy": 0.664167916041979, | |
| "signed_gap": -0.3077471815664014 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1160, | |
| "average_confidence": 0.4550788116095395, | |
| "empirical_accuracy": 0.7741379310344828, | |
| "signed_gap": -0.3190591194249433 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4972, | |
| "average_confidence": 0.5200908452703985, | |
| "empirical_accuracy": 0.6351568785197104, | |
| "signed_gap": -0.11506603324931186 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1678, | |
| "average_confidence": 0.6489259644685684, | |
| "empirical_accuracy": 0.9713945172824792, | |
| "signed_gap": -0.32246855281391074 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1540, | |
| "average_confidence": 0.7498124199641215, | |
| "empirical_accuracy": 0.9935064935064936, | |
| "signed_gap": -0.24369407354237205 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1412, | |
| "average_confidence": 0.8490157142154894, | |
| "empirical_accuracy": 0.9936260623229461, | |
| "signed_gap": -0.14461034810745677 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2534, | |
| "average_confidence": 0.9746775966087663, | |
| "empirical_accuracy": 0.9984214680347278, | |
| "signed_gap": -0.02374387142596146 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8600382680178584, | |
| "accuracy": 0.844676994067238, | |
| "risk": 0.15532300593276205, | |
| "sample_count": 12136 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5076890369215505, | |
| "accuracy": 0.9900893355667225, | |
| "risk": 0.009910664433277483, | |
| "sample_count": 7164 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.3887747147615336, | |
| "accuracy": 0.9958075100255195, | |
| "risk": 0.004192489974480518, | |
| "sample_count": 5486 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.279639997165332, | |
| "accuracy": 0.996705524581855, | |
| "risk": 0.003294475418144982, | |
| "sample_count": 3946 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.17957621713556798, | |
| "accuracy": 0.9984214680347278, | |
| "risk": 0.0015785319652722452, | |
| "sample_count": 2534 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.13273332860888667, | |
| "accuracy": 0.9983982915109451, | |
| "risk": 0.0016017084890549427, | |
| "sample_count": 1873 | |
| } | |
| ] | |
| }, | |
| "validation_temperature_scaled": { | |
| "count": 14111, | |
| "top1_accuracy": 0.8263057189426688, | |
| "nll": 0.6840468913421749, | |
| "brier_score": 0.011582336771387319, | |
| "ece": 0.16813608304214336, | |
| "signed_calibration_error": -0.16813608304214334, | |
| "kl_divergence": 0.023099982948858303, | |
| "confidence_analysis": { | |
| "correct": { | |
| "count": 11660, | |
| "mean": 0.694681208951386, | |
| "median": 0.6747364841480057, | |
| "std": 0.19693845421787254, | |
| "p25": 0.5021719095333224, | |
| "p75": 0.8689303683324948 | |
| }, | |
| "incorrect": { | |
| "count": 2451, | |
| "mean": 0.4844752492122222, | |
| "median": 0.5003890293824186, | |
| "std": 0.07420726284342961, | |
| "p25": 0.500027128879432, | |
| "p75": 0.5009866390613054 | |
| } | |
| }, | |
| "confidence_histogram": [ | |
| { | |
| "bucket": "0.0-0.1", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.1-0.2", | |
| "sample_count": 0, | |
| "average_confidence": null, | |
| "empirical_accuracy": null, | |
| "signed_gap": null | |
| }, | |
| { | |
| "bucket": "0.2-0.3", | |
| "sample_count": 151, | |
| "average_confidence": 0.2751785209954829, | |
| "empirical_accuracy": 0.46357615894039733, | |
| "signed_gap": -0.18839763794491443 | |
| }, | |
| { | |
| "bucket": "0.3-0.4", | |
| "sample_count": 674, | |
| "average_confidence": 0.35612814661984515, | |
| "empirical_accuracy": 0.6646884272997032, | |
| "signed_gap": -0.3085602806798581 | |
| }, | |
| { | |
| "bucket": "0.4-0.5", | |
| "sample_count": 1168, | |
| "average_confidence": 0.45463179276212934, | |
| "empirical_accuracy": 0.7782534246575342, | |
| "signed_gap": -0.3236216318954049 | |
| }, | |
| { | |
| "bucket": "0.5-0.6", | |
| "sample_count": 4987, | |
| "average_confidence": 0.5201110232818913, | |
| "empirical_accuracy": 0.6360537397232805, | |
| "signed_gap": -0.1159427164413892 | |
| }, | |
| { | |
| "bucket": "0.6-0.7", | |
| "sample_count": 1673, | |
| "average_confidence": 0.6487813857971743, | |
| "empirical_accuracy": 0.9719067543335326, | |
| "signed_gap": -0.3231253685363583 | |
| }, | |
| { | |
| "bucket": "0.7-0.8", | |
| "sample_count": 1546, | |
| "average_confidence": 0.7495579579669318, | |
| "empirical_accuracy": 0.9935316946959897, | |
| "signed_gap": -0.24397373672905787 | |
| }, | |
| { | |
| "bucket": "0.8-0.9", | |
| "sample_count": 1409, | |
| "average_confidence": 0.8491268025105129, | |
| "empirical_accuracy": 0.99361249112846, | |
| "signed_gap": -0.14448568861794708 | |
| }, | |
| { | |
| "bucket": "0.9-1.0", | |
| "sample_count": 2503, | |
| "average_confidence": 0.9749893213960854, | |
| "empirical_accuracy": 0.9984019176987615, | |
| "signed_gap": -0.023412596302676114 | |
| } | |
| ], | |
| "selective_accuracy": [ | |
| { | |
| "threshold": 0.5, | |
| "coverage": 0.8587626674225781, | |
| "accuracy": 0.8444462782637399, | |
| "risk": 0.15555372173626014, | |
| "sample_count": 12118 | |
| }, | |
| { | |
| "threshold": 0.6, | |
| "coverage": 0.5053504358302033, | |
| "accuracy": 0.9901837049502173, | |
| "risk": 0.009816295049782653, | |
| "sample_count": 7131 | |
| }, | |
| { | |
| "threshold": 0.7, | |
| "coverage": 0.38679044716887534, | |
| "accuracy": 0.9957860021986076, | |
| "risk": 0.004213997801392422, | |
| "sample_count": 5458 | |
| }, | |
| { | |
| "threshold": 0.8, | |
| "coverage": 0.277230529374247, | |
| "accuracy": 0.996676891615542, | |
| "risk": 0.00332310838445804, | |
| "sample_count": 3912 | |
| }, | |
| { | |
| "threshold": 0.9, | |
| "coverage": 0.1773793494436964, | |
| "accuracy": 0.9984019176987615, | |
| "risk": 0.0015980823012384793, | |
| "sample_count": 2503 | |
| }, | |
| { | |
| "threshold": 0.95, | |
| "coverage": 0.131599461413082, | |
| "accuracy": 0.9983844911147012, | |
| "risk": 0.001615508885298822, | |
| "sample_count": 1857 | |
| } | |
| ] | |
| } | |
| } |