{ "checkpoint": "runs/mbert-large-v4-calibrated", "calibration": { "temperature": 1.0000000000000004, "bias": [ 1.0, 0.0, 0.0, 0.0, 0.9 ] }, "val_raw": { "accuracy": 0.9775280898876404, "macro_f1": 0.9775743091117549, "macro_f1_supported": 0.9775743091117549, "weighted_f1": 0.9775158976678895, "nll": 0.1288159340620041, "ece": 0.011444486660903795, "brier": 0.04359452426433563, "aurc": 0.012627429970769228, "mean_conf": 0.9716665148735046, "confident_error_rate": 0.8941255807876587, "selective": { "0.5": { "coverage": 0.49887640449438203, "accuracy": 0.990990990990991, "confidence_cutoff": 0.9896456003189087 }, "0.8": { "coverage": 0.8, "accuracy": 0.9831460674157303, "confidence_cutoff": 0.9393740892410278 }, "0.9": { "coverage": 0.898876404494382, "accuracy": 0.985, "confidence_cutoff": 0.9332247376441956 } }, "per_class": { "compliant": { "precision": 1.0, "recall": 0.946236559139785, "f1": 0.9723756906077349, "support": 93 }, "full_refusal": { "precision": 0.9883720930232558, "recall": 0.9659090909090909, "f1": 0.9770114942528736, "support": 88 }, "partial_refusal": { "precision": 0.945054945054945, "recall": 0.9772727272727273, "f1": 0.9608938547486033, "support": 88 }, "capability_limitation": { "precision": 0.9887640449438202, "recall": 1.0, "f1": 0.9943502824858756, "support": 88 }, "clarification": { "precision": 0.967032967032967, "recall": 1.0, "f1": 0.9832402234636871, "support": 88 } }, "confusion": [ [ 88, 0, 2, 1, 2 ], [ 0, 85, 3, 0, 0 ], [ 0, 1, 86, 0, 1 ], [ 0, 0, 0, 88, 0 ], [ 0, 0, 0, 0, 88 ] ], "support": 445 }, "val_calibrated": { "accuracy": 0.9797752808988764, "macro_f1": 0.9797832226111749, "macro_f1_supported": 0.9797832226111749, "weighted_f1": 0.9797634333461278, "nll": 0.12827310616382437, "ece": 0.03314757731706912, "brier": 0.043282836501022375, "aurc": 0.010907583758805439, "mean_conf": 0.9682135336223163, "confident_error_rate": 0.9188248039158842, "selective": { "0.5": { "coverage": 0.49887640449438203, "accuracy": 0.9819819819819819, "confidence_cutoff": 0.9916877198340319 }, "0.8": { "coverage": 0.8, "accuracy": 0.9831460674157303, "confidence_cutoff": 0.913997891114871 }, "0.9": { "coverage": 0.898876404494382, "accuracy": 0.985, "confidence_cutoff": 0.9059301601030256 } }, "per_class": { "compliant": { "precision": 1.0, "recall": 0.956989247311828, "f1": 0.978021978021978, "support": 93 }, "full_refusal": { "precision": 0.9883720930232558, "recall": 0.9659090909090909, "f1": 0.9770114942528736, "support": 88 }, "partial_refusal": { "precision": 0.9555555555555556, "recall": 0.9772727272727273, "f1": 0.9662921348314608, "support": 88 }, "capability_limitation": { "precision": 0.9887640449438202, "recall": 1.0, "f1": 0.9943502824858756, "support": 88 }, "clarification": { "precision": 0.967032967032967, "recall": 1.0, "f1": 0.9832402234636871, "support": 88 } }, "confusion": [ [ 89, 0, 1, 1, 2 ], [ 0, 85, 3, 0, 0 ], [ 0, 1, 86, 0, 1 ], [ 0, 0, 0, 88, 0 ], [ 0, 0, 0, 0, 88 ] ], "support": 445 }, "test_raw": { "accuracy": 0.9910112359550561, "macro_f1": 0.9910740397730361, "macro_f1_supported": 0.9910740397730361, "weighted_f1": 0.9909994244508226, "nll": 0.06652859598398209, "ece": 0.01684493764062944, "brier": 0.017338627949357033, "aurc": 0.008423663643542165, "mean_conf": 0.9749369025230408, "confident_error_rate": 0.8797094821929932, "selective": { "0.5": { "coverage": 0.49887640449438203, "accuracy": 0.990990990990991, "confidence_cutoff": 0.989537239074707 }, "0.8": { "coverage": 0.8, "accuracy": 0.9915730337078652, "confidence_cutoff": 0.9396082162857056 }, "0.9": { "coverage": 0.898876404494382, "accuracy": 0.9925, "confidence_cutoff": 0.9324463605880737 } }, "per_class": { "compliant": { "precision": 0.989247311827957, "recall": 0.9787234042553191, "f1": 0.983957219251337, "support": 94 }, "full_refusal": { "precision": 1.0, "recall": 0.9886363636363636, "f1": 0.9942857142857142, "support": 88 }, "partial_refusal": { "precision": 0.9883720930232558, "recall": 0.9883720930232558, "f1": 0.9883720930232558, "support": 86 }, "capability_limitation": { "precision": 0.9885057471264368, "recall": 1.0, "f1": 0.9942196531791908, "support": 86 }, "clarification": { "precision": 0.9891304347826086, "recall": 1.0, "f1": 0.994535519125683, "support": 91 } }, "confusion": [ [ 92, 0, 0, 1, 1 ], [ 0, 87, 1, 0, 0 ], [ 1, 0, 85, 0, 0 ], [ 0, 0, 0, 86, 0 ], [ 0, 0, 0, 0, 91 ] ], "support": 445 }, "test_calibrated": { "accuracy": 0.9910112359550561, "macro_f1": 0.9910740397730361, "macro_f1_supported": 0.9910740397730361, "weighted_f1": 0.9909994244508226, "nll": 0.06978580453913351, "ece": 0.022631346186655556, "brier": 0.018303621149644102, "aurc": 0.004501822525535942, "mean_conf": 0.971843404608016, "confident_error_rate": 0.8869774945028713, "selective": { "0.5": { "coverage": 0.49887640449438203, "accuracy": 0.9954954954954955, "confidence_cutoff": 0.9919190574804833 }, "0.8": { "coverage": 0.8, "accuracy": 0.9915730337078652, "confidence_cutoff": 0.9185372280015965 }, "0.9": { "coverage": 0.898876404494382, "accuracy": 0.9925, "confidence_cutoff": 0.9058025500487228 } }, "per_class": { "compliant": { "precision": 0.989247311827957, "recall": 0.9787234042553191, "f1": 0.983957219251337, "support": 94 }, "full_refusal": { "precision": 1.0, "recall": 0.9886363636363636, "f1": 0.9942857142857142, "support": 88 }, "partial_refusal": { "precision": 0.9883720930232558, "recall": 0.9883720930232558, "f1": 0.9883720930232558, "support": 86 }, "capability_limitation": { "precision": 0.9885057471264368, "recall": 1.0, "f1": 0.9942196531791908, "support": 86 }, "clarification": { "precision": 0.9891304347826086, "recall": 1.0, "f1": 0.994535519125683, "support": 91 } }, "confusion": [ [ 92, 0, 0, 1, 1 ], [ 0, 87, 1, 0, 0 ], [ 1, 0, 85, 0, 0 ], [ 0, 0, 0, 86, 0 ], [ 0, 0, 0, 0, 91 ] ], "support": 445 } }