vantablack / metrics.json
saidutta69's picture
VantaBlack on dataset v5: calibrated ModernBERT-large, macro F1 0.9911
dcf6697 verified
Raw History Blame Contribute Delete
8.52 kB
{
"checkpoint": "runs/mbert-large-v4-calibrated",
"calibration": {
"temperature": 1.0000000000000004,
"bias": [
1.0,
0.0,
0.0,
0.0,
0.9
]
},
"val_raw": {
"accuracy": 0.9775280898876404,
"macro_f1": 0.9775743091117549,
"macro_f1_supported": 0.9775743091117549,
"weighted_f1": 0.9775158976678895,
"nll": 0.1288159340620041,
"ece": 0.011444486660903795,
"brier": 0.04359452426433563,
"aurc": 0.012627429970769228,
"mean_conf": 0.9716665148735046,
"confident_error_rate": 0.8941255807876587,
"selective": {
"0.5": {
"coverage": 0.49887640449438203,
"accuracy": 0.990990990990991,
"confidence_cutoff": 0.9896456003189087
},
"0.8": {
"coverage": 0.8,
"accuracy": 0.9831460674157303,
"confidence_cutoff": 0.9393740892410278
},
"0.9": {
"coverage": 0.898876404494382,
"accuracy": 0.985,
"confidence_cutoff": 0.9332247376441956
}
},
"per_class": {
"compliant": {
"precision": 1.0,
"recall": 0.946236559139785,
"f1": 0.9723756906077349,
"support": 93
},
"full_refusal": {
"precision": 0.9883720930232558,
"recall": 0.9659090909090909,
"f1": 0.9770114942528736,
"support": 88
},
"partial_refusal": {
"precision": 0.945054945054945,
"recall": 0.9772727272727273,
"f1": 0.9608938547486033,
"support": 88
},
"capability_limitation": {
"precision": 0.9887640449438202,
"recall": 1.0,
"f1": 0.9943502824858756,
"support": 88
},
"clarification": {
"precision": 0.967032967032967,
"recall": 1.0,
"f1": 0.9832402234636871,
"support": 88
}
},
"confusion": [
[
88,
0,
2,
1,
2
],
[
0,
85,
3,
0,
0
],
[
0,
1,
86,
0,
1
],
[
0,
0,
0,
88,
0
],
[
0,
0,
0,
0,
88
]
],
"support": 445
},
"val_calibrated": {
"accuracy": 0.9797752808988764,
"macro_f1": 0.9797832226111749,
"macro_f1_supported": 0.9797832226111749,
"weighted_f1": 0.9797634333461278,
"nll": 0.12827310616382437,
"ece": 0.03314757731706912,
"brier": 0.043282836501022375,
"aurc": 0.010907583758805439,
"mean_conf": 0.9682135336223163,
"confident_error_rate": 0.9188248039158842,
"selective": {
"0.5": {
"coverage": 0.49887640449438203,
"accuracy": 0.9819819819819819,
"confidence_cutoff": 0.9916877198340319
},
"0.8": {
"coverage": 0.8,
"accuracy": 0.9831460674157303,
"confidence_cutoff": 0.913997891114871
},
"0.9": {
"coverage": 0.898876404494382,
"accuracy": 0.985,
"confidence_cutoff": 0.9059301601030256
}
},
"per_class": {
"compliant": {
"precision": 1.0,
"recall": 0.956989247311828,
"f1": 0.978021978021978,
"support": 93
},
"full_refusal": {
"precision": 0.9883720930232558,
"recall": 0.9659090909090909,
"f1": 0.9770114942528736,
"support": 88
},
"partial_refusal": {
"precision": 0.9555555555555556,
"recall": 0.9772727272727273,
"f1": 0.9662921348314608,
"support": 88
},
"capability_limitation": {
"precision": 0.9887640449438202,
"recall": 1.0,
"f1": 0.9943502824858756,
"support": 88
},
"clarification": {
"precision": 0.967032967032967,
"recall": 1.0,
"f1": 0.9832402234636871,
"support": 88
}
},
"confusion": [
[
89,
0,
1,
1,
2
],
[
0,
85,
3,
0,
0
],
[
0,
1,
86,
0,
1
],
[
0,
0,
0,
88,
0
],
[
0,
0,
0,
0,
88
]
],
"support": 445
},
"test_raw": {
"accuracy": 0.9910112359550561,
"macro_f1": 0.9910740397730361,
"macro_f1_supported": 0.9910740397730361,
"weighted_f1": 0.9909994244508226,
"nll": 0.06652859598398209,
"ece": 0.01684493764062944,
"brier": 0.017338627949357033,
"aurc": 0.008423663643542165,
"mean_conf": 0.9749369025230408,
"confident_error_rate": 0.8797094821929932,
"selective": {
"0.5": {
"coverage": 0.49887640449438203,
"accuracy": 0.990990990990991,
"confidence_cutoff": 0.989537239074707
},
"0.8": {
"coverage": 0.8,
"accuracy": 0.9915730337078652,
"confidence_cutoff": 0.9396082162857056
},
"0.9": {
"coverage": 0.898876404494382,
"accuracy": 0.9925,
"confidence_cutoff": 0.9324463605880737
}
},
"per_class": {
"compliant": {
"precision": 0.989247311827957,
"recall": 0.9787234042553191,
"f1": 0.983957219251337,
"support": 94
},
"full_refusal": {
"precision": 1.0,
"recall": 0.9886363636363636,
"f1": 0.9942857142857142,
"support": 88
},
"partial_refusal": {
"precision": 0.9883720930232558,
"recall": 0.9883720930232558,
"f1": 0.9883720930232558,
"support": 86
},
"capability_limitation": {
"precision": 0.9885057471264368,
"recall": 1.0,
"f1": 0.9942196531791908,
"support": 86
},
"clarification": {
"precision": 0.9891304347826086,
"recall": 1.0,
"f1": 0.994535519125683,
"support": 91
}
},
"confusion": [
[
92,
0,
0,
1,
1
],
[
0,
87,
1,
0,
0
],
[
1,
0,
85,
0,
0
],
[
0,
0,
0,
86,
0
],
[
0,
0,
0,
0,
91
]
],
"support": 445
},
"test_calibrated": {
"accuracy": 0.9910112359550561,
"macro_f1": 0.9910740397730361,
"macro_f1_supported": 0.9910740397730361,
"weighted_f1": 0.9909994244508226,
"nll": 0.06978580453913351,
"ece": 0.022631346186655556,
"brier": 0.018303621149644102,
"aurc": 0.004501822525535942,
"mean_conf": 0.971843404608016,
"confident_error_rate": 0.8869774945028713,
"selective": {
"0.5": {
"coverage": 0.49887640449438203,
"accuracy": 0.9954954954954955,
"confidence_cutoff": 0.9919190574804833
},
"0.8": {
"coverage": 0.8,
"accuracy": 0.9915730337078652,
"confidence_cutoff": 0.9185372280015965
},
"0.9": {
"coverage": 0.898876404494382,
"accuracy": 0.9925,
"confidence_cutoff": 0.9058025500487228
}
},
"per_class": {
"compliant": {
"precision": 0.989247311827957,
"recall": 0.9787234042553191,
"f1": 0.983957219251337,
"support": 94
},
"full_refusal": {
"precision": 1.0,
"recall": 0.9886363636363636,
"f1": 0.9942857142857142,
"support": 88
},
"partial_refusal": {
"precision": 0.9883720930232558,
"recall": 0.9883720930232558,
"f1": 0.9883720930232558,
"support": 86
},
"capability_limitation": {
"precision": 0.9885057471264368,
"recall": 1.0,
"f1": 0.9942196531791908,
"support": 86
},
"clarification": {
"precision": 0.9891304347826086,
"recall": 1.0,
"f1": 0.994535519125683,
"support": 91
}
},
"confusion": [
[
92,
0,
0,
1,
1
],
[
0,
87,
1,
0,
0
],
[
1,
0,
85,
0,
0
],
[
0,
0,
0,
86,
0
],
[
0,
0,
0,
0,
91
]
],
"support": 445
}
}