modalityscan / evaluation.json
vectorsense's picture
Upload evaluation.json with huggingface_hub
07ac1d1 verified
Raw History Blame Contribute Delete
19.1 kB
{
"split": "test",
"backend": "onnx",
"rows": 15712,
"taxonomy_version": "1.0.0",
"policy": "balanced",
"calibration_status": "calibrated",
"temperature": {
"modality": 0.6848,
"family": 0.6492
},
"modality": {
"accuracy": 0.7446,
"macro_f1": 0.9067,
"classes_scored": 3,
"per_class": {
"CT": {
"precision": 0.8469,
"recall": 0.8059,
"f1": 0.8259,
"support": 2684
},
"MR": {
"precision": 0.2476,
"recall": 1.0,
"f1": 0.3969,
"support": 52
},
"US": {
"precision": 0.861,
"recall": 0.9558,
"f1": 0.9059,
"support": 3123
},
"CR": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DX": {
"precision": 0.9956,
"recall": 0.9814,
"f1": 0.9884,
"support": 3442
},
"MG": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"RF": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"XA": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"PT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"NM": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"OCT": {
"precision": 0.9977,
"recall": 1.0,
"f1": 0.9989,
"support": 881
},
"FUNDUS": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 246
},
"DERMOSCOPY": {
"precision": 0.5773,
"recall": 0.9943,
"f1": 0.7305,
"support": 879
},
"ENDOSCOPY": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"HISTOPATHOLOGY": {
"precision": 0.2853,
"recall": 1.0,
"f1": 0.4439,
"support": 926
},
"MICROSCOPY": {
"precision": 0.9749,
"recall": 0.0558,
"f1": 0.1055,
"support": 3479
},
"PHOTO": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"RENDER_3D": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DOCUMENT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"WAVEFORM": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
}
},
"confusion": [
[
2163,
10,
390,
0,
15,
0,
0,
0,
0,
0,
0,
0,
0,
0,
101,
5,
0,
0,
0,
0
],
[
0,
52,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
18,
118,
2985,
0,
0,
0,
0,
0,
0,
0,
2,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
3,
0,
57,
0,
3378,
0,
0,
0,
0,
0,
0,
0,
0,
0,
4,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
881,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
246,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
874,
0,
4,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
926,
0,
0,
0,
0,
0
],
[
370,
29,
35,
0,
0,
0,
0,
0,
0,
0,
0,
0,
640,
0,
2211,
194,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
],
"confusion_labels": [
"CT",
"MR",
"US",
"CR",
"DX",
"MG",
"RF",
"XA",
"PT",
"NM",
"OCT",
"FUNDUS",
"DERMOSCOPY",
"ENDOSCOPY",
"HISTOPATHOLOGY",
"MICROSCOPY",
"PHOTO",
"RENDER_3D",
"DOCUMENT",
"WAVEFORM"
]
},
"family": {
"accuracy": 0.8282,
"macro_f1": 0.828,
"classes_scored": 5,
"per_class": {
"projection_radiography": {
"precision": 0.9976,
"recall": 0.9741,
"f1": 0.9857,
"support": 3442
},
"cross_sectional": {
"precision": 0.7003,
"recall": 0.8154,
"f1": 0.7535,
"support": 2736
},
"ultrasound": {
"precision": 0.7559,
"recall": 0.9789,
"f1": 0.8531,
"support": 3123
},
"optical": {
"precision": 0.7521,
"recall": 0.9965,
"f1": 0.8572,
"support": 2006
},
"microscopy": {
"precision": 0.9631,
"recall": 0.5385,
"f1": 0.6907,
"support": 4405
},
"non_diagnostic": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
}
},
"confusion": [
[
3353,
3,
84,
0,
2,
0
],
[
8,
2231,
414,
0,
83,
0
],
[
0,
66,
3057,
0,
0,
0
],
[
0,
1,
0,
1999,
6,
0
],
[
0,
885,
489,
659,
2372,
0
],
[
0,
0,
0,
0,
0,
0
]
],
"confusion_labels": [
"projection_radiography",
"cross_sectional",
"ultrasound",
"optical",
"microscopy",
"non_diagnostic"
]
},
"coarse": {
"coarse_accuracy": 0.7446,
"fine_accuracy": 0.7446,
"gap": 0.0,
"groups": {
"XR_PLAIN": [
"CR",
"DX"
],
"XR_FLUORO": [
"RF",
"XA"
],
"NUCLEAR": [
"PT",
"NM"
]
}
},
"confusable_pairs": {
"CR->DX": {
"rate": null,
"support": 0
},
"DX->CR": {
"rate": 0.0,
"support": 3442
},
"RF->XA": {
"rate": null,
"support": 0
},
"XA->RF": {
"rate": null,
"support": 0
},
"PT->NM": {
"rate": null,
"support": 0
},
"NM->PT": {
"rate": null,
"support": 0
},
"CT->MR": {
"rate": 0.0037,
"support": 2684
},
"MR->CT": {
"rate": 0.0,
"support": 52
},
"MG->DX": {
"rate": null,
"support": 0
},
"DX->MG": {
"rate": 0.0,
"support": 3442
},
"OCT->US": {
"rate": 0.0,
"support": 881
},
"US->OCT": {
"rate": 0.0006,
"support": 3123
},
"RENDER_3D->CT": {
"rate": null,
"support": 0
},
"CT->RENDER_3D": {
"rate": 0.0,
"support": 2684
},
"PHOTO->DERMOSCOPY": {
"rate": null,
"support": 0
},
"DERMOSCOPY->PHOTO": {
"rate": 0.0,
"support": 879
},
"DOCUMENT->WAVEFORM": {
"rate": null,
"support": 0
},
"WAVEFORM->DOCUMENT": {
"rate": null,
"support": 0
}
},
"by_tier": {
"held_out_source": {
"rows": 11000,
"accuracy": 0.6618,
"macro_f1": 0.9021,
"per_class": {
"CT": {
"precision": 0.8987,
"recall": 0.741,
"f1": 0.8123,
"support": 2000
},
"MR": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"US": {
"precision": 0.8649,
"recall": 0.954,
"f1": 0.9073,
"support": 3000
},
"CR": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DX": {
"precision": 0.9949,
"recall": 0.9787,
"f1": 0.9867,
"support": 3000
},
"MG": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"RF": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"XA": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"PT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"NM": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"OCT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"FUNDUS": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DERMOSCOPY": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"ENDOSCOPY": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"HISTOPATHOLOGY": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"MICROSCOPY": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 3000
},
"PHOTO": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"RENDER_3D": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DOCUMENT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"WAVEFORM": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
}
}
},
"in_distribution": {
"rows": 4712,
"accuracy": 0.9378,
"macro_f1": 0.9109,
"per_class": {
"CT": {
"precision": 0.7525,
"recall": 0.9956,
"f1": 0.8571,
"support": 684
},
"MR": {
"precision": 0.7879,
"recall": 1.0,
"f1": 0.8814,
"support": 52
},
"US": {
"precision": 0.7785,
"recall": 1.0,
"f1": 0.8754,
"support": 123
},
"CR": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DX": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 442
},
"MG": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"RF": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"XA": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"PT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"NM": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"OCT": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 881
},
"FUNDUS": {
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"support": 246
},
"DERMOSCOPY": {
"precision": 1.0,
"recall": 0.9943,
"f1": 0.9971,
"support": 879
},
"ENDOSCOPY": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"HISTOPATHOLOGY": {
"precision": 0.9789,
"recall": 1.0,
"f1": 0.9893,
"support": 926
},
"MICROSCOPY": {
"precision": 1.0,
"recall": 0.405,
"f1": 0.5765,
"support": 479
},
"PHOTO": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"RENDER_3D": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"DOCUMENT": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
},
"WAVEFORM": {
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"support": 0
}
}
}
},
"by_source": {
"amos22": {
"rows": 404,
"accuracy": 1.0
},
"fetal-planes-db": {
"rows": 3000,
"accuracy": 0.954
},
"medmnist-plus-bloodmnist-224": {
"rows": 3000,
"accuracy": 0.0
},
"medmnist-plus-breastmnist-224": {
"rows": 123,
"accuracy": 1.0
},
"medmnist-plus-chestmnist-224": {
"rows": 442,
"accuracy": 1.0
},
"medmnist-plus-dermamnist-224": {
"rows": 879,
"accuracy": 0.9943
},
"medmnist-plus-octmnist-224": {
"rows": 881,
"accuracy": 1.0
},
"medmnist-plus-organamnist-224": {
"rows": 2000,
"accuracy": 0.741
},
"medmnist-plus-pathmnist-224": {
"rows": 926,
"accuracy": 1.0
},
"medmnist-plus-pneumoniamnist-224": {
"rows": 3000,
"accuracy": 0.9787
},
"medmnist-plus-retinamnist-224": {
"rows": 246,
"accuracy": 1.0
},
"medmnist-plus-tissuemnist-224": {
"rows": 479,
"accuracy": 0.405
},
"totalsegmentator-ct-v300": {
"rows": 332,
"accuracy": 0.991
}
},
"series": {
"series": 184,
"frame_accuracy": 0.9959,
"series_accuracy": 1.0,
"gain": 0.0041
},
"calibration": {
"expected_calibration_error": 0.0991
},
"open_set": {
"rows": 0,
"energy_auroc": null,
"energy_percentiles_in_distribution": {
"1": -8.4258,
"5": -7.5222,
"50": -5.787,
"95": -2.5901,
"99": -2.4627
}
}
}