Multimodel_setup / scripts /system_audit /step5_metrics.json
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
6.96 kB
{
"held_out_test_103": {
"Model A (ConvNeXt)": {
"TP": 6,
"FP": 14,
"TN": 37,
"FN": 46,
"Accuracy": 0.4174757281553398,
"Precision": 0.3,
"Recall": 0.11538461538461539,
"F1": 0.16666666666666666,
"ROC-AUC": 0.33333333333333337,
"PR-AUC": 0.4059174195371763,
"EER": 0.607843137255571,
"ECE": 0.5388236019417476
},
"Model B (UnivFD)": {
"TP": 11,
"FP": 1,
"TN": 50,
"FN": 41,
"Accuracy": 0.5922330097087378,
"Precision": 0.9166666666666666,
"Recall": 0.21153846153846154,
"F1": 0.34375,
"ROC-AUC": 0.6506410256410257,
"PR-AUC": 0.6953476667472117,
"EER": 0.42307692307668304,
"ECE": 0.36694775728155343
},
"Model C (ViT)": {
"TP": 50,
"FP": 0,
"TN": 51,
"FN": 2,
"Accuracy": 0.9805825242718447,
"Precision": 1.0,
"Recall": 0.9615384615384616,
"F1": 0.9803921568627451,
"ROC-AUC": 0.9992458521870287,
"PR-AUC": 0.9992877492877494,
"EER": 0.01923076923085128,
"ECE": 0.016270077669902985
},
"Simple Average": {
"TP": 16,
"FP": 0,
"TN": 51,
"FN": 36,
"Accuracy": 0.6504854368932039,
"Precision": 1.0,
"Recall": 0.3076923076923077,
"F1": 0.47058823529411764,
"ROC-AUC": 0.9581447963800904,
"PR-AUC": 0.9551236587001712,
"EER": 0.058823529411953686,
"ECE": 0.26600742718446607
},
"Central Analyzer (ViT-Priority)": {
"TP": 50,
"FP": 0,
"TN": 51,
"FN": 2,
"Accuracy": 0.9805825242718447,
"Precision": 1.0,
"Recall": 0.9615384615384616,
"F1": 0.9803921568627451,
"ROC-AUC": 0.9992458521870287,
"PR-AUC": 0.9992877492877494,
"EER": 0.01923076923085128,
"ECE": 0.016270077669902985
}
},
"development_val_50": {
"Model A (ConvNeXt)": {
"TP": 6,
"FP": 2,
"TN": 23,
"FN": 19,
"Accuracy": 0.58,
"Precision": 0.75,
"Recall": 0.24,
"F1": 0.36363636363636365,
"ROC-AUC": 0.6815999999999999,
"PR-AUC": 0.6887120429338933,
"EER": 0.36,
"ECE": 0.37706333999999997
},
"Model B (UnivFD)": {
"TP": 8,
"FP": 1,
"TN": 24,
"FN": 17,
"Accuracy": 0.64,
"Precision": 0.8888888888888888,
"Recall": 0.32,
"F1": 0.47058823529411764,
"ROC-AUC": 0.7343999999999999,
"PR-AUC": 0.7668867912021051,
"EER": 0.3200000000000001,
"ECE": 0.28705824
},
"Model C (ViT)": {
"TP": 25,
"FP": 0,
"TN": 25,
"FN": 0,
"Accuracy": 1.0,
"Precision": 1.0,
"Recall": 1.0,
"F1": 1.0,
"ROC-AUC": 1.0,
"PR-AUC": 1.0,
"EER": 0.0,
"ECE": 0.0019278000000000155
},
"Simple Average": {
"TP": 13,
"FP": 0,
"TN": 25,
"FN": 12,
"Accuracy": 0.76,
"Precision": 1.0,
"Recall": 0.52,
"F1": 0.6842105263157895,
"ROC-AUC": 0.9904,
"PR-AUC": 0.9897327976371455,
"EER": 0.04000000000107677,
"ECE": 0.23633806
},
"Central Analyzer (ViT-Priority)": {
"TP": 25,
"FP": 0,
"TN": 25,
"FN": 0,
"Accuracy": 1.0,
"Precision": 1.0,
"Recall": 1.0,
"F1": 1.0,
"ROC-AUC": 1.0,
"PR-AUC": 1.0,
"EER": 0.0,
"ECE": 0.0019278000000000155
}
},
"phase6_test_200": {
"Model A (ConvNeXt)": {
"TP": 16,
"FP": 14,
"TN": 86,
"FN": 84,
"Accuracy": 0.51,
"Precision": 0.5333333333333333,
"Recall": 0.16,
"F1": 0.24615384615384617,
"ROC-AUC": 0.49765000000000004,
"PR-AUC": 0.5184827417295357,
"EER": 0.4899999999985364,
"ECE": 0.4266103999999999
},
"Model B (UnivFD)": {
"TP": 17,
"FP": 14,
"TN": 86,
"FN": 83,
"Accuracy": 0.515,
"Precision": 0.5483870967741935,
"Recall": 0.17,
"F1": 0.2595419847328244,
"ROC-AUC": 0.5503000000000001,
"PR-AUC": 0.5587964761407023,
"EER": 0.4700000000006478,
"ECE": 0.38979305499999994
},
"Model C (ViT)": {
"TP": 100,
"FP": 4,
"TN": 96,
"FN": 0,
"Accuracy": 0.98,
"Precision": 0.9615384615384616,
"Recall": 1.0,
"F1": 0.9803921568627451,
"ROC-AUC": 0.9994999999999999,
"PR-AUC": 0.9994948463981681,
"EER": 0.010000000001510646,
"ECE": 0.02271725000000001
},
"Simple Average": {
"TP": 31,
"FP": 4,
"TN": 96,
"FN": 69,
"Accuracy": 0.635,
"Precision": 0.8857142857142857,
"Recall": 0.31,
"F1": 0.45925925925925926,
"ROC-AUC": 0.9519000000000001,
"PR-AUC": 0.8835832815963003,
"EER": 0.07999999999930718,
"ECE": 0.28019790499999997
},
"Central Analyzer (ViT-Priority)": {
"TP": 100,
"FP": 4,
"TN": 96,
"FN": 0,
"Accuracy": 0.98,
"Precision": 0.9615384615384616,
"Recall": 1.0,
"F1": 0.9803921568627451,
"ROC-AUC": 0.9994999999999999,
"PR-AUC": 0.9994948463981681,
"EER": 0.010000000001510646,
"ECE": 0.02271725000000001
}
},
"phase7_test_200": {
"Model A (ConvNeXt)": {
"TP": 13,
"FP": 0,
"TN": 100,
"FN": 87,
"Accuracy": 0.565,
"Precision": 1.0,
"Recall": 0.13,
"F1": 0.23008849557522124,
"ROC-AUC": 0.7926000000000001,
"PR-AUC": 0.8118848908196705,
"EER": 0.3000000000000137,
"ECE": 0.41736847499999996
},
"Model B (UnivFD)": {
"TP": 15,
"FP": 0,
"TN": 100,
"FN": 85,
"Accuracy": 0.575,
"Precision": 1.0,
"Recall": 0.15,
"F1": 0.2608695652173913,
"ROC-AUC": 0.6662,
"PR-AUC": 0.7415217378672225,
"EER": 0.4199999999999896,
"ECE": 0.40551109999999996
},
"Model C (ViT)": {
"TP": 18,
"FP": 14,
"TN": 86,
"FN": 82,
"Accuracy": 0.52,
"Precision": 0.5625,
"Recall": 0.18,
"F1": 0.2727272727272727,
"ROC-AUC": 0.4673,
"PR-AUC": 0.5252072002901083,
"EER": 0.529999999999551,
"ECE": 0.26795644999999996
},
"Simple Average": {
"TP": 5,
"FP": 0,
"TN": 100,
"FN": 95,
"Accuracy": 0.525,
"Precision": 1.0,
"Recall": 0.05,
"F1": 0.09523809523809523,
"ROC-AUC": 0.7181000000000001,
"PR-AUC": 0.7800443586455368,
"EER": 0.37999999999999995,
"ECE": 0.34207653
},
"Central Analyzer (ViT-Priority)": {
"TP": 18,
"FP": 14,
"TN": 86,
"FN": 82,
"Accuracy": 0.52,
"Precision": 0.5625,
"Recall": 0.18,
"F1": 0.2727272727272727,
"ROC-AUC": 0.4673,
"PR-AUC": 0.5252072002901083,
"EER": 0.529999999999551,
"ECE": 0.26795644999999996
}
}
}