Multimodel_setup / scripts /system_audit /step6_diagnostics.json
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
17.9 kB
{
"held_out_test_103": {
"score_distributions": {
"Model A (ConvNeXt)": {
"REAL": {
"count": 51,
"mean": 0.2711376470588236,
"std": 0.36621063906653845,
"min": 2.2e-05,
"p10": 3.3e-05,
"p25": 0.002192,
"p50_median": 0.023724,
"p75": 0.578281,
"p90": 0.885986,
"max": 0.999193,
"near_0_pct": 56.86274509803921,
"near_1_pct": 7.8431372549019605
},
"FAKE": {
"count": 52,
"mean": 0.1074628653846154,
"std": 0.25073037072643695,
"min": 7e-06,
"p10": 1.7400000000000006e-05,
"p25": 9.3e-05,
"p50_median": 0.0030935,
"p75": 0.029103499999999997,
"p90": 0.4924598999999998,
"max": 0.998509,
"near_0_pct": 80.76923076923077,
"near_1_pct": 3.8461538461538463
}
},
"Model B (UnivFD)": {
"REAL": {
"count": 51,
"mean": 0.07302660784313725,
"std": 0.15597365995905427,
"min": 1.8e-05,
"p10": 0.000396,
"p25": 0.0011415,
"p50_median": 0.016223,
"p75": 0.055903,
"p90": 0.189683,
"max": 0.903896,
"near_0_pct": 72.54901960784314,
"near_1_pct": 0.0
},
"FAKE": {
"count": 52,
"mean": 0.23877465384615387,
"std": 0.327038693165061,
"min": 6.4e-05,
"p10": 0.0011891,
"p25": 0.00572075,
"p50_median": 0.0463535,
"p75": 0.40061525,
"p90": 0.8324995,
"max": 0.996067,
"near_0_pct": 51.92307692307693,
"near_1_pct": 3.8461538461538463
}
},
"Model C (ViT)": {
"REAL": {
"count": 51,
"mean": 0.008010019607843137,
"std": 0.02963891407505739,
"min": 0.000499,
"p10": 0.000598,
"p25": 0.0008275,
"p50_median": 0.001152,
"p75": 0.0016365,
"p90": 0.004124,
"max": 0.197797,
"near_0_pct": 96.07843137254902,
"near_1_pct": 0.0
},
"FAKE": {
"count": 52,
"mean": 0.9675243269230768,
"std": 0.15705170755956105,
"min": 0.049064,
"p10": 0.9967457000000001,
"p25": 0.998827,
"p50_median": 0.9993,
"p75": 0.9995075,
"p90": 0.999567,
"max": 0.999627,
"near_0_pct": 1.9230769230769231,
"near_1_pct": 96.15384615384616
}
},
"Simple Average": {
"REAL": {
"count": 51,
"mean": 0.11739145098039215,
"std": 0.1298655932854613,
"min": 0.00075,
"p10": 0.002424,
"p25": 0.0123935,
"p50_median": 0.043415,
"p75": 0.22734349999999998,
"p90": 0.306038,
"max": 0.475567,
"near_0_pct": 50.98039215686274,
"near_1_pct": 0.0
},
"FAKE": {
"count": 52,
"mean": 0.43792059615384615,
"std": 0.15897050347545846,
"min": 0.029099,
"p10": 0.3336774,
"p25": 0.33666050000000003,
"p50_median": 0.36632549999999997,
"p75": 0.5391900000000001,
"p90": 0.664677,
"max": 0.861989,
"near_0_pct": 1.9230769230769231,
"near_1_pct": 0.0
}
},
"Central Analyzer": {
"REAL": {
"count": 51,
"mean": 0.008010019607843137,
"std": 0.02963891407505739,
"min": 0.000499,
"p10": 0.000598,
"p25": 0.0008275,
"p50_median": 0.001152,
"p75": 0.0016365,
"p90": 0.004124,
"max": 0.197797,
"near_0_pct": 96.07843137254902,
"near_1_pct": 0.0
},
"FAKE": {
"count": 52,
"mean": 0.9675243269230768,
"std": 0.15705170755956105,
"min": 0.049064,
"p10": 0.9967457000000001,
"p25": 0.998827,
"p50_median": 0.9993,
"p75": 0.9995075,
"p90": 0.999567,
"max": 0.999627,
"near_0_pct": 1.9230769230769231,
"near_1_pct": 96.15384615384616
}
}
},
"pearson_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
-0.059888633258731776,
-0.23817519707037244
],
[
-0.05988863325873178,
1.0,
0.3262249738389083
],
[
-0.23817519707037244,
0.32622497383890836,
1.0
]
]
},
"spearman_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
-0.04613335896010032,
-0.26599725977096617
],
[
-0.04613335896010032,
1.0,
0.2818420445726783
],
[
-0.2659972597709661,
0.2818420445726783,
1.0
]
]
},
"error_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
0.5021423220864053,
0.11912774295651779
],
[
0.5021423220864053,
1.0,
0.16958779100816418
],
[
0.11912774295651779,
0.16958779100816418,
1.0
]
]
},
"error_cooccurrence": {
"total_samples": 103,
"all_3_correct": 38,
"only_1_wrong": 28,
"only_2_wrong": 35,
"all_3_wrong": 2,
"breakdown_1_wrong": {
"convnext_only": 23,
"univfd_only": 5,
"vit_only": 0
},
"breakdown_2_wrong": {
"convnext_and_univfd": 35,
"convnext_and_vit": 0,
"univfd_and_vit": 0
}
}
},
"phase6_test_200": {
"score_distributions": {
"Model A (ConvNeXt)": {
"REAL": {
"count": 100,
"mean": 0.17080352000000001,
"std": 0.28986336303905264,
"min": 7e-06,
"p10": 3e-05,
"p25": 0.00036375000000000003,
"p50_median": 0.0111675,
"p75": 0.204376,
"p90": 0.6961787000000005,
"max": 0.999191,
"near_0_pct": 61.0,
"near_1_pct": 4.0
},
"FAKE": {
"count": 100,
"mean": 0.17723096000000002,
"std": 0.2909793071167405,
"min": 6e-06,
"p10": 2.57e-05,
"p25": 0.00022075,
"p50_median": 0.0143455,
"p75": 0.20633325,
"p90": 0.7488821,
"max": 0.999312,
"near_0_pct": 60.0,
"near_1_pct": 5.0
}
},
"Model B (UnivFD)": {
"REAL": {
"count": 100,
"mean": 0.17072465999999997,
"std": 0.26593859610580106,
"min": 2e-06,
"p10": 0.0004388,
"p25": 0.0039127499999999996,
"p50_median": 0.040763999999999995,
"p75": 0.19577699999999998,
"p90": 0.5991829000000012,
"max": 0.963984,
"near_0_pct": 55.00000000000001,
"near_1_pct": 1.0
},
"FAKE": {
"count": 100,
"mean": 0.20455361,
"std": 0.2878792947794924,
"min": 4.5e-05,
"p10": 0.0018044,
"p25": 0.010398,
"p50_median": 0.0450465,
"p75": 0.319957,
"p90": 0.6149121000000005,
"max": 0.999498,
"near_0_pct": 51.0,
"near_1_pct": 4.0
}
},
"Model C (ViT)": {
"REAL": {
"count": 100,
"mean": 0.0466233,
"std": 0.18436697457915283,
"min": 0.000442,
"p10": 0.0006684999999999999,
"p25": 0.0008385,
"p50_median": 0.001536,
"p75": 0.00387075,
"p90": 0.015150500000000018,
"max": 0.997184,
"near_0_pct": 93.0,
"near_1_pct": 3.0
},
"FAKE": {
"count": 100,
"mean": 0.9988112000000001,
"std": 0.0021933990243455472,
"min": 0.981055,
"p10": 0.9981588,
"p25": 0.9990224999999999,
"p50_median": 0.999409,
"p75": 0.9995364999999999,
"p90": 0.999575,
"max": 0.999614,
"near_0_pct": 0.0,
"near_1_pct": 100.0
}
},
"Simple Average": {
"REAL": {
"count": 100,
"mean": 0.12938381999999998,
"std": 0.16391796753854532,
"min": 0.000341,
"p10": 0.0015264,
"p25": 0.0070685,
"p50_median": 0.0651555,
"p75": 0.21157025000000002,
"p90": 0.3199618,
"max": 0.85426,
"near_0_pct": 46.0,
"near_1_pct": 0.0
},
"FAKE": {
"count": 100,
"mean": 0.46019858999999996,
"std": 0.13406834891234357,
"min": 0.332731,
"p10": 0.33618359999999997,
"p25": 0.35098175,
"p50_median": 0.3986105,
"p75": 0.55853175,
"p90": 0.6606218,
"max": 0.844168,
"near_0_pct": 0.0,
"near_1_pct": 0.0
}
},
"Central Analyzer": {
"REAL": {
"count": 100,
"mean": 0.0466233,
"std": 0.18436697457915283,
"min": 0.000442,
"p10": 0.0006684999999999999,
"p25": 0.0008385,
"p50_median": 0.001536,
"p75": 0.00387075,
"p90": 0.015150500000000018,
"max": 0.997184,
"near_0_pct": 93.0,
"near_1_pct": 3.0
},
"FAKE": {
"count": 100,
"mean": 0.9988112000000001,
"std": 0.0021933990243455472,
"min": 0.981055,
"p10": 0.9981588,
"p25": 0.9990224999999999,
"p50_median": 0.999409,
"p75": 0.9995364999999999,
"p90": 0.999575,
"max": 0.999614,
"near_0_pct": 0.0,
"near_1_pct": 100.0
}
}
},
"pearson_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
-0.020652075851289746,
0.08992674816708252
],
[
-0.020652075851289746,
0.9999999999999998,
0.07469369779799365
],
[
0.0899267481670825,
0.07469369779799365,
1.0
]
]
},
"spearman_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
0.005545663597115109,
0.01993564043246151
],
[
0.005545663597115109,
1.0,
0.15843374555807604
],
[
0.01993564043246151,
0.15843374555807604,
1.0
]
]
},
"error_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
0.48971833222960126,
0.0743005758872006
],
[
0.4897183322296013,
1.0,
-0.0671730918385222
],
[
0.0743005758872006,
-0.0671730918385222,
1.0
]
]
},
"error_cooccurrence": {
"total_samples": 200,
"all_3_correct": 76,
"only_1_wrong": 50,
"only_2_wrong": 73,
"all_3_wrong": 1,
"breakdown_1_wrong": {
"convnext_only": 24,
"univfd_only": 25,
"vit_only": 1
},
"breakdown_2_wrong": {
"convnext_and_univfd": 71,
"convnext_and_vit": 2,
"univfd_and_vit": 0
}
}
},
"phase7_test_200": {
"score_distributions": {
"Model A (ConvNeXt)": {
"REAL": {
"count": 100,
"mean": 0.00890046,
"std": 0.031971518845503734,
"min": 4e-06,
"p10": 1.2e-05,
"p25": 2.9e-05,
"p50_median": 0.000137,
"p75": 0.001581,
"p90": 0.016218800000000005,
"max": 0.257813,
"near_0_pct": 95.0,
"near_1_pct": 0.0
},
"FAKE": {
"count": 100,
"mean": 0.15636259,
"std": 0.27693753136005583,
"min": 1e-05,
"p10": 9.81e-05,
"p25": 0.00066175,
"p50_median": 0.018803,
"p75": 0.144734,
"p90": 0.6999319,
"max": 0.998442,
"near_0_pct": 63.0,
"near_1_pct": 3.0
}
},
"Model B (UnivFD)": {
"REAL": {
"count": 100,
"mean": 0.015169800000000002,
"std": 0.04782356724022163,
"min": 3e-06,
"p10": 3.94e-05,
"p25": 0.00018375,
"p50_median": 0.0014965,
"p75": 0.00960425,
"p90": 0.02680290000000006,
"max": 0.417461,
"near_0_pct": 93.0,
"near_1_pct": 0.0
},
"FAKE": {
"count": 100,
"mean": 0.17380799999999996,
"std": 0.2881503140643438,
"min": 0.0,
"p10": 5.72e-05,
"p25": 0.000969,
"p50_median": 0.009413,
"p75": 0.24630575000000002,
"p90": 0.7440687,
"max": 0.994512,
"near_0_pct": 61.0,
"near_1_pct": 2.0
}
},
"Model C (ViT)": {
"REAL": {
"count": 100,
"mean": 0.30094564,
"std": 0.18129056772184923,
"min": 0.00192,
"p10": 0.0462809,
"p25": 0.1576235,
"p50_median": 0.29376,
"p75": 0.43560125,
"p90": 0.5227547,
"max": 0.883132,
"near_0_pct": 11.0,
"near_1_pct": 0.0
},
"FAKE": {
"count": 100,
"mean": 0.29235422000000005,
"std": 0.21904917972951096,
"min": 0.001902,
"p10": 0.0080444,
"p25": 0.114321,
"p50_median": 0.27071350000000005,
"p75": 0.41478400000000004,
"p90": 0.6168822,
"max": 0.991048,
"near_0_pct": 17.0,
"near_1_pct": 1.0
}
},
"Simple Average": {
"REAL": {
"count": 100,
"mean": 0.10833867999999999,
"std": 0.06376218944294182,
"min": 0.000722,
"p10": 0.0214146,
"p25": 0.05384,
"p50_median": 0.110046,
"p75": 0.15990700000000002,
"p90": 0.1793799,
"max": 0.295399,
"near_0_pct": 23.0,
"near_1_pct": 0.0
},
"FAKE": {
"count": 100,
"mean": 0.20750825999999997,
"std": 0.1377042004855785,
"min": 0.001996,
"p10": 0.055523,
"p25": 0.10482825,
"p50_median": 0.1794775,
"p75": 0.3017395,
"p90": 0.3818748,
"max": 0.679002,
"near_0_pct": 9.0,
"near_1_pct": 0.0
}
},
"Central Analyzer": {
"REAL": {
"count": 100,
"mean": 0.30094564,
"std": 0.18129056772184923,
"min": 0.00192,
"p10": 0.0462809,
"p25": 0.1576235,
"p50_median": 0.29376,
"p75": 0.43560125,
"p90": 0.5227547,
"max": 0.883132,
"near_0_pct": 11.0,
"near_1_pct": 0.0
},
"FAKE": {
"count": 100,
"mean": 0.29235422000000005,
"std": 0.21904917972951096,
"min": 0.001902,
"p10": 0.0080444,
"p25": 0.114321,
"p50_median": 0.27071350000000005,
"p75": 0.41478400000000004,
"p90": 0.6168822,
"max": 0.991048,
"near_0_pct": 17.0,
"near_1_pct": 1.0
}
}
},
"pearson_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
0.022705723059898385,
-0.03282118140059127
],
[
0.022705723059898385,
1.0,
-0.08169545067238367
],
[
-0.03282118140059127,
-0.08169545067238368,
1.0
]
]
},
"spearman_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
0.12810530705951026,
-0.07601825566657862
],
[
0.12810530705951026,
1.0,
-0.02539944213486247
],
[
-0.07601825566657862,
-0.02539944213486247,
1.0
]
]
},
"error_correlation": {
"models": [
"ConvNeXt",
"UnivFD",
"ViT"
],
"matrix": [
[
1.0,
0.7349820861314256,
0.5700901716346122
],
[
0.7349820861314257,
1.0,
0.5911561035156886
],
[
0.5700901716346121,
0.5911561035156887,
1.0
]
]
},
"error_cooccurrence": {
"total_samples": 200,
"all_3_correct": 86,
"only_1_wrong": 19,
"only_2_wrong": 36,
"all_3_wrong": 59,
"breakdown_1_wrong": {
"convnext_only": 3,
"univfd_only": 1,
"vit_only": 15
},
"breakdown_2_wrong": {
"convnext_and_univfd": 14,
"convnext_and_vit": 11,
"univfd_and_vit": 11
}
}
}
}