File size: 10,697 Bytes
151f388
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
"""
Step 6 & 7 Diagnostics and Failure Analysis Script
"""

import os
import sys
import json
import hashlib
import numpy as np
import pandas as pd
from scipy.stats import pearsonr, spearmanr
from PIL import Image

PROJECT_ROOT = r"C:\Users\user\Desktop\multi model connection"
sys.path.insert(0, PROJECT_ROOT)

def compute_distribution_stats(series):
    """Compute summary statistics and percentiles for a series of scores."""
    arr = series.dropna().values
    return {
        "count": int(len(arr)),
        "mean": float(np.mean(arr)),
        "std": float(np.std(arr)),
        "min": float(np.min(arr)),
        "p10": float(np.percentile(arr, 10)),
        "p25": float(np.percentile(arr, 25)),
        "p50_median": float(np.percentile(arr, 50)),
        "p75": float(np.percentile(arr, 75)),
        "p90": float(np.percentile(arr, 90)),
        "max": float(np.max(arr)),
        "near_0_pct": float(np.mean(arr < 0.05) * 100.0),
        "near_1_pct": float(np.mean(arr > 0.95) * 100.0),
    }

def run_diagnostics():
    output_dir = os.path.join(PROJECT_ROOT, "scripts", "system_audit")
    os.makedirs(output_dir, exist_ok=True)

    print("=== Step 6: Running Diagnostics ===")
    
    # We will run diagnostics primarily on Phase 6 (in-domain faces, 200 samples)
    # and Phase 7 (out-of-domain GenImage, 200 samples), as well as held-out test (103 samples).
    
    # 1. Load Phase 6
    df6 = pd.read_csv(os.path.join(PROJECT_ROOT, "benchmark", "results", "phase6_combined_predictions.csv"))
    
    # 2. Load Phase 7
    df7 = pd.read_csv(os.path.join(PROJECT_ROOT, "benchmark", "results", "phase7_combined_predictions.csv"))
    
    # 3. Load Phase 4 held-out test (103 samples)
    p4_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_predictions.csv")
    df_p4_long = pd.read_csv(p4_file)
    df_p4_long = df_p4_long[df_p4_long["status"] == "SUCCESS"].copy()
    
    pivoted_p4 = []
    for sid in df_p4_long["sample_id"].unique():
        rows = df_p4_long[df_p4_long["sample_id"] == sid]
        sample_meta = rows.iloc[0]
        gt = sample_meta["ground_truth"]
        if gt not in ("REAL", "FAKE"):
            continue
        c_row = rows[rows["detector_key"] == "chen2024_convnext"]
        u_row = rows[rows["detector_key"] == "univfd"]
        v_row = rows[rows["detector_key"] == "vit"]
        if c_row.empty or u_row.empty or v_row.empty:
            continue
        c_p = float(c_row["fake_probability"].iloc[0])
        u_p = float(u_row["fake_probability"].iloc[0])
        v_p = float(v_row["fake_probability"].iloc[0])
        c_pred = c_row["prediction"].iloc[0]
        u_pred = u_row["prediction"].iloc[0]
        v_pred = v_row["prediction"].iloc[0]
        mean_p = round((c_p + u_p + v_p) / 3.0, 6)
        mean_pred = "FAKE" if mean_p >= 0.50 else "REAL"
        
        # analyzer
        analyzer_pred = v_pred
        analyzer_prob = v_p
        
        pivoted_p4.append({
            "sample_id": sid,
            "original_filename": sample_meta["file_path"].replace("\\", "/").split("/")[-1],
            "ground_truth": gt,
            "category": sample_meta["category"],
            "convnext_fake_prob": c_p,
            "convnext_pred": c_pred,
            "univfd_fake_prob": u_p,
            "univfd_pred": u_pred,
            "vit_fake_prob": v_p,
            "vit_pred": v_pred,
            "mean_prob_fake_prob": mean_p,
            "mean_prob_pred": mean_pred,
            "vit_priority_fake_prob": analyzer_prob,
            "vit_priority_pred": analyzer_pred,
        })
    df_p4 = pd.DataFrame(pivoted_p4)

    diagnostics_result = {}

    for name, df in [("held_out_test_103", df_p4), ("phase6_test_200", df6), ("phase7_test_200", df7)]:
        print(f"\n--- Processing {name} ---")
        real_df = df[df["ground_truth"] == "REAL"]
        fake_df = df[df["ground_truth"] == "FAKE"]
        
        # a) Score distributions
        score_dist = {}
        for model_col, model_label in [
            ("convnext_fake_prob", "Model A (ConvNeXt)"),
            ("univfd_fake_prob", "Model B (UnivFD)"),
            ("vit_fake_prob", "Model C (ViT)"),
            ("mean_prob_fake_prob", "Simple Average"),
            ("vit_priority_fake_prob", "Central Analyzer"),
        ]:
            score_dist[model_label] = {
                "REAL": compute_distribution_stats(real_df[model_col]),
                "FAKE": compute_distribution_stats(fake_df[model_col]),
            }
            
        # b) Correlation matrix between 3 models' scores
        scores = df[["convnext_fake_prob", "univfd_fake_prob", "vit_fake_prob"]].values
        pearson_mat = np.corrcoef(scores.T)
        
        # Spearman correlation
        spearman_mat = np.zeros((3, 3))
        for i in range(3):
            for j in range(3):
                spearman_mat[i, j] = spearmanr(scores[:, i], scores[:, j])[0]
                
        # Error correlation
        y_true = (df["ground_truth"] == "FAKE").astype(int).values
        c_err = ((df["convnext_pred"] == "FAKE").astype(int).values != y_true).astype(int)
        u_err = ((df["univfd_pred"] == "FAKE").astype(int).values != y_true).astype(int)
        v_err = ((df["vit_pred"] == "FAKE").astype(int).values != y_true).astype(int)
        
        err_matrix = np.column_stack([c_err, u_err, v_err])
        # Error correlation
        err_corr = np.corrcoef(err_matrix.T)
        
        # Error co-occurrence counts
        total_errs_per_sample = c_err + u_err + v_err
        all_3_correct = int(np.sum(total_errs_per_sample == 0))
        only_1_wrong = int(np.sum(total_errs_per_sample == 1))
        only_2_wrong = int(np.sum(total_errs_per_sample == 2))
        all_3_wrong = int(np.sum(total_errs_per_sample == 3))
        
        breakdown_1_wrong = {
            "convnext_only": int(np.sum((c_err == 1) & (u_err == 0) & (v_err == 0))),
            "univfd_only": int(np.sum((c_err == 0) & (u_err == 1) & (v_err == 0))),
            "vit_only": int(np.sum((c_err == 0) & (u_err == 0) & (v_err == 1))),
        }
        breakdown_2_wrong = {
            "convnext_and_univfd": int(np.sum((c_err == 1) & (u_err == 1) & (v_err == 0))),
            "convnext_and_vit": int(np.sum((c_err == 1) & (u_err == 0) & (v_err == 1))),
            "univfd_and_vit": int(np.sum((c_err == 0) & (u_err == 1) & (v_err == 1))),
        }

        diagnostics_result[name] = {
            "score_distributions": score_dist,
            "pearson_correlation": {
                "models": ["ConvNeXt", "UnivFD", "ViT"],
                "matrix": pearson_mat.tolist(),
            },
            "spearman_correlation": {
                "models": ["ConvNeXt", "UnivFD", "ViT"],
                "matrix": spearman_mat.tolist(),
            },
            "error_correlation": {
                "models": ["ConvNeXt", "UnivFD", "ViT"],
                "matrix": err_corr.tolist(),
            },
            "error_cooccurrence": {
                "total_samples": len(df),
                "all_3_correct": all_3_correct,
                "only_1_wrong": only_1_wrong,
                "only_2_wrong": only_2_wrong,
                "all_3_wrong": all_3_wrong,
                "breakdown_1_wrong": breakdown_1_wrong,
                "breakdown_2_wrong": breakdown_2_wrong,
            }
        }
        
    # Save diagnostics
    with open(os.path.join(output_dir, "step6_diagnostics.json"), "w") as f:
        json.dump(diagnostics_result, f, indent=2)
    print("Saved step6_diagnostics.json")

    # Save 10 misclassified examples per type (FP and FN) for Step 7
    # We will pull from Phase 6 (StyleGAN faces) and Phase 7 (GenImage scenes)
    failures = {}
    for dname, df in [("phase6_facial", df6), ("phase7_general_scene", df7), ("held_out_test", df_p4)]:
        y_true = (df["ground_truth"] == "FAKE").astype(int).values
        # Analyzer predictions
        ana_pred = (df["vit_priority_pred"] == "FAKE").astype(int).values
        
        fp_indices = np.where((y_true == 0) & (ana_pred == 1))[0]
        fn_indices = np.where((y_true == 1) & (ana_pred == 0))[0]
        
        # Also check where ANY base model failed
        c_pred = (df["convnext_pred"] == "FAKE").astype(int).values
        u_pred = (df["univfd_pred"] == "FAKE").astype(int).values
        v_pred = (df["vit_pred"] == "FAKE").astype(int).values
        
        # Gather samples
        fp_samples = []
        for idx in fp_indices[:10]:
            row = df.iloc[idx]
            fp_samples.append({
                "sample_id": str(row["sample_id"]),
                "filename": str(row.get("original_filename", row["sample_id"])),
                "ground_truth": "REAL",
                "convnext_prob": float(row["convnext_fake_prob"]),
                "convnext_pred": str(row["convnext_pred"]),
                "univfd_prob": float(row["univfd_fake_prob"]),
                "univfd_pred": str(row["univfd_pred"]),
                "vit_prob": float(row["vit_fake_prob"]),
                "vit_pred": str(row["vit_pred"]),
                "simple_avg_prob": float(row["mean_prob_fake_prob"]),
                "analyzer_pred": str(row["vit_priority_pred"]),
                "analyzer_prob": float(row["vit_priority_fake_prob"]),
            })
            
        fn_samples = []
        for idx in fn_indices[:10]:
            row = df.iloc[idx]
            fn_samples.append({
                "sample_id": str(row["sample_id"]),
                "filename": str(row.get("original_filename", row["sample_id"])),
                "ground_truth": "FAKE",
                "generator": str(row.get("generator_family", "StyleGAN")),
                "convnext_prob": float(row["convnext_fake_prob"]),
                "convnext_pred": str(row["convnext_pred"]),
                "univfd_prob": float(row["univfd_fake_prob"]),
                "univfd_pred": str(row["univfd_pred"]),
                "vit_prob": float(row["vit_fake_prob"]),
                "vit_pred": str(row["vit_pred"]),
                "simple_avg_prob": float(row["mean_prob_fake_prob"]),
                "analyzer_pred": str(row["vit_priority_pred"]),
                "analyzer_prob": float(row["vit_priority_fake_prob"]),
            })
            
        failures[dname] = {
            "false_positives_count": len(fp_indices),
            "false_negatives_count": len(fn_indices),
            "fp_examples_top10": fp_samples,
            "fn_examples_top10": fn_samples,
        }
        
    with open(os.path.join(output_dir, "step7_failures.json"), "w") as f:
        json.dump(failures, f, indent=2)
    print("Saved step7_failures.json")

if __name__ == "__main__":
    run_diagnostics()