Multimodel_setup / scripts /system_audit /audit_metrics.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
12.2 kB
"""
Comprehensive Diagnostic and Metrics Audit Script for Multi-Model Connection
Computes all metrics and diagnostics required for Step 5, Step 6, and Step 7.
"""
import os
import sys
import json
import math
import numpy as np
import pandas as pd
from scipy.optimize import brentq
from scipy.interpolate import interp1d
from sklearn.metrics import (
accuracy_score,
precision_score,
recall_score,
f1_score,
roc_auc_score,
average_precision_score,
confusion_matrix,
roc_curve,
)
# Project paths
PROJECT_ROOT = r"C:\Users\user\Desktop\multi model connection"
sys.path.insert(0, PROJECT_ROOT)
from src.decision_fusion import DecisionFusionPipeline, STRATEGY_VIT_PRIORITY, STRATEGY_MEAN_PROBABILITY
from src.detector_schema import DetectorResult
def compute_eer(y_true, y_score):
"""Compute Equal Error Rate (EER) and the optimal threshold."""
fpr, tpr, thresholds = roc_curve(y_true, y_score, pos_label=1)
fnr = 1 - tpr
# Find the point where fpr == fnr
try:
eer = brentq(lambda x: 1.0 - interp1d(fpr, tpr)(x) - x, 0.0, 1.0)
except Exception:
# Fallback to closest point
diff = np.abs(fpr - fnr)
min_idx = np.argmin(diff)
eer = (fpr[min_idx] + fnr[min_idx]) / 2.0
return float(eer)
def compute_ece(y_true, y_score, n_bins=10):
"""Compute Expected Calibration Error (ECE)."""
bin_boundaries = np.linspace(0, 1, n_bins + 1)
ece = 0.0
total_samples = len(y_true)
bin_details = []
for i in range(n_bins):
bin_lower = bin_boundaries[i]
bin_upper = bin_boundaries[i + 1]
# Bin inclusion rule: [lower, upper] for last bin, [lower, upper) for others
if i == n_bins - 1:
in_bin = (y_score >= bin_lower) & (y_score <= bin_upper)
else:
in_bin = (y_score >= bin_lower) & (y_score < bin_upper)
prop_in_bin = np.mean(in_bin)
if prop_in_bin > 0:
bin_acc = np.mean(y_true[in_bin])
bin_conf = np.mean(y_score[in_bin])
bin_error = np.abs(bin_acc - bin_conf)
ece += bin_error * prop_in_bin
bin_details.append({
"bin": i,
"range": f"[{bin_lower:.1f}, {bin_upper:.1f}]",
"samples": int(np.sum(in_bin)),
"acc": float(bin_acc),
"conf": float(bin_conf),
"abs_diff": float(bin_error),
})
return float(ece), bin_details
def compute_metrics_table(y_true, y_pred, y_score):
"""Compute standard evaluation metrics."""
tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()
acc = accuracy_score(y_true, y_pred)
prec = precision_score(y_true, y_pred, zero_division=0)
rec = recall_score(y_true, y_pred, zero_division=0)
f1 = f1_score(y_true, y_pred, zero_division=0)
# Check if multiple classes are present in y_true for AUC
if len(np.unique(y_true)) > 1:
roc_auc = roc_auc_score(y_true, y_score)
pr_auc = average_precision_score(y_true, y_score)
eer = compute_eer(y_true, y_score)
else:
roc_auc = float('nan')
pr_auc = float('nan')
eer = float('nan')
return {
"TP": int(tp),
"FP": int(fp),
"TN": int(tn),
"FN": int(fn),
"Accuracy": float(acc),
"Precision": float(prec),
"Recall": float(rec),
"F1": float(f1),
"ROC-AUC": float(roc_auc),
"PR-AUC": float(pr_auc),
"EER": float(eer),
}
def analyze_dataset_predictions(df, ground_truth_col, score_cols, pred_cols):
"""Analyze predictions across models on a dataset."""
results = {}
y_true = (df[ground_truth_col] == "FAKE").astype(int).values
for entity_name in score_cols:
y_score = df[score_cols[entity_name]].values
if entity_name in pred_cols:
y_pred = (df[pred_cols[entity_name]] == "FAKE").astype(int).values
else:
y_pred = (y_score >= 0.50).astype(int)
m = compute_metrics_table(y_true, y_pred, y_score)
ece, _ = compute_ece(y_true, y_score)
m["ECE"] = ece
results[entity_name] = m
return results
def main():
fusion_pipe = DecisionFusionPipeline()
output_dir = os.path.join(PROJECT_ROOT, "scripts", "system_audit")
os.makedirs(output_dir, exist_ok=True)
print("=== Step 5 & 6 Evaluation Audit Runner ===")
# 1. HELD-OUT TEST SET (Phase 4 predictions, 103 classification samples)
p4_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_predictions.csv")
df_p4_long = pd.read_csv(p4_file)
# Filter to eligible samples
df_p4_long = df_p4_long[df_p4_long["status"] == "SUCCESS"].copy()
# Pivot to wide format: one row per sample_id
pivoted_p4 = []
sample_ids = df_p4_long["sample_id"].unique()
for sid in sample_ids:
rows = df_p4_long[df_p4_long["sample_id"] == sid]
sample_meta = rows.iloc[0]
gt = sample_meta["ground_truth"]
if gt not in ("REAL", "FAKE"):
continue
c_row = rows[rows["detector_key"] == "chen2024_convnext"]
u_row = rows[rows["detector_key"] == "univfd"]
v_row = rows[rows["detector_key"] == "vit"]
if c_row.empty or u_row.empty or v_row.empty:
continue
c_prob = float(c_row["fake_probability"].iloc[0])
u_prob = float(u_row["fake_probability"].iloc[0])
v_prob = float(v_row["fake_probability"].iloc[0])
c_pred = c_row["prediction"].iloc[0]
u_pred = u_row["prediction"].iloc[0]
v_pred = v_row["prediction"].iloc[0]
# Simple average
mean_p = round((c_prob + u_prob + v_prob) / 3.0, 6)
mean_pred = "FAKE" if mean_p >= 0.50 else "REAL"
# Central analyzer (ViT Priority)
det_dict = {
"chen2024_convnext": {"prediction": c_pred, "fake_probability": c_prob},
"univfd": {"prediction": u_pred, "fake_probability": u_prob},
"vit": {"prediction": v_pred, "fake_probability": v_prob},
}
fused = fusion_pipe.fuse(det_dict, strategy=STRATEGY_VIT_PRIORITY)
pivoted_p4.append({
"sample_id": sid,
"ground_truth": gt,
"category": sample_meta["category"],
"domain": sample_meta["domain"],
"file_path": sample_meta["file_path"],
"convnext_prob": c_prob,
"convnext_pred": c_pred,
"univfd_prob": u_prob,
"univfd_pred": u_pred,
"vit_prob": v_prob,
"vit_pred": v_pred,
"simple_avg_prob": mean_p,
"simple_avg_pred": mean_pred,
"analyzer_prob": fused.fake_probability,
"analyzer_pred": fused.prediction,
})
df_p4 = pd.DataFrame(pivoted_p4)
print(f"Held-out test set processed: {len(df_p4)} samples")
score_cols_p4 = {
"Model A (ConvNeXt)": "convnext_prob",
"Model B (UnivFD)": "univfd_prob",
"Model C (ViT)": "vit_prob",
"Simple Average": "simple_avg_prob",
"Central Analyzer (ViT-Priority)": "analyzer_prob",
}
pred_cols_p4 = {
"Model A (ConvNeXt)": "convnext_pred",
"Model B (UnivFD)": "univfd_pred",
"Model C (ViT)": "vit_pred",
"Simple Average": "simple_avg_pred",
"Central Analyzer (ViT-Priority)": "analyzer_pred",
}
metrics_held_out = analyze_dataset_predictions(df_p4, "ground_truth", score_cols_p4, pred_cols_p4)
# 2. VALIDATION SET (Phase 5, 50 samples)
p5_val_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase5_val_raw_predictions.csv")
df_p5_long = pd.read_csv(p5_val_file)
pivoted_p5 = []
for sid in df_p5_long["sample_id"].unique():
rows = df_p5_long[df_p5_long["sample_id"] == sid]
sample_meta = rows.iloc[0]
gt = sample_meta["ground_truth"]
c_row = rows[rows["detector_key"] == "chen2024_convnext"]
u_row = rows[rows["detector_key"] == "univfd"]
v_row = rows[rows["detector_key"] == "vit"]
c_prob = float(c_row["fake_probability"].iloc[0])
u_prob = float(u_row["fake_probability"].iloc[0])
v_prob = float(v_row["fake_probability"].iloc[0])
c_pred = c_row["prediction"].iloc[0]
u_pred = u_row["prediction"].iloc[0]
v_pred = v_row["prediction"].iloc[0]
mean_p = round((c_prob + u_prob + v_prob) / 3.0, 6)
mean_pred = "FAKE" if mean_p >= 0.50 else "REAL"
det_dict = {
"chen2024_convnext": {"prediction": c_pred, "fake_probability": c_prob},
"univfd": {"prediction": u_pred, "fake_probability": u_prob},
"vit": {"prediction": v_pred, "fake_probability": v_prob},
}
fused = fusion_pipe.fuse(det_dict, strategy=STRATEGY_VIT_PRIORITY)
pivoted_p5.append({
"sample_id": sid,
"ground_truth": gt,
"convnext_prob": c_prob,
"convnext_pred": c_pred,
"univfd_prob": u_prob,
"univfd_pred": u_pred,
"vit_prob": v_prob,
"vit_pred": v_pred,
"simple_avg_prob": mean_p,
"simple_avg_pred": mean_pred,
"analyzer_prob": fused.fake_probability,
"analyzer_pred": fused.prediction,
})
df_p5 = pd.DataFrame(pivoted_p5)
print(f"Validation set processed: {len(df_p5)} samples")
metrics_val = analyze_dataset_predictions(df_p5, "ground_truth", score_cols_p4, pred_cols_p4)
# 3. PHASE 6 TEST SET (200 samples)
p6_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase6_combined_predictions.csv")
df_p6 = pd.read_csv(p6_file)
score_cols_p6 = {
"Model A (ConvNeXt)": "convnext_fake_prob",
"Model B (UnivFD)": "univfd_fake_prob",
"Model C (ViT)": "vit_fake_prob",
"Simple Average": "mean_prob_fake_prob",
"Central Analyzer (ViT-Priority)": "vit_priority_fake_prob",
}
pred_cols_p6 = {
"Model A (ConvNeXt)": "convnext_pred",
"Model B (UnivFD)": "univfd_pred",
"Model C (ViT)": "vit_pred",
"Simple Average": "mean_prob_pred",
"Central Analyzer (ViT-Priority)": "vit_priority_pred",
}
metrics_p6 = analyze_dataset_predictions(df_p6, "ground_truth", score_cols_p6, pred_cols_p6)
print(f"Phase 6 test set processed: {len(df_p6)} samples")
# 4. PHASE 7 TEST SET (200 samples)
p7_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase7_combined_predictions.csv")
df_p7 = pd.read_csv(p7_file)
metrics_p7 = analyze_dataset_predictions(df_p7, "ground_truth", score_cols_p6, pred_cols_p6)
print(f"Phase 7 test set processed: {len(df_p7)} samples")
# Save all tables to JSON
audit_results = {
"held_out_test_103": metrics_held_out,
"development_val_50": metrics_val,
"phase6_test_200": metrics_p6,
"phase7_test_200": metrics_p7,
}
with open(os.path.join(output_dir, "step5_metrics.json"), "w") as f:
json.dump(audit_results, f, indent=2)
print("Saved step5_metrics.json")
# Save formatted markdown tables for easy reading
print("\n--- HELD-OUT TEST (103 samples) ---")
print(pd.DataFrame(metrics_held_out).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]])
print("\n--- PHASE 6 TEST (200 samples) ---")
print(pd.DataFrame(metrics_p6).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]])
print("\n--- VALIDATION SET (50 samples) ---")
print(pd.DataFrame(metrics_val).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]])
print("\n--- PHASE 7 TEST (200 samples) ---")
print(pd.DataFrame(metrics_p7).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]])
if __name__ == "__main__":
main()