Multimodel_setup / scripts /system_audit /audit_robustness.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
6.07 kB
"""
Robustness Quick Test for Step 6g
Evaluates 20 test samples (10 Real, 10 Fake from Phase 6 manifest) under:
1. Baseline clean images
2. JPEG compression (quality = 75)
3. Downscaling (50% scale, resized back via bicubic)
Runs sequentially with memory-safe model loading/unloading.
"""
import os
import io
import sys
import gc
import json
import torch
import numpy as np
import pandas as pd
from PIL import Image
PROJECT_ROOT = r"C:\Users\user\Desktop\multi model connection"
sys.path.insert(0, PROJECT_ROOT)
from src.detector_registry import get_detector
from src.decision_fusion import DecisionFusionPipeline, STRATEGY_VIT_PRIORITY
def main():
print("=== Step 6g: Robustness Evaluation Runner ===")
manifest_path = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv")
df = pd.read_csv(manifest_path)
# Select first 10 Real and first 10 Fake
real_subset = df[df["ground_truth"] == "REAL"].head(10)
fake_subset = df[df["ground_truth"] == "FAKE"].head(10)
eval_df = pd.concat([real_subset, fake_subset], ignore_index=True)
perturbations = ["clean", "jpeg_75", "downscale_half"]
# Store predictions: {det_key: {perturbation: [{sample_id, gt, prob, pred}]}}
raw_results = {k: {p: [] for p in perturbations} for k in ["chen2024_convnext", "univfd", "vit"]}
for det_key in ["chen2024_convnext", "univfd", "vit"]:
print(f"\nEvaluating detector: {det_key}")
adapter = get_detector(det_key, lazy_load=False, singleton=False)
for p in perturbations:
print(f" Condition: {p}")
for _, row in eval_df.iterrows():
img_path = row["absolute_path"]
gt = row["ground_truth"]
sid = row["sample_id"]
with Image.open(img_path) as img:
img_rgb = img.convert("RGB")
if p == "clean":
test_img = img_rgb
elif p == "jpeg_75":
buf = io.BytesIO()
img_rgb.save(buf, format="JPEG", quality=75)
buf.seek(0)
test_img = Image.open(buf).convert("RGB")
elif p == "downscale_half":
w, h = img_rgb.size
# Downscale by 50% then upscale back to original size
small = img_rgb.resize((w // 2, h // 2), Image.Resampling.BILINEAR)
test_img = small.resize((w, h), Image.Resampling.BICUBIC)
res = adapter.predict(test_img)
raw_results[det_key][p].append({
"sample_id": sid,
"ground_truth": gt,
"prediction": res.prediction,
"fake_probability": res.fake_probability,
})
# Explicitly unload and free memory
adapter.unload_model()
del adapter
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
# Now compute metrics across perturbations including simple average and central analyzer
fusion_pipe = DecisionFusionPipeline()
metrics_summary = {}
for p in perturbations:
metrics_summary[p] = {}
# Reconstruct rows for this perturbation
fused_rows = []
for i in range(len(eval_df)):
sid = eval_df.iloc[i]["sample_id"]
gt = eval_df.iloc[i]["ground_truth"]
c_res = raw_results["chen2024_convnext"][p][i]
u_res = raw_results["univfd"][p][i]
v_res = raw_results["vit"][p][i]
c_prob = c_res["fake_probability"]
u_prob = u_res["fake_probability"]
v_prob = v_res["fake_probability"]
mean_p = round((c_prob + u_prob + v_prob) / 3.0, 6)
mean_pred = "FAKE" if mean_p >= 0.50 else "REAL"
det_dict = {
"chen2024_convnext": {"prediction": c_res["prediction"], "fake_probability": c_prob},
"univfd": {"prediction": u_res["prediction"], "fake_probability": u_prob},
"vit": {"prediction": v_res["prediction"], "fake_probability": v_prob},
}
fused = fusion_pipe.fuse(det_dict, strategy=STRATEGY_VIT_PRIORITY)
fused_rows.append({
"sample_id": sid,
"ground_truth": gt,
"convnext_pred": c_res["prediction"],
"univfd_pred": u_res["prediction"],
"vit_pred": v_res["prediction"],
"simple_avg_pred": mean_pred,
"analyzer_pred": fused.prediction,
})
df_p = pd.DataFrame(fused_rows)
y_true = (df_p["ground_truth"] == "FAKE").astype(int).values
for ent_name, col in [
("Model A (ConvNeXt)", "convnext_pred"),
("Model B (UnivFD)", "univfd_pred"),
("Model C (ViT)", "vit_pred"),
("Simple Average", "simple_avg_pred"),
("Central Analyzer (ViT-Priority)", "analyzer_pred"),
]:
y_pred = (df_p[col] == "FAKE").astype(int).values
acc = float(np.mean(y_true == y_pred))
real_acc = float(np.mean((y_true == 0) == (y_pred == 0)))
fake_acc = float(np.mean((y_true == 1) == (y_pred == 1)))
metrics_summary[p][ent_name] = {
"accuracy": acc,
"real_recall": real_acc,
"fake_recall": fake_acc,
}
out_file = os.path.join(PROJECT_ROOT, "scripts", "system_audit", "robustness_results.json")
with open(out_file, "w") as f:
json.dump(metrics_summary, f, indent=2)
print(f"\nRobustness test finished! Results saved to {out_file}")
print(json.dumps(metrics_summary, indent=2))
if __name__ == "__main__":
main()