Spaces:
Running on Zero
Running on Zero
File size: 6,069 Bytes
151f388 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """
Robustness Quick Test for Step 6g
Evaluates 20 test samples (10 Real, 10 Fake from Phase 6 manifest) under:
1. Baseline clean images
2. JPEG compression (quality = 75)
3. Downscaling (50% scale, resized back via bicubic)
Runs sequentially with memory-safe model loading/unloading.
"""
import os
import io
import sys
import gc
import json
import torch
import numpy as np
import pandas as pd
from PIL import Image
PROJECT_ROOT = r"C:\Users\user\Desktop\multi model connection"
sys.path.insert(0, PROJECT_ROOT)
from src.detector_registry import get_detector
from src.decision_fusion import DecisionFusionPipeline, STRATEGY_VIT_PRIORITY
def main():
print("=== Step 6g: Robustness Evaluation Runner ===")
manifest_path = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv")
df = pd.read_csv(manifest_path)
# Select first 10 Real and first 10 Fake
real_subset = df[df["ground_truth"] == "REAL"].head(10)
fake_subset = df[df["ground_truth"] == "FAKE"].head(10)
eval_df = pd.concat([real_subset, fake_subset], ignore_index=True)
perturbations = ["clean", "jpeg_75", "downscale_half"]
# Store predictions: {det_key: {perturbation: [{sample_id, gt, prob, pred}]}}
raw_results = {k: {p: [] for p in perturbations} for k in ["chen2024_convnext", "univfd", "vit"]}
for det_key in ["chen2024_convnext", "univfd", "vit"]:
print(f"\nEvaluating detector: {det_key}")
adapter = get_detector(det_key, lazy_load=False, singleton=False)
for p in perturbations:
print(f" Condition: {p}")
for _, row in eval_df.iterrows():
img_path = row["absolute_path"]
gt = row["ground_truth"]
sid = row["sample_id"]
with Image.open(img_path) as img:
img_rgb = img.convert("RGB")
if p == "clean":
test_img = img_rgb
elif p == "jpeg_75":
buf = io.BytesIO()
img_rgb.save(buf, format="JPEG", quality=75)
buf.seek(0)
test_img = Image.open(buf).convert("RGB")
elif p == "downscale_half":
w, h = img_rgb.size
# Downscale by 50% then upscale back to original size
small = img_rgb.resize((w // 2, h // 2), Image.Resampling.BILINEAR)
test_img = small.resize((w, h), Image.Resampling.BICUBIC)
res = adapter.predict(test_img)
raw_results[det_key][p].append({
"sample_id": sid,
"ground_truth": gt,
"prediction": res.prediction,
"fake_probability": res.fake_probability,
})
# Explicitly unload and free memory
adapter.unload_model()
del adapter
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
# Now compute metrics across perturbations including simple average and central analyzer
fusion_pipe = DecisionFusionPipeline()
metrics_summary = {}
for p in perturbations:
metrics_summary[p] = {}
# Reconstruct rows for this perturbation
fused_rows = []
for i in range(len(eval_df)):
sid = eval_df.iloc[i]["sample_id"]
gt = eval_df.iloc[i]["ground_truth"]
c_res = raw_results["chen2024_convnext"][p][i]
u_res = raw_results["univfd"][p][i]
v_res = raw_results["vit"][p][i]
c_prob = c_res["fake_probability"]
u_prob = u_res["fake_probability"]
v_prob = v_res["fake_probability"]
mean_p = round((c_prob + u_prob + v_prob) / 3.0, 6)
mean_pred = "FAKE" if mean_p >= 0.50 else "REAL"
det_dict = {
"chen2024_convnext": {"prediction": c_res["prediction"], "fake_probability": c_prob},
"univfd": {"prediction": u_res["prediction"], "fake_probability": u_prob},
"vit": {"prediction": v_res["prediction"], "fake_probability": v_prob},
}
fused = fusion_pipe.fuse(det_dict, strategy=STRATEGY_VIT_PRIORITY)
fused_rows.append({
"sample_id": sid,
"ground_truth": gt,
"convnext_pred": c_res["prediction"],
"univfd_pred": u_res["prediction"],
"vit_pred": v_res["prediction"],
"simple_avg_pred": mean_pred,
"analyzer_pred": fused.prediction,
})
df_p = pd.DataFrame(fused_rows)
y_true = (df_p["ground_truth"] == "FAKE").astype(int).values
for ent_name, col in [
("Model A (ConvNeXt)", "convnext_pred"),
("Model B (UnivFD)", "univfd_pred"),
("Model C (ViT)", "vit_pred"),
("Simple Average", "simple_avg_pred"),
("Central Analyzer (ViT-Priority)", "analyzer_pred"),
]:
y_pred = (df_p[col] == "FAKE").astype(int).values
acc = float(np.mean(y_true == y_pred))
real_acc = float(np.mean((y_true == 0) == (y_pred == 0)))
fake_acc = float(np.mean((y_true == 1) == (y_pred == 1)))
metrics_summary[p][ent_name] = {
"accuracy": acc,
"real_recall": real_acc,
"fake_recall": fake_acc,
}
out_file = os.path.join(PROJECT_ROOT, "scripts", "system_audit", "robustness_results.json")
with open(out_file, "w") as f:
json.dump(metrics_summary, f, indent=2)
print(f"\nRobustness test finished! Results saved to {out_file}")
print(json.dumps(metrics_summary, indent=2))
if __name__ == "__main__":
main()
|