Multimodel_setup / scripts /evaluate_phase7_pipeline.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
27.7 kB
"""
Phase 7 External Generalization Evaluation Pipeline Runner
Evaluates all three integrated deepfake detectors (ViT, UnivFD, ConvNeXt/DRCT)
independently against benchmark/external_generalization_manifest.csv
(200 GenImage validation samples: 100 REAL ImageNet photographs, 50 FAKE Midjourney, 50 FAKE ADM).
Then evaluates ensemble combination strategies (Majority Vote, Mean Fake Probability, ViT-Priority Fallback).
Memory Safety:
- Sequential subprocess isolation per detector guarantees 100% OS-level reclamation of virtual memory.
- Complete garbage collection and instance resetting between runs.
"""
import os
import sys
import json
import csv
import time
import math
import argparse
import subprocess
import tempfile
import gc
from typing import Dict, Any, List, Tuple, Optional
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
import torch
from sklearn.metrics import roc_auc_score
from src.detector_registry import (
get_detector,
get_detector_info,
reset_instances,
)
from src.detector_schema import (
DetectorResult,
PREDICTION_REAL,
PREDICTION_FAKE,
PREDICTION_ERROR,
PREDICTION_UNCERTAIN,
)
from src.decision_fusion import (
DecisionFusionPipeline,
strategy_vit_only,
strategy_majority_vote,
strategy_mean_fake_probability,
strategy_vit_priority,
STRATEGY_VIT_ONLY,
STRATEGY_MAJORITY_VOTE,
STRATEGY_MEAN_PROBABILITY,
STRATEGY_VIT_PRIORITY,
)
MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv")
RESULTS_DIR = os.path.join(PROJECT_ROOT, "benchmark", "results")
OUTPUT_VIT_CSV = os.path.join(RESULTS_DIR, "phase7_vit_predictions.csv")
OUTPUT_UNIVFD_CSV = os.path.join(RESULTS_DIR, "phase7_univfd_predictions.csv")
OUTPUT_CONVNEXT_CSV = os.path.join(RESULTS_DIR, "phase7_convnext_predictions.csv")
OUTPUT_COMBINED_CSV = os.path.join(RESULTS_DIR, "phase7_combined_predictions.csv")
OUTPUT_METRICS_JSON = os.path.join(RESULTS_DIR, "phase7_combined_metrics.json")
OUTPUT_COMPARISON_CSV = os.path.join(RESULTS_DIR, "phase7_model_comparison.csv")
DETECTOR_KEYS = ["vit", "univfd", "chen2024_convnext"]
def wilson_score_interval(k: int, n: int, confidence: float = 0.95) -> List[float]:
"""Computes Wilson score confidence interval for binomial proportion k / n."""
if n <= 0:
return [0.0, 0.0]
p_hat = k / n
z = 1.959963984540054
z2 = z * z
denom = 1.0 + z2 / n
center = (p_hat + z2 / (2.0 * n)) / denom
margin = z * math.sqrt((p_hat * (1.0 - p_hat) / n) + (z2 / (4.0 * n * n))) / denom
lower = max(0.0, center - margin)
upper = min(1.0, center + margin)
return [round(lower, 4), round(upper, 4)]
def compute_cohen_kappa(both_fake: int, both_real: int, m1_f_m2_r: int, m1_r_m2_f: int) -> float:
"""Computes Cohen's Kappa for 2x2 agreement matrix."""
n = both_fake + both_real + m1_f_m2_r + m1_r_m2_f
if n == 0:
return 0.0
po = (both_fake + both_real) / n
p_f1 = (both_fake + m1_f_m2_r) / n
p_r1 = (m1_r_m2_f + both_real) / n
p_f2 = (both_fake + m1_r_m2_f) / n
p_r2 = (m1_f_m2_r + both_real) / n
pe = (p_f1 * p_f2) + (p_r1 * p_r2)
if abs(1.0 - pe) < 1e-9:
return 1.0 if abs(po - 1.0) < 1e-9 else 0.0
kappa = (po - pe) / (1.0 - pe)
return round(kappa, 4)
def run_worker(det_key: str, input_json_path: str, output_json_path: str) -> None:
"""Worker function executed in isolated child process for a single detector."""
print(f" [Worker Process] Initializing detector '{det_key}' in clean subprocess...")
with open(input_json_path, "r", encoding="utf-8") as f:
samples = json.load(f)
info = get_detector_info(det_key)
meta = info.get("metadata", {})
fallback_det_name = meta.get("detector_name", det_key)
fallback_model_name = meta.get("model_name", "Unknown")
adapter = None
load_err = None
t0 = time.perf_counter()
try:
adapter = get_detector(det_key, lazy_load=False, singleton=False)
load_time_sec = round(time.perf_counter() - t0, 3)
print(f" [Worker Process] Detector '{det_key}' loaded successfully in {load_time_sec}s.")
except Exception as e:
load_err = str(e)
load_time_sec = round(time.perf_counter() - t0, 3)
print(f" [Worker Process] ERROR loading detector '{det_key}': {load_err}")
results = []
latencies = []
for idx, sample in enumerate(samples, 1):
file_path = sample["file_path"]
sample_id = sample["sample_id"]
sample_t0 = time.perf_counter()
if load_err is not None or adapter is None:
err_res = DetectorResult.create_error(
detector_name=fallback_det_name,
model_name=fallback_model_name,
error_message=f"Model initialization failed: {load_err}",
)
res_dict = err_res.to_dict()
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
elif not os.path.exists(file_path):
err_res = DetectorResult.create_error(
detector_name=fallback_det_name,
model_name=fallback_model_name,
error_message=f"File not found on disk: {file_path}",
)
res_dict = err_res.to_dict()
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
else:
try:
res = adapter.predict(file_path)
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
res_dict = res.to_dict()
except Exception as inf_exc:
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
err_res = DetectorResult.create_error(
detector_name=fallback_det_name,
model_name=fallback_model_name,
error_message=f"Inference exception: {str(inf_exc)}",
)
res_dict = err_res.to_dict()
res_dict["latency_ms"] = sample_latency_ms
latencies.append(sample_latency_ms)
record = {
"sample_id": sample_id,
"detector_key": det_key,
**res_dict
}
results.append(record)
if idx % 50 == 0 or idx == len(samples):
print(f" [Worker Process] Progress: {idx}/{len(samples)} samples processed.")
# Explicit cleanup
if adapter is not None and hasattr(adapter, "unload_model"):
try:
adapter.unload_model()
except Exception:
pass
del adapter
reset_instances()
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
with open(output_json_path, "w", encoding="utf-8") as f:
json.dump({
"results": results,
"load_time_sec": load_time_sec,
"mean_latency_ms": round(sum(latencies) / len(latencies), 2) if latencies else 0.0
}, f, indent=2)
print(f" [Worker Process] Successfully completed all {len(results)} samples for '{det_key}'.")
def run_detector_subprocess(det_key: str, samples: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], float, float]:
"""Runs a detector in an isolated subprocess to guarantee memory reclamation."""
with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_in:
json.dump(samples, f_in)
in_path = f_in.name
with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_out:
out_path = f_out.name
cmd = [
sys.executable,
os.path.abspath(__file__),
"--worker",
"--worker-detector", det_key,
"--worker-input", in_path,
"--worker-output", out_path,
]
try:
proc = subprocess.run(cmd, capture_output=True, text=True, check=True)
print(proc.stdout.strip())
with open(out_path, "r", encoding="utf-8") as f:
data = json.load(f)
return data["results"], data["load_time_sec"], data["mean_latency_ms"]
except subprocess.CalledProcessError as exc:
print(f" [ERROR] Subprocess for {det_key} failed with return code {exc.returncode}")
print(" --- Subprocess STDOUT ---")
print(exc.stdout)
print(" --- Subprocess STDERR ---")
print(exc.stderr)
raise RuntimeError(f"Detector evaluation subprocess failed for {det_key}") from exc
finally:
for p in [in_path, out_path]:
if os.path.exists(p):
try:
os.remove(p)
except Exception:
pass
def compute_binary_metrics(predictions: List[str], ground_truths: List[str], probabilities: Optional[List[float]] = None) -> Dict[str, Any]:
"""Computes binary classification metrics, Wilson CIs, and ROC-AUC."""
tp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_FAKE)
fp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_REAL)
tn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_REAL)
fn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_FAKE)
error_or_uncertain = sum(1 for p in predictions if p not in (PREDICTION_REAL, PREDICTION_FAKE))
total = len(predictions)
valid_total = tp + fp + tn + fn
actual_pos = tp + fn
actual_neg = tn + fp
accuracy = round((tp + tn) / valid_total, 4) if valid_total > 0 else 0.0
accuracy_ci = wilson_score_interval(tp + tn, valid_total) if valid_total > 0 else [0.0, 0.0]
recall = round(tp / actual_pos, 4) if actual_pos > 0 else 0.0
recall_ci = wilson_score_interval(tp, actual_pos) if actual_pos > 0 else [0.0, 0.0]
specificity = round(tn / actual_neg, 4) if actual_neg > 0 else 0.0
specificity_ci = wilson_score_interval(tn, actual_neg) if actual_neg > 0 else [0.0, 0.0]
precision = round(tp / (tp + fp), 4) if (tp + fp) > 0 else 0.0
precision_ci = wilson_score_interval(tp, tp + fp) if (tp + fp) > 0 else [0.0, 0.0]
balanced_acc = round(0.5 * (recall + specificity), 4)
if (precision + recall) > 0:
f1 = round(2.0 * precision * recall / (precision + recall), 4)
else:
f1 = 0.0
roc_auc = None
if probabilities is not None and len(probabilities) == len(ground_truths) and actual_pos > 0 and actual_neg > 0:
try:
y_binary = [1 if g == PREDICTION_FAKE else 0 for g in ground_truths]
# Filter non-null probs
valid_pairs = [(y, p) for y, p in zip(y_binary, probabilities) if p is not None]
if len(valid_pairs) == len(ground_truths):
roc_auc = round(float(roc_auc_score([y for y, _ in valid_pairs], [p for _, p in valid_pairs])), 4)
except Exception:
roc_auc = None
return {
"total_samples": total,
"valid_evaluations": valid_total,
"non_decisive_or_error": error_or_uncertain,
"confusion_matrix": {"tp": tp, "fp": fp, "tn": tn, "fn": fn},
"accuracy": accuracy,
"accuracy_ci_95": accuracy_ci,
"balanced_accuracy": balanced_acc,
"precision": precision,
"precision_ci_95": precision_ci,
"recall": recall,
"recall_ci_95": recall_ci,
"specificity": specificity,
"specificity_ci_95": specificity_ci,
"f1_score": f1,
"roc_auc": roc_auc,
}
def main():
parser = argparse.ArgumentParser(description="Phase 7 External Generalization Evaluation Pipeline")
parser.add_argument("--worker", action="store_true", help="Internal worker mode")
parser.add_argument("--worker-detector", type=str, help="Detector key for worker")
parser.add_argument("--worker-input", type=str, help="Input JSON for worker")
parser.add_argument("--worker-output", type=str, help="Output JSON for worker")
parser.add_argument("--skip-smoke", action="store_true", help="Skip preliminary smoke tests")
args = parser.parse_args()
if args.worker:
run_worker(args.worker_detector, args.worker_input, args.worker_output)
return
print("=" * 80)
print("PHASE 7: EXTERNAL AI-GENERATED IMAGE GENERALIZATION EVALUATION")
print("=" * 80)
os.makedirs(RESULTS_DIR, exist_ok=True)
# 1. Load manifest
print(f"Loading external generalization manifest from: {MANIFEST_PATH}")
if not os.path.exists(MANIFEST_PATH):
raise FileNotFoundError(f"Manifest not found: {MANIFEST_PATH}")
samples = []
with open(MANIFEST_PATH, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
for row in reader:
samples.append({
"sample_id": row["sample_id"],
"original_filename": row["original_filename"],
"file_path": row["absolute_path"],
"ground_truth": row["label"],
"generator_family": row["generator_family"],
"category": row.get("category", ""),
})
total_samples = len(samples)
real_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_REAL)
fake_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_FAKE)
midjourney_count = sum(1 for s in samples if s["generator_family"] == "midjourney")
adm_count = sum(1 for s in samples if s["generator_family"] == "adm")
print(f"Manifest verified: {total_samples} samples total (REAL={real_count}, FAKE={fake_count} [Midjourney={midjourney_count}, ADM={adm_count}])")
assert total_samples == 200, f"Expected 200 samples, got {total_samples}"
assert real_count == 100, f"Expected 100 REAL samples, got {real_count}"
assert fake_count == 100, f"Expected 100 FAKE samples, got {fake_count}"
# 2. Preliminary Smoke Tests
if not args.skip_smoke:
print("\n--- Step 1: Preliminary Single-Sample Smoke Tests ---")
smoke_sample = [samples[0]]
for det_key in DETECTOR_KEYS:
print(f"Testing detector '{det_key}' smoke execution...")
smoke_res, load_t, lat_t = run_detector_subprocess(det_key, smoke_sample)
print(f" -> '{det_key}' smoke passed! Pred: {smoke_res[0].get('prediction')}, Latency: {lat_t}ms")
# 3. Sequential Subprocess Inference across 200 samples
print("\n--- Step 2: Sequential Detector Inference on 200 Samples ---")
detector_outputs: Dict[str, List[Dict[str, Any]]] = {}
detector_perf: Dict[str, Dict[str, float]] = {}
for det_key in DETECTOR_KEYS:
print(f"\nEvaluating detector '{det_key}' on 200 samples...")
t_start = time.perf_counter()
results, load_time, mean_lat = run_detector_subprocess(det_key, samples)
t_total = time.perf_counter() - t_start
print(f"Completed '{det_key}' in {t_total:.2f}s (Load: {load_time}s, Mean Latency: {mean_lat}ms)")
detector_outputs[det_key] = results
detector_perf[det_key] = {
"load_time_sec": load_time,
"mean_latency_ms": mean_lat,
"total_time_sec": round(t_total, 2),
}
# 4. Save Individual Prediction CSVs
print("\n--- Step 3: Exporting Individual Model Predictions ---")
pred_fieldnames = [
"sample_id", "original_filename", "ground_truth", "generator_family",
"detector_key", "detector_name", "model_name", "prediction", "predicted_class",
"fake_probability", "real_probability", "confidence", "raw_logit",
"threshold", "status", "error_message", "latency_ms", "file_path"
]
for det_key, out_path in [
("vit", OUTPUT_VIT_CSV),
("univfd", OUTPUT_UNIVFD_CSV),
("chen2024_convnext", OUTPUT_CONVNEXT_CSV),
]:
res_list = detector_outputs[det_key]
with open(out_path, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=pred_fieldnames)
writer.writeheader()
for r, s in zip(res_list, samples):
row_dict = {
"sample_id": s["sample_id"],
"original_filename": s["original_filename"],
"ground_truth": s["ground_truth"],
"generator_family": s["generator_family"],
"detector_key": det_key,
"detector_name": r.get("detector_name", ""),
"model_name": r.get("model_name", ""),
"prediction": r.get("prediction", ""),
"predicted_class": r.get("predicted_class", r.get("prediction", "")),
"fake_probability": r.get("fake_probability", ""),
"real_probability": r.get("real_probability", ""),
"confidence": r.get("confidence", ""),
"raw_logit": r.get("raw_logit", ""),
"threshold": r.get("threshold", 0.5),
"status": r.get("status", ""),
"error_message": r.get("error", "") or r.get("error_message", ""),
"latency_ms": r.get("latency_ms", ""),
"file_path": s["file_path"],
}
writer.writerow(row_dict)
print(f" Written {len(res_list)} rows to: {out_path}")
# 5. Combined Classifier & Ensemble Predictions
print("\n--- Step 4: Running Decision Fusion Across All Samples ---")
vit_preds = [r.get("prediction") for r in detector_outputs["vit"]]
univfd_preds = [r.get("prediction") for r in detector_outputs["univfd"]]
convnext_preds = [r.get("prediction") for r in detector_outputs["chen2024_convnext"]]
vit_probs = [r.get("fake_probability") for r in detector_outputs["vit"]]
univfd_probs = [r.get("fake_probability") for r in detector_outputs["univfd"]]
convnext_probs = [r.get("fake_probability") for r in detector_outputs["chen2024_convnext"]]
combined_rows = []
maj_preds = []
maj_probs = []
mean_preds = []
mean_probs = []
vit_priority_preds = []
vit_priority_probs = []
agree_3_count = 0
for i, s in enumerate(samples):
record_map = {
"vit": detector_outputs["vit"][i],
"univfd": detector_outputs["univfd"][i],
"chen2024_convnext": detector_outputs["chen2024_convnext"][i],
}
res_maj = strategy_majority_vote(record_map, tie_breaker="vit_or_real")
res_mean = strategy_mean_fake_probability(record_map, threshold=0.50)
res_prio = strategy_vit_priority(record_map, vit_key="vit")
maj_preds.append(res_maj.prediction)
maj_probs.append(res_maj.fake_probability)
mean_preds.append(res_mean.prediction)
mean_probs.append(res_mean.fake_probability)
vit_priority_preds.append(res_prio.prediction)
vit_priority_probs.append(res_prio.fake_probability)
all_three_agree = (vit_preds[i] == univfd_preds[i] == convnext_preds[i])
if all_three_agree:
agree_3_count += 1
combined_rows.append({
"sample_id": s["sample_id"],
"original_filename": s["original_filename"],
"ground_truth": s["ground_truth"],
"generator_family": s["generator_family"],
"vit_pred": vit_preds[i],
"vit_fake_prob": vit_probs[i],
"univfd_pred": univfd_preds[i],
"univfd_fake_prob": univfd_probs[i],
"convnext_pred": convnext_preds[i],
"convnext_fake_prob": convnext_probs[i],
"majority_vote_pred": res_maj.prediction,
"majority_vote_fake_prob": res_maj.fake_probability,
"mean_prob_pred": res_mean.prediction,
"mean_prob_fake_prob": res_mean.fake_probability,
"vit_priority_pred": res_prio.prediction,
"vit_priority_fake_prob": res_prio.fake_probability,
"all_three_agree": all_three_agree,
"majority_explanation": res_maj.explanation,
"mean_explanation": res_mean.explanation,
"priority_explanation": res_prio.explanation,
})
with open(OUTPUT_COMBINED_CSV, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(combined_rows[0].keys()))
writer.writeheader()
writer.writerows(combined_rows)
print(f" Written combined predictions ({len(combined_rows)} rows) to: {OUTPUT_COMBINED_CSV}")
# 6. Compute Comprehensive Metrics (Aggregate + Per-Generator)
print("\n--- Step 5: Computing Statistical Metrics (Aggregate & Per-Generator) ---")
ground_truths = [s["ground_truth"] for s in samples]
targets = {
"vit": {"name": "ViT (dima806)", "preds": vit_preds, "probs": vit_probs, "lat": detector_perf["vit"]["mean_latency_ms"]},
"univfd": {"name": "UnivFD (CLIP ViT-L/14)", "preds": univfd_preds, "probs": univfd_probs, "lat": detector_perf["univfd"]["mean_latency_ms"]},
"chen2024_convnext": {"name": "ConvNeXt (DRCT / RAID)", "preds": convnext_preds, "probs": convnext_probs, "lat": detector_perf["chen2024_convnext"]["mean_latency_ms"]},
"ensemble_majority_vote": {"name": "Ensemble (Majority Vote)", "preds": maj_preds, "probs": maj_probs, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)},
"ensemble_mean_prob": {"name": "Ensemble (Mean Fake Probability)", "preds": mean_preds, "probs": mean_probs, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)},
"ensemble_vit_priority": {"name": "Ensemble (ViT Priority Fallback)", "preds": vit_priority_preds, "probs": vit_priority_probs, "lat": detector_perf["vit"]["mean_latency_ms"]},
}
# Slice indices
idx_real = [i for i, s in enumerate(samples) if s["ground_truth"] == PREDICTION_REAL]
idx_midjourney = [i for i, s in enumerate(samples) if s["generator_family"] == "midjourney"]
idx_adm = [i for i, s in enumerate(samples) if s["generator_family"] == "adm"]
slice_mj = idx_real + idx_midjourney
slice_adm = idx_real + idx_adm
metrics_report: Dict[str, Any] = {
"metadata": {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"manifest_path": os.path.relpath(MANIFEST_PATH, PROJECT_ROOT).replace("\\", "/"),
"dataset_source": "GenImage (TheKernel01/Tiny-GenImage validation split)",
"total_samples": total_samples,
"real_samples": real_count,
"fake_samples": fake_count,
"generator_counts": {"midjourney": midjourney_count, "adm": adm_count},
"environment": {
"python_version": sys.version,
"torch_version": torch.__version__,
"platform": sys.platform,
}
},
"aggregate_performance": {},
"midjourney_subset_performance": {},
"adm_subset_performance": {},
"agreement_analysis": {},
}
comparison_rows = []
print("\n--- AGGREGATE PERFORMANCE (200 SAMPLES: 100 Real, 50 Midjourney, 50 ADM) ---")
for key, t in targets.items():
m_agg = compute_binary_metrics(t["preds"], ground_truths, t["probs"])
m_agg["mean_latency_ms"] = t["lat"]
m_agg["display_name"] = t["name"]
metrics_report["aggregate_performance"][key] = m_agg
# Midjourney subset (150 samples)
mj_gt = [ground_truths[i] for i in slice_mj]
mj_preds = [t["preds"][i] for i in slice_mj]
mj_probs = [t["probs"][i] for i in slice_mj]
m_mj = compute_binary_metrics(mj_preds, mj_gt, mj_probs)
metrics_report["midjourney_subset_performance"][key] = m_mj
# ADM subset (150 samples)
adm_gt = [ground_truths[i] for i in slice_adm]
adm_preds = [t["preds"][i] for i in slice_adm]
adm_probs = [t["probs"][i] for i in slice_adm]
m_adm = compute_binary_metrics(adm_preds, adm_gt, adm_probs)
metrics_report["adm_subset_performance"][key] = m_adm
auc_str = f"{m_agg['roc_auc']:.4f}" if m_agg['roc_auc'] is not None else "N/A"
print(f" {t['name']:35s} | Acc: {m_agg['accuracy']*100:6.2f}% ({m_agg['accuracy_ci_95'][0]*100:.1f}-{m_agg['accuracy_ci_95'][1]*100:.1f}%) | "
f"BalAcc: {m_agg['balanced_accuracy']*100:6.2f}% | Prec: {m_agg['precision']:5.3f} | Rec: {m_agg['recall']:5.3f} | "
f"Spec: {m_agg['specificity']:5.3f} | F1: {m_agg['f1_score']:5.3f} | AUC: {auc_str}")
comparison_rows.append({
"Model / Strategy": t["name"],
"Aggregate Acc (%)": f"{m_agg['accuracy'] * 100:.2f}",
"Aggregate 95% CI": f"[{m_agg['accuracy_ci_95'][0]*100:.2f}%, {m_agg['accuracy_ci_95'][1]*100:.2f}%]",
"Aggregate BalAcc (%)": f"{m_agg['balanced_accuracy'] * 100:.2f}",
"Aggregate Precision": f"{m_agg['precision']:.4f}",
"Aggregate Recall": f"{m_agg['recall']:.4f}",
"Aggregate Specificity": f"{m_agg['specificity']:.4f}",
"Aggregate F1": f"{m_agg['f1_score']:.4f}",
"Aggregate ROC-AUC": auc_str,
"Midjourney Acc (%)": f"{m_mj['accuracy'] * 100:.2f}",
"Midjourney Recall (%)": f"{m_mj['recall'] * 100:.2f}",
"ADM Acc (%)": f"{m_adm['accuracy'] * 100:.2f}",
"ADM Recall (%)": f"{m_adm['recall'] * 100:.2f}",
"TP": m_agg["confusion_matrix"]["tp"],
"FP": m_agg["confusion_matrix"]["fp"],
"TN": m_agg["confusion_matrix"]["tn"],
"FN": m_agg["confusion_matrix"]["fn"],
"Mean Latency (ms)": f"{t['lat']:.1f}",
})
# Cross-model agreement
def get_pairwise_kappa(p1, p2):
both_fake = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_FAKE)
both_real = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_REAL)
m1_f_m2_r = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_REAL)
m1_r_m2_f = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_FAKE)
raw_agree = (both_fake + both_real) / len(p1)
kappa = compute_cohen_kappa(both_fake, both_real, m1_f_m2_r, m1_r_m2_f)
return {
"raw_agreement_rate": round(raw_agree, 4),
"cohen_kappa": kappa,
"both_fake": both_fake,
"both_real": both_real,
"m1_fake_m2_real": m1_f_m2_r,
"m1_real_m2_fake": m1_r_m2_f,
}
metrics_report["agreement_analysis"] = {
"all_three_agree_count": agree_3_count,
"all_three_agree_fraction": round(agree_3_count / total_samples, 4),
"vit_vs_univfd": get_pairwise_kappa(vit_preds, univfd_preds),
"vit_vs_convnext": get_pairwise_kappa(vit_preds, convnext_preds),
"univfd_vs_convnext": get_pairwise_kappa(univfd_preds, convnext_preds),
}
# Save metrics JSON
with open(OUTPUT_METRICS_JSON, "w", encoding="utf-8") as f:
json.dump(metrics_report, f, indent=2)
print(f"\n Saved Phase 7 metrics JSON to: {OUTPUT_METRICS_JSON}")
# Save comparison CSV
with open(OUTPUT_COMPARISON_CSV, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(comparison_rows[0].keys()))
writer.writeheader()
writer.writerows(comparison_rows)
print(f" Saved Phase 7 model comparison CSV to: {OUTPUT_COMPARISON_CSV}")
print("=" * 80)
print("PHASE 7 EVALUATION PIPELINE COMPLETED SUCCESSFULLY!")
print("=" * 80)
if __name__ == "__main__":
main()