Spaces:
Running on Zero
Running on Zero
Download scripts/evaluate_phase7_pipeline.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 27.7 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/evaluate_phase7_pipeline.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/evaluate_phase7_pipeline.py
-
curl -L -o evaluate_phase7_pipeline.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/evaluate_phase7_pipeline.py
27.7 kB
| """ | |
| Phase 7 External Generalization Evaluation Pipeline Runner | |
| Evaluates all three integrated deepfake detectors (ViT, UnivFD, ConvNeXt/DRCT) | |
| independently against benchmark/external_generalization_manifest.csv | |
| (200 GenImage validation samples: 100 REAL ImageNet photographs, 50 FAKE Midjourney, 50 FAKE ADM). | |
| Then evaluates ensemble combination strategies (Majority Vote, Mean Fake Probability, ViT-Priority Fallback). | |
| Memory Safety: | |
| - Sequential subprocess isolation per detector guarantees 100% OS-level reclamation of virtual memory. | |
| - Complete garbage collection and instance resetting between runs. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import csv | |
| import time | |
| import math | |
| import argparse | |
| import subprocess | |
| import tempfile | |
| import gc | |
| from typing import Dict, Any, List, Tuple, Optional | |
| PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if PROJECT_ROOT not in sys.path: | |
| sys.path.insert(0, PROJECT_ROOT) | |
| import torch | |
| from sklearn.metrics import roc_auc_score | |
| from src.detector_registry import ( | |
| get_detector, | |
| get_detector_info, | |
| reset_instances, | |
| ) | |
| from src.detector_schema import ( | |
| DetectorResult, | |
| PREDICTION_REAL, | |
| PREDICTION_FAKE, | |
| PREDICTION_ERROR, | |
| PREDICTION_UNCERTAIN, | |
| ) | |
| from src.decision_fusion import ( | |
| DecisionFusionPipeline, | |
| strategy_vit_only, | |
| strategy_majority_vote, | |
| strategy_mean_fake_probability, | |
| strategy_vit_priority, | |
| STRATEGY_VIT_ONLY, | |
| STRATEGY_MAJORITY_VOTE, | |
| STRATEGY_MEAN_PROBABILITY, | |
| STRATEGY_VIT_PRIORITY, | |
| ) | |
| MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv") | |
| RESULTS_DIR = os.path.join(PROJECT_ROOT, "benchmark", "results") | |
| OUTPUT_VIT_CSV = os.path.join(RESULTS_DIR, "phase7_vit_predictions.csv") | |
| OUTPUT_UNIVFD_CSV = os.path.join(RESULTS_DIR, "phase7_univfd_predictions.csv") | |
| OUTPUT_CONVNEXT_CSV = os.path.join(RESULTS_DIR, "phase7_convnext_predictions.csv") | |
| OUTPUT_COMBINED_CSV = os.path.join(RESULTS_DIR, "phase7_combined_predictions.csv") | |
| OUTPUT_METRICS_JSON = os.path.join(RESULTS_DIR, "phase7_combined_metrics.json") | |
| OUTPUT_COMPARISON_CSV = os.path.join(RESULTS_DIR, "phase7_model_comparison.csv") | |
| DETECTOR_KEYS = ["vit", "univfd", "chen2024_convnext"] | |
| def wilson_score_interval(k: int, n: int, confidence: float = 0.95) -> List[float]: | |
| """Computes Wilson score confidence interval for binomial proportion k / n.""" | |
| if n <= 0: | |
| return [0.0, 0.0] | |
| p_hat = k / n | |
| z = 1.959963984540054 | |
| z2 = z * z | |
| denom = 1.0 + z2 / n | |
| center = (p_hat + z2 / (2.0 * n)) / denom | |
| margin = z * math.sqrt((p_hat * (1.0 - p_hat) / n) + (z2 / (4.0 * n * n))) / denom | |
| lower = max(0.0, center - margin) | |
| upper = min(1.0, center + margin) | |
| return [round(lower, 4), round(upper, 4)] | |
| def compute_cohen_kappa(both_fake: int, both_real: int, m1_f_m2_r: int, m1_r_m2_f: int) -> float: | |
| """Computes Cohen's Kappa for 2x2 agreement matrix.""" | |
| n = both_fake + both_real + m1_f_m2_r + m1_r_m2_f | |
| if n == 0: | |
| return 0.0 | |
| po = (both_fake + both_real) / n | |
| p_f1 = (both_fake + m1_f_m2_r) / n | |
| p_r1 = (m1_r_m2_f + both_real) / n | |
| p_f2 = (both_fake + m1_r_m2_f) / n | |
| p_r2 = (m1_f_m2_r + both_real) / n | |
| pe = (p_f1 * p_f2) + (p_r1 * p_r2) | |
| if abs(1.0 - pe) < 1e-9: | |
| return 1.0 if abs(po - 1.0) < 1e-9 else 0.0 | |
| kappa = (po - pe) / (1.0 - pe) | |
| return round(kappa, 4) | |
| def run_worker(det_key: str, input_json_path: str, output_json_path: str) -> None: | |
| """Worker function executed in isolated child process for a single detector.""" | |
| print(f" [Worker Process] Initializing detector '{det_key}' in clean subprocess...") | |
| with open(input_json_path, "r", encoding="utf-8") as f: | |
| samples = json.load(f) | |
| info = get_detector_info(det_key) | |
| meta = info.get("metadata", {}) | |
| fallback_det_name = meta.get("detector_name", det_key) | |
| fallback_model_name = meta.get("model_name", "Unknown") | |
| adapter = None | |
| load_err = None | |
| t0 = time.perf_counter() | |
| try: | |
| adapter = get_detector(det_key, lazy_load=False, singleton=False) | |
| load_time_sec = round(time.perf_counter() - t0, 3) | |
| print(f" [Worker Process] Detector '{det_key}' loaded successfully in {load_time_sec}s.") | |
| except Exception as e: | |
| load_err = str(e) | |
| load_time_sec = round(time.perf_counter() - t0, 3) | |
| print(f" [Worker Process] ERROR loading detector '{det_key}': {load_err}") | |
| results = [] | |
| latencies = [] | |
| for idx, sample in enumerate(samples, 1): | |
| file_path = sample["file_path"] | |
| sample_id = sample["sample_id"] | |
| sample_t0 = time.perf_counter() | |
| if load_err is not None or adapter is None: | |
| err_res = DetectorResult.create_error( | |
| detector_name=fallback_det_name, | |
| model_name=fallback_model_name, | |
| error_message=f"Model initialization failed: {load_err}", | |
| ) | |
| res_dict = err_res.to_dict() | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| elif not os.path.exists(file_path): | |
| err_res = DetectorResult.create_error( | |
| detector_name=fallback_det_name, | |
| model_name=fallback_model_name, | |
| error_message=f"File not found on disk: {file_path}", | |
| ) | |
| res_dict = err_res.to_dict() | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| else: | |
| try: | |
| res = adapter.predict(file_path) | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| res_dict = res.to_dict() | |
| except Exception as inf_exc: | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| err_res = DetectorResult.create_error( | |
| detector_name=fallback_det_name, | |
| model_name=fallback_model_name, | |
| error_message=f"Inference exception: {str(inf_exc)}", | |
| ) | |
| res_dict = err_res.to_dict() | |
| res_dict["latency_ms"] = sample_latency_ms | |
| latencies.append(sample_latency_ms) | |
| record = { | |
| "sample_id": sample_id, | |
| "detector_key": det_key, | |
| **res_dict | |
| } | |
| results.append(record) | |
| if idx % 50 == 0 or idx == len(samples): | |
| print(f" [Worker Process] Progress: {idx}/{len(samples)} samples processed.") | |
| # Explicit cleanup | |
| if adapter is not None and hasattr(adapter, "unload_model"): | |
| try: | |
| adapter.unload_model() | |
| except Exception: | |
| pass | |
| del adapter | |
| reset_instances() | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| with open(output_json_path, "w", encoding="utf-8") as f: | |
| json.dump({ | |
| "results": results, | |
| "load_time_sec": load_time_sec, | |
| "mean_latency_ms": round(sum(latencies) / len(latencies), 2) if latencies else 0.0 | |
| }, f, indent=2) | |
| print(f" [Worker Process] Successfully completed all {len(results)} samples for '{det_key}'.") | |
| def run_detector_subprocess(det_key: str, samples: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], float, float]: | |
| """Runs a detector in an isolated subprocess to guarantee memory reclamation.""" | |
| with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_in: | |
| json.dump(samples, f_in) | |
| in_path = f_in.name | |
| with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_out: | |
| out_path = f_out.name | |
| cmd = [ | |
| sys.executable, | |
| os.path.abspath(__file__), | |
| "--worker", | |
| "--worker-detector", det_key, | |
| "--worker-input", in_path, | |
| "--worker-output", out_path, | |
| ] | |
| try: | |
| proc = subprocess.run(cmd, capture_output=True, text=True, check=True) | |
| print(proc.stdout.strip()) | |
| with open(out_path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| return data["results"], data["load_time_sec"], data["mean_latency_ms"] | |
| except subprocess.CalledProcessError as exc: | |
| print(f" [ERROR] Subprocess for {det_key} failed with return code {exc.returncode}") | |
| print(" --- Subprocess STDOUT ---") | |
| print(exc.stdout) | |
| print(" --- Subprocess STDERR ---") | |
| print(exc.stderr) | |
| raise RuntimeError(f"Detector evaluation subprocess failed for {det_key}") from exc | |
| finally: | |
| for p in [in_path, out_path]: | |
| if os.path.exists(p): | |
| try: | |
| os.remove(p) | |
| except Exception: | |
| pass | |
| def compute_binary_metrics(predictions: List[str], ground_truths: List[str], probabilities: Optional[List[float]] = None) -> Dict[str, Any]: | |
| """Computes binary classification metrics, Wilson CIs, and ROC-AUC.""" | |
| tp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_FAKE) | |
| fp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_REAL) | |
| tn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_REAL) | |
| fn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_FAKE) | |
| error_or_uncertain = sum(1 for p in predictions if p not in (PREDICTION_REAL, PREDICTION_FAKE)) | |
| total = len(predictions) | |
| valid_total = tp + fp + tn + fn | |
| actual_pos = tp + fn | |
| actual_neg = tn + fp | |
| accuracy = round((tp + tn) / valid_total, 4) if valid_total > 0 else 0.0 | |
| accuracy_ci = wilson_score_interval(tp + tn, valid_total) if valid_total > 0 else [0.0, 0.0] | |
| recall = round(tp / actual_pos, 4) if actual_pos > 0 else 0.0 | |
| recall_ci = wilson_score_interval(tp, actual_pos) if actual_pos > 0 else [0.0, 0.0] | |
| specificity = round(tn / actual_neg, 4) if actual_neg > 0 else 0.0 | |
| specificity_ci = wilson_score_interval(tn, actual_neg) if actual_neg > 0 else [0.0, 0.0] | |
| precision = round(tp / (tp + fp), 4) if (tp + fp) > 0 else 0.0 | |
| precision_ci = wilson_score_interval(tp, tp + fp) if (tp + fp) > 0 else [0.0, 0.0] | |
| balanced_acc = round(0.5 * (recall + specificity), 4) | |
| if (precision + recall) > 0: | |
| f1 = round(2.0 * precision * recall / (precision + recall), 4) | |
| else: | |
| f1 = 0.0 | |
| roc_auc = None | |
| if probabilities is not None and len(probabilities) == len(ground_truths) and actual_pos > 0 and actual_neg > 0: | |
| try: | |
| y_binary = [1 if g == PREDICTION_FAKE else 0 for g in ground_truths] | |
| # Filter non-null probs | |
| valid_pairs = [(y, p) for y, p in zip(y_binary, probabilities) if p is not None] | |
| if len(valid_pairs) == len(ground_truths): | |
| roc_auc = round(float(roc_auc_score([y for y, _ in valid_pairs], [p for _, p in valid_pairs])), 4) | |
| except Exception: | |
| roc_auc = None | |
| return { | |
| "total_samples": total, | |
| "valid_evaluations": valid_total, | |
| "non_decisive_or_error": error_or_uncertain, | |
| "confusion_matrix": {"tp": tp, "fp": fp, "tn": tn, "fn": fn}, | |
| "accuracy": accuracy, | |
| "accuracy_ci_95": accuracy_ci, | |
| "balanced_accuracy": balanced_acc, | |
| "precision": precision, | |
| "precision_ci_95": precision_ci, | |
| "recall": recall, | |
| "recall_ci_95": recall_ci, | |
| "specificity": specificity, | |
| "specificity_ci_95": specificity_ci, | |
| "f1_score": f1, | |
| "roc_auc": roc_auc, | |
| } | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Phase 7 External Generalization Evaluation Pipeline") | |
| parser.add_argument("--worker", action="store_true", help="Internal worker mode") | |
| parser.add_argument("--worker-detector", type=str, help="Detector key for worker") | |
| parser.add_argument("--worker-input", type=str, help="Input JSON for worker") | |
| parser.add_argument("--worker-output", type=str, help="Output JSON for worker") | |
| parser.add_argument("--skip-smoke", action="store_true", help="Skip preliminary smoke tests") | |
| args = parser.parse_args() | |
| if args.worker: | |
| run_worker(args.worker_detector, args.worker_input, args.worker_output) | |
| return | |
| print("=" * 80) | |
| print("PHASE 7: EXTERNAL AI-GENERATED IMAGE GENERALIZATION EVALUATION") | |
| print("=" * 80) | |
| os.makedirs(RESULTS_DIR, exist_ok=True) | |
| # 1. Load manifest | |
| print(f"Loading external generalization manifest from: {MANIFEST_PATH}") | |
| if not os.path.exists(MANIFEST_PATH): | |
| raise FileNotFoundError(f"Manifest not found: {MANIFEST_PATH}") | |
| samples = [] | |
| with open(MANIFEST_PATH, "r", encoding="utf-8") as f: | |
| reader = csv.DictReader(f) | |
| for row in reader: | |
| samples.append({ | |
| "sample_id": row["sample_id"], | |
| "original_filename": row["original_filename"], | |
| "file_path": row["absolute_path"], | |
| "ground_truth": row["label"], | |
| "generator_family": row["generator_family"], | |
| "category": row.get("category", ""), | |
| }) | |
| total_samples = len(samples) | |
| real_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_REAL) | |
| fake_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_FAKE) | |
| midjourney_count = sum(1 for s in samples if s["generator_family"] == "midjourney") | |
| adm_count = sum(1 for s in samples if s["generator_family"] == "adm") | |
| print(f"Manifest verified: {total_samples} samples total (REAL={real_count}, FAKE={fake_count} [Midjourney={midjourney_count}, ADM={adm_count}])") | |
| assert total_samples == 200, f"Expected 200 samples, got {total_samples}" | |
| assert real_count == 100, f"Expected 100 REAL samples, got {real_count}" | |
| assert fake_count == 100, f"Expected 100 FAKE samples, got {fake_count}" | |
| # 2. Preliminary Smoke Tests | |
| if not args.skip_smoke: | |
| print("\n--- Step 1: Preliminary Single-Sample Smoke Tests ---") | |
| smoke_sample = [samples[0]] | |
| for det_key in DETECTOR_KEYS: | |
| print(f"Testing detector '{det_key}' smoke execution...") | |
| smoke_res, load_t, lat_t = run_detector_subprocess(det_key, smoke_sample) | |
| print(f" -> '{det_key}' smoke passed! Pred: {smoke_res[0].get('prediction')}, Latency: {lat_t}ms") | |
| # 3. Sequential Subprocess Inference across 200 samples | |
| print("\n--- Step 2: Sequential Detector Inference on 200 Samples ---") | |
| detector_outputs: Dict[str, List[Dict[str, Any]]] = {} | |
| detector_perf: Dict[str, Dict[str, float]] = {} | |
| for det_key in DETECTOR_KEYS: | |
| print(f"\nEvaluating detector '{det_key}' on 200 samples...") | |
| t_start = time.perf_counter() | |
| results, load_time, mean_lat = run_detector_subprocess(det_key, samples) | |
| t_total = time.perf_counter() - t_start | |
| print(f"Completed '{det_key}' in {t_total:.2f}s (Load: {load_time}s, Mean Latency: {mean_lat}ms)") | |
| detector_outputs[det_key] = results | |
| detector_perf[det_key] = { | |
| "load_time_sec": load_time, | |
| "mean_latency_ms": mean_lat, | |
| "total_time_sec": round(t_total, 2), | |
| } | |
| # 4. Save Individual Prediction CSVs | |
| print("\n--- Step 3: Exporting Individual Model Predictions ---") | |
| pred_fieldnames = [ | |
| "sample_id", "original_filename", "ground_truth", "generator_family", | |
| "detector_key", "detector_name", "model_name", "prediction", "predicted_class", | |
| "fake_probability", "real_probability", "confidence", "raw_logit", | |
| "threshold", "status", "error_message", "latency_ms", "file_path" | |
| ] | |
| for det_key, out_path in [ | |
| ("vit", OUTPUT_VIT_CSV), | |
| ("univfd", OUTPUT_UNIVFD_CSV), | |
| ("chen2024_convnext", OUTPUT_CONVNEXT_CSV), | |
| ]: | |
| res_list = detector_outputs[det_key] | |
| with open(out_path, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=pred_fieldnames) | |
| writer.writeheader() | |
| for r, s in zip(res_list, samples): | |
| row_dict = { | |
| "sample_id": s["sample_id"], | |
| "original_filename": s["original_filename"], | |
| "ground_truth": s["ground_truth"], | |
| "generator_family": s["generator_family"], | |
| "detector_key": det_key, | |
| "detector_name": r.get("detector_name", ""), | |
| "model_name": r.get("model_name", ""), | |
| "prediction": r.get("prediction", ""), | |
| "predicted_class": r.get("predicted_class", r.get("prediction", "")), | |
| "fake_probability": r.get("fake_probability", ""), | |
| "real_probability": r.get("real_probability", ""), | |
| "confidence": r.get("confidence", ""), | |
| "raw_logit": r.get("raw_logit", ""), | |
| "threshold": r.get("threshold", 0.5), | |
| "status": r.get("status", ""), | |
| "error_message": r.get("error", "") or r.get("error_message", ""), | |
| "latency_ms": r.get("latency_ms", ""), | |
| "file_path": s["file_path"], | |
| } | |
| writer.writerow(row_dict) | |
| print(f" Written {len(res_list)} rows to: {out_path}") | |
| # 5. Combined Classifier & Ensemble Predictions | |
| print("\n--- Step 4: Running Decision Fusion Across All Samples ---") | |
| vit_preds = [r.get("prediction") for r in detector_outputs["vit"]] | |
| univfd_preds = [r.get("prediction") for r in detector_outputs["univfd"]] | |
| convnext_preds = [r.get("prediction") for r in detector_outputs["chen2024_convnext"]] | |
| vit_probs = [r.get("fake_probability") for r in detector_outputs["vit"]] | |
| univfd_probs = [r.get("fake_probability") for r in detector_outputs["univfd"]] | |
| convnext_probs = [r.get("fake_probability") for r in detector_outputs["chen2024_convnext"]] | |
| combined_rows = [] | |
| maj_preds = [] | |
| maj_probs = [] | |
| mean_preds = [] | |
| mean_probs = [] | |
| vit_priority_preds = [] | |
| vit_priority_probs = [] | |
| agree_3_count = 0 | |
| for i, s in enumerate(samples): | |
| record_map = { | |
| "vit": detector_outputs["vit"][i], | |
| "univfd": detector_outputs["univfd"][i], | |
| "chen2024_convnext": detector_outputs["chen2024_convnext"][i], | |
| } | |
| res_maj = strategy_majority_vote(record_map, tie_breaker="vit_or_real") | |
| res_mean = strategy_mean_fake_probability(record_map, threshold=0.50) | |
| res_prio = strategy_vit_priority(record_map, vit_key="vit") | |
| maj_preds.append(res_maj.prediction) | |
| maj_probs.append(res_maj.fake_probability) | |
| mean_preds.append(res_mean.prediction) | |
| mean_probs.append(res_mean.fake_probability) | |
| vit_priority_preds.append(res_prio.prediction) | |
| vit_priority_probs.append(res_prio.fake_probability) | |
| all_three_agree = (vit_preds[i] == univfd_preds[i] == convnext_preds[i]) | |
| if all_three_agree: | |
| agree_3_count += 1 | |
| combined_rows.append({ | |
| "sample_id": s["sample_id"], | |
| "original_filename": s["original_filename"], | |
| "ground_truth": s["ground_truth"], | |
| "generator_family": s["generator_family"], | |
| "vit_pred": vit_preds[i], | |
| "vit_fake_prob": vit_probs[i], | |
| "univfd_pred": univfd_preds[i], | |
| "univfd_fake_prob": univfd_probs[i], | |
| "convnext_pred": convnext_preds[i], | |
| "convnext_fake_prob": convnext_probs[i], | |
| "majority_vote_pred": res_maj.prediction, | |
| "majority_vote_fake_prob": res_maj.fake_probability, | |
| "mean_prob_pred": res_mean.prediction, | |
| "mean_prob_fake_prob": res_mean.fake_probability, | |
| "vit_priority_pred": res_prio.prediction, | |
| "vit_priority_fake_prob": res_prio.fake_probability, | |
| "all_three_agree": all_three_agree, | |
| "majority_explanation": res_maj.explanation, | |
| "mean_explanation": res_mean.explanation, | |
| "priority_explanation": res_prio.explanation, | |
| }) | |
| with open(OUTPUT_COMBINED_CSV, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=list(combined_rows[0].keys())) | |
| writer.writeheader() | |
| writer.writerows(combined_rows) | |
| print(f" Written combined predictions ({len(combined_rows)} rows) to: {OUTPUT_COMBINED_CSV}") | |
| # 6. Compute Comprehensive Metrics (Aggregate + Per-Generator) | |
| print("\n--- Step 5: Computing Statistical Metrics (Aggregate & Per-Generator) ---") | |
| ground_truths = [s["ground_truth"] for s in samples] | |
| targets = { | |
| "vit": {"name": "ViT (dima806)", "preds": vit_preds, "probs": vit_probs, "lat": detector_perf["vit"]["mean_latency_ms"]}, | |
| "univfd": {"name": "UnivFD (CLIP ViT-L/14)", "preds": univfd_preds, "probs": univfd_probs, "lat": detector_perf["univfd"]["mean_latency_ms"]}, | |
| "chen2024_convnext": {"name": "ConvNeXt (DRCT / RAID)", "preds": convnext_preds, "probs": convnext_probs, "lat": detector_perf["chen2024_convnext"]["mean_latency_ms"]}, | |
| "ensemble_majority_vote": {"name": "Ensemble (Majority Vote)", "preds": maj_preds, "probs": maj_probs, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)}, | |
| "ensemble_mean_prob": {"name": "Ensemble (Mean Fake Probability)", "preds": mean_preds, "probs": mean_probs, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)}, | |
| "ensemble_vit_priority": {"name": "Ensemble (ViT Priority Fallback)", "preds": vit_priority_preds, "probs": vit_priority_probs, "lat": detector_perf["vit"]["mean_latency_ms"]}, | |
| } | |
| # Slice indices | |
| idx_real = [i for i, s in enumerate(samples) if s["ground_truth"] == PREDICTION_REAL] | |
| idx_midjourney = [i for i, s in enumerate(samples) if s["generator_family"] == "midjourney"] | |
| idx_adm = [i for i, s in enumerate(samples) if s["generator_family"] == "adm"] | |
| slice_mj = idx_real + idx_midjourney | |
| slice_adm = idx_real + idx_adm | |
| metrics_report: Dict[str, Any] = { | |
| "metadata": { | |
| "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), | |
| "manifest_path": os.path.relpath(MANIFEST_PATH, PROJECT_ROOT).replace("\\", "/"), | |
| "dataset_source": "GenImage (TheKernel01/Tiny-GenImage validation split)", | |
| "total_samples": total_samples, | |
| "real_samples": real_count, | |
| "fake_samples": fake_count, | |
| "generator_counts": {"midjourney": midjourney_count, "adm": adm_count}, | |
| "environment": { | |
| "python_version": sys.version, | |
| "torch_version": torch.__version__, | |
| "platform": sys.platform, | |
| } | |
| }, | |
| "aggregate_performance": {}, | |
| "midjourney_subset_performance": {}, | |
| "adm_subset_performance": {}, | |
| "agreement_analysis": {}, | |
| } | |
| comparison_rows = [] | |
| print("\n--- AGGREGATE PERFORMANCE (200 SAMPLES: 100 Real, 50 Midjourney, 50 ADM) ---") | |
| for key, t in targets.items(): | |
| m_agg = compute_binary_metrics(t["preds"], ground_truths, t["probs"]) | |
| m_agg["mean_latency_ms"] = t["lat"] | |
| m_agg["display_name"] = t["name"] | |
| metrics_report["aggregate_performance"][key] = m_agg | |
| # Midjourney subset (150 samples) | |
| mj_gt = [ground_truths[i] for i in slice_mj] | |
| mj_preds = [t["preds"][i] for i in slice_mj] | |
| mj_probs = [t["probs"][i] for i in slice_mj] | |
| m_mj = compute_binary_metrics(mj_preds, mj_gt, mj_probs) | |
| metrics_report["midjourney_subset_performance"][key] = m_mj | |
| # ADM subset (150 samples) | |
| adm_gt = [ground_truths[i] for i in slice_adm] | |
| adm_preds = [t["preds"][i] for i in slice_adm] | |
| adm_probs = [t["probs"][i] for i in slice_adm] | |
| m_adm = compute_binary_metrics(adm_preds, adm_gt, adm_probs) | |
| metrics_report["adm_subset_performance"][key] = m_adm | |
| auc_str = f"{m_agg['roc_auc']:.4f}" if m_agg['roc_auc'] is not None else "N/A" | |
| print(f" {t['name']:35s} | Acc: {m_agg['accuracy']*100:6.2f}% ({m_agg['accuracy_ci_95'][0]*100:.1f}-{m_agg['accuracy_ci_95'][1]*100:.1f}%) | " | |
| f"BalAcc: {m_agg['balanced_accuracy']*100:6.2f}% | Prec: {m_agg['precision']:5.3f} | Rec: {m_agg['recall']:5.3f} | " | |
| f"Spec: {m_agg['specificity']:5.3f} | F1: {m_agg['f1_score']:5.3f} | AUC: {auc_str}") | |
| comparison_rows.append({ | |
| "Model / Strategy": t["name"], | |
| "Aggregate Acc (%)": f"{m_agg['accuracy'] * 100:.2f}", | |
| "Aggregate 95% CI": f"[{m_agg['accuracy_ci_95'][0]*100:.2f}%, {m_agg['accuracy_ci_95'][1]*100:.2f}%]", | |
| "Aggregate BalAcc (%)": f"{m_agg['balanced_accuracy'] * 100:.2f}", | |
| "Aggregate Precision": f"{m_agg['precision']:.4f}", | |
| "Aggregate Recall": f"{m_agg['recall']:.4f}", | |
| "Aggregate Specificity": f"{m_agg['specificity']:.4f}", | |
| "Aggregate F1": f"{m_agg['f1_score']:.4f}", | |
| "Aggregate ROC-AUC": auc_str, | |
| "Midjourney Acc (%)": f"{m_mj['accuracy'] * 100:.2f}", | |
| "Midjourney Recall (%)": f"{m_mj['recall'] * 100:.2f}", | |
| "ADM Acc (%)": f"{m_adm['accuracy'] * 100:.2f}", | |
| "ADM Recall (%)": f"{m_adm['recall'] * 100:.2f}", | |
| "TP": m_agg["confusion_matrix"]["tp"], | |
| "FP": m_agg["confusion_matrix"]["fp"], | |
| "TN": m_agg["confusion_matrix"]["tn"], | |
| "FN": m_agg["confusion_matrix"]["fn"], | |
| "Mean Latency (ms)": f"{t['lat']:.1f}", | |
| }) | |
| # Cross-model agreement | |
| def get_pairwise_kappa(p1, p2): | |
| both_fake = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_FAKE) | |
| both_real = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_REAL) | |
| m1_f_m2_r = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_REAL) | |
| m1_r_m2_f = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_FAKE) | |
| raw_agree = (both_fake + both_real) / len(p1) | |
| kappa = compute_cohen_kappa(both_fake, both_real, m1_f_m2_r, m1_r_m2_f) | |
| return { | |
| "raw_agreement_rate": round(raw_agree, 4), | |
| "cohen_kappa": kappa, | |
| "both_fake": both_fake, | |
| "both_real": both_real, | |
| "m1_fake_m2_real": m1_f_m2_r, | |
| "m1_real_m2_fake": m1_r_m2_f, | |
| } | |
| metrics_report["agreement_analysis"] = { | |
| "all_three_agree_count": agree_3_count, | |
| "all_three_agree_fraction": round(agree_3_count / total_samples, 4), | |
| "vit_vs_univfd": get_pairwise_kappa(vit_preds, univfd_preds), | |
| "vit_vs_convnext": get_pairwise_kappa(vit_preds, convnext_preds), | |
| "univfd_vs_convnext": get_pairwise_kappa(univfd_preds, convnext_preds), | |
| } | |
| # Save metrics JSON | |
| with open(OUTPUT_METRICS_JSON, "w", encoding="utf-8") as f: | |
| json.dump(metrics_report, f, indent=2) | |
| print(f"\n Saved Phase 7 metrics JSON to: {OUTPUT_METRICS_JSON}") | |
| # Save comparison CSV | |
| with open(OUTPUT_COMPARISON_CSV, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=list(comparison_rows[0].keys())) | |
| writer.writeheader() | |
| writer.writerows(comparison_rows) | |
| print(f" Saved Phase 7 model comparison CSV to: {OUTPUT_COMPARISON_CSV}") | |
| print("=" * 80) | |
| print("PHASE 7 EVALUATION PIPELINE COMPLETED SUCCESSFULLY!") | |
| print("=" * 80) | |
| if __name__ == "__main__": | |
| main() | |