Spaces:
Running on Zero
Running on Zero
Download scripts/evaluate_phase6_pipeline.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 25.5 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/evaluate_phase6_pipeline.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/evaluate_phase6_pipeline.py
-
curl -L -o evaluate_phase6_pipeline.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/evaluate_phase6_pipeline.py
25.5 kB
| """ | |
| Phase 6 Evaluation Pipeline Runner | |
| Evaluates all three integrated deepfake detectors (ViT, UnivFD, ConvNeXt/DRCT) | |
| independently against benchmark/phase6_test_manifest.csv (200 pristine test images: 100 REAL, 100 FAKE). | |
| Then evaluates ensemble combination strategies (Majority Vote, Mean Fake Probability, ViT-Priority Fallback). | |
| Memory Safety: | |
| - Sequential subprocess isolation per detector guarantees 100% OS-level reclamation of virtual memory. | |
| - Complete garbage collection and CUDA cache clearing before and after each detector run. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import csv | |
| import time | |
| import math | |
| import argparse | |
| import subprocess | |
| import tempfile | |
| import gc | |
| from typing import Dict, Any, List, Tuple, Optional | |
| # Ensure project root is in sys.path | |
| PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if PROJECT_ROOT not in sys.path: | |
| sys.path.insert(0, PROJECT_ROOT) | |
| import torch | |
| from src.detector_registry import ( | |
| get_detector, | |
| get_detector_info, | |
| reset_instances, | |
| ) | |
| from src.detector_schema import ( | |
| DetectorResult, | |
| PREDICTION_REAL, | |
| PREDICTION_FAKE, | |
| PREDICTION_ERROR, | |
| PREDICTION_UNCERTAIN, | |
| ) | |
| from src.decision_fusion import ( | |
| DecisionFusionPipeline, | |
| strategy_vit_only, | |
| strategy_majority_vote, | |
| strategy_mean_fake_probability, | |
| strategy_vit_priority, | |
| STRATEGY_VIT_ONLY, | |
| STRATEGY_MAJORITY_VOTE, | |
| STRATEGY_MEAN_PROBABILITY, | |
| STRATEGY_VIT_PRIORITY, | |
| ) | |
| MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv") | |
| RESULTS_DIR = os.path.join(PROJECT_ROOT, "benchmark", "results") | |
| OUTPUT_VIT_CSV = os.path.join(RESULTS_DIR, "phase6_vit_predictions.csv") | |
| OUTPUT_UNIVFD_CSV = os.path.join(RESULTS_DIR, "phase6_univfd_predictions.csv") | |
| OUTPUT_CONVNEXT_CSV = os.path.join(RESULTS_DIR, "phase6_convnext_predictions.csv") | |
| OUTPUT_COMBINED_CSV = os.path.join(RESULTS_DIR, "phase6_combined_predictions.csv") | |
| OUTPUT_METRICS_JSON = os.path.join(RESULTS_DIR, "phase6_combined_metrics.json") | |
| OUTPUT_COMPARISON_CSV = os.path.join(RESULTS_DIR, "phase6_model_comparison.csv") | |
| DETECTOR_KEYS = ["vit", "univfd", "chen2024_convnext"] | |
| def wilson_score_interval(k: int, n: int, confidence: float = 0.95) -> List[float]: | |
| """ | |
| Computes Wilson score confidence interval for binomial proportion k / n. | |
| Standard z for 95% confidence is 1.959963984540054. | |
| Returns [lower_bound, upper_bound] rounded to 4 decimal places. | |
| """ | |
| if n <= 0: | |
| return [0.0, 0.0] | |
| p_hat = k / n | |
| z = 1.959963984540054 | |
| z2 = z * z | |
| denom = 1.0 + z2 / n | |
| center = (p_hat + z2 / (2.0 * n)) / denom | |
| margin = z * math.sqrt((p_hat * (1.0 - p_hat) / n) + (z2 / (4.0 * n * n))) / denom | |
| lower = max(0.0, center - margin) | |
| upper = min(1.0, center + margin) | |
| return [round(lower, 4), round(upper, 4)] | |
| def compute_cohen_kappa(both_fake: int, both_real: int, m1_f_m2_r: int, m1_r_m2_f: int) -> float: | |
| """Computes Cohen's Kappa for 2x2 agreement matrix.""" | |
| n = both_fake + both_real + m1_f_m2_r + m1_r_m2_f | |
| if n == 0: | |
| return 0.0 | |
| po = (both_fake + both_real) / n | |
| p_f1 = (both_fake + m1_f_m2_r) / n | |
| p_r1 = (m1_r_m2_f + both_real) / n | |
| p_f2 = (both_fake + m1_r_m2_f) / n | |
| p_r2 = (m1_f_m2_r + both_real) / n | |
| pe = (p_f1 * p_f2) + (p_r1 * p_r2) | |
| if abs(1.0 - pe) < 1e-9: | |
| return 1.0 if abs(po - 1.0) < 1e-9 else 0.0 | |
| kappa = (po - pe) / (1.0 - pe) | |
| return round(kappa, 4) | |
| def run_worker(det_key: str, input_json_path: str, output_json_path: str) -> None: | |
| """ | |
| Worker function executed in an isolated child process for a single detector. | |
| Evaluates all samples in input_json_path and writes results to output_json_path. | |
| """ | |
| print(f" [Worker Process] Initializing detector '{det_key}' in clean subprocess...") | |
| with open(input_json_path, "r", encoding="utf-8") as f: | |
| samples = json.load(f) | |
| info = get_detector_info(det_key) | |
| meta = info.get("metadata", {}) | |
| fallback_det_name = meta.get("detector_name", det_key) | |
| fallback_model_name = meta.get("model_name", "Unknown") | |
| adapter = None | |
| load_err = None | |
| t0 = time.perf_counter() | |
| try: | |
| adapter = get_detector(det_key, lazy_load=False, singleton=False) | |
| load_time_sec = round(time.perf_counter() - t0, 3) | |
| print(f" [Worker Process] Detector '{det_key}' loaded successfully in {load_time_sec}s.") | |
| except Exception as e: | |
| load_err = str(e) | |
| load_time_sec = round(time.perf_counter() - t0, 3) | |
| print(f" [Worker Process] ERROR loading detector '{det_key}': {load_err}") | |
| results = [] | |
| latencies = [] | |
| for idx, sample in enumerate(samples, 1): | |
| file_path = sample["file_path"] | |
| sample_id = sample["sample_id"] | |
| sample_t0 = time.perf_counter() | |
| if load_err is not None or adapter is None: | |
| err_res = DetectorResult.create_error( | |
| detector_name=fallback_det_name, | |
| model_name=fallback_model_name, | |
| error_message=f"Model initialization failed: {load_err}", | |
| ) | |
| res_dict = err_res.to_dict() | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| elif not os.path.exists(file_path): | |
| err_res = DetectorResult.create_error( | |
| detector_name=fallback_det_name, | |
| model_name=fallback_model_name, | |
| error_message=f"File not found on disk: {file_path}", | |
| ) | |
| res_dict = err_res.to_dict() | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| else: | |
| try: | |
| res = adapter.predict(file_path) | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| res_dict = res.to_dict() | |
| except Exception as inf_exc: | |
| sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2) | |
| err_res = DetectorResult.create_error( | |
| detector_name=fallback_det_name, | |
| model_name=fallback_model_name, | |
| error_message=f"Inference exception: {str(inf_exc)}", | |
| ) | |
| res_dict = err_res.to_dict() | |
| res_dict["latency_ms"] = sample_latency_ms | |
| latencies.append(sample_latency_ms) | |
| record = { | |
| "sample_id": sample_id, | |
| "detector_key": det_key, | |
| **res_dict | |
| } | |
| results.append(record) | |
| if idx % 50 == 0 or idx == len(samples): | |
| print(f" [Worker Process] Progress: {idx}/{len(samples)} samples processed.") | |
| # Explicit cleanup | |
| if adapter is not None and hasattr(adapter, "unload_model"): | |
| try: | |
| adapter.unload_model() | |
| except Exception: | |
| pass | |
| del adapter | |
| reset_instances() | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| with open(output_json_path, "w", encoding="utf-8") as f: | |
| json.dump({ | |
| "results": results, | |
| "load_time_sec": load_time_sec, | |
| "mean_latency_ms": round(sum(latencies) / len(latencies), 2) if latencies else 0.0 | |
| }, f, indent=2) | |
| print(f" [Worker Process] Successfully completed all {len(results)} samples for '{det_key}'.") | |
| def run_detector_subprocess(det_key: str, samples: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], float, float]: | |
| """ | |
| Executes a detector in an isolated child process to ensure memory safety. | |
| Returns (results, load_time_sec, mean_latency_ms). | |
| """ | |
| with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_in: | |
| json.dump(samples, f_in) | |
| in_path = f_in.name | |
| with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_out: | |
| out_path = f_out.name | |
| cmd = [ | |
| sys.executable, | |
| os.path.abspath(__file__), | |
| "--worker", | |
| "--worker-detector", det_key, | |
| "--worker-input", in_path, | |
| "--worker-output", out_path, | |
| ] | |
| try: | |
| proc = subprocess.run(cmd, capture_output=True, text=True, check=True) | |
| print(proc.stdout.strip()) | |
| with open(out_path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| return data["results"], data["load_time_sec"], data["mean_latency_ms"] | |
| except subprocess.CalledProcessError as exc: | |
| print(f" [ERROR] Subprocess for {det_key} failed with return code {exc.returncode}") | |
| print(" --- Subprocess STDOUT ---") | |
| print(exc.stdout) | |
| print(" --- Subprocess STDERR ---") | |
| print(exc.stderr) | |
| raise RuntimeError(f"Detector evaluation subprocess failed for {det_key}") from exc | |
| finally: | |
| for p in [in_path, out_path]: | |
| if os.path.exists(p): | |
| try: | |
| os.remove(p) | |
| except Exception: | |
| pass | |
| def compute_binary_metrics(predictions: List[str], ground_truths: List[str]) -> Dict[str, Any]: | |
| """ | |
| Computes confusion matrix, accuracy, balanced accuracy, precision, recall, specificity, F1, and Wilson 95% CIs. | |
| """ | |
| tp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_FAKE) | |
| fp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_REAL) | |
| tn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_REAL) | |
| fn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_FAKE) | |
| error_or_uncertain = sum(1 for p in predictions if p not in (PREDICTION_REAL, PREDICTION_FAKE)) | |
| total = len(predictions) | |
| valid_total = tp + fp + tn + fn | |
| actual_pos = tp + fn | |
| actual_neg = tn + fp | |
| accuracy = round((tp + tn) / valid_total, 4) if valid_total > 0 else 0.0 | |
| accuracy_ci = wilson_score_interval(tp + tn, valid_total) if valid_total > 0 else [0.0, 0.0] | |
| recall = round(tp / actual_pos, 4) if actual_pos > 0 else 0.0 | |
| recall_ci = wilson_score_interval(tp, actual_pos) if actual_pos > 0 else [0.0, 0.0] | |
| specificity = round(tn / actual_neg, 4) if actual_neg > 0 else 0.0 | |
| specificity_ci = wilson_score_interval(tn, actual_neg) if actual_neg > 0 else [0.0, 0.0] | |
| precision = round(tp / (tp + fp), 4) if (tp + fp) > 0 else 0.0 | |
| precision_ci = wilson_score_interval(tp, tp + fp) if (tp + fp) > 0 else [0.0, 0.0] | |
| balanced_acc = round(0.5 * (recall + specificity), 4) | |
| if (precision + recall) > 0: | |
| f1 = round(2.0 * precision * recall / (precision + recall), 4) | |
| else: | |
| f1 = 0.0 | |
| return { | |
| "total_samples": total, | |
| "valid_evaluations": valid_total, | |
| "non_decisive_or_error": error_or_uncertain, | |
| "confusion_matrix": {"tp": tp, "fp": fp, "tn": tn, "fn": fn}, | |
| "accuracy": accuracy, | |
| "accuracy_ci_95": accuracy_ci, | |
| "balanced_accuracy": balanced_acc, | |
| "precision": precision, | |
| "precision_ci_95": precision_ci, | |
| "recall": recall, | |
| "recall_ci_95": recall_ci, | |
| "specificity": specificity, | |
| "specificity_ci_95": specificity_ci, | |
| "f1_score": f1, | |
| } | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Phase 6 Evaluation Pipeline") | |
| parser.add_argument("--worker", action="store_true", help="Internal worker mode") | |
| parser.add_argument("--worker-detector", type=str, help="Detector key for worker") | |
| parser.add_argument("--worker-input", type=str, help="Input JSON for worker") | |
| parser.add_argument("--worker-output", type=str, help="Output JSON for worker") | |
| parser.add_argument("--skip-smoke", action="store_true", help="Skip preliminary smoke tests") | |
| args = parser.parse_args() | |
| if args.worker: | |
| run_worker(args.worker_detector, args.worker_input, args.worker_output) | |
| return | |
| print("================================================================================") | |
| print("PHASE 6: AUTONOMOUS EVALUATION PIPELINE & COMBINED CLASSIFIER") | |
| print("================================================================================") | |
| os.makedirs(RESULTS_DIR, exist_ok=True) | |
| # 1. Load manifest | |
| print(f"Loading manifest from: {MANIFEST_PATH}") | |
| if not os.path.exists(MANIFEST_PATH): | |
| raise FileNotFoundError(f"Manifest not found: {MANIFEST_PATH}") | |
| samples = [] | |
| with open(MANIFEST_PATH, "r", encoding="utf-8") as f: | |
| reader = csv.DictReader(f) | |
| for row in reader: | |
| samples.append({ | |
| "sample_id": row["sample_id"], | |
| "original_filename": row["original_filename"], | |
| "file_path": row["absolute_path"], | |
| "ground_truth": row["ground_truth"], | |
| "category": row.get("category", ""), | |
| "domain": row.get("domain", ""), | |
| }) | |
| total_samples = len(samples) | |
| real_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_REAL) | |
| fake_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_FAKE) | |
| print(f"Manifest verified: {total_samples} samples total (REAL={real_count}, FAKE={fake_count})") | |
| assert total_samples == 200, f"Expected 200 samples, got {total_samples}" | |
| assert real_count == 100, f"Expected 100 REAL samples, got {real_count}" | |
| assert fake_count == 100, f"Expected 100 FAKE samples, got {fake_count}" | |
| # 2. Preliminary Smoke Test | |
| if not args.skip_smoke: | |
| print("\n--- Step 1: Preliminary Single-Sample Smoke Tests ---") | |
| smoke_sample = [samples[0]] # first real image | |
| for det_key in DETECTOR_KEYS: | |
| print(f"Testing detector '{det_key}' smoke execution...") | |
| smoke_res, load_t, lat_t = run_detector_subprocess(det_key, smoke_sample) | |
| print(f" -> '{det_key}' smoke passed! Pred: {smoke_res[0].get('prediction')}, Latency: {lat_t}ms") | |
| # 3. Sequential Subprocess Inference across 200 samples | |
| print("\n--- Step 2: Sequential Detector Inference on 200 Samples ---") | |
| detector_outputs: Dict[str, List[Dict[str, Any]]] = {} | |
| detector_perf: Dict[str, Dict[str, float]] = {} | |
| for det_key in DETECTOR_KEYS: | |
| print(f"\nEvaluating detector '{det_key}' on 200 samples...") | |
| t_start = time.perf_counter() | |
| results, load_time, mean_lat = run_detector_subprocess(det_key, samples) | |
| t_total = time.perf_counter() - t_start | |
| print(f"Completed '{det_key}' in {t_total:.2f}s (Load: {load_time}s, Mean Sample Latency: {mean_lat}ms)") | |
| detector_outputs[det_key] = results | |
| detector_perf[det_key] = { | |
| "load_time_sec": load_time, | |
| "mean_latency_ms": mean_lat, | |
| "total_time_sec": round(t_total, 2), | |
| } | |
| # 4. Save Individual Prediction CSVs | |
| print("\n--- Step 3: Exporting Individual Model Predictions ---") | |
| pred_fieldnames = [ | |
| "sample_id", "original_filename", "ground_truth", "detector_key", | |
| "detector_name", "model_name", "prediction", "predicted_class", | |
| "fake_probability", "real_probability", "confidence", "raw_logit", | |
| "threshold", "status", "error_message", "latency_ms", "file_path" | |
| ] | |
| for det_key, out_path in [ | |
| ("vit", OUTPUT_VIT_CSV), | |
| ("univfd", OUTPUT_UNIVFD_CSV), | |
| ("chen2024_convnext", OUTPUT_CONVNEXT_CSV), | |
| ]: | |
| res_list = detector_outputs[det_key] | |
| with open(out_path, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=pred_fieldnames) | |
| writer.writeheader() | |
| for r, s in zip(res_list, samples): | |
| row_dict = { | |
| "sample_id": s["sample_id"], | |
| "original_filename": s["original_filename"], | |
| "ground_truth": s["ground_truth"], | |
| "detector_key": det_key, | |
| "detector_name": r.get("detector_name", ""), | |
| "model_name": r.get("model_name", ""), | |
| "prediction": r.get("prediction", ""), | |
| "predicted_class": r.get("predicted_class", r.get("prediction", "")), | |
| "fake_probability": r.get("fake_probability", ""), | |
| "real_probability": r.get("real_probability", ""), | |
| "confidence": r.get("confidence", ""), | |
| "raw_logit": r.get("raw_logit", ""), | |
| "threshold": r.get("threshold", 0.5), | |
| "status": r.get("status", ""), | |
| "error_message": r.get("error", "") or r.get("error_message", ""), | |
| "latency_ms": r.get("latency_ms", ""), | |
| "file_path": s["file_path"], | |
| } | |
| writer.writerow(row_dict) | |
| print(f" Written {len(res_list)} rows to: {out_path}") | |
| # 5. Combined Classifier & Ensemble Predictions | |
| print("\n--- Step 4: Running Combined Classifier & Fusion Strategies ---") | |
| ground_truths = [s["ground_truth"] for s in samples] | |
| vit_preds = [r.get("prediction") for r in detector_outputs["vit"]] | |
| univfd_preds = [r.get("prediction") for r in detector_outputs["univfd"]] | |
| convnext_preds = [r.get("prediction") for r in detector_outputs["chen2024_convnext"]] | |
| vit_probs = [r.get("fake_probability") for r in detector_outputs["vit"]] | |
| univfd_probs = [r.get("fake_probability") for r in detector_outputs["univfd"]] | |
| convnext_probs = [r.get("fake_probability") for r in detector_outputs["chen2024_convnext"]] | |
| combined_rows = [] | |
| maj_preds = [] | |
| maj_probs = [] | |
| mean_preds = [] | |
| mean_probs = [] | |
| vit_priority_preds = [] | |
| vit_priority_probs = [] | |
| agree_3_count = 0 | |
| agree_maj_count = 0 | |
| for i, s in enumerate(samples): | |
| # Prepare dictionary for fusion | |
| record_map = { | |
| "vit": detector_outputs["vit"][i], | |
| "univfd": detector_outputs["univfd"][i], | |
| "chen2024_convnext": detector_outputs["chen2024_convnext"][i], | |
| } | |
| # Run strategies | |
| res_maj = strategy_majority_vote(record_map, tie_breaker="vit_or_real") | |
| res_mean = strategy_mean_fake_probability(record_map, threshold=0.50) | |
| res_prio = strategy_vit_priority(record_map, vit_key="vit") | |
| maj_preds.append(res_maj.prediction) | |
| maj_probs.append(res_maj.fake_probability) | |
| mean_preds.append(res_mean.prediction) | |
| mean_probs.append(res_mean.fake_probability) | |
| vit_priority_preds.append(res_prio.prediction) | |
| vit_priority_probs.append(res_prio.fake_probability) | |
| all_three_agree = (vit_preds[i] == univfd_preds[i] == convnext_preds[i]) | |
| if all_three_agree: | |
| agree_3_count += 1 | |
| combined_rows.append({ | |
| "sample_id": s["sample_id"], | |
| "original_filename": s["original_filename"], | |
| "ground_truth": s["ground_truth"], | |
| "vit_pred": vit_preds[i], | |
| "vit_fake_prob": vit_probs[i], | |
| "univfd_pred": univfd_preds[i], | |
| "univfd_fake_prob": univfd_probs[i], | |
| "convnext_pred": convnext_preds[i], | |
| "convnext_fake_prob": convnext_probs[i], | |
| "majority_vote_pred": res_maj.prediction, | |
| "majority_vote_fake_prob": res_maj.fake_probability, | |
| "mean_prob_pred": res_mean.prediction, | |
| "mean_prob_fake_prob": res_mean.fake_probability, | |
| "vit_priority_pred": res_prio.prediction, | |
| "vit_priority_fake_prob": res_prio.fake_probability, | |
| "all_three_agree": all_three_agree, | |
| "majority_explanation": res_maj.explanation, | |
| "mean_explanation": res_mean.explanation, | |
| "priority_explanation": res_prio.explanation, | |
| }) | |
| # Write combined predictions CSV | |
| comb_fieldnames = list(combined_rows[0].keys()) | |
| with open(OUTPUT_COMBINED_CSV, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=comb_fieldnames) | |
| writer.writeheader() | |
| writer.writerows(combined_rows) | |
| print(f" Written combined predictions ({len(combined_rows)} rows) to: {OUTPUT_COMBINED_CSV}") | |
| # 6. Compute Comprehensive Metrics | |
| print("\n--- Step 5: Computing Statistical Metrics & Cross-Model Agreement ---") | |
| metrics_report: Dict[str, Any] = { | |
| "metadata": { | |
| "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), | |
| "manifest_path": os.path.relpath(MANIFEST_PATH, PROJECT_ROOT).replace("\\", "/"), | |
| "dataset_source": "Kaggle 140k Real and Fake Faces (FFHQ / StyleGAN)", | |
| "total_test_samples": total_samples, | |
| "class_distribution": {"REAL": real_count, "FAKE": fake_count}, | |
| "environment": { | |
| "python_version": sys.version, | |
| "torch_version": torch.__version__, | |
| "platform": sys.platform, | |
| } | |
| }, | |
| "model_performance": {}, | |
| "agreement_analysis": {}, | |
| } | |
| # Evaluate each individual detector and strategy | |
| eval_targets = { | |
| "vit": {"name": "ViT (dima806)", "preds": vit_preds, "lat": detector_perf["vit"]["mean_latency_ms"]}, | |
| "univfd": {"name": "UnivFD (CLIP ViT-L/14)", "preds": univfd_preds, "lat": detector_perf["univfd"]["mean_latency_ms"]}, | |
| "chen2024_convnext": {"name": "ConvNeXt (DRCT / RAID)", "preds": convnext_preds, "lat": detector_perf["chen2024_convnext"]["mean_latency_ms"]}, | |
| "ensemble_majority_vote": {"name": "Ensemble (Majority Vote)", "preds": maj_preds, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)}, | |
| "ensemble_mean_prob": {"name": "Ensemble (Mean Fake Probability)", "preds": mean_preds, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)}, | |
| "ensemble_vit_priority": {"name": "Ensemble (ViT Priority Fallback)", "preds": vit_priority_preds, "lat": detector_perf["vit"]["mean_latency_ms"]}, | |
| } | |
| comparison_rows = [] | |
| for key, target in eval_targets.items(): | |
| m = compute_binary_metrics(target["preds"], ground_truths) | |
| m["mean_latency_ms"] = target["lat"] | |
| m["display_name"] = target["name"] | |
| metrics_report["model_performance"][key] = m | |
| print(f" {target['name']:35s} | Acc: {m['accuracy']*100:6.2f}% ({m['accuracy_ci_95'][0]*100:.1f}-{m['accuracy_ci_95'][1]*100:.1f}%) | " | |
| f"BalAcc: {m['balanced_accuracy']*100:6.2f}% | Prec: {m['precision']:5.3f} | Rec: {m['recall']:5.3f} | " | |
| f"Spec: {m['specificity']:5.3f} | F1: {m['f1_score']:5.3f}") | |
| comparison_rows.append({ | |
| "Model / Strategy": target["name"], | |
| "Accuracy (%)": f"{m['accuracy'] * 100:.2f}", | |
| "95% CI (Accuracy)": f"[{m['accuracy_ci_95'][0]*100:.2f}%, {m['accuracy_ci_95'][1]*100:.2f}%]", | |
| "Balanced Acc (%)": f"{m['balanced_accuracy'] * 100:.2f}", | |
| "Precision": f"{m['precision']:.4f}", | |
| "Recall": f"{m['recall']:.4f}", | |
| "Specificity": f"{m['specificity']:.4f}", | |
| "F1-Score": f"{m['f1_score']:.4f}", | |
| "TP": m["confusion_matrix"]["tp"], | |
| "FP": m["confusion_matrix"]["fp"], | |
| "TN": m["confusion_matrix"]["tn"], | |
| "FN": m["confusion_matrix"]["fn"], | |
| "Mean Latency (ms)": f"{target['lat']:.1f}", | |
| }) | |
| # Agreement matrices & Cohen's Kappa | |
| def get_pairwise_kappa(p1, p2): | |
| both_fake = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_FAKE) | |
| both_real = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_REAL) | |
| m1_f_m2_r = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_REAL) | |
| m1_r_m2_f = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_FAKE) | |
| raw_agree = (both_fake + both_real) / len(p1) | |
| kappa = compute_cohen_kappa(both_fake, both_real, m1_f_m2_r, m1_r_m2_f) | |
| return { | |
| "raw_agreement_rate": round(raw_agree, 4), | |
| "cohen_kappa": kappa, | |
| "both_fake": both_fake, | |
| "both_real": both_real, | |
| "m1_fake_m2_real": m1_f_m2_r, | |
| "m1_real_m2_fake": m1_r_m2_f, | |
| } | |
| metrics_report["agreement_analysis"] = { | |
| "all_three_agree_count": agree_3_count, | |
| "all_three_agree_fraction": round(agree_3_count / total_samples, 4), | |
| "vit_vs_univfd": get_pairwise_kappa(vit_preds, univfd_preds), | |
| "vit_vs_convnext": get_pairwise_kappa(vit_preds, convnext_preds), | |
| "univfd_vs_convnext": get_pairwise_kappa(univfd_preds, convnext_preds), | |
| } | |
| # Save metrics JSON | |
| with open(OUTPUT_METRICS_JSON, "w", encoding="utf-8") as f: | |
| json.dump(metrics_report, f, indent=2) | |
| print(f"\n Saved comprehensive metrics JSON to: {OUTPUT_METRICS_JSON}") | |
| # Save comparison CSV | |
| with open(OUTPUT_COMPARISON_CSV, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=list(comparison_rows[0].keys())) | |
| writer.writeheader() | |
| writer.writerows(comparison_rows) | |
| print(f" Saved model comparison summary CSV to: {OUTPUT_COMPARISON_CSV}") | |
| print("\n================================================================================") | |
| print("PHASE 6 PIPELINE EVALUATION COMPLETED SUCCESSFULLY!") | |
| print("================================================================================") | |
| if __name__ == "__main__": | |
| main() | |