Multimodel_setup / scripts /evaluate_phase6_pipeline.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
25.5 kB
"""
Phase 6 Evaluation Pipeline Runner
Evaluates all three integrated deepfake detectors (ViT, UnivFD, ConvNeXt/DRCT)
independently against benchmark/phase6_test_manifest.csv (200 pristine test images: 100 REAL, 100 FAKE).
Then evaluates ensemble combination strategies (Majority Vote, Mean Fake Probability, ViT-Priority Fallback).
Memory Safety:
- Sequential subprocess isolation per detector guarantees 100% OS-level reclamation of virtual memory.
- Complete garbage collection and CUDA cache clearing before and after each detector run.
"""
import os
import sys
import json
import csv
import time
import math
import argparse
import subprocess
import tempfile
import gc
from typing import Dict, Any, List, Tuple, Optional
# Ensure project root is in sys.path
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
import torch
from src.detector_registry import (
get_detector,
get_detector_info,
reset_instances,
)
from src.detector_schema import (
DetectorResult,
PREDICTION_REAL,
PREDICTION_FAKE,
PREDICTION_ERROR,
PREDICTION_UNCERTAIN,
)
from src.decision_fusion import (
DecisionFusionPipeline,
strategy_vit_only,
strategy_majority_vote,
strategy_mean_fake_probability,
strategy_vit_priority,
STRATEGY_VIT_ONLY,
STRATEGY_MAJORITY_VOTE,
STRATEGY_MEAN_PROBABILITY,
STRATEGY_VIT_PRIORITY,
)
MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv")
RESULTS_DIR = os.path.join(PROJECT_ROOT, "benchmark", "results")
OUTPUT_VIT_CSV = os.path.join(RESULTS_DIR, "phase6_vit_predictions.csv")
OUTPUT_UNIVFD_CSV = os.path.join(RESULTS_DIR, "phase6_univfd_predictions.csv")
OUTPUT_CONVNEXT_CSV = os.path.join(RESULTS_DIR, "phase6_convnext_predictions.csv")
OUTPUT_COMBINED_CSV = os.path.join(RESULTS_DIR, "phase6_combined_predictions.csv")
OUTPUT_METRICS_JSON = os.path.join(RESULTS_DIR, "phase6_combined_metrics.json")
OUTPUT_COMPARISON_CSV = os.path.join(RESULTS_DIR, "phase6_model_comparison.csv")
DETECTOR_KEYS = ["vit", "univfd", "chen2024_convnext"]
def wilson_score_interval(k: int, n: int, confidence: float = 0.95) -> List[float]:
"""
Computes Wilson score confidence interval for binomial proportion k / n.
Standard z for 95% confidence is 1.959963984540054.
Returns [lower_bound, upper_bound] rounded to 4 decimal places.
"""
if n <= 0:
return [0.0, 0.0]
p_hat = k / n
z = 1.959963984540054
z2 = z * z
denom = 1.0 + z2 / n
center = (p_hat + z2 / (2.0 * n)) / denom
margin = z * math.sqrt((p_hat * (1.0 - p_hat) / n) + (z2 / (4.0 * n * n))) / denom
lower = max(0.0, center - margin)
upper = min(1.0, center + margin)
return [round(lower, 4), round(upper, 4)]
def compute_cohen_kappa(both_fake: int, both_real: int, m1_f_m2_r: int, m1_r_m2_f: int) -> float:
"""Computes Cohen's Kappa for 2x2 agreement matrix."""
n = both_fake + both_real + m1_f_m2_r + m1_r_m2_f
if n == 0:
return 0.0
po = (both_fake + both_real) / n
p_f1 = (both_fake + m1_f_m2_r) / n
p_r1 = (m1_r_m2_f + both_real) / n
p_f2 = (both_fake + m1_r_m2_f) / n
p_r2 = (m1_f_m2_r + both_real) / n
pe = (p_f1 * p_f2) + (p_r1 * p_r2)
if abs(1.0 - pe) < 1e-9:
return 1.0 if abs(po - 1.0) < 1e-9 else 0.0
kappa = (po - pe) / (1.0 - pe)
return round(kappa, 4)
def run_worker(det_key: str, input_json_path: str, output_json_path: str) -> None:
"""
Worker function executed in an isolated child process for a single detector.
Evaluates all samples in input_json_path and writes results to output_json_path.
"""
print(f" [Worker Process] Initializing detector '{det_key}' in clean subprocess...")
with open(input_json_path, "r", encoding="utf-8") as f:
samples = json.load(f)
info = get_detector_info(det_key)
meta = info.get("metadata", {})
fallback_det_name = meta.get("detector_name", det_key)
fallback_model_name = meta.get("model_name", "Unknown")
adapter = None
load_err = None
t0 = time.perf_counter()
try:
adapter = get_detector(det_key, lazy_load=False, singleton=False)
load_time_sec = round(time.perf_counter() - t0, 3)
print(f" [Worker Process] Detector '{det_key}' loaded successfully in {load_time_sec}s.")
except Exception as e:
load_err = str(e)
load_time_sec = round(time.perf_counter() - t0, 3)
print(f" [Worker Process] ERROR loading detector '{det_key}': {load_err}")
results = []
latencies = []
for idx, sample in enumerate(samples, 1):
file_path = sample["file_path"]
sample_id = sample["sample_id"]
sample_t0 = time.perf_counter()
if load_err is not None or adapter is None:
err_res = DetectorResult.create_error(
detector_name=fallback_det_name,
model_name=fallback_model_name,
error_message=f"Model initialization failed: {load_err}",
)
res_dict = err_res.to_dict()
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
elif not os.path.exists(file_path):
err_res = DetectorResult.create_error(
detector_name=fallback_det_name,
model_name=fallback_model_name,
error_message=f"File not found on disk: {file_path}",
)
res_dict = err_res.to_dict()
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
else:
try:
res = adapter.predict(file_path)
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
res_dict = res.to_dict()
except Exception as inf_exc:
sample_latency_ms = round((time.perf_counter() - sample_t0) * 1000.0, 2)
err_res = DetectorResult.create_error(
detector_name=fallback_det_name,
model_name=fallback_model_name,
error_message=f"Inference exception: {str(inf_exc)}",
)
res_dict = err_res.to_dict()
res_dict["latency_ms"] = sample_latency_ms
latencies.append(sample_latency_ms)
record = {
"sample_id": sample_id,
"detector_key": det_key,
**res_dict
}
results.append(record)
if idx % 50 == 0 or idx == len(samples):
print(f" [Worker Process] Progress: {idx}/{len(samples)} samples processed.")
# Explicit cleanup
if adapter is not None and hasattr(adapter, "unload_model"):
try:
adapter.unload_model()
except Exception:
pass
del adapter
reset_instances()
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
with open(output_json_path, "w", encoding="utf-8") as f:
json.dump({
"results": results,
"load_time_sec": load_time_sec,
"mean_latency_ms": round(sum(latencies) / len(latencies), 2) if latencies else 0.0
}, f, indent=2)
print(f" [Worker Process] Successfully completed all {len(results)} samples for '{det_key}'.")
def run_detector_subprocess(det_key: str, samples: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], float, float]:
"""
Executes a detector in an isolated child process to ensure memory safety.
Returns (results, load_time_sec, mean_latency_ms).
"""
with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_in:
json.dump(samples, f_in)
in_path = f_in.name
with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as f_out:
out_path = f_out.name
cmd = [
sys.executable,
os.path.abspath(__file__),
"--worker",
"--worker-detector", det_key,
"--worker-input", in_path,
"--worker-output", out_path,
]
try:
proc = subprocess.run(cmd, capture_output=True, text=True, check=True)
print(proc.stdout.strip())
with open(out_path, "r", encoding="utf-8") as f:
data = json.load(f)
return data["results"], data["load_time_sec"], data["mean_latency_ms"]
except subprocess.CalledProcessError as exc:
print(f" [ERROR] Subprocess for {det_key} failed with return code {exc.returncode}")
print(" --- Subprocess STDOUT ---")
print(exc.stdout)
print(" --- Subprocess STDERR ---")
print(exc.stderr)
raise RuntimeError(f"Detector evaluation subprocess failed for {det_key}") from exc
finally:
for p in [in_path, out_path]:
if os.path.exists(p):
try:
os.remove(p)
except Exception:
pass
def compute_binary_metrics(predictions: List[str], ground_truths: List[str]) -> Dict[str, Any]:
"""
Computes confusion matrix, accuracy, balanced accuracy, precision, recall, specificity, F1, and Wilson 95% CIs.
"""
tp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_FAKE)
fp = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_FAKE and g == PREDICTION_REAL)
tn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_REAL)
fn = sum(1 for p, g in zip(predictions, ground_truths) if p == PREDICTION_REAL and g == PREDICTION_FAKE)
error_or_uncertain = sum(1 for p in predictions if p not in (PREDICTION_REAL, PREDICTION_FAKE))
total = len(predictions)
valid_total = tp + fp + tn + fn
actual_pos = tp + fn
actual_neg = tn + fp
accuracy = round((tp + tn) / valid_total, 4) if valid_total > 0 else 0.0
accuracy_ci = wilson_score_interval(tp + tn, valid_total) if valid_total > 0 else [0.0, 0.0]
recall = round(tp / actual_pos, 4) if actual_pos > 0 else 0.0
recall_ci = wilson_score_interval(tp, actual_pos) if actual_pos > 0 else [0.0, 0.0]
specificity = round(tn / actual_neg, 4) if actual_neg > 0 else 0.0
specificity_ci = wilson_score_interval(tn, actual_neg) if actual_neg > 0 else [0.0, 0.0]
precision = round(tp / (tp + fp), 4) if (tp + fp) > 0 else 0.0
precision_ci = wilson_score_interval(tp, tp + fp) if (tp + fp) > 0 else [0.0, 0.0]
balanced_acc = round(0.5 * (recall + specificity), 4)
if (precision + recall) > 0:
f1 = round(2.0 * precision * recall / (precision + recall), 4)
else:
f1 = 0.0
return {
"total_samples": total,
"valid_evaluations": valid_total,
"non_decisive_or_error": error_or_uncertain,
"confusion_matrix": {"tp": tp, "fp": fp, "tn": tn, "fn": fn},
"accuracy": accuracy,
"accuracy_ci_95": accuracy_ci,
"balanced_accuracy": balanced_acc,
"precision": precision,
"precision_ci_95": precision_ci,
"recall": recall,
"recall_ci_95": recall_ci,
"specificity": specificity,
"specificity_ci_95": specificity_ci,
"f1_score": f1,
}
def main():
parser = argparse.ArgumentParser(description="Phase 6 Evaluation Pipeline")
parser.add_argument("--worker", action="store_true", help="Internal worker mode")
parser.add_argument("--worker-detector", type=str, help="Detector key for worker")
parser.add_argument("--worker-input", type=str, help="Input JSON for worker")
parser.add_argument("--worker-output", type=str, help="Output JSON for worker")
parser.add_argument("--skip-smoke", action="store_true", help="Skip preliminary smoke tests")
args = parser.parse_args()
if args.worker:
run_worker(args.worker_detector, args.worker_input, args.worker_output)
return
print("================================================================================")
print("PHASE 6: AUTONOMOUS EVALUATION PIPELINE & COMBINED CLASSIFIER")
print("================================================================================")
os.makedirs(RESULTS_DIR, exist_ok=True)
# 1. Load manifest
print(f"Loading manifest from: {MANIFEST_PATH}")
if not os.path.exists(MANIFEST_PATH):
raise FileNotFoundError(f"Manifest not found: {MANIFEST_PATH}")
samples = []
with open(MANIFEST_PATH, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
for row in reader:
samples.append({
"sample_id": row["sample_id"],
"original_filename": row["original_filename"],
"file_path": row["absolute_path"],
"ground_truth": row["ground_truth"],
"category": row.get("category", ""),
"domain": row.get("domain", ""),
})
total_samples = len(samples)
real_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_REAL)
fake_count = sum(1 for s in samples if s["ground_truth"] == PREDICTION_FAKE)
print(f"Manifest verified: {total_samples} samples total (REAL={real_count}, FAKE={fake_count})")
assert total_samples == 200, f"Expected 200 samples, got {total_samples}"
assert real_count == 100, f"Expected 100 REAL samples, got {real_count}"
assert fake_count == 100, f"Expected 100 FAKE samples, got {fake_count}"
# 2. Preliminary Smoke Test
if not args.skip_smoke:
print("\n--- Step 1: Preliminary Single-Sample Smoke Tests ---")
smoke_sample = [samples[0]] # first real image
for det_key in DETECTOR_KEYS:
print(f"Testing detector '{det_key}' smoke execution...")
smoke_res, load_t, lat_t = run_detector_subprocess(det_key, smoke_sample)
print(f" -> '{det_key}' smoke passed! Pred: {smoke_res[0].get('prediction')}, Latency: {lat_t}ms")
# 3. Sequential Subprocess Inference across 200 samples
print("\n--- Step 2: Sequential Detector Inference on 200 Samples ---")
detector_outputs: Dict[str, List[Dict[str, Any]]] = {}
detector_perf: Dict[str, Dict[str, float]] = {}
for det_key in DETECTOR_KEYS:
print(f"\nEvaluating detector '{det_key}' on 200 samples...")
t_start = time.perf_counter()
results, load_time, mean_lat = run_detector_subprocess(det_key, samples)
t_total = time.perf_counter() - t_start
print(f"Completed '{det_key}' in {t_total:.2f}s (Load: {load_time}s, Mean Sample Latency: {mean_lat}ms)")
detector_outputs[det_key] = results
detector_perf[det_key] = {
"load_time_sec": load_time,
"mean_latency_ms": mean_lat,
"total_time_sec": round(t_total, 2),
}
# 4. Save Individual Prediction CSVs
print("\n--- Step 3: Exporting Individual Model Predictions ---")
pred_fieldnames = [
"sample_id", "original_filename", "ground_truth", "detector_key",
"detector_name", "model_name", "prediction", "predicted_class",
"fake_probability", "real_probability", "confidence", "raw_logit",
"threshold", "status", "error_message", "latency_ms", "file_path"
]
for det_key, out_path in [
("vit", OUTPUT_VIT_CSV),
("univfd", OUTPUT_UNIVFD_CSV),
("chen2024_convnext", OUTPUT_CONVNEXT_CSV),
]:
res_list = detector_outputs[det_key]
with open(out_path, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=pred_fieldnames)
writer.writeheader()
for r, s in zip(res_list, samples):
row_dict = {
"sample_id": s["sample_id"],
"original_filename": s["original_filename"],
"ground_truth": s["ground_truth"],
"detector_key": det_key,
"detector_name": r.get("detector_name", ""),
"model_name": r.get("model_name", ""),
"prediction": r.get("prediction", ""),
"predicted_class": r.get("predicted_class", r.get("prediction", "")),
"fake_probability": r.get("fake_probability", ""),
"real_probability": r.get("real_probability", ""),
"confidence": r.get("confidence", ""),
"raw_logit": r.get("raw_logit", ""),
"threshold": r.get("threshold", 0.5),
"status": r.get("status", ""),
"error_message": r.get("error", "") or r.get("error_message", ""),
"latency_ms": r.get("latency_ms", ""),
"file_path": s["file_path"],
}
writer.writerow(row_dict)
print(f" Written {len(res_list)} rows to: {out_path}")
# 5. Combined Classifier & Ensemble Predictions
print("\n--- Step 4: Running Combined Classifier & Fusion Strategies ---")
ground_truths = [s["ground_truth"] for s in samples]
vit_preds = [r.get("prediction") for r in detector_outputs["vit"]]
univfd_preds = [r.get("prediction") for r in detector_outputs["univfd"]]
convnext_preds = [r.get("prediction") for r in detector_outputs["chen2024_convnext"]]
vit_probs = [r.get("fake_probability") for r in detector_outputs["vit"]]
univfd_probs = [r.get("fake_probability") for r in detector_outputs["univfd"]]
convnext_probs = [r.get("fake_probability") for r in detector_outputs["chen2024_convnext"]]
combined_rows = []
maj_preds = []
maj_probs = []
mean_preds = []
mean_probs = []
vit_priority_preds = []
vit_priority_probs = []
agree_3_count = 0
agree_maj_count = 0
for i, s in enumerate(samples):
# Prepare dictionary for fusion
record_map = {
"vit": detector_outputs["vit"][i],
"univfd": detector_outputs["univfd"][i],
"chen2024_convnext": detector_outputs["chen2024_convnext"][i],
}
# Run strategies
res_maj = strategy_majority_vote(record_map, tie_breaker="vit_or_real")
res_mean = strategy_mean_fake_probability(record_map, threshold=0.50)
res_prio = strategy_vit_priority(record_map, vit_key="vit")
maj_preds.append(res_maj.prediction)
maj_probs.append(res_maj.fake_probability)
mean_preds.append(res_mean.prediction)
mean_probs.append(res_mean.fake_probability)
vit_priority_preds.append(res_prio.prediction)
vit_priority_probs.append(res_prio.fake_probability)
all_three_agree = (vit_preds[i] == univfd_preds[i] == convnext_preds[i])
if all_three_agree:
agree_3_count += 1
combined_rows.append({
"sample_id": s["sample_id"],
"original_filename": s["original_filename"],
"ground_truth": s["ground_truth"],
"vit_pred": vit_preds[i],
"vit_fake_prob": vit_probs[i],
"univfd_pred": univfd_preds[i],
"univfd_fake_prob": univfd_probs[i],
"convnext_pred": convnext_preds[i],
"convnext_fake_prob": convnext_probs[i],
"majority_vote_pred": res_maj.prediction,
"majority_vote_fake_prob": res_maj.fake_probability,
"mean_prob_pred": res_mean.prediction,
"mean_prob_fake_prob": res_mean.fake_probability,
"vit_priority_pred": res_prio.prediction,
"vit_priority_fake_prob": res_prio.fake_probability,
"all_three_agree": all_three_agree,
"majority_explanation": res_maj.explanation,
"mean_explanation": res_mean.explanation,
"priority_explanation": res_prio.explanation,
})
# Write combined predictions CSV
comb_fieldnames = list(combined_rows[0].keys())
with open(OUTPUT_COMBINED_CSV, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=comb_fieldnames)
writer.writeheader()
writer.writerows(combined_rows)
print(f" Written combined predictions ({len(combined_rows)} rows) to: {OUTPUT_COMBINED_CSV}")
# 6. Compute Comprehensive Metrics
print("\n--- Step 5: Computing Statistical Metrics & Cross-Model Agreement ---")
metrics_report: Dict[str, Any] = {
"metadata": {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"manifest_path": os.path.relpath(MANIFEST_PATH, PROJECT_ROOT).replace("\\", "/"),
"dataset_source": "Kaggle 140k Real and Fake Faces (FFHQ / StyleGAN)",
"total_test_samples": total_samples,
"class_distribution": {"REAL": real_count, "FAKE": fake_count},
"environment": {
"python_version": sys.version,
"torch_version": torch.__version__,
"platform": sys.platform,
}
},
"model_performance": {},
"agreement_analysis": {},
}
# Evaluate each individual detector and strategy
eval_targets = {
"vit": {"name": "ViT (dima806)", "preds": vit_preds, "lat": detector_perf["vit"]["mean_latency_ms"]},
"univfd": {"name": "UnivFD (CLIP ViT-L/14)", "preds": univfd_preds, "lat": detector_perf["univfd"]["mean_latency_ms"]},
"chen2024_convnext": {"name": "ConvNeXt (DRCT / RAID)", "preds": convnext_preds, "lat": detector_perf["chen2024_convnext"]["mean_latency_ms"]},
"ensemble_majority_vote": {"name": "Ensemble (Majority Vote)", "preds": maj_preds, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)},
"ensemble_mean_prob": {"name": "Ensemble (Mean Fake Probability)", "preds": mean_preds, "lat": sum(detector_perf[k]["mean_latency_ms"] for k in DETECTOR_KEYS)},
"ensemble_vit_priority": {"name": "Ensemble (ViT Priority Fallback)", "preds": vit_priority_preds, "lat": detector_perf["vit"]["mean_latency_ms"]},
}
comparison_rows = []
for key, target in eval_targets.items():
m = compute_binary_metrics(target["preds"], ground_truths)
m["mean_latency_ms"] = target["lat"]
m["display_name"] = target["name"]
metrics_report["model_performance"][key] = m
print(f" {target['name']:35s} | Acc: {m['accuracy']*100:6.2f}% ({m['accuracy_ci_95'][0]*100:.1f}-{m['accuracy_ci_95'][1]*100:.1f}%) | "
f"BalAcc: {m['balanced_accuracy']*100:6.2f}% | Prec: {m['precision']:5.3f} | Rec: {m['recall']:5.3f} | "
f"Spec: {m['specificity']:5.3f} | F1: {m['f1_score']:5.3f}")
comparison_rows.append({
"Model / Strategy": target["name"],
"Accuracy (%)": f"{m['accuracy'] * 100:.2f}",
"95% CI (Accuracy)": f"[{m['accuracy_ci_95'][0]*100:.2f}%, {m['accuracy_ci_95'][1]*100:.2f}%]",
"Balanced Acc (%)": f"{m['balanced_accuracy'] * 100:.2f}",
"Precision": f"{m['precision']:.4f}",
"Recall": f"{m['recall']:.4f}",
"Specificity": f"{m['specificity']:.4f}",
"F1-Score": f"{m['f1_score']:.4f}",
"TP": m["confusion_matrix"]["tp"],
"FP": m["confusion_matrix"]["fp"],
"TN": m["confusion_matrix"]["tn"],
"FN": m["confusion_matrix"]["fn"],
"Mean Latency (ms)": f"{target['lat']:.1f}",
})
# Agreement matrices & Cohen's Kappa
def get_pairwise_kappa(p1, p2):
both_fake = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_FAKE)
both_real = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_REAL)
m1_f_m2_r = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_FAKE and b == PREDICTION_REAL)
m1_r_m2_f = sum(1 for a, b in zip(p1, p2) if a == PREDICTION_REAL and b == PREDICTION_FAKE)
raw_agree = (both_fake + both_real) / len(p1)
kappa = compute_cohen_kappa(both_fake, both_real, m1_f_m2_r, m1_r_m2_f)
return {
"raw_agreement_rate": round(raw_agree, 4),
"cohen_kappa": kappa,
"both_fake": both_fake,
"both_real": both_real,
"m1_fake_m2_real": m1_f_m2_r,
"m1_real_m2_fake": m1_r_m2_f,
}
metrics_report["agreement_analysis"] = {
"all_three_agree_count": agree_3_count,
"all_three_agree_fraction": round(agree_3_count / total_samples, 4),
"vit_vs_univfd": get_pairwise_kappa(vit_preds, univfd_preds),
"vit_vs_convnext": get_pairwise_kappa(vit_preds, convnext_preds),
"univfd_vs_convnext": get_pairwise_kappa(univfd_preds, convnext_preds),
}
# Save metrics JSON
with open(OUTPUT_METRICS_JSON, "w", encoding="utf-8") as f:
json.dump(metrics_report, f, indent=2)
print(f"\n Saved comprehensive metrics JSON to: {OUTPUT_METRICS_JSON}")
# Save comparison CSV
with open(OUTPUT_COMPARISON_CSV, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(comparison_rows[0].keys()))
writer.writeheader()
writer.writerows(comparison_rows)
print(f" Saved model comparison summary CSV to: {OUTPUT_COMPARISON_CSV}")
print("\n================================================================================")
print("PHASE 6 PIPELINE EVALUATION COMPLETED SUCCESSFULLY!")
print("================================================================================")
if __name__ == "__main__":
main()