Spaces:
Running on Zero
Running on Zero
Download scripts/system_audit/audit_metrics.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/system_audit/audit_metrics.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/system_audit/audit_metrics.py
-
curl -L -o audit_metrics.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/system_audit/audit_metrics.py
12.2 kB
| """ | |
| Comprehensive Diagnostic and Metrics Audit Script for Multi-Model Connection | |
| Computes all metrics and diagnostics required for Step 5, Step 6, and Step 7. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import math | |
| import numpy as np | |
| import pandas as pd | |
| from scipy.optimize import brentq | |
| from scipy.interpolate import interp1d | |
| from sklearn.metrics import ( | |
| accuracy_score, | |
| precision_score, | |
| recall_score, | |
| f1_score, | |
| roc_auc_score, | |
| average_precision_score, | |
| confusion_matrix, | |
| roc_curve, | |
| ) | |
| # Project paths | |
| PROJECT_ROOT = r"C:\Users\user\Desktop\multi model connection" | |
| sys.path.insert(0, PROJECT_ROOT) | |
| from src.decision_fusion import DecisionFusionPipeline, STRATEGY_VIT_PRIORITY, STRATEGY_MEAN_PROBABILITY | |
| from src.detector_schema import DetectorResult | |
| def compute_eer(y_true, y_score): | |
| """Compute Equal Error Rate (EER) and the optimal threshold.""" | |
| fpr, tpr, thresholds = roc_curve(y_true, y_score, pos_label=1) | |
| fnr = 1 - tpr | |
| # Find the point where fpr == fnr | |
| try: | |
| eer = brentq(lambda x: 1.0 - interp1d(fpr, tpr)(x) - x, 0.0, 1.0) | |
| except Exception: | |
| # Fallback to closest point | |
| diff = np.abs(fpr - fnr) | |
| min_idx = np.argmin(diff) | |
| eer = (fpr[min_idx] + fnr[min_idx]) / 2.0 | |
| return float(eer) | |
| def compute_ece(y_true, y_score, n_bins=10): | |
| """Compute Expected Calibration Error (ECE).""" | |
| bin_boundaries = np.linspace(0, 1, n_bins + 1) | |
| ece = 0.0 | |
| total_samples = len(y_true) | |
| bin_details = [] | |
| for i in range(n_bins): | |
| bin_lower = bin_boundaries[i] | |
| bin_upper = bin_boundaries[i + 1] | |
| # Bin inclusion rule: [lower, upper] for last bin, [lower, upper) for others | |
| if i == n_bins - 1: | |
| in_bin = (y_score >= bin_lower) & (y_score <= bin_upper) | |
| else: | |
| in_bin = (y_score >= bin_lower) & (y_score < bin_upper) | |
| prop_in_bin = np.mean(in_bin) | |
| if prop_in_bin > 0: | |
| bin_acc = np.mean(y_true[in_bin]) | |
| bin_conf = np.mean(y_score[in_bin]) | |
| bin_error = np.abs(bin_acc - bin_conf) | |
| ece += bin_error * prop_in_bin | |
| bin_details.append({ | |
| "bin": i, | |
| "range": f"[{bin_lower:.1f}, {bin_upper:.1f}]", | |
| "samples": int(np.sum(in_bin)), | |
| "acc": float(bin_acc), | |
| "conf": float(bin_conf), | |
| "abs_diff": float(bin_error), | |
| }) | |
| return float(ece), bin_details | |
| def compute_metrics_table(y_true, y_pred, y_score): | |
| """Compute standard evaluation metrics.""" | |
| tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel() | |
| acc = accuracy_score(y_true, y_pred) | |
| prec = precision_score(y_true, y_pred, zero_division=0) | |
| rec = recall_score(y_true, y_pred, zero_division=0) | |
| f1 = f1_score(y_true, y_pred, zero_division=0) | |
| # Check if multiple classes are present in y_true for AUC | |
| if len(np.unique(y_true)) > 1: | |
| roc_auc = roc_auc_score(y_true, y_score) | |
| pr_auc = average_precision_score(y_true, y_score) | |
| eer = compute_eer(y_true, y_score) | |
| else: | |
| roc_auc = float('nan') | |
| pr_auc = float('nan') | |
| eer = float('nan') | |
| return { | |
| "TP": int(tp), | |
| "FP": int(fp), | |
| "TN": int(tn), | |
| "FN": int(fn), | |
| "Accuracy": float(acc), | |
| "Precision": float(prec), | |
| "Recall": float(rec), | |
| "F1": float(f1), | |
| "ROC-AUC": float(roc_auc), | |
| "PR-AUC": float(pr_auc), | |
| "EER": float(eer), | |
| } | |
| def analyze_dataset_predictions(df, ground_truth_col, score_cols, pred_cols): | |
| """Analyze predictions across models on a dataset.""" | |
| results = {} | |
| y_true = (df[ground_truth_col] == "FAKE").astype(int).values | |
| for entity_name in score_cols: | |
| y_score = df[score_cols[entity_name]].values | |
| if entity_name in pred_cols: | |
| y_pred = (df[pred_cols[entity_name]] == "FAKE").astype(int).values | |
| else: | |
| y_pred = (y_score >= 0.50).astype(int) | |
| m = compute_metrics_table(y_true, y_pred, y_score) | |
| ece, _ = compute_ece(y_true, y_score) | |
| m["ECE"] = ece | |
| results[entity_name] = m | |
| return results | |
| def main(): | |
| fusion_pipe = DecisionFusionPipeline() | |
| output_dir = os.path.join(PROJECT_ROOT, "scripts", "system_audit") | |
| os.makedirs(output_dir, exist_ok=True) | |
| print("=== Step 5 & 6 Evaluation Audit Runner ===") | |
| # 1. HELD-OUT TEST SET (Phase 4 predictions, 103 classification samples) | |
| p4_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase4_baseline_predictions.csv") | |
| df_p4_long = pd.read_csv(p4_file) | |
| # Filter to eligible samples | |
| df_p4_long = df_p4_long[df_p4_long["status"] == "SUCCESS"].copy() | |
| # Pivot to wide format: one row per sample_id | |
| pivoted_p4 = [] | |
| sample_ids = df_p4_long["sample_id"].unique() | |
| for sid in sample_ids: | |
| rows = df_p4_long[df_p4_long["sample_id"] == sid] | |
| sample_meta = rows.iloc[0] | |
| gt = sample_meta["ground_truth"] | |
| if gt not in ("REAL", "FAKE"): | |
| continue | |
| c_row = rows[rows["detector_key"] == "chen2024_convnext"] | |
| u_row = rows[rows["detector_key"] == "univfd"] | |
| v_row = rows[rows["detector_key"] == "vit"] | |
| if c_row.empty or u_row.empty or v_row.empty: | |
| continue | |
| c_prob = float(c_row["fake_probability"].iloc[0]) | |
| u_prob = float(u_row["fake_probability"].iloc[0]) | |
| v_prob = float(v_row["fake_probability"].iloc[0]) | |
| c_pred = c_row["prediction"].iloc[0] | |
| u_pred = u_row["prediction"].iloc[0] | |
| v_pred = v_row["prediction"].iloc[0] | |
| # Simple average | |
| mean_p = round((c_prob + u_prob + v_prob) / 3.0, 6) | |
| mean_pred = "FAKE" if mean_p >= 0.50 else "REAL" | |
| # Central analyzer (ViT Priority) | |
| det_dict = { | |
| "chen2024_convnext": {"prediction": c_pred, "fake_probability": c_prob}, | |
| "univfd": {"prediction": u_pred, "fake_probability": u_prob}, | |
| "vit": {"prediction": v_pred, "fake_probability": v_prob}, | |
| } | |
| fused = fusion_pipe.fuse(det_dict, strategy=STRATEGY_VIT_PRIORITY) | |
| pivoted_p4.append({ | |
| "sample_id": sid, | |
| "ground_truth": gt, | |
| "category": sample_meta["category"], | |
| "domain": sample_meta["domain"], | |
| "file_path": sample_meta["file_path"], | |
| "convnext_prob": c_prob, | |
| "convnext_pred": c_pred, | |
| "univfd_prob": u_prob, | |
| "univfd_pred": u_pred, | |
| "vit_prob": v_prob, | |
| "vit_pred": v_pred, | |
| "simple_avg_prob": mean_p, | |
| "simple_avg_pred": mean_pred, | |
| "analyzer_prob": fused.fake_probability, | |
| "analyzer_pred": fused.prediction, | |
| }) | |
| df_p4 = pd.DataFrame(pivoted_p4) | |
| print(f"Held-out test set processed: {len(df_p4)} samples") | |
| score_cols_p4 = { | |
| "Model A (ConvNeXt)": "convnext_prob", | |
| "Model B (UnivFD)": "univfd_prob", | |
| "Model C (ViT)": "vit_prob", | |
| "Simple Average": "simple_avg_prob", | |
| "Central Analyzer (ViT-Priority)": "analyzer_prob", | |
| } | |
| pred_cols_p4 = { | |
| "Model A (ConvNeXt)": "convnext_pred", | |
| "Model B (UnivFD)": "univfd_pred", | |
| "Model C (ViT)": "vit_pred", | |
| "Simple Average": "simple_avg_pred", | |
| "Central Analyzer (ViT-Priority)": "analyzer_pred", | |
| } | |
| metrics_held_out = analyze_dataset_predictions(df_p4, "ground_truth", score_cols_p4, pred_cols_p4) | |
| # 2. VALIDATION SET (Phase 5, 50 samples) | |
| p5_val_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase5_val_raw_predictions.csv") | |
| df_p5_long = pd.read_csv(p5_val_file) | |
| pivoted_p5 = [] | |
| for sid in df_p5_long["sample_id"].unique(): | |
| rows = df_p5_long[df_p5_long["sample_id"] == sid] | |
| sample_meta = rows.iloc[0] | |
| gt = sample_meta["ground_truth"] | |
| c_row = rows[rows["detector_key"] == "chen2024_convnext"] | |
| u_row = rows[rows["detector_key"] == "univfd"] | |
| v_row = rows[rows["detector_key"] == "vit"] | |
| c_prob = float(c_row["fake_probability"].iloc[0]) | |
| u_prob = float(u_row["fake_probability"].iloc[0]) | |
| v_prob = float(v_row["fake_probability"].iloc[0]) | |
| c_pred = c_row["prediction"].iloc[0] | |
| u_pred = u_row["prediction"].iloc[0] | |
| v_pred = v_row["prediction"].iloc[0] | |
| mean_p = round((c_prob + u_prob + v_prob) / 3.0, 6) | |
| mean_pred = "FAKE" if mean_p >= 0.50 else "REAL" | |
| det_dict = { | |
| "chen2024_convnext": {"prediction": c_pred, "fake_probability": c_prob}, | |
| "univfd": {"prediction": u_pred, "fake_probability": u_prob}, | |
| "vit": {"prediction": v_pred, "fake_probability": v_prob}, | |
| } | |
| fused = fusion_pipe.fuse(det_dict, strategy=STRATEGY_VIT_PRIORITY) | |
| pivoted_p5.append({ | |
| "sample_id": sid, | |
| "ground_truth": gt, | |
| "convnext_prob": c_prob, | |
| "convnext_pred": c_pred, | |
| "univfd_prob": u_prob, | |
| "univfd_pred": u_pred, | |
| "vit_prob": v_prob, | |
| "vit_pred": v_pred, | |
| "simple_avg_prob": mean_p, | |
| "simple_avg_pred": mean_pred, | |
| "analyzer_prob": fused.fake_probability, | |
| "analyzer_pred": fused.prediction, | |
| }) | |
| df_p5 = pd.DataFrame(pivoted_p5) | |
| print(f"Validation set processed: {len(df_p5)} samples") | |
| metrics_val = analyze_dataset_predictions(df_p5, "ground_truth", score_cols_p4, pred_cols_p4) | |
| # 3. PHASE 6 TEST SET (200 samples) | |
| p6_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase6_combined_predictions.csv") | |
| df_p6 = pd.read_csv(p6_file) | |
| score_cols_p6 = { | |
| "Model A (ConvNeXt)": "convnext_fake_prob", | |
| "Model B (UnivFD)": "univfd_fake_prob", | |
| "Model C (ViT)": "vit_fake_prob", | |
| "Simple Average": "mean_prob_fake_prob", | |
| "Central Analyzer (ViT-Priority)": "vit_priority_fake_prob", | |
| } | |
| pred_cols_p6 = { | |
| "Model A (ConvNeXt)": "convnext_pred", | |
| "Model B (UnivFD)": "univfd_pred", | |
| "Model C (ViT)": "vit_pred", | |
| "Simple Average": "mean_prob_pred", | |
| "Central Analyzer (ViT-Priority)": "vit_priority_pred", | |
| } | |
| metrics_p6 = analyze_dataset_predictions(df_p6, "ground_truth", score_cols_p6, pred_cols_p6) | |
| print(f"Phase 6 test set processed: {len(df_p6)} samples") | |
| # 4. PHASE 7 TEST SET (200 samples) | |
| p7_file = os.path.join(PROJECT_ROOT, "benchmark", "results", "phase7_combined_predictions.csv") | |
| df_p7 = pd.read_csv(p7_file) | |
| metrics_p7 = analyze_dataset_predictions(df_p7, "ground_truth", score_cols_p6, pred_cols_p6) | |
| print(f"Phase 7 test set processed: {len(df_p7)} samples") | |
| # Save all tables to JSON | |
| audit_results = { | |
| "held_out_test_103": metrics_held_out, | |
| "development_val_50": metrics_val, | |
| "phase6_test_200": metrics_p6, | |
| "phase7_test_200": metrics_p7, | |
| } | |
| with open(os.path.join(output_dir, "step5_metrics.json"), "w") as f: | |
| json.dump(audit_results, f, indent=2) | |
| print("Saved step5_metrics.json") | |
| # Save formatted markdown tables for easy reading | |
| print("\n--- HELD-OUT TEST (103 samples) ---") | |
| print(pd.DataFrame(metrics_held_out).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]]) | |
| print("\n--- PHASE 6 TEST (200 samples) ---") | |
| print(pd.DataFrame(metrics_p6).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]]) | |
| print("\n--- VALIDATION SET (50 samples) ---") | |
| print(pd.DataFrame(metrics_val).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]]) | |
| print("\n--- PHASE 7 TEST (200 samples) ---") | |
| print(pd.DataFrame(metrics_p7).T[["Accuracy", "Precision", "Recall", "F1", "ROC-AUC", "PR-AUC", "EER", "TP", "FP", "TN", "FN"]]) | |
| if __name__ == "__main__": | |
| main() | |