Spaces:
Sleeping
Sleeping
Download src/evaluation/metrics.py from APEXPRIT/deception-analysis-api: direct link, hf CLI and curl.
- Browser
- Download file 13.3 kB
-
https://huggingface.co/spaces/APEXPRIT/deception-analysis-api/resolve/main/src/evaluation/metrics.py
- Command line
-
hf download hf://spaces/APEXPRIT/deception-analysis-api/src/evaluation/metrics.py
-
curl -L -o metrics.py https://huggingface.co/spaces/APEXPRIT/deception-analysis-api/resolve/main/src/evaluation/metrics.py
13.3 kB
| """Comprehensive evaluation metrics for deception detection. | |
| This module provides a DeceptionMetrics class that computes a wide range of | |
| classification metrics, fairness metrics, and confidence intervals for | |
| evaluating deception detection models. | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| from typing import Any, Dict, List, Optional, Tuple, Union | |
| import numpy as np | |
| import pandas as pd | |
| from sklearn.metrics import ( | |
| accuracy_score, | |
| precision_score, | |
| recall_score, | |
| f1_score, | |
| roc_auc_score, | |
| average_precision_score, | |
| confusion_matrix, | |
| classification_report, | |
| balanced_accuracy_score, | |
| ) | |
| from sklearn.utils import resample | |
| from scipy import stats | |
| logger = logging.getLogger(__name__) | |
| class DeceptionMetrics: | |
| """Compute and aggregate evaluation metrics for deception detection. | |
| This class provides methods for computing standard classification metrics, | |
| per-subject metrics, fairness metrics, and bootstrap confidence intervals. | |
| Attributes: | |
| random_state: Random seed for reproducibility. | |
| n_bootstrap: Number of bootstrap samples for confidence intervals. | |
| ci_level: Confidence level for intervals (default 0.95). | |
| """ | |
| def __init__( | |
| self, | |
| random_state: int = 42, | |
| n_bootstrap: int = 1000, | |
| ci_level: float = 0.95, | |
| ) -> None: | |
| """Initialize metrics calculator. | |
| Args: | |
| random_state: Random seed for reproducibility. | |
| n_bootstrap: Number of bootstrap samples for confidence intervals. | |
| ci_level: Confidence level for intervals (default 0.95). | |
| """ | |
| self.random_state = random_state | |
| self.n_bootstrap = n_bootstrap | |
| self.ci_level = ci_level | |
| np.random.seed(random_state) | |
| def compute_all_metrics( | |
| self, | |
| y_true: Union[np.ndarray, List[int]], | |
| y_pred: Union[np.ndarray, List[int]], | |
| y_prob: Optional[Union[np.ndarray, List[float]]] = None, | |
| ) -> Dict[str, Any]: | |
| """Compute comprehensive classification metrics. | |
| Args: | |
| y_true: Ground truth binary labels (0 = truthful, 1 = deceptive). | |
| y_pred: Predicted binary labels. | |
| y_prob: Predicted probabilities for the positive class (deceptive). | |
| If provided, ROC‑AUC and average precision are computed. | |
| Returns: | |
| Dictionary containing: | |
| - accuracy, precision, recall, f1, specificity | |
| - balanced_accuracy | |
| - auc_roc, average_precision (if y_prob provided) | |
| - confusion_matrix (as 2x2 array) | |
| - classification_report (as dict) | |
| """ | |
| y_true = np.asarray(y_true) | |
| y_pred = np.asarray(y_pred) | |
| if y_true.ndim != 1: | |
| raise ValueError(f"y_true must be 1‑D, got shape {y_true.shape}") | |
| if y_pred.ndim != 1: | |
| raise ValueError(f"y_pred must be 1‑D, got shape {y_pred.shape}") | |
| if len(y_true) != len(y_pred): | |
| raise ValueError( | |
| f"Length mismatch: y_true ({len(y_true)}) vs y_pred ({len(y_pred)})" | |
| ) | |
| # Basic metrics | |
| accuracy = accuracy_score(y_true, y_pred) | |
| precision = precision_score(y_true, y_pred, zero_division=0) | |
| recall = recall_score(y_true, y_pred, zero_division=0) | |
| f1 = f1_score(y_true, y_pred, zero_division=0) | |
| balanced_acc = balanced_accuracy_score(y_true, y_pred) | |
| # Specificity (true negative rate) | |
| tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel() | |
| specificity = tn / (tn + fp) if (tn + fp) > 0 else 0.0 | |
| # Probability‑based metrics | |
| auc_roc = None | |
| avg_precision = None | |
| if y_prob is not None: | |
| y_prob = np.asarray(y_prob) | |
| if y_prob.ndim == 2 and y_prob.shape[1] == 2: | |
| y_prob = y_prob[:, 1] # take positive class probabilities | |
| try: | |
| auc_roc = roc_auc_score(y_true, y_prob) | |
| except ValueError as e: | |
| logger.warning(f"Could not compute ROC‑AUC: {e}") | |
| auc_roc = None | |
| try: | |
| avg_precision = average_precision_score(y_true, y_prob) | |
| except ValueError as e: | |
| logger.warning(f"Could not compute average precision: {e}") | |
| avg_precision = None | |
| # Confusion matrix | |
| cm = confusion_matrix(y_true, y_pred) | |
| # Classification report as dict | |
| report = classification_report( | |
| y_true, y_pred, output_dict=True, zero_division=0 | |
| ) | |
| metrics = { | |
| "accuracy": float(accuracy), | |
| "precision": float(precision), | |
| "recall": float(recall), | |
| "f1": float(f1), | |
| "specificity": float(specificity), | |
| "balanced_accuracy": float(balanced_acc), | |
| "confusion_matrix": cm.tolist(), | |
| "classification_report": report, | |
| "support": int(len(y_true)), | |
| } | |
| if auc_roc is not None: | |
| metrics["auc_roc"] = float(auc_roc) | |
| if avg_precision is not None: | |
| metrics["average_precision"] = float(avg_precision) | |
| logger.info( | |
| f"Computed metrics: accuracy={accuracy:.3f}, f1={f1:.3f}, " | |
| f"recall={recall:.3f}, precision={precision:.3f}" | |
| ) | |
| return metrics | |
| def compute_per_subject_metrics( | |
| self, | |
| y_true: Union[np.ndarray, List[int]], | |
| y_pred: Union[np.ndarray, List[int]], | |
| subject_ids: Union[np.ndarray, List[str]], | |
| y_prob: Optional[Union[np.ndarray, List[float]]] = None, | |
| ) -> Dict[str, Dict[str, Any]]: | |
| """Compute metrics separately for each subject. | |
| Args: | |
| y_true: Ground truth binary labels. | |
| y_pred: Predicted binary labels. | |
| subject_ids: Subject identifier for each sample. | |
| y_prob: Predicted probabilities (optional). | |
| Returns: | |
| Dictionary mapping subject_id -> metrics dict. | |
| """ | |
| y_true = np.asarray(y_true) | |
| y_pred = np.asarray(y_pred) | |
| subject_ids = np.asarray(subject_ids) | |
| if len(y_true) != len(subject_ids): | |
| raise ValueError( | |
| f"Length mismatch: y_true ({len(y_true)}) vs subject_ids ({len(subject_ids)})" | |
| ) | |
| unique_subjects = np.unique(subject_ids) | |
| subject_metrics = {} | |
| for subj in unique_subjects: | |
| mask = subject_ids == subj | |
| if np.sum(mask) == 0: | |
| continue | |
| subj_y_true = y_true[mask] | |
| subj_y_pred = y_pred[mask] | |
| subj_y_prob = y_prob[mask] if y_prob is not None else None | |
| try: | |
| metrics = self.compute_all_metrics( | |
| subj_y_true, subj_y_pred, subj_y_prob | |
| ) | |
| subject_metrics[subj] = metrics | |
| except Exception as e: | |
| logger.warning(f"Failed to compute metrics for subject {subj}: {e}") | |
| subject_metrics[subj] = {"error": str(e)} | |
| logger.info(f"Computed per‑subject metrics for {len(subject_metrics)} subjects") | |
| return subject_metrics | |
| def compute_fairness_metrics( | |
| self, | |
| y_true: Union[np.ndarray, List[int]], | |
| y_pred: Union[np.ndarray, List[int]], | |
| sensitive_groups: Union[np.ndarray, List[str]], | |
| ) -> Dict[str, float]: | |
| """Compute fairness metrics across sensitive groups. | |
| Currently implemented: | |
| - Demographic parity difference: max difference in positive rate across groups. | |
| - Equalized odds difference: max difference in TPR and FPR across groups. | |
| Args: | |
| y_true: Ground truth binary labels. | |
| y_pred: Predicted binary labels. | |
| sensitive_groups: Group membership for each sample (e.g., gender, ethnicity). | |
| Returns: | |
| Dictionary with fairness metrics. | |
| """ | |
| y_true = np.asarray(y_true) | |
| y_pred = np.asarray(y_pred) | |
| sensitive_groups = np.asarray(sensitive_groups) | |
| unique_groups = np.unique(sensitive_groups) | |
| group_stats = {} | |
| for group in unique_groups: | |
| mask = sensitive_groups == group | |
| if np.sum(mask) == 0: | |
| continue | |
| group_y_true = y_true[mask] | |
| group_y_pred = y_pred[mask] | |
| # Positive rate (predicted) | |
| pos_rate = np.mean(group_y_pred) | |
| # True positive rate and false positive rate | |
| tn, fp, fn, tp = confusion_matrix(group_y_true, group_y_pred).ravel() | |
| tpr = tp / (tp + fn) if (tp + fn) > 0 else 0.0 | |
| fpr = fp / (fp + tn) if (fp + tn) > 0 else 0.0 | |
| group_stats[group] = { | |
| "positive_rate": pos_rate, | |
| "tpr": tpr, | |
| "fpr": fpr, | |
| "support": int(np.sum(mask)), | |
| } | |
| # Demographic parity difference | |
| pos_rates = [stats["positive_rate"] for stats in group_stats.values()] | |
| demographic_parity_diff = max(pos_rates) - min(pos_rates) if pos_rates else 0.0 | |
| # Equalized odds differences (max TPR diff, max FPR diff) | |
| tprs = [stats["tpr"] for stats in group_stats.values()] | |
| fprs = [stats["fpr"] for stats in group_stats.values()] | |
| equalized_odds_tpr_diff = max(tprs) - min(tprs) if tprs else 0.0 | |
| equalized_odds_fpr_diff = max(fprs) - min(fprs) if fprs else 0.0 | |
| fairness = { | |
| "demographic_parity_difference": float(demographic_parity_diff), | |
| "equalized_odds_tpr_difference": float(equalized_odds_tpr_diff), | |
| "equalized_odds_fpr_difference": float(equalized_odds_fpr_diff), | |
| "group_stats": group_stats, | |
| } | |
| logger.info( | |
| f"Fairness metrics: demographic parity diff = {demographic_parity_diff:.3f}, " | |
| f"equalized odds TPR diff = {equalized_odds_tpr_diff:.3f}" | |
| ) | |
| return fairness | |
| def bootstrap_confidence_intervals( | |
| self, | |
| y_true: Union[np.ndarray, List[int]], | |
| y_pred: Union[np.ndarray, List[int]], | |
| y_prob: Optional[Union[np.ndarray, List[float]]] = None, | |
| n_bootstrap: Optional[int] = None, | |
| ci: Optional[float] = None, | |
| ) -> Dict[str, Dict[str, float]]: | |
| """Compute bootstrap confidence intervals for metrics. | |
| Args: | |
| y_true: Ground truth binary labels. | |
| y_pred: Predicted binary labels. | |
| y_prob: Predicted probabilities (optional). | |
| n_bootstrap: Override default number of bootstrap samples. | |
| ci: Override default confidence level. | |
| Returns: | |
| Dictionary mapping metric name -> {'lower': ..., 'upper': ..., 'mean': ...}. | |
| """ | |
| n_bootstrap = n_bootstrap or self.n_bootstrap | |
| ci_level = ci or self.ci_level | |
| y_true = np.asarray(y_true) | |
| y_pred = np.asarray(y_pred) | |
| y_prob = np.asarray(y_prob) if y_prob is not None else None | |
| n_samples = len(y_true) | |
| metric_names = [ | |
| "accuracy", | |
| "precision", | |
| "recall", | |
| "f1", | |
| "specificity", | |
| "balanced_accuracy", | |
| ] | |
| if y_prob is not None: | |
| metric_names.extend(["auc_roc", "average_precision"]) | |
| bootstrap_metrics = {name: [] for name in metric_names} | |
| for i in range(n_bootstrap): | |
| indices = resample( | |
| np.arange(n_samples), | |
| replace=True, | |
| n_samples=n_samples, | |
| random_state=self.random_state + i, | |
| ) | |
| b_y_true = y_true[indices] | |
| b_y_pred = y_pred[indices] | |
| b_y_prob = y_prob[indices] if y_prob is not None else None | |
| try: | |
| metrics = self.compute_all_metrics(b_y_true, b_y_pred, b_y_prob) | |
| for name in metric_names: | |
| if name in metrics: | |
| bootstrap_metrics[name].append(metrics[name]) | |
| except Exception as e: | |
| logger.warning(f"Bootstrap iteration {i} failed: {e}") | |
| # Compute confidence intervals | |
| ci_results = {} | |
| alpha = (1 - ci_level) / 2 | |
| for name, values in bootstrap_metrics.items(): | |
| if len(values) == 0: | |
| ci_results[name] = {"lower": np.nan, "upper": np.nan, "mean": np.nan} | |
| continue | |
| lower = np.percentile(values, 100 * alpha) | |
| upper = np.percentile(values, 100 * (1 - alpha)) | |
| mean = np.mean(values) | |
| ci_results[name] = { | |
| "lower": float(lower), | |
| "upper": float(upper), | |
| "mean": float(mean), | |
| } | |
| logger.info( | |
| f"Computed bootstrap CIs for {len(ci_results)} metrics " | |
| f"({n_bootstrap} samples, CI={ci_level})" | |
| ) | |
| return ci_results |