APEXPRIT's picture
Upload folder using huggingface_hub
59f824e verified
Raw History Blame Contribute Delete
13.3 kB
"""Comprehensive evaluation metrics for deception detection.
This module provides a DeceptionMetrics class that computes a wide range of
classification metrics, fairness metrics, and confidence intervals for
evaluating deception detection models.
"""
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional, Tuple, Union
import numpy as np
import pandas as pd
from sklearn.metrics import (
accuracy_score,
precision_score,
recall_score,
f1_score,
roc_auc_score,
average_precision_score,
confusion_matrix,
classification_report,
balanced_accuracy_score,
)
from sklearn.utils import resample
from scipy import stats
logger = logging.getLogger(__name__)
class DeceptionMetrics:
"""Compute and aggregate evaluation metrics for deception detection.
This class provides methods for computing standard classification metrics,
per-subject metrics, fairness metrics, and bootstrap confidence intervals.
Attributes:
random_state: Random seed for reproducibility.
n_bootstrap: Number of bootstrap samples for confidence intervals.
ci_level: Confidence level for intervals (default 0.95).
"""
def __init__(
self,
random_state: int = 42,
n_bootstrap: int = 1000,
ci_level: float = 0.95,
) -> None:
"""Initialize metrics calculator.
Args:
random_state: Random seed for reproducibility.
n_bootstrap: Number of bootstrap samples for confidence intervals.
ci_level: Confidence level for intervals (default 0.95).
"""
self.random_state = random_state
self.n_bootstrap = n_bootstrap
self.ci_level = ci_level
np.random.seed(random_state)
def compute_all_metrics(
self,
y_true: Union[np.ndarray, List[int]],
y_pred: Union[np.ndarray, List[int]],
y_prob: Optional[Union[np.ndarray, List[float]]] = None,
) -> Dict[str, Any]:
"""Compute comprehensive classification metrics.
Args:
y_true: Ground truth binary labels (0 = truthful, 1 = deceptive).
y_pred: Predicted binary labels.
y_prob: Predicted probabilities for the positive class (deceptive).
If provided, ROC‑AUC and average precision are computed.
Returns:
Dictionary containing:
- accuracy, precision, recall, f1, specificity
- balanced_accuracy
- auc_roc, average_precision (if y_prob provided)
- confusion_matrix (as 2x2 array)
- classification_report (as dict)
"""
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
if y_true.ndim != 1:
raise ValueError(f"y_true must be 1‑D, got shape {y_true.shape}")
if y_pred.ndim != 1:
raise ValueError(f"y_pred must be 1‑D, got shape {y_pred.shape}")
if len(y_true) != len(y_pred):
raise ValueError(
f"Length mismatch: y_true ({len(y_true)}) vs y_pred ({len(y_pred)})"
)
# Basic metrics
accuracy = accuracy_score(y_true, y_pred)
precision = precision_score(y_true, y_pred, zero_division=0)
recall = recall_score(y_true, y_pred, zero_division=0)
f1 = f1_score(y_true, y_pred, zero_division=0)
balanced_acc = balanced_accuracy_score(y_true, y_pred)
# Specificity (true negative rate)
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
specificity = tn / (tn + fp) if (tn + fp) > 0 else 0.0
# Probability‑based metrics
auc_roc = None
avg_precision = None
if y_prob is not None:
y_prob = np.asarray(y_prob)
if y_prob.ndim == 2 and y_prob.shape[1] == 2:
y_prob = y_prob[:, 1] # take positive class probabilities
try:
auc_roc = roc_auc_score(y_true, y_prob)
except ValueError as e:
logger.warning(f"Could not compute ROC‑AUC: {e}")
auc_roc = None
try:
avg_precision = average_precision_score(y_true, y_prob)
except ValueError as e:
logger.warning(f"Could not compute average precision: {e}")
avg_precision = None
# Confusion matrix
cm = confusion_matrix(y_true, y_pred)
# Classification report as dict
report = classification_report(
y_true, y_pred, output_dict=True, zero_division=0
)
metrics = {
"accuracy": float(accuracy),
"precision": float(precision),
"recall": float(recall),
"f1": float(f1),
"specificity": float(specificity),
"balanced_accuracy": float(balanced_acc),
"confusion_matrix": cm.tolist(),
"classification_report": report,
"support": int(len(y_true)),
}
if auc_roc is not None:
metrics["auc_roc"] = float(auc_roc)
if avg_precision is not None:
metrics["average_precision"] = float(avg_precision)
logger.info(
f"Computed metrics: accuracy={accuracy:.3f}, f1={f1:.3f}, "
f"recall={recall:.3f}, precision={precision:.3f}"
)
return metrics
def compute_per_subject_metrics(
self,
y_true: Union[np.ndarray, List[int]],
y_pred: Union[np.ndarray, List[int]],
subject_ids: Union[np.ndarray, List[str]],
y_prob: Optional[Union[np.ndarray, List[float]]] = None,
) -> Dict[str, Dict[str, Any]]:
"""Compute metrics separately for each subject.
Args:
y_true: Ground truth binary labels.
y_pred: Predicted binary labels.
subject_ids: Subject identifier for each sample.
y_prob: Predicted probabilities (optional).
Returns:
Dictionary mapping subject_id -> metrics dict.
"""
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
subject_ids = np.asarray(subject_ids)
if len(y_true) != len(subject_ids):
raise ValueError(
f"Length mismatch: y_true ({len(y_true)}) vs subject_ids ({len(subject_ids)})"
)
unique_subjects = np.unique(subject_ids)
subject_metrics = {}
for subj in unique_subjects:
mask = subject_ids == subj
if np.sum(mask) == 0:
continue
subj_y_true = y_true[mask]
subj_y_pred = y_pred[mask]
subj_y_prob = y_prob[mask] if y_prob is not None else None
try:
metrics = self.compute_all_metrics(
subj_y_true, subj_y_pred, subj_y_prob
)
subject_metrics[subj] = metrics
except Exception as e:
logger.warning(f"Failed to compute metrics for subject {subj}: {e}")
subject_metrics[subj] = {"error": str(e)}
logger.info(f"Computed per‑subject metrics for {len(subject_metrics)} subjects")
return subject_metrics
def compute_fairness_metrics(
self,
y_true: Union[np.ndarray, List[int]],
y_pred: Union[np.ndarray, List[int]],
sensitive_groups: Union[np.ndarray, List[str]],
) -> Dict[str, float]:
"""Compute fairness metrics across sensitive groups.
Currently implemented:
- Demographic parity difference: max difference in positive rate across groups.
- Equalized odds difference: max difference in TPR and FPR across groups.
Args:
y_true: Ground truth binary labels.
y_pred: Predicted binary labels.
sensitive_groups: Group membership for each sample (e.g., gender, ethnicity).
Returns:
Dictionary with fairness metrics.
"""
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
sensitive_groups = np.asarray(sensitive_groups)
unique_groups = np.unique(sensitive_groups)
group_stats = {}
for group in unique_groups:
mask = sensitive_groups == group
if np.sum(mask) == 0:
continue
group_y_true = y_true[mask]
group_y_pred = y_pred[mask]
# Positive rate (predicted)
pos_rate = np.mean(group_y_pred)
# True positive rate and false positive rate
tn, fp, fn, tp = confusion_matrix(group_y_true, group_y_pred).ravel()
tpr = tp / (tp + fn) if (tp + fn) > 0 else 0.0
fpr = fp / (fp + tn) if (fp + tn) > 0 else 0.0
group_stats[group] = {
"positive_rate": pos_rate,
"tpr": tpr,
"fpr": fpr,
"support": int(np.sum(mask)),
}
# Demographic parity difference
pos_rates = [stats["positive_rate"] for stats in group_stats.values()]
demographic_parity_diff = max(pos_rates) - min(pos_rates) if pos_rates else 0.0
# Equalized odds differences (max TPR diff, max FPR diff)
tprs = [stats["tpr"] for stats in group_stats.values()]
fprs = [stats["fpr"] for stats in group_stats.values()]
equalized_odds_tpr_diff = max(tprs) - min(tprs) if tprs else 0.0
equalized_odds_fpr_diff = max(fprs) - min(fprs) if fprs else 0.0
fairness = {
"demographic_parity_difference": float(demographic_parity_diff),
"equalized_odds_tpr_difference": float(equalized_odds_tpr_diff),
"equalized_odds_fpr_difference": float(equalized_odds_fpr_diff),
"group_stats": group_stats,
}
logger.info(
f"Fairness metrics: demographic parity diff = {demographic_parity_diff:.3f}, "
f"equalized odds TPR diff = {equalized_odds_tpr_diff:.3f}"
)
return fairness
def bootstrap_confidence_intervals(
self,
y_true: Union[np.ndarray, List[int]],
y_pred: Union[np.ndarray, List[int]],
y_prob: Optional[Union[np.ndarray, List[float]]] = None,
n_bootstrap: Optional[int] = None,
ci: Optional[float] = None,
) -> Dict[str, Dict[str, float]]:
"""Compute bootstrap confidence intervals for metrics.
Args:
y_true: Ground truth binary labels.
y_pred: Predicted binary labels.
y_prob: Predicted probabilities (optional).
n_bootstrap: Override default number of bootstrap samples.
ci: Override default confidence level.
Returns:
Dictionary mapping metric name -> {'lower': ..., 'upper': ..., 'mean': ...}.
"""
n_bootstrap = n_bootstrap or self.n_bootstrap
ci_level = ci or self.ci_level
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
y_prob = np.asarray(y_prob) if y_prob is not None else None
n_samples = len(y_true)
metric_names = [
"accuracy",
"precision",
"recall",
"f1",
"specificity",
"balanced_accuracy",
]
if y_prob is not None:
metric_names.extend(["auc_roc", "average_precision"])
bootstrap_metrics = {name: [] for name in metric_names}
for i in range(n_bootstrap):
indices = resample(
np.arange(n_samples),
replace=True,
n_samples=n_samples,
random_state=self.random_state + i,
)
b_y_true = y_true[indices]
b_y_pred = y_pred[indices]
b_y_prob = y_prob[indices] if y_prob is not None else None
try:
metrics = self.compute_all_metrics(b_y_true, b_y_pred, b_y_prob)
for name in metric_names:
if name in metrics:
bootstrap_metrics[name].append(metrics[name])
except Exception as e:
logger.warning(f"Bootstrap iteration {i} failed: {e}")
# Compute confidence intervals
ci_results = {}
alpha = (1 - ci_level) / 2
for name, values in bootstrap_metrics.items():
if len(values) == 0:
ci_results[name] = {"lower": np.nan, "upper": np.nan, "mean": np.nan}
continue
lower = np.percentile(values, 100 * alpha)
upper = np.percentile(values, 100 * (1 - alpha))
mean = np.mean(values)
ci_results[name] = {
"lower": float(lower),
"upper": float(upper),
"mean": float(mean),
}
logger.info(
f"Computed bootstrap CIs for {len(ci_results)} metrics "
f"({n_bootstrap} samples, CI={ci_level})"
)
return ci_results