File size: 10,836 Bytes
59f824e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
"""Cross‑validation strategies for deception detection.



This module provides subject‑aware and stratified cross‑validation

splits suitable for multimodal deception analysis.

"""

from __future__ import annotations

import logging
from typing import Any, Dict, List, Optional, Tuple, Union

import numpy as np
import pandas as pd
from sklearn.model_selection import (
    StratifiedKFold,
    GroupKFold,
    LeaveOneGroupOut,
    cross_val_predict,
)
from sklearn.base import BaseEstimator

from .metrics import DeceptionMetrics


logger = logging.getLogger(__name__)


class CrossValidator:
    """Cross‑validation with stratification and subject‑aware splits.



    This class provides methods for performing k‑fold cross‑validation

    while respecting subject boundaries (no data leakage) and maintaining

    class balance.



    Attributes:

        random_state: Random seed for reproducibility.

        n_splits: Default number of folds.

        metrics: Instance of DeceptionMetrics for evaluation.

    """

    def __init__(

        self,

        random_state: int = 42,

        n_splits: int = 5,

        metrics: Optional[DeceptionMetrics] = None,

    ) -> None:
        """Initialize cross‑validator.



        Args:

            random_state: Random seed for reproducibility.

            n_splits: Default number of folds.

            metrics: Optional DeceptionMetrics instance. If None, a default is created.

        """
        self.random_state = random_state
        self.n_splits = n_splits
        self.metrics = metrics or DeceptionMetrics(random_state=random_state)

    def stratified_k_fold(

        self,

        X: Union[np.ndarray, pd.DataFrame],

        y: Union[np.ndarray, List[int]],

        n_splits: Optional[int] = None,

        random_state: Optional[int] = None,

    ) -> List[Tuple[np.ndarray, np.ndarray]]:
        """Generate stratified k‑fold indices.



        Args:

            X: Feature matrix (any shape, only length matters).

            y: Target labels used for stratification.

            n_splits: Override default number of folds.

            random_state: Override default random seed.



        Returns:

            List of (train_indices, test_indices) for each fold.

        """
        n_splits = n_splits or self.n_splits
        random_state = random_state or self.random_state

        y = np.asarray(y)
        skf = StratifiedKFold(
            n_splits=n_splits, shuffle=True, random_state=random_state
        )
        splits = []
        for train_idx, test_idx in skf.split(X, y):
            splits.append((train_idx, test_idx))

        logger.info(
            f"Generated {n_splits} stratified folds "
            f"(random_state={random_state}, n_samples={len(y)})"
        )
        return splits

    def subject_aware_split(

        self,

        X: Union[np.ndarray, pd.DataFrame],

        y: Union[np.ndarray, List[int]],

        subject_ids: Union[np.ndarray, List[str]],

        n_splits: Optional[int] = None,

    ) -> List[Tuple[np.ndarray, np.ndarray]]:
        """Generate k‑fold splits where the same subject does not appear in both train and test.



        Uses GroupKFold with subject_ids as groups.



        Args:

            X: Feature matrix.

            y: Target labels (unused for grouping, kept for compatibility).

            subject_ids: Subject identifier for each sample.

            n_splits: Override default number of folds.



        Returns:

            List of (train_indices, test_indices) for each fold.

        """
        n_splits = n_splits or self.n_splits
        subject_ids = np.asarray(subject_ids)

        # Ensure we have enough unique groups
        unique_subjects = np.unique(subject_ids)
        if len(unique_subjects) < n_splits:
            logger.warning(
                f"Only {len(unique_subjects)} unique subjects, reducing n_splits to {len(unique_subjects)}"
            )
            n_splits = len(unique_subjects)

        gkf = GroupKFold(n_splits=n_splits)
        splits = []
        for train_idx, test_idx in gkf.split(X, y, groups=subject_ids):
            splits.append((train_idx, test_idx))

        logger.info(
            f"Generated {n_splits} subject‑aware folds "
            f"(unique subjects={len(unique_subjects)})"
        )
        return splits

    def leave_one_subject_out(

        self,

        X: Union[np.ndarray, pd.DataFrame],

        y: Union[np.ndarray, List[int]],

        subject_ids: Union[np.ndarray, List[str]],

    ) -> List[Tuple[np.ndarray, np.ndarray]]:
        """Leave‑One‑Subject‑Out (LOSO) cross‑validation generator.



        Each fold uses all subjects except one as training, and the left‑out

        subject as test.



        Args:

            X: Feature matrix.

            y: Target labels.

            subject_ids: Subject identifier for each sample.



        Returns:

            List of (train_indices, test_indices) for each fold.

        """
        subject_ids = np.asarray(subject_ids)
        unique_subjects = np.unique(subject_ids)

        splits = []
        for subj in unique_subjects:
            test_mask = subject_ids == subj
            train_mask = ~test_mask
            train_idx = np.where(train_mask)[0]
            test_idx = np.where(test_mask)[0]
            splits.append((train_idx, test_idx))

        logger.info(
            f"Generated {len(splits)} LOSO folds (unique subjects={len(unique_subjects)})"
        )
        return splits

    def run_cross_validation(

        self,

        classifier: BaseEstimator,

        X: Union[np.ndarray, pd.DataFrame],

        y: Union[np.ndarray, List[int]],

        n_splits: Optional[int] = None,

        subject_ids: Optional[Union[np.ndarray, List[str]]] = None,

        method: str = "stratified",

        return_predictions: bool = True,

    ) -> Dict[str, Any]:
        """Run cross‑validation and compute metrics across folds.



        Args:

            classifier: A scikit‑learn compatible classifier.

            X: Feature matrix.

            y: Target labels.

            n_splits: Override default number of folds.

            subject_ids: Required for 'subject_aware' and 'loso' methods.

            method: One of 'stratified', 'subject_aware', 'loso'.

            return_predictions: If True, aggregate predictions across folds.



        Returns:

            Dictionary with:

                - mean_metrics: dict of mean metric values across folds

                - std_metrics: dict of standard deviations

                - per_fold_metrics: list of metric dicts for each fold

                - aggregated_predictions: dict with y_true, y_pred, y_prob (if return_predictions)

                - fold_indices: list of (train_idx, test_idx) for each fold

        """
        n_splits = n_splits or self.n_splits
        y = np.asarray(y)

        # Generate fold indices
        if method == "stratified":
            folds = self.stratified_k_fold(X, y, n_splits=n_splits)
        elif method == "subject_aware":
            if subject_ids is None:
                raise ValueError("subject_ids required for subject_aware method")
            folds = self.subject_aware_split(X, y, subject_ids, n_splits=n_splits)
        elif method == "loso":
            if subject_ids is None:
                raise ValueError("subject_ids required for loso method")
            folds = self.leave_one_subject_out(X, y, subject_ids)
        else:
            raise ValueError(
                f"Unknown method '{method}'. Choose from 'stratified', 'subject_aware', 'loso'."
            )

        per_fold_metrics = []
        all_y_true = []
        all_y_pred = []
        all_y_prob = []
        fold_indices = []

        for fold_idx, (train_idx, test_idx) in enumerate(folds):
            logger.info(f"Fold {fold_idx+1}/{len(folds)}: train={len(train_idx)}, test={len(test_idx)}")

            X_train, X_test = X[train_idx], X[test_idx]
            y_train, y_test = y[train_idx], y[test_idx]

            # Train classifier
            classifier.fit(X_train, y_train)

            # Predict
            y_pred = classifier.predict(X_test)
            y_prob = (
                classifier.predict_proba(X_test)[:, 1]
                if hasattr(classifier, "predict_proba")
                else None
            )

            # Compute metrics
            metrics = self.metrics.compute_all_metrics(y_test, y_pred, y_prob)
            per_fold_metrics.append(metrics)

            # Aggregate for overall analysis
            all_y_true.extend(y_test.tolist())
            all_y_pred.extend(y_pred.tolist())
            if y_prob is not None:
                all_y_prob.extend(y_prob.tolist())

            fold_indices.append((train_idx.tolist(), test_idx.tolist()))

        # Compute mean and std across folds
        metric_names = [
            "accuracy",
            "precision",
            "recall",
            "f1",
            "specificity",
            "balanced_accuracy",
        ]
        if any("auc_roc" in m for m in per_fold_metrics):
            metric_names.append("auc_roc")
        if any("average_precision" in m for m in per_fold_metrics):
            metric_names.append("average_precision")

        mean_metrics = {}
        std_metrics = {}
        for name in metric_names:
            values = []
            for fold_metrics in per_fold_metrics:
                if name in fold_metrics:
                    values.append(fold_metrics[name])
            if values:
                mean_metrics[name] = float(np.mean(values))
                std_metrics[name] = float(np.std(values))
            else:
                mean_metrics[name] = np.nan
                std_metrics[name] = np.nan

        # Aggregate predictions
        aggregated = {}
        if return_predictions:
            aggregated["y_true"] = all_y_true
            aggregated["y_pred"] = all_y_pred
            if all_y_prob:
                aggregated["y_prob"] = all_y_prob

        result = {
            "mean_metrics": mean_metrics,
            "std_metrics": std_metrics,
            "per_fold_metrics": per_fold_metrics,
            "aggregated_predictions": aggregated,
            "fold_indices": fold_indices,
            "method": method,
            "n_splits": len(folds),
        }

        logger.info(
            f"Cross‑validation completed ({method}, {len(folds)} folds). "
            f"Mean accuracy = {mean_metrics.get('accuracy', 0):.3f} ± {std_metrics.get('accuracy', 0):.3f}"
        )
        return result