File size: 3,549 Bytes
04ae9bb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
"""Patient-level and cohort-held-out splits."""

from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List

import numpy as np
import pandas as pd
from sklearn.model_selection import KFold, StratifiedKFold


def make_folds(patients: pd.DataFrame, cfg: Dict[str, Any]) -> pd.DataFrame:
    scheme = cfg.get("scheme", "patient_kfold")
    seed = int(cfg.get("seed", 42))
    df = patients.copy()
    train_patients = df[df["is_supplemental"] == 0].reset_index(drop=True)
    supp_patients = df[df["is_supplemental"] == 1].reset_index(drop=True)

    if scheme == "patient_kfold":
        n_splits = int(cfg.get("n_splits", 5))
        strat = cfg.get("stratify_by", "cohort")
        y = train_patients[strat].astype(str).values if strat in train_patients.columns else None
        counts = pd.Series(y).value_counts() if y is not None else None
        if y is not None and counts is not None and counts.min() >= n_splits:
            splitter = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)
            splits = list(splitter.split(train_patients, y))
        else:
            splitter = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
            splits = list(splitter.split(train_patients))
        train_patients["fold"] = -1
        for fold, (_, val_idx) in enumerate(splits):
            train_patients.loc[val_idx, "fold"] = fold
    elif scheme == "cohort_heldout":
        cohorts = sorted(train_patients["cohort"].unique().tolist())
        holdout = cfg.get("holdout_cohorts")
        if holdout:
            holdout = [c for c in cohorts if c in set(holdout)]
        else:
            holdout = cohorts
        mapping = {c: i for i, c in enumerate(holdout)}
        train_patients["fold"] = train_patients["cohort"].map(mapping).fillna(-1).astype(int)
    elif scheme == "single":
        frac = float(cfg.get("single_val_fraction", 0.2))
        rng = np.random.default_rng(seed)
        idx = np.arange(len(train_patients))
        rng.shuffle(idx)
        n_val = max(1, int(round(len(idx) * frac)))
        train_patients["fold"] = 1
        train_patients.loc[idx[:n_val], "fold"] = 0
    else:
        raise ValueError(f"Unknown cv scheme: {scheme}")

    out = train_patients[["uid", "patient_id", "cohort", "fold", "is_supplemental"]].copy()
    if len(supp_patients):
        supp_patients = supp_patients.copy()
        supp_patients["fold"] = -2
        out = pd.concat(
            [out, supp_patients[["uid", "patient_id", "cohort", "fold", "is_supplemental"]]],
            ignore_index=True,
        )
    out = out.sort_values(["is_supplemental", "cohort", "uid"]).reset_index(drop=True)
    return out


def fold_ids(folds: pd.DataFrame) -> List[int]:
    return sorted(int(x) for x in folds.loc[folds["fold"] >= 0, "fold"].unique())


def save_folds(folds: pd.DataFrame, path: str | Path) -> None:
    p = Path(path)
    p.parent.mkdir(parents=True, exist_ok=True)
    folds.to_csv(p, index=False)


def fold_summary(folds: pd.DataFrame) -> pd.DataFrame:
    rows = []
    for fold, g in folds[folds["fold"] >= 0].groupby("fold"):
        rows.append(
            {
                "fold": int(fold),
                "n_patients": int(len(g)),
                "n_control": int((g["cohort"] == "control").sum()),
                "n_frequent": int((g["cohort"] == "frequent").sum()),
                "n_annual": int((g["cohort"] == "annual").sum()),
            }
        )
    return pd.DataFrame(rows).sort_values("fold").reset_index(drop=True)