File size: 5,392 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""Deterministic inner-validation selection for the split files.

Rule (recorded verbatim in every split file):

1. The test folds are copied verbatim from the source fold file.
2. Per fold, ``n_val = floor(0.15 * n_train + 0.5)`` of the fold's training
   volunteers become inner-validation volunteers (pool33: 4, pool27: 3).
3. Strata: volunteers are grouped by whether a majority of their S1-S8
   recordings list each *strata modality* in metadata/recordings.csv (pool33:
   pressure, so the 7 volunteers without pressure are represented).  ``n_val``
   is split over strata in proportion to their size among the fold's training
   volunteers (largest remainder; ties by stratum key).
4. Eligibility: an inner-validation volunteer must have all 8 scenarios S1-S8
   with every *required modality* listed (and dense labels, if required), so
   the inner-validation set covers all scenario classes.  Volunteers with
   partial coverage always stay in the training set.
5. Within a stratum, eligible volunteers are ranked by (number of earlier folds
   in which they were already inner-validation, sha256 of
   ``"pulse-inner-val/<pool>/fold<k>/<vol>"``) and the first ones are taken, so
   picks are spread over volunteers and fully reproducible.
"""
from __future__ import annotations

import hashlib
import math
from collections import defaultdict
from typing import Dict, Iterable, List, Sequence, Tuple

from .paths import vol_number
from .data.recordings import SCENARIOS_TASK, Recording

RULE_TEXT = __doc__.split("Rule (recorded verbatim in every split file):", 1)[1].strip()


def round_half_up(x: float) -> int:
    return int(math.floor(x + 0.5))


def volunteer_profiles(recs: Dict[str, Recording], volunteers: Iterable[str],
                       required: Sequence[str], strata_modalities: Sequence[str],
                       require_annotations: bool) -> Dict[str, dict]:
    out = {}
    for v in volunteers:
        task = [r for r in recs.values() if r.volunteer == v and r.scenario in SCENARIOS_TASK]
        full = [r for r in task
                if all(r.lists(m) for m in required)
                and (r.has_dense_annotations or not require_annotations)]
        strata = tuple(m for m in strata_modalities
                       if task and sum(r.lists(m) for r in task) * 2 > len(task))
        out[v] = {
            "n_task_recordings": len(task),
            "n_full_recordings": len(full),
            "eligible": len({r.scenario for r in full}) == len(SCENARIOS_TASK),
            "stratum": "+".join(strata) if strata else "none",
        }
    return out


def _rank_key(pool: str, fold: int, vol: str, used: Dict[str, int]) -> Tuple[int, str]:
    h = hashlib.sha256(f"pulse-inner-val/{pool}/fold{fold}/{vol}".encode()).hexdigest()
    return used.get(vol, 0), h


def allocate(n: int, sizes: Dict[str, int]) -> Dict[str, int]:
    """Largest-remainder apportionment of ``n`` over strata of ``sizes``."""
    total = sum(sizes.values())
    if total == 0:
        return {k: 0 for k in sizes}
    quotas = {k: n * s / total for k, s in sizes.items()}
    alloc = {k: int(math.floor(q)) for k, q in quotas.items()}
    rest = n - sum(alloc.values())
    order = sorted(sizes, key=lambda k: (-(quotas[k] - alloc[k]), k))
    for k in order[:rest]:
        alloc[k] += 1
    return alloc


def choose_inner_val(pool: str, volunteers: Sequence[str], test_folds: Sequence[Sequence[str]],
                     profiles: Dict[str, dict], fraction: float = 0.15) -> List[dict]:
    used: Dict[str, int] = defaultdict(int)
    folds = []
    for k, test in enumerate(test_folds):
        train_all = [v for v in volunteers if v not in set(test)]
        n_val = round_half_up(fraction * len(train_all))
        by_stratum: Dict[str, List[str]] = defaultdict(list)
        for v in train_all:
            by_stratum[profiles[v]["stratum"]].append(v)
        quota = allocate(n_val, {s: len(vs) for s, vs in by_stratum.items()})
        chosen: List[str] = []
        shortfall = 0
        for s in sorted(by_stratum):
            cands = sorted((v for v in by_stratum[s] if profiles[v]["eligible"]),
                           key=lambda v: _rank_key(pool, k, v, used))
            take = cands[:quota[s]]
            shortfall += quota[s] - len(take)
            chosen.extend(take)
        if shortfall:  # not enough eligible volunteers in a stratum: fill from the rest
            rest = sorted((v for v in train_all if profiles[v]["eligible"] and v not in chosen),
                          key=lambda v: _rank_key(pool, k, v, used))
            chosen.extend(rest[:shortfall])
        if len(chosen) != n_val:
            raise RuntimeError(f"{pool} fold {k}: only {len(chosen)} eligible inner-val volunteers")
        for v in chosen:
            used[v] += 1
        iv = sorted(chosen, key=vol_number)
        tr = sorted((v for v in train_all if v not in set(chosen)), key=vol_number)
        folds.append({
            "fold": k, "test": list(test), "inner_val": iv, "train": tr,
            "n": {"test": len(test), "inner_val": len(iv), "train": len(tr)},
            "inner_val_strata": {s: sorted((v for v in iv if profiles[v]["stratum"] == s),
                                           key=vol_number) for s in sorted(by_stratum)},
            "train_strata_sizes": {s: len(by_stratum[s]) for s in sorted(by_stratum)},
        })
    return folds