File size: 5,392 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """Deterministic inner-validation selection for the split files.
Rule (recorded verbatim in every split file):
1. The test folds are copied verbatim from the source fold file.
2. Per fold, ``n_val = floor(0.15 * n_train + 0.5)`` of the fold's training
volunteers become inner-validation volunteers (pool33: 4, pool27: 3).
3. Strata: volunteers are grouped by whether a majority of their S1-S8
recordings list each *strata modality* in metadata/recordings.csv (pool33:
pressure, so the 7 volunteers without pressure are represented). ``n_val``
is split over strata in proportion to their size among the fold's training
volunteers (largest remainder; ties by stratum key).
4. Eligibility: an inner-validation volunteer must have all 8 scenarios S1-S8
with every *required modality* listed (and dense labels, if required), so
the inner-validation set covers all scenario classes. Volunteers with
partial coverage always stay in the training set.
5. Within a stratum, eligible volunteers are ranked by (number of earlier folds
in which they were already inner-validation, sha256 of
``"pulse-inner-val/<pool>/fold<k>/<vol>"``) and the first ones are taken, so
picks are spread over volunteers and fully reproducible.
"""
from __future__ import annotations
import hashlib
import math
from collections import defaultdict
from typing import Dict, Iterable, List, Sequence, Tuple
from .paths import vol_number
from .data.recordings import SCENARIOS_TASK, Recording
RULE_TEXT = __doc__.split("Rule (recorded verbatim in every split file):", 1)[1].strip()
def round_half_up(x: float) -> int:
return int(math.floor(x + 0.5))
def volunteer_profiles(recs: Dict[str, Recording], volunteers: Iterable[str],
required: Sequence[str], strata_modalities: Sequence[str],
require_annotations: bool) -> Dict[str, dict]:
out = {}
for v in volunteers:
task = [r for r in recs.values() if r.volunteer == v and r.scenario in SCENARIOS_TASK]
full = [r for r in task
if all(r.lists(m) for m in required)
and (r.has_dense_annotations or not require_annotations)]
strata = tuple(m for m in strata_modalities
if task and sum(r.lists(m) for r in task) * 2 > len(task))
out[v] = {
"n_task_recordings": len(task),
"n_full_recordings": len(full),
"eligible": len({r.scenario for r in full}) == len(SCENARIOS_TASK),
"stratum": "+".join(strata) if strata else "none",
}
return out
def _rank_key(pool: str, fold: int, vol: str, used: Dict[str, int]) -> Tuple[int, str]:
h = hashlib.sha256(f"pulse-inner-val/{pool}/fold{fold}/{vol}".encode()).hexdigest()
return used.get(vol, 0), h
def allocate(n: int, sizes: Dict[str, int]) -> Dict[str, int]:
"""Largest-remainder apportionment of ``n`` over strata of ``sizes``."""
total = sum(sizes.values())
if total == 0:
return {k: 0 for k in sizes}
quotas = {k: n * s / total for k, s in sizes.items()}
alloc = {k: int(math.floor(q)) for k, q in quotas.items()}
rest = n - sum(alloc.values())
order = sorted(sizes, key=lambda k: (-(quotas[k] - alloc[k]), k))
for k in order[:rest]:
alloc[k] += 1
return alloc
def choose_inner_val(pool: str, volunteers: Sequence[str], test_folds: Sequence[Sequence[str]],
profiles: Dict[str, dict], fraction: float = 0.15) -> List[dict]:
used: Dict[str, int] = defaultdict(int)
folds = []
for k, test in enumerate(test_folds):
train_all = [v for v in volunteers if v not in set(test)]
n_val = round_half_up(fraction * len(train_all))
by_stratum: Dict[str, List[str]] = defaultdict(list)
for v in train_all:
by_stratum[profiles[v]["stratum"]].append(v)
quota = allocate(n_val, {s: len(vs) for s, vs in by_stratum.items()})
chosen: List[str] = []
shortfall = 0
for s in sorted(by_stratum):
cands = sorted((v for v in by_stratum[s] if profiles[v]["eligible"]),
key=lambda v: _rank_key(pool, k, v, used))
take = cands[:quota[s]]
shortfall += quota[s] - len(take)
chosen.extend(take)
if shortfall: # not enough eligible volunteers in a stratum: fill from the rest
rest = sorted((v for v in train_all if profiles[v]["eligible"] and v not in chosen),
key=lambda v: _rank_key(pool, k, v, used))
chosen.extend(rest[:shortfall])
if len(chosen) != n_val:
raise RuntimeError(f"{pool} fold {k}: only {len(chosen)} eligible inner-val volunteers")
for v in chosen:
used[v] += 1
iv = sorted(chosen, key=vol_number)
tr = sorted((v for v in train_all if v not in set(chosen)), key=vol_number)
folds.append({
"fold": k, "test": list(test), "inner_val": iv, "train": tr,
"n": {"test": len(test), "inner_val": len(iv), "train": len(tr)},
"inner_val_strata": {s: sorted((v for v in iv if profiles[v]["stratum"] == s),
key=vol_number) for s in sorted(by_stratum)},
"train_strata_sizes": {s: len(by_stratum[s]) for s in sorted(by_stratum)},
})
return folds
|