PULSE-code / pulse /splits.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
3.78 kB
"""Volunteer split files (``splits/pool33.json``, ``splits/pool27.json``).
A split file lists the pool, and for each of the 5 folds the held-out test
volunteers, the inner-validation volunteers (taken from that fold's training
volunteers; used for epoch selection, early stopping and LR scheduling) and the
remaining training volunteers. The test fold is evaluated once.
Schema ``pulse-split/1``::
{"schema": "pulse-split/1", "name": "pool33", "volunteers": [...],
"folds": [{"fold": 0, "test": [...], "inner_val": [...], "train": [...]}, ...],
"source": {...}, "inner_val_rule": {...}}
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from pathlib import Path
from typing import List, Tuple
from .errors import SplitError
SCHEMA = "pulse-split/1"
def file_md5(path) -> str:
h = hashlib.md5()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
@dataclass(frozen=True)
class FoldSplit:
pool: str
fold: int
train: Tuple[str, ...]
inner_val: Tuple[str, ...]
test: Tuple[str, ...]
split_file: str = ""
split_md5: str = ""
def as_dict(self) -> dict:
return {"pool": self.pool, "fold": self.fold, "train": list(self.train),
"inner_val": list(self.inner_val), "test": list(self.test),
"split_file": self.split_file, "split_md5": self.split_md5}
@dataclass
class SplitFile:
data: dict
path: str
md5: str
@property
def name(self) -> str:
return self.data["name"]
@property
def volunteers(self) -> List[str]:
return list(self.data["volunteers"])
@property
def n_folds(self) -> int:
return len(self.data["folds"])
def fold(self, k: int) -> FoldSplit:
if not 0 <= k < self.n_folds:
raise SplitError(f"{self.name}: fold {k} out of range 0..{self.n_folds - 1}")
f = self.data["folds"][k]
return FoldSplit(self.name, k, tuple(f["train"]), tuple(f["inner_val"]),
tuple(f["test"]), self.path, self.md5)
def validate_split(d: dict) -> None:
if d.get("schema") != SCHEMA:
raise SplitError(f"not a split file (schema={d.get('schema')!r})")
vols = d["volunteers"]
if len(set(vols)) != len(vols):
raise SplitError("duplicate volunteers in pool")
pool = set(vols)
seen_test: List[str] = []
for i, f in enumerate(d["folds"]):
if f.get("fold") != i:
raise SplitError(f"fold entry {i} has fold={f.get('fold')}")
tr, iv, te = set(f["train"]), set(f["inner_val"]), set(f["test"])
for name, part in (("train", f["train"]), ("inner_val", f["inner_val"]), ("test", f["test"])):
if len(set(part)) != len(part):
raise SplitError(f"fold {i}: duplicate volunteers in {name}")
if tr & iv or tr & te or iv & te:
raise SplitError(f"fold {i}: partitions overlap")
if tr | iv | te != pool:
raise SplitError(f"fold {i}: partitions do not cover the pool "
f"(missing {sorted(pool - (tr | iv | te))}, "
f"extra {sorted((tr | iv | te) - pool)})")
if not iv:
raise SplitError(f"fold {i}: empty inner_val")
seen_test.extend(f["test"])
if sorted(seen_test) != sorted(vols):
raise SplitError("test folds are not a partition of the pool")
def load_split(path) -> SplitFile:
p = Path(path)
with open(p, encoding="utf-8") as f:
d = json.load(f)
validate_split(d)
return SplitFile(d, p.as_posix(), file_md5(p))
def load_fold(path, k: int) -> FoldSplit:
return load_split(path).fold(k)