PULSE-code / scripts /make_splits.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
5.37 kB
#!/usr/bin/env python
"""Generate a camera-ready split file from a legacy 5-fold file.
The test folds are copied verbatim from ``--source`` (the fold file the
original CV runs used; its md5 is recorded, its ``data_root`` field is not).
Inner-validation volunteers are added with the deterministic rule in
``pulse/split_builder.py``.
Commands used for the release::
python scripts/make_splits.py --root $PULSE_ROOT --source <t1_cv/cv_folds.json> \
--name pool33 --used-by T1,T4,T7 --required emg,mocap --strata pressure \
--source-name "results/t1_cv/cv_folds.json" --out splits/pool33.json
python scripts/make_splits.py --root $PULSE_ROOT --source <t2_cv/cv_folds.json> \
--name pool27 --used-by T2,T3,T5,T6 --required emg,pressure,mocap \
--require-annotations --strata pressure,mocap \
--source-name "results/t2_cv/cv_folds.json" --out splits/pool27.json
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pulse import DATA_TAG # noqa: E402
from pulse.paths import DataRoot, vol_number # noqa: E402
from pulse.data.recordings import load_recordings # noqa: E402
from pulse.split_builder import RULE_TEXT, choose_inner_val, volunteer_profiles # noqa: E402
from pulse.splits import SCHEMA, file_md5, validate_split # noqa: E402
GENERATORS = {
"pool33": "round-robin over the pool sorted by volunteer number: fold k = pool[k::5] "
"(re-derivable; documented in the original CV notes)",
"pool27": "seed-42 random partition; the generator script was not kept, so the folds are "
"published verbatim",
}
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", default=None, help="dataset root (default $PULSE_ROOT)")
ap.add_argument("--source", required=True, help="legacy cv_folds.json")
ap.add_argument("--source-name", required=True, help="how the source is cited in the output")
ap.add_argument("--name", required=True)
ap.add_argument("--used-by", required=True, help="comma-separated task ids")
ap.add_argument("--required", required=True,
help="modalities an inner-val volunteer must have in all S1-S8")
ap.add_argument("--strata", default="", help="modalities that define strata")
ap.add_argument("--require-annotations", action="store_true")
ap.add_argument("--fraction", type=float, default=0.15)
ap.add_argument("--out", required=True)
args = ap.parse_args(argv)
dr = DataRoot.from_arg(args.root)
recs = load_recordings(dr)
with open(args.source, encoding="utf-8") as f:
src = json.load(f)
src_md5 = file_md5(args.source)
pool = sorted(src["all"], key=vol_number)
test_folds = [list(t) for t in src["folds"]]
flat = [v for t in test_folds for v in t]
if sorted(flat, key=vol_number) != pool or len(flat) != len(set(flat)):
raise SystemExit("source folds are not a partition of source 'all'")
if src.get("pool_size") not in (None, len(pool)):
raise SystemExit("source pool_size disagrees with 'all'")
known = {r.volunteer for r in recs.values()}
if set(pool) - known:
raise SystemExit(f"volunteers not in recordings.csv: {sorted(set(pool) - known)}")
required = [m for m in args.required.split(",") if m]
strata = [m for m in args.strata.split(",") if m]
prof = volunteer_profiles(recs, pool, required, strata, args.require_annotations)
folds = choose_inner_val(args.name, pool, test_folds, prof, args.fraction)
out = {
"schema": SCHEMA,
"name": args.name,
"data_tag": DATA_TAG,
"used_by": [t for t in args.used_by.split(",") if t],
"n_volunteers": len(pool),
"volunteers": pool,
"n_folds": len(folds),
"folds": folds,
"source": {
"file": args.source_name,
"md5": src_md5,
"note": "test folds copied verbatim (order preserved); the source's data_root "
"field is not reproduced",
"fold_sizes": [len(t) for t in test_folds],
"generator": GENERATORS.get(args.name, ""),
},
"inner_val_rule": {
"fraction": args.fraction,
"required_modalities": required,
"strata_modalities": strata,
"require_annotations": bool(args.require_annotations),
"metadata": "metadata/recordings.csv (aligned_modalities, has_dense_annotations)",
"rule": RULE_TEXT,
},
"volunteer_profiles": {v: prof[v] for v in pool},
}
validate_split(out)
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
with open(args.out, "w", encoding="utf-8", newline="\n") as f:
json.dump(out, f, indent=2, ensure_ascii=False)
f.write("\n")
print(f"{args.name}: {len(pool)} volunteers, source md5 {src_md5}")
for fo in folds:
print(f" fold {fo['fold']}: test {fo['n']['test']} {fo['test']} inner_val "
f"{fo['inner_val']} train {fo['n']['train']} strata {fo['inner_val_strata']}")
print(f"wrote {args.out} (md5 {file_md5(args.out)})")
return 0
if __name__ == "__main__":
sys.exit(main())