File size: 5,368 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#!/usr/bin/env python
"""Generate a camera-ready split file from a legacy 5-fold file.

The test folds are copied verbatim from ``--source`` (the fold file the
original CV runs used; its md5 is recorded, its ``data_root`` field is not).
Inner-validation volunteers are added with the deterministic rule in
``pulse/split_builder.py``.

Commands used for the release::

    python scripts/make_splits.py --root $PULSE_ROOT --source <t1_cv/cv_folds.json> \
        --name pool33 --used-by T1,T4,T7 --required emg,mocap --strata pressure \
        --source-name "results/t1_cv/cv_folds.json" --out splits/pool33.json
    python scripts/make_splits.py --root $PULSE_ROOT --source <t2_cv/cv_folds.json> \
        --name pool27 --used-by T2,T3,T5,T6 --required emg,pressure,mocap \
        --require-annotations --strata pressure,mocap \
        --source-name "results/t2_cv/cv_folds.json" --out splits/pool27.json
"""
from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from pulse import DATA_TAG  # noqa: E402
from pulse.paths import DataRoot, vol_number  # noqa: E402
from pulse.data.recordings import load_recordings  # noqa: E402
from pulse.split_builder import RULE_TEXT, choose_inner_val, volunteer_profiles  # noqa: E402
from pulse.splits import SCHEMA, file_md5, validate_split  # noqa: E402

GENERATORS = {
    "pool33": "round-robin over the pool sorted by volunteer number: fold k = pool[k::5] "
              "(re-derivable; documented in the original CV notes)",
    "pool27": "seed-42 random partition; the generator script was not kept, so the folds are "
              "published verbatim",
}


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--root", default=None, help="dataset root (default $PULSE_ROOT)")
    ap.add_argument("--source", required=True, help="legacy cv_folds.json")
    ap.add_argument("--source-name", required=True, help="how the source is cited in the output")
    ap.add_argument("--name", required=True)
    ap.add_argument("--used-by", required=True, help="comma-separated task ids")
    ap.add_argument("--required", required=True,
                    help="modalities an inner-val volunteer must have in all S1-S8")
    ap.add_argument("--strata", default="", help="modalities that define strata")
    ap.add_argument("--require-annotations", action="store_true")
    ap.add_argument("--fraction", type=float, default=0.15)
    ap.add_argument("--out", required=True)
    args = ap.parse_args(argv)

    dr = DataRoot.from_arg(args.root)
    recs = load_recordings(dr)
    with open(args.source, encoding="utf-8") as f:
        src = json.load(f)
    src_md5 = file_md5(args.source)
    pool = sorted(src["all"], key=vol_number)
    test_folds = [list(t) for t in src["folds"]]
    flat = [v for t in test_folds for v in t]
    if sorted(flat, key=vol_number) != pool or len(flat) != len(set(flat)):
        raise SystemExit("source folds are not a partition of source 'all'")
    if src.get("pool_size") not in (None, len(pool)):
        raise SystemExit("source pool_size disagrees with 'all'")
    known = {r.volunteer for r in recs.values()}
    if set(pool) - known:
        raise SystemExit(f"volunteers not in recordings.csv: {sorted(set(pool) - known)}")

    required = [m for m in args.required.split(",") if m]
    strata = [m for m in args.strata.split(",") if m]
    prof = volunteer_profiles(recs, pool, required, strata, args.require_annotations)
    folds = choose_inner_val(args.name, pool, test_folds, prof, args.fraction)

    out = {
        "schema": SCHEMA,
        "name": args.name,
        "data_tag": DATA_TAG,
        "used_by": [t for t in args.used_by.split(",") if t],
        "n_volunteers": len(pool),
        "volunteers": pool,
        "n_folds": len(folds),
        "folds": folds,
        "source": {
            "file": args.source_name,
            "md5": src_md5,
            "note": "test folds copied verbatim (order preserved); the source's data_root "
                    "field is not reproduced",
            "fold_sizes": [len(t) for t in test_folds],
            "generator": GENERATORS.get(args.name, ""),
        },
        "inner_val_rule": {
            "fraction": args.fraction,
            "required_modalities": required,
            "strata_modalities": strata,
            "require_annotations": bool(args.require_annotations),
            "metadata": "metadata/recordings.csv (aligned_modalities, has_dense_annotations)",
            "rule": RULE_TEXT,
        },
        "volunteer_profiles": {v: prof[v] for v in pool},
    }
    validate_split(out)
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    with open(args.out, "w", encoding="utf-8", newline="\n") as f:
        json.dump(out, f, indent=2, ensure_ascii=False)
        f.write("\n")
    print(f"{args.name}: {len(pool)} volunteers, source md5 {src_md5}")
    for fo in folds:
        print(f"  fold {fo['fold']}: test {fo['n']['test']} {fo['test']}  inner_val "
              f"{fo['inner_val']}  train {fo['n']['train']}  strata {fo['inner_val_strata']}")
    print(f"wrote {args.out} (md5 {file_md5(args.out)})")
    return 0


if __name__ == "__main__":
    sys.exit(main())