velvet-pine-22 commited on
Commit
dffbc1f
·
verified ·
1 Parent(s): 230aab3

PULSE-code: benchmark code for the PULSE dataset (camera-ready release)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +5 -35
  2. .gitignore +8 -0
  3. LICENSE +21 -0
  4. README.md +588 -0
  5. configs/datacheck/emg.yaml +3 -0
  6. configs/datacheck/emg_gate_pressure.yaml +4 -0
  7. configs/datacheck/emg_pressure.yaml +6 -0
  8. configs/datacheck/mocap.yaml +3 -0
  9. configs/f0/README.md +43 -0
  10. configs/f0/grasp_timing.yaml +20 -0
  11. configs/f5/README.md +94 -0
  12. configs/f5/base.yaml +43 -0
  13. configs/f5/segment.yaml +3 -0
  14. configs/f5/segment/s9pre_f005.yaml +4 -0
  15. configs/f5/segment/s9pre_f010.yaml +4 -0
  16. configs/f5/segment/s9pre_f025.yaml +4 -0
  17. configs/f5/segment/s9pre_f050.yaml +4 -0
  18. configs/f5/segment/s9pre_f100.yaml +4 -0
  19. configs/f5/segment/scratch_f005.yaml +4 -0
  20. configs/f5/segment/scratch_f010.yaml +4 -0
  21. configs/f5/segment/scratch_f025.yaml +4 -0
  22. configs/f5/segment/scratch_f050.yaml +4 -0
  23. configs/f5/segment/scratch_f100.yaml +4 -0
  24. configs/f5/sliding.yaml +3 -0
  25. configs/f5/sliding/s9pre_f005.yaml +4 -0
  26. configs/f5/sliding/s9pre_f010.yaml +4 -0
  27. configs/f5/sliding/s9pre_f025.yaml +4 -0
  28. configs/f5/sliding/s9pre_f050.yaml +4 -0
  29. configs/f5/sliding/s9pre_f100.yaml +4 -0
  30. configs/f5/sliding/scratch_f005.yaml +4 -0
  31. configs/f5/sliding/scratch_f010.yaml +4 -0
  32. configs/f5/sliding/scratch_f025.yaml +4 -0
  33. configs/f5/sliding/scratch_f050.yaml +4 -0
  34. configs/f5/sliding/scratch_f100.yaml +4 -0
  35. configs/manifests.yaml +20 -0
  36. configs/slurm.yaml +17 -0
  37. configs/smoke.yaml +49 -0
  38. configs/t1/actionsense.yaml +4 -0
  39. configs/t1/actionsense_m4_gate_pressure.yaml +5 -0
  40. configs/t1/actionsense_m5.yaml +4 -0
  41. configs/t1/base.yaml +25 -0
  42. configs/t1/ctrgcn.yaml +5 -0
  43. configs/t1/emg_cnn.yaml +4 -0
  44. configs/t1/exp1/base.yaml +9 -0
  45. configs/t1/exp1/bb_cnn_emg.yaml +4 -0
  46. configs/t1/exp1/bb_cnn_imu.yaml +4 -0
  47. configs/t1/exp1/bb_cnn_mocap.yaml +4 -0
  48. configs/t1/exp1/bb_lstm_emg.yaml +4 -0
  49. configs/t1/exp1/bb_lstm_imu.yaml +4 -0
  50. configs/t1/exp1/bb_lstm_mocap.yaml +4 -0
.gitattributes CHANGED
@@ -1,35 +1,5 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ * text=auto eol=lf
2
+ *.png binary
3
+ *.pdf binary
4
+ *.npz binary
5
+ *.pt binary
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .pytest_cache/
5
+ .cache/
6
+ runs/
7
+ *.tmp
8
+ .DS_Store
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The PULSE authors (Wolin Liang, Yang Gao, Peiyu Yan, Yuexiang Hu, Yingjing Xiao, Xiongfeng Ying, Yingnian Guo, Cheng Zhang, Zhanpeng Jin)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
README.md ADDED
@@ -0,0 +1,588 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ tags:
4
+ - multimodal
5
+ - human-activity-recognition
6
+ - emg
7
+ - motion-capture
8
+ - eye-tracking
9
+ - tactile
10
+ - benchmark
11
+ datasets:
12
+ - velvet-pine-22/PULSE
13
+ ---
14
+
15
+ # PULSE benchmark code
16
+
17
+ Reference code of the paper **"PULSE: A Synchronized Five-Modality Dataset for
18
+ Sensorimotor Coordination in Long-Horizon Daily Activities"** (NeurIPS 2026
19
+ Evaluations and Datasets Track, spotlight). It loads the public PULSE dataset,
20
+ defines the volunteer-stratified 5-fold cross-validation splits, trains every
21
+ baseline of the benchmark (T1-T7, F5), runs the F0 case study, and regenerates the
22
+ benchmark tables, figures and numbers of the paper from the run results.
23
+
24
+ | | |
25
+ |---|---|
26
+ | Paper | https://openreview.net/forum?id=AoU576r9O2 |
27
+ | Dataset | https://huggingface.co/datasets/velvet-pine-22/PULSE (CC BY-NC 4.0), DOI [10.57967/hf/11314](https://doi.org/10.57967/hf/11314) |
28
+ | Small sample of the dataset | https://huggingface.co/datasets/velvet-pine-22/PULSE-sample |
29
+ | Code (this repository) | https://huggingface.co/velvet-pine-22/PULSE-code (MIT, see `LICENSE`) |
30
+ | Contact | Yang Gao (corresponding author, gaoyang2025@scut.edu.cn); maintainer Wolin Liang (202610196903@mail.scut.edu.cn) |
31
+
32
+ The benchmark data are the dataset's tag **`v1.1`** (commit
33
+ `e27b1bb6bb35f714705ae8685507f71773c065eb`, 3,001 files, 22.0 GB: sensor streams,
34
+ annotations, metadata). All numbers in the paper were computed on that tag with
35
+ the commands in section 3.
36
+
37
+ ## Contents
38
+
39
+ ```
40
+ pulse/ Python package: paths, data layer, splits, taxonomy, metrics, statistics,
41
+ provenance, aggregation, SLURM launcher, models/, tasks/ (one module per task)
42
+ configs/ tables.yaml (registry of every table and its rows), slurm.yaml, per-task configs
43
+ splits/ pool33.json, pool27.json (volunteer-disjoint 5-fold CV with inner validation)
44
+ taxonomy/ taxonomy_v1.1.json (T2 / F5 label vocabulary)
45
+ manifests/ hf_PULSE_v1.1_files.tsv (file list of dataset tag v1.1), availability_v1.1.json,
46
+ expected_v1.1.json
47
+ scripts/ run_table, aggregate, build_outputs, smoke_suite, verify_data, fetch_hf_manifest,
48
+ make_splits, build_taxonomy, build_availability, build_manifests
49
+ figures/ figure scripts (T1, T2, T5, F0, F5)
50
+ prompts/ VLM annotation prompt templates used to draft the action labels (see prompts/README.md)
51
+ slurm/ sbatch templates
52
+ tests/ unit tests (synthetic mini dataset + optional checks on the real data)
53
+ environment.yml the environment every number was produced with
54
+ ```
55
+
56
+ ## 1. Setup
57
+
58
+ ```bash
59
+ git clone https://huggingface.co/velvet-pine-22/PULSE-code && cd PULSE-code
60
+ conda env create -f environment.yml # Python 3.12.13, torch 2.4.1+cu121
61
+ conda activate pulse
62
+
63
+ # dataset, pinned to the benchmark tag (22.0 GB, no videos)
64
+ hf download velvet-pine-22/PULSE --repo-type dataset --revision v1.1 --local-dir PULSE
65
+ export PULSE_ROOT=$PWD/PULSE
66
+
67
+ # check every file against the tag's file list (sha256 for LFS files, git blob id otherwise)
68
+ python scripts/verify_data.py --root $PULSE_ROOT --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 8
69
+ python -m pytest -q tests # unit tests on CPU; a few also read $PULSE_ROOT
70
+ ```
71
+
72
+ `hf` is the Hugging Face CLI (installed with `huggingface_hub`, which the
73
+ environment contains); with older versions use `huggingface-cli download` with the
74
+ same arguments. Tasks read the dataset from `--root` or `$PULSE_ROOT`. Run every
75
+ command from the repository root (or `pip install -e .` to import `pulse` from
76
+ elsewhere; the configs, splits and manifests are still read from the checkout).
77
+
78
+ **Dataset revisions.** The `main` branch of the dataset contains everything in
79
+ `v1.1` plus the first-person videos (622 files, about 63 GB), the S9
80
+ motion-primitive labels (`annotations/vX/s9.json`, 30 files, and
81
+ `annotations_flat/segments_s9.csv`) and updated documentation. The sensor
82
+ streams and the S1-S8 annotations are identical; `metadata/recordings.csv`
83
+ differs only in the line endings and in marking the scene video of v5s8 and
84
+ v18s2 as missing (not a benchmark modality). The benchmark needs only `v1.1`;
85
+ F5 additionally needs the S9 labels (section 3, F5). Against a full download of
86
+ `main`, `verify_data.py` reports those differences and the extra files; it
87
+ passes with:
88
+
89
+ ```bash
90
+ python scripts/verify_data.py --root $PULSE_ROOT --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 8 \
91
+ --expect-different README.md --expect-different DATASHEET.md --expect-different LICENSE \
92
+ --expect-different CODE_LICENSE --expect-different croissant.json \
93
+ --expect-different metadata/recordings.csv --expect-different docs/upload_to_huggingface.md \
94
+ --ignore-extra '*.mp4' --ignore-extra 'annotations/*/s9.json' --ignore-extra annotations_flat/segments_s9.csv
95
+ ```
96
+
97
+ ## 2. Getting started
98
+
99
+ Load one recording and its action segments, and iterate the folds of a split
100
+ (save as a file in the repository root and run it there):
101
+
102
+ ```python
103
+ # Run from the root of a PULSE-code checkout, with $PULSE_ROOT = the dataset (revision v1.1).
104
+ from pulse.paths import DataRoot
105
+ from pulse.splits import load_split
106
+ from pulse.data import load_recording
107
+ from pulse.data.annotations import load_segments
108
+ from pulse.data.availability import Availability
109
+ from pulse.data.manifest import expected_manifest
110
+
111
+ dr = DataRoot.from_arg() # reads $PULSE_ROOT
112
+ av = Availability.load("manifests/availability_v1.1.json") # usable streams per recording
113
+ mods = ["mocap", "emg", "eyetrack", "imu", "pressure"]
114
+ split = load_split("splits/pool27.json") # volunteer-disjoint 5-fold CV
115
+
116
+ # (b) iterate the folds: which recordings each partition uses for these modalities
117
+ for k in range(split.n_folds):
118
+ fold = split.fold(k)
119
+ man = expected_manifest(av, fold, mods, require_annotation=True)
120
+ print(f"fold {k}: test volunteers {list(fold.test)}; {man.summary()}")
121
+
122
+ # (a) one test recording of the last fold: aligned 100 Hz arrays + action segments
123
+ rid = man.partitions["test"][0]
124
+ rec = load_recording(dr, rid, mods, align_trimmed=True)
125
+ print(rid, {m: x.shape for m, x in rec.arrays.items()}) # (T, features), common T
126
+ for s in load_segments(dr, rid)[:3]:
127
+ print(f"{s.start_sec:7.2f} {s.end_sec:7.2f} {s.verb} {s.noun} ({s.hand})")
128
+ ```
129
+
130
+ Output on `v1.1`:
131
+
132
+ ```
133
+ fold 0: test volunteers ['v25', 'v28', 'v11', 'v38', 'v5', 'v35']; pool27 fold 0 [mocap+emg+eyetrack+imu+pressure; s1,s2,s3,s4,s5,s6,s7,s8; labelled]: train 128 / inner_val 24 / test 40 recordings (excluded 11/0/8)
134
+ fold 1: test volunteers ['v3', 'v32', 'v2', 'v13', 'v36', 'v23']; pool27 fold 1 [...]: train 123 / inner_val 24 / test 45 recordings (excluded 16/0/3)
135
+ fold 2: test volunteers ['v26', 'v33', 'v22', 'v41', 'v24']; pool27 fold 2 [...]: train 128 / inner_val 24 / test 40 recordings (excluded 19/0/0)
136
+ fold 3: test volunteers ['v37', 'v29', 'v12', 'v39', 'v27']; pool27 fold 3 [...]: train 141 / inner_val 23 / test 28 recordings (excluded 11/1/7)
137
+ fold 4: test volunteers ['v1', 'v40', 'v31', 'v34', 'v30']; pool27 fold 4 [...]: train 134 / inner_val 19 / test 39 recordings (excluded 13/5/1)
138
+ v1s2 {'mocap': (5645, 620), 'emg': (5645, 8), 'eyetrack': (5645, 24), 'imu': (5645, 180), 'pressure': (5645, 50)}
139
+ 0.00 5.00 open box (both)
140
+ 11.00 14.00 close box (both)
141
+ 14.00 16.00 open box (both)
142
+ ```
143
+
144
+ Every benchmark run has the same interface and writes one `results.json`
145
+ (metrics, per-recording statistics, provenance):
146
+
147
+ ```bash
148
+ python -m pulse.tasks.<task> --root $PULSE_ROOT --split splits/<pool>.json \
149
+ --fold <k> --seed <s> --config configs/<...>.yaml [--out DIR] [--set key=value] [--device auto|cpu|cuda]
150
+ python -m pulse.tasks.<task> ... --print-manifest # list the recordings a configuration uses
151
+ ```
152
+
153
+ A quick CPU check of the whole chain (nearest-centroid scenario classifier, not
154
+ in the paper):
155
+
156
+ ```bash
157
+ python -m pulse.tasks.datacheck --root $PULSE_ROOT --split splits/pool33.json \
158
+ --fold 0 --seed 42 --config configs/datacheck/emg_pressure.yaml
159
+ ```
160
+
161
+ ## 3. Reproducing the paper
162
+
163
+ `configs/tables.yaml` registers every table: its task, split, seeds and rows
164
+ (one config per row). `scripts/run_table.py` turns a table into one run per
165
+ (row, fold, seed) and `scripts/aggregate.py` turns the runs into LaTeX tables and
166
+ number macros:
167
+
168
+ ```bash
169
+ python scripts/run_table.py --table 3 # print the commands (135 runs)
170
+ python scripts/run_table.py --table 3 --local # run them here, one after another
171
+ python scripts/run_table.py --table 3 --slurm # or as a SLURM job array (section 7)
172
+ python scripts/aggregate.py --table 3 # -> tables/tab3.tex, tab3_common.tex, tab3_paired.tex, numbers.tex
173
+ python scripts/build_outputs.py --runs runs --out out --f0 --fig7 # every table, report and figure at once
174
+ ```
175
+
176
+ Runs whose `results.json` exists are skipped, so an interrupted table can be
177
+ resumed. The registry ids below are internal names; the registry titles, config
178
+ comments and docstrings use the table numbers of the submitted version of the
179
+ paper, and the table below gives the camera-ready numbers.
180
+
181
+ | Camera-ready paper | Registry table(s) | Task | Split | Runs | Outputs |
182
+ |---|---|---|---|---|---|
183
+ | Table 3, Fig. 3 (T1 scenario recognition, each method's own modalities) | `3` | `t1` | pool33 | 135 | `tab3*.tex`; Fig. 3: `python figures/t1/fig3_confusion.py --runs runs --table 3 --row syncfuse` |
184
+ | Table 4 (T1 matched inputs: IMU only, 4 modalities, 5 modalities, 4 -> 5 modalities on the same recordings) | `3imu`, `3m4`, `3m5`, `3gain`, `3gain-as` | `t1` | pool33 | 120 | `tab3imu*.tex`, ... (rows marked `reuse:` read the runs of the identical row of another table) |
185
+ | Table 5 (T1 on the right-wrist accelerometer: harnet5 vs TinyHAR) | `3harnet` | `t1` | pool33 | 45 | `tab3harnet*.tex` |
186
+ | Table 6, Table 27 (T2 action recognition, sliding windows; full per-head metrics; training-majority row) | `4`, `4maj` | `t2` | pool27 | 120 | `tab4*.tex`, `tab4maj*.tex` |
187
+ | Table 7 (T2, segment-aligned protocol) | `4seg` (`4segmaj`: majority rates, not in the paper) | `t2` | pool27 | 120 | `tab4seg*.tex` |
188
+ | Table 8 (T2 matched inputs: all five / IMU only) | `4m5`, `4imu` | `t2` | pool27 | 240 | `tab4m5*.tex`, `tab4imu*.tex` |
189
+ | Table 9, Fig. 4 (F5 motion-primitive pre-training -> T2) | `5` (`5seg`: segment-aligned variant, not in the paper) | `f5` | pool27 | 150 | `tab5*.tex`; Fig. 4: `python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out` |
190
+ | Table 10 (T3 grasp-onset anticipation) | `6` | `t3` | pool27 | 90 | `tab6*.tex`; per-fold counts: `python -m pulse.tasks.t3_report --table 6` |
191
+ | Table 11 (T4 missing modalities, 4 modalities) | `7` | `t4` | pool33 | 30 | `tab7*.tex` |
192
+ | Table 12 (T4 with five modalities; 5 vs 4 modalities on the same recordings) | `7m5`, `7m5-matched` | `t4` | pool33 | 45 | `tab7m5*.tex`, `tab7m5-matched*.tex` |
193
+ | Tables 13, 14, 15, 28; Figs. 7, 8 (T5 grasp state, EMG vs EMG + pressure) | `8` | `t5` | pool27 | 105 | `python -m pulse.tasks.t5_report --table 8 --table 8full --runs runs --out tables` -> `t5_tab8_lift.tex` (13), `_classes.tex` (14), `_events.tex` (15), `_backbones.tex` (28); Fig. 7: `python figures/t5/fig_label_derivation.py --root $PULSE_ROOT --out figures/t5`; Fig. 8: `python figures/t5/fig_sustained_lift.py --report tables/t5_tab8.json --out figures/t5` |
194
+ | Table 16 (T5 with the full non-tactile stack +- pressure) | `8full` | `t5` | pool27 | 90 | `t5_tab8full_lift.tex` (same report command) |
195
+ | Table 17 (T6a / T6b pressure prediction, single-seed and within-fold 3-seed ensemble columns; Ghorbani GRU with four modalities, marked with a dagger) | `11a`, `11b`, `11a-ens`, `11b-ens`, `11b-b19` | `t6`, `t6ens` | pool27 | 120 GPU + 35 CPU | `tab11a*.tex`, `tab11b*.tex`, `tab11a-ens*.tex`, `tab11b-ens*.tex`, `tab11b-b19*.tex` |
196
+ | Table 18 (T6a budget-matched: EMG vs four modalities on the same recordings) | `11a-b7-pham`, `11a-b7-transformer`, `11a-b7-bao` | `t6` | pool27 | 90 | `tab11a-b7-*.tex` |
197
+ | Table 19 (T7 EMG -> right-hand pose) | `12` | `t7` | pool33 | 75 | `tab12*.tex` |
198
+ | Table 23 (appendix backbone ablation, 1 seed) | `16` | `t1` | pool33 | 45 | `tab16*.tex` |
199
+ | Table 24 (appendix fusion ablation, 1 seed) | `17` | `t1` | pool33 | 45 | `tab17*.tex` |
200
+ | Figs. 5, 6 and the F0 numbers (anticipatory coordination case study, CPU) | -- | `f0` | all recordings with EMG + pressure + MoCap | 1 | `python -m pulse.tasks.f0 --root $PULSE_ROOT --out runs/f0 --workers 16`, then `python figures/f0/plot_f0.py --in runs/f0 --out figures/out --tables tables` (`f0_grasp_phase_timing.pdf` = Fig. 5, `f0_delay_distributions.pdf` = Fig. 6, `numbers_f0.tex`); see `configs/f0/README.md` |
201
+ | Table 25 (verb rollup), Table 26 (test recordings per input configuration) | -- | -- | -- | -- | `taxonomy/taxonomy_v1.1.json` (`scripts/build_taxonomy.py`); `manifests/expected_v1.1.json` (`python scripts/build_manifests.py --check`) |
202
+
203
+ Main tables use 5 folds x 3 seeds (42, 43, 44); the appendix ablations one seed.
204
+ The T2 confusion-matrix script (`figures/t2/confusion.py`) draws a figure that is
205
+ not in the paper.
206
+
207
+ **Result provenance.** This repository is published as a single snapshot; its
208
+ development history is not public. The numbers in the paper come from runs of
209
+ development commits `e792638` (T1-T6 and the appendix ablations), `cb55d98` (T7
210
+ and F0) and `487a753` (F5), all from clean checkouts on data tag `v1.1` (each run
211
+ records its commit id in its `results.json`). Between `e792638`
212
+ and `487a753` the T1-T6 code paths are unchanged except the label of one T6a
213
+ baseline (the multi-branch CNN of Hajian et al. 2021) and documentation. The
214
+ benchmark code in this snapshot is identical to `487a753`; the snapshot only adds
215
+ documentation, packaging metadata, the licence and `prompts/`.
216
+ Every `results.json` records the commit (and whether the checkout had uncommitted
217
+ changes), the data tag and the md5 of the split file, taxonomy, availability table
218
+ and expected manifest, so a rerun can be compared run by run. GPU runs are not
219
+ bit-reproducible across hardware and library versions; expect small differences
220
+ from the paper's numbers within the reported fold-to-fold spread.
221
+
222
+ **F5 needs the S9 labels**, which are not in tag `v1.1`. They are published on
223
+ the dataset's `main` branch (identical to the reviewed labels the paper's F5 runs
224
+ used):
225
+
226
+ ```bash
227
+ hf download velvet-pine-22/PULSE --repo-type dataset --revision main \
228
+ --include "annotations/*/s9.json" --local-dir PULSE_S9
229
+ export PULSE_S9_ANNOTATIONS=$PWD/PULSE_S9/annotations
230
+ python scripts/run_table.py --table 5 --local # 10 rows x 5 folds x 3 seeds
231
+ ```
232
+
233
+ Pre-training uses volunteers v9 and v16-v21 (none of them is in pool27); v20s9's
234
+ IMU fails the |x| > 1e6 rule, so 6 recordings are used. Details:
235
+ `configs/f5/README.md`.
236
+
237
+ **harnet5 rows** (`3harnet`, and the harnet5 row of `3imu`) load
238
+ `OxWearables/ssl-wearables` at commit `150550ea` with `torch.hub` (University of
239
+ Oxford Academic Use Licence, not vendored; checkpoint SHA-256 verified). On
240
+ machines without internet access fetch it once and point
241
+ `$PULSE_SSL_WEARABLES_DIR` at it:
242
+
243
+ ```bash
244
+ python -m pulse.models.t1.harnet --fetch ~/ext/ssl-wearables # once, with internet
245
+ export PULSE_SSL_WEARABLES_DIR=~/ext/ssl-wearables/ssl-wearables-150550ea5d41800229c95e36f88f5bf0d2e7cf04
246
+ ```
247
+
248
+ Launch paper runs from a checkout nobody commits to and keep `--runs` outside
249
+ it: every task refuses to write results if the checkout changes while it runs,
250
+ and the aggregator refuses to merge runs from different commits, configurations,
251
+ `--set` overrides, device types or torch versions.
252
+
253
+ ## 4. Protocol
254
+
255
+ **Volunteer pools and folds** (`splits/`). Two split files, volunteer-disjoint 5-fold CV:
256
+
257
+ | File | Volunteers | Test folds | Used by |
258
+ |---|---|---|---|
259
+ | `splits/pool33.json` | 33 (all with EMG + skeleton MoCap) | 7/7/7/6/6, round-robin by volunteer number | T1, T4, T7, appendix ablations |
260
+ | `splits/pool27.json` | 27 (all with EMG + pressure) | 6/6/5/5/5, seed-42 random partition | T2, T3, T5, T6, F5 |
261
+
262
+ Within a configuration only recordings that have all its modalities are used
263
+ (e.g. 26 of the 33 pool33 volunteers have pressure; v5 has no MoCap); paper
264
+ Table 26 lists the resulting test recordings per input configuration.
265
+
266
+ **Inner validation.** Each fold also lists inner-validation volunteers:
267
+ `round(0.15 x n_train)` training volunteers (pool33: 4, pool27: 3), chosen
268
+ deterministically (strata by modality coverage, full S1-S8 coverage required,
269
+ each volunteer at most once across folds; the exact rule is stored in the
270
+ split file). Epoch selection, early stopping and LR scheduling use only the
271
+ inner-validation volunteers; the test fold is evaluated once.
272
+
273
+ **Seeds and statistics.** Main tables: seeds 42, 43, 44. Per fold the metric
274
+ is averaged over seeds; tables report mean +- SD (ddof = 1) over the 5 fold
275
+ means. Rows are compared on the *common test set* (per fold, the recordings
276
+ every compared row loaded) with paired t-tests over fold means (n = 5) and over
277
+ fold x seed pairs (n = 15), plus the number of folds where the row is better.
278
+ The T6 within-fold ensemble averages all 3 seeds.
279
+
280
+ **Labels (T2, F5).** v1.1 human-reviewed annotations, frozen in
281
+ `taxonomy/taxonomy_v1.1.json` by `scripts/build_taxonomy.py`: 17 fine verbs
282
+ (the single `support` segment is dropped), 6 composite verbs, 33 nouns with
283
+ >= 50 segments over all 36 annotated volunteers (`cup` as named in v1.1), 3
284
+ hands; 7,076 of 7,356 segments kept. Uniform chance: verb@1 5.9 %, noun@1 3.0 %,
285
+ action@5 (top-5 verbs x top-5 nouns) 25/561 = 4.5 %.
286
+
287
+ **T1 (activity-scenario recognition) and T4 (missing modalities).** One
288
+ sample = one whole S1-S8 recording, label = its scenario (8 classes; no action
289
+ annotations), pool33. The configured modalities are cut to their common
290
+ length (the three re-trimmed recordings v2s6, v13s2, v13s3 aligned first,
291
+ `align_trimmed`), concatenated in config order, decimated to 20 Hz (longest
292
+ sequence 4,844 steps; every positional encoding holds 8,192) and z-scored
293
+ with training-frame statistics; batches are zero-padded with a frame mask.
294
+ Training (`configs/t1/base.yaml`, `configs/t4/base.yaml`): Adam lr 1e-3, weight
295
+ decay 1e-4, batch 16 (ST-GCN / CTR-GCN 4), cross-entropy with inverse-frequency
296
+ class weights and label smoothing 0.1, gradient clipping 1.0, ReduceLROnPlateau
297
+ on the inner-validation loss, early stopping after 15 epochs; the epoch is
298
+ selected on inner-validation macro F1 (T1, at most 80 epochs) or
299
+ inner-validation loss (T4 and the appendix ablations, at most 100 epochs); the
300
+ test fold is evaluated once. SyncFuse is `pulse/models/t1/syncfuse.py`; its
301
+ modality dropout (p = 0.3) is drawn **per sample**, and its parameter count
302
+ (about 3.1M with five modalities) is stored as `n_params` in every results.json.
303
+ "Transformer+Late" (per-modality Transformers, mean of logits) is trained from
304
+ scratch. LIMU-BERT is the supervised-only variant. ST-GCN and CTR-GCN
305
+ average over real frames only (the frame mask follows the stride-2 blocks), so a
306
+ recording's prediction does not depend on the other recordings in its
307
+ evaluation batch. InceptionTime has no frame mask, so it is evaluated one
308
+ recording at a time (`train.eval_batch_size: 1`). T1/T4 refuse a split file
309
+ whose `used_by` does not list them. T4 trains the late-fusion Transformer with
310
+ per-sample input dropout (p = 0 / 0.3) and evaluates it on full, leave-one-out
311
+ and singleton inputs (zeroed z-scored inputs), all on the same test recordings;
312
+ `gate_modalities: [pressure]` restricts a 4-modality run to the 5-modality
313
+ recordings (tables `3gain*`, `7m5-matched`). harnet5 (`3harnet`): right-wrist
314
+ (WT8) acceleration in g resampled to 30 Hz, 5 s windows every 5 s, recording
315
+ prediction = mean of window logits; AdamW lr 1e-3, batch 64, 80 epochs,
316
+ patience 15.
317
+
318
+ **T2 (fine-grained action recognition).** One config per model
319
+ (`configs/t2/<model>.yaml`) serves every T2 table. Primary protocol (paper
320
+ Table 6, registry `4`, `protocol: sliding`): 4 s windows at 1 s stride over each
321
+ recording; a window takes the labels of the kept segment that contains its
322
+ centre and is skipped otherwise; models are trained, inner-validated and tested
323
+ on such windows. Secondary (paper Table 7, registry `4seg`,
324
+ `--set protocol=segment`): one sample per kept segment, cut at its annotated
325
+ [start, end) (the 63 of 5,312 pool27 segments longer than 12.8 s are
326
+ centre-cropped to 256 frames). Signals are decimated to 20 Hz and z-scored with
327
+ training-sample statistics. Windows and segments lie in the recording's
328
+ *evaluation span* -- the frames covered by every usable benchmark modality,
329
+ computed from the availability table -- so two rows that load a recording score
330
+ identical samples; the re-trimmed EMG / eye-tracking / pressure files of v2s6,
331
+ v13s2 and v13s3 are placed at their release offset
332
+ (`mocap_padding.front_samples`) on the annotation timeline. Training: AdamW
333
+ (lr 3e-4, weight decay 1e-4), batch 32, at most 40 epochs, 5-epoch linear
334
+ warm-up then cosine decay (fixed schedule), label smoothing 0.05, no class
335
+ weighting, loss weights verb 1 / composite 0.5 / noun 1 / hand 0.5; the epoch
336
+ is selected and training stopped (patience 25) on inner-validation `act_vn@5`;
337
+ the test fold is evaluated once. Every T2 input set contains IMU or MoCap, so
338
+ v5 (72-column IMU, no MoCap) never enters T2: the effective pool is 26
339
+ volunteers (of the 211 annotated S1-S8 pool27 recordings, 198 have IMU, 198
340
+ MoCap, 192 all five modalities). Metrics: `act_vn@k` = verb in the top-k verbs
341
+ and noun in the top-k nouns (chance at k = 5: 25/561 = 4.5 %), verb / noun top-1
342
+ and top-5, composite-verb and hand top-1, per-head macro-F1, and the accuracy of
343
+ the training-majority class (`4maj`, `4segmaj`). DAF = DailyActFormer:
344
+ per-modality multi-scale conv stems, whole-modality dropout p = 0.3 drawn once
345
+ per batch, concat + 1x1 fusion, 3-layer Transformer, learnable-query pooling,
346
+ joint verb-noun head (17 x 33). "ActionLLM" is a sensor surrogate trained from
347
+ scratch (conv stem + 2-layer Transformer; no language model). The matched-input
348
+ tables (paper Table 8) are `4m5` (all five modalities) and `4imu` (IMU only);
349
+ rows marked `reuse:` in `configs/tables.yaml` read the runs of the identical
350
+ row of table `4`.
351
+
352
+ **T3 (grasp-onset anticipation).** Protocol in the docstring of
353
+ `pulse/tasks/t3.py` and `configs/t3/base.yaml`; per-fold sample counts and the
354
+ positive fraction: `python -m pulse.tasks.t3_report --table 6`.
355
+
356
+ **T5 (tactile-driven grasp-state recognition).** At anchors every 0.25 s the
357
+ past 1 s of the inputs (20 Hz) predicts the state of the next 0.5 s: NoGrasp
358
+ unless >= 50 % of it lies in v1.1 segments with a grasp verb (grasp, hold,
359
+ pick_up, move, place, put_down, pull, rotate, insert, remove); otherwise
360
+ SustainedGrasp if its longest contact run is >= 0.3 s (contact = >= 3 of the 50
361
+ pressure cells > 10 g), else AttemptedGrasp. Anchors are also stratified by
362
+ past / future contact (non-contact, pre-contact, steady-grip, release; paper
363
+ Table 15). All rows of a table load the same streams -- table `8`: EMG +
364
+ pressure (pressure defines the labels; the EMG-only rows do not see it), table
365
+ `8full`: all five -- so all rows score identical anchors (checked by the
366
+ report); test recordings over the 5 folds: 210 (`8`; v5s8 has no pressure) and
367
+ 192 (`8full`). Backbones DAF, FUTR, DeepConvLSTM (`pulse/models/t5`).
368
+ Training: each class capped at 10,000 anchors, class-weighted cross-entropy,
369
+ AdamW (wd 1e-3), fixed cosine schedule, d_model 64, dropout 0.3; DAF / FUTR lr
370
+ 3e-4, 30 epochs, patience 6; DeepConvLSTM lr 1e-4, 50 epochs, patience 12; the
371
+ epoch is chosen on inner-validation macro F1 and the test fold is evaluated
372
+ once. Lift = with - without pressure, paired per (fold, seed); per backbone
373
+ paired t over the 5 fold means and the 15 fold x seed runs; 3-backbone aggregate
374
+ over fold means (n = 5) and the 15 fold x backbone pairs. In v2s6, v13s2 and
375
+ v13s3 the labels are shifted by the re-trim offset
376
+ (`mocap_padding.mocap_sq.front_samples` = 1048 / 393 / 336 frames): their
377
+ annotations are on the full timeline (the last segments end 1.5 / 3.8 / 0.3 s
378
+ before the end of the re-trimmed streams on that timeline, but 9.0 / 0.1 /
379
+ 3.1 s after it without the shift). Details: docstrings of `pulse/tasks/t5.py`
380
+ and `pulse/tasks/t5_data.py`.
381
+
382
+ **T6 (cross-modal pressure prediction).** T6a regresses the right-hand
383
+ pressure sum (`R1..R25`, 1-D; the Myo side), T6b the 50 cells; metrics are
384
+ pooled over all scored test frames of a fold, in grams (T6a `pearson`, T6b
385
+ `pearson_max` = r_max, the best of the 50 per-cell r, and `pearson_mean`).
386
+ Only active segments are trained on and scored: pressure total > 100 g,
387
+ dilated by 1.0 s, 1.0 s context on each side, overlapping segments merged.
388
+ Inputs are z-scored per subject from a calibration window only -- the first 30 s
389
+ of the subject's recordings, S1 first (std floored at 0.1 x the pooled training
390
+ std, clipped at +-10); calibration frames are not scored; no statistic of the
391
+ rest of a test recording is used. EMG is a 200 ms RMS envelope; T6b MoCap is
392
+ the 460-column hand set (both hands and forearms, wrist-relative, + velocity),
393
+ the T6a 4-modality rows use the full skeleton. Reported model = SWA average of
394
+ epochs 30-60 (ReduceLROnPlateau on the inner-validation Huber loss before
395
+ that); the test fold is evaluated once. The ensemble tables
396
+ (`pulse.tasks.t6ens`, CPU) average the per-frame test predictions of seeds 42,
397
+ 43 and 44 of a row. The budget-matched rows (paper Table 18) are trained and
398
+ tested on identical recordings (the EMG-only rows are gated on the 4-modality
399
+ recordings); `11b-b19` is the 4-modality Ghorbani GRU row of paper Table 17. A
400
+ T6 run refuses to write results if the checkout changes while it runs;
401
+ `t6ens` refuses a subset of the table's seeds and a `predictions.npz` that does
402
+ not reproduce its member's `results.json`. Protocol details: docstring of
403
+ `pulse/tasks/t6.py`.
404
+
405
+ **T7 (EMG -> right-hand pose).** Input: the 8 Myo channels, pooled to 20 Hz
406
+ by block RMS (5 frames per block); target: the right hand as 21 joints x 3 =
407
+ 63-D in the wrist frame (mm) from the skeleton TSV -- wrist + per finger
408
+ `RightHand{f}1` (MCP), `{f}2`, `{f}3` and `Right{f}Tip` (the usual 21-keypoint
409
+ layout). MPJPE is over the 21 joints, fingertip MPJPE over the 5 tips, and
410
+ the angle error is the bone-direction error (`atan2(|u x v|, u . v)`) over the
411
+ 20 hand-skeleton bones; `finger_angle_err_deg` (15 finger bones, without the 5
412
+ palm bones that are fixed in the wrist frame) is also written to
413
+ `results.json`. MoCap dropout frames (zero wrist quaternion, NaN) are masked.
414
+ Definitions: `pulse/tasks/t7_pose.py`. Loss (`train.loss`): `mpjpe` (default;
415
+ masked MPJPE in mm, fingertips weighted `train.tip_weight` = 2), or `huber` /
416
+ `mse` on the z-scored target. Rows: "MS-CNN+Transformer" (multi-scale CNN +
417
+ Transformer; named "EMG2Pose" in the submitted version, but it is not the TDS +
418
+ LSTM vemg2pose of the emg2pose paper), NeuroPose, Transformer, TDS (the emg2pose
419
+ featurizer) and the training-set mean pose (reference row). MS-CNN+Transformer
420
+ uses its position head: the velocity-integration head drifts over whole
421
+ recordings (fold-0 inner-validation MPJPE 138.5 mm after 12 epochs vs 21.6 mm;
422
+ `configs/t7/mscnn_transformer_velocity.yaml`, not in the table). Training
423
+ batches are whole recordings padded to the longest one; the BatchNorm layers of
424
+ MS-CNN+Transformer and NeuroPose take their statistics from real frames only,
425
+ and NeuroPose resets padded positions to 0 after every block. RMS pooling is
426
+ used instead of decimation (`emg[::5]`): on fold-0 inner-validation volunteers a
427
+ ridge probe on decimated EMG equals the mean pose (21.95 vs 21.95 mm) while RMS
428
+ blocks give 20.99 mm. Both choices (position head, RMS pooling) were made once,
429
+ on the fold-0 inner-validation volunteers v16/v25/v33/v36, who are test
430
+ volunteers in folds 1-4; they are fixed for all folds.
431
+
432
+ ```bash
433
+ python -m pulse.tasks.t7 --root $PULSE_ROOT --split splits/pool33.json \
434
+ --fold 0 --seed 42 --config configs/t7/mscnn_transformer.yaml # one run
435
+ ```
436
+
437
+ ## 5. Data layer (`pulse/paths.py`, `pulse/data/`)
438
+
439
+ * Modalities: `emg` (Myo, 8 ch, **right forearm for every volunteer**), `imu`
440
+ (10 IMUs x 18 = 180 columns; subsets `imu_wrist`, `imu_finger`), `eyetrack`
441
+ (24 Dikablis core columns; subsets `eyetrack_gaze`, `eyetrack_ocular`),
442
+ `pressure` (25 cells per hand), `mocap` (skeleton TSV: hip-relative positions
443
+ + position velocity, 620 columns), `mocap_markers`, `myo_quat`. All streams
444
+ are on the dataset's common 100 Hz timebase.
445
+ * The skeleton TSV `aligned_vXsY_s_Q.tsv` is in `data/vX/sY/raw/` for 237
446
+ recordings and at the recording root for 10 (v1s5, v2s2, v2s6, v3s1, v3s2,
447
+ v3s5, v3s6, v11s8, v13s2, v13s3); both are supported.
448
+ * Nothing is skipped silently. A missing file raises `MissingFileError`; a file
449
+ that does not provide the modality raises `ModalityUnavailableError` (v5's IMU
450
+ file has 72 columns -> treated as a missing modality; one-eye eye-tracking
451
+ exports lack core columns); a corrupted stream raises `DataQualityError`.
452
+ Select recordings through the availability table (as in section 2) rather
453
+ than loading every recording id.
454
+ * `manifests/availability_v1.1.json` (built by `scripts/build_availability.py`)
455
+ records, per recording and modality, whether it is usable. For each task
456
+ configuration and fold, `pulse.data.manifest.expected_manifest` lists exactly
457
+ which recordings train / inner-val / test must load; every task checks what it
458
+ loaded against it before writing results. `manifests/expected_v1.1.json`
459
+ (`scripts/build_manifests.py`) freezes these lists for reference
460
+ configurations, e.g. pool33 MoCap+EMG+eye+IMU: 235 test recordings over the
461
+ 5 folds; with pressure (or gated on pressure): 192.
462
+ * Documented exclusions found by the scan of v1.1 (no data errors):
463
+
464
+ | Recording(s) | Effect |
465
+ |---|---|
466
+ | v5 (s1-s9) | IMU file has 72 columns -> no `imu`; no MoCap |
467
+ | v14s1, v20s3-s9, v29s4-s9 | IMU values with abs > 1e6 -> no `imu` (`quality:extreme_values`) |
468
+ | v1s1, v14s8 | one-eye eye-tracking export -> no `eyetrack` |
469
+ | v23s3 | scene-camera gaze columns > 90 % zero -> no `eyetrack_gaze` (full `eyetrack` is fine) |
470
+ | v11s6 | marker CSV corrupted -> no `mocap_markers` (skeleton `mocap` is fine) |
471
+ | v4s1, v4s4 | listed in recordings.csv but ship no files |
472
+
473
+ Notes: v17s6 has an aligned skeleton TSV although recordings.csv lists no
474
+ `mocap` (that column tracks the marker CSV), so it is used; for 7 of the 10
475
+ root-level TSVs `alignment_metadata.json` does not list MoCap but the streams
476
+ have exactly the recording's sample count; in v2s6, v13s2 and v13s3 the EMG,
477
+ eye-tracking and pressure streams are shorter than the others (re-trimmed at
478
+ release) and multi-modal inputs are cut to the shortest stream. The short
479
+ files cover frames `[front, front + n)` of the IMU / MoCap timeline
480
+ (`mocap_padding.mocap_sq.front_samples` / `n_frames` in
481
+ `alignment_metadata.json`), so cutting from the start shifts IMU / MoCap
482
+ against EMG / eye / pressure by 10.5 s, 3.9 s and 3.4 s.
483
+ `load_recording(..., align_trimmed=True)` slices the full-timeline streams to
484
+ that window first (off by default in the loader; T1 / T4 turn it on through
485
+ `data.align_trimmed`, T3 always uses it). T6 and T7 use the same window
486
+ (`release_trim_window`); T2, T5, F5 and F0 place the short streams at their
487
+ release offset on the annotation timeline (`recording_timeline`).
488
+
489
+ ## 6. Results files
490
+
491
+ `results.json` (schema `pulse-results/1`, see `pulse/provenance.py`) holds the
492
+ test metrics, per-recording sufficient statistics (confusion matrices, top-k
493
+ hits, Pearson sums, ...), from which every metric can be recomputed on any
494
+ recording subset (the aggregator does this for the common test set; per-volunteer
495
+ metrics can be computed the same way), and provenance: git commit (+ dirty
496
+ flag), data tag `v1.1`, split file md5, taxonomy md5, availability md5,
497
+ manifest md5, seed, device, train / inner-val / test volunteers, loaded
498
+ recording ids and the full config. `scripts/aggregate.py` writes
499
+ `tables/tab<N>.tex`, `tab<N>_common.tex`, `tab<N>_paired.tex`, `tab<N>.json` and
500
+ `tables/numbers.tex` (macros `\pulsenum{<table>/<row>/<metric>/<stat>}`).
501
+
502
+ ## 7. Running on a SLURM cluster
503
+
504
+ `scripts/run_table.py --slurm` writes `<runs>/_jobs/<name>-<time>/{commands.txt,job.sbatch,job.json}`
505
+ and submits one job array (one GPU, 8 CPUs and 60 GB per task, 8 h limit;
506
+ `configs/slurm.yaml`); repeat `--table` to put several tables into one array.
507
+ The launcher was written for the authors' cluster: the GPU partition name is
508
+ hard-coded in `pulse/slurm.py` (`GPU_PARTITION`) and
509
+ `slurm/gpu_array.sbatch.in`, and the CPU partition is set in
510
+ `configs/slurm.yaml`; change these for another cluster, or use `--local` /
511
+ `--dry-run` (writes the job files without submitting). The user's running and
512
+ pending GPUs on that partition are counted first and the array is throttled to
513
+ stay within a cap (default 4, `MAXGPU=1..8`). `--after JOBID[,JOBID]` adds
514
+ `--dependency=afterany:...`. GPU arrays are refused from a checkout with
515
+ uncommitted changes, and every `--set` key (CLI and the rows' `set:`) must exist
516
+ in the row's config (`--allow-new-keys` for smoke-test keys such as
517
+ `train.max_train_batches`). The variables the rows need are checked before
518
+ submission and exported in the job script: `PULSE_ROOT`,
519
+ `PULSE_SSL_WEARABLES_DIR` (harnet rows) and `PULSE_S9_ANNOTATIONS` (F5 rows with
520
+ S9 pre-training). The account is `$PULSE_SLURM_ACCOUNT`, else `account` in
521
+ `configs/slurm.yaml`, else the user's default. `--cpu` runs on the CPU
522
+ partition. Set `PULSE_PYTHON` (the environment's python) before submitting.
523
+
524
+ ```bash
525
+ # CPU smoke suite: every run row of every table, fold 0, seed 42, 1 short epoch (configs/smoke.yaml)
526
+ python scripts/smoke_suite.py --runs SMOKE/runs --submit
527
+ # GPU smoke before the full runs: GPU tables only, 2 full epochs
528
+ python scripts/smoke_suite.py --runs GPUSMOKE/runs --gpu --submit
529
+ # every table, numbers.tex, report and figure from one runs directory (with a LaTeX build check)
530
+ python scripts/build_outputs.py --runs SMOKE/runs --out SMOKE/out --allow-incomplete --latex
531
+ ```
532
+
533
+ ## 8. Compute
534
+
535
+ The 1,515 GPU runs behind the T1-T7 tables of the paper (including the appendix
536
+ ablations and the matched-input comparisons) took about 53 GPU-hours on NVIDIA
537
+ A800 80 GB GPUs, one GPU per run (median about 2 minutes, at most 22 minutes per
538
+ run); the F5 runs (150 for paper Table 9) are not included in that figure. The
539
+ T6 seed ensembles and F0 run on CPU. Peak host memory of a run was about 34 GB
540
+ (T4). On CPU, MulT at batch 16 needs more than 64 GB (no fused attention over
541
+ 4,844 steps); the CPU smoke suite runs it at batch 4.
542
+
543
+ ## 9. Known issues and naming
544
+
545
+ * v5: IMU has 4 sensors (72 columns) and is treated as missing; no MoCap.
546
+ * 10 skeleton TSVs sit at the recording root (handled, see section 5).
547
+ * F5 reads the S9 labels from `--s9-annotations` / `$PULSE_S9_ANNOTATIONS`
548
+ (`vX/s9.json`, published on the dataset's `main` branch); a missing label file
549
+ raises `MissingFileError`. v20s9's IMU fails the |x| > 1e6 rule, so
550
+ pre-training uses the other 6 recordings.
551
+ * "Transformer+Late" (T1) is trained from scratch; the earlier name
552
+ "Transformer+Late+Pretrain" is dropped because no pretraining is performed.
553
+ * `metadata/recordings.csv` column `split` is a legacy fixed hold-out and is
554
+ not used; all results use the 5-fold split files in `splits/`.
555
+ * T7: the TSV fingertip points `Right{f}Tip` were added in post-processing
556
+ (`alignment_metadata.json`: `tips_added`, `tip_scale` 0.75) and the MCP
557
+ joints are rigid in the wrist frame (fixed bone lengths per volunteer).
558
+ * Registry titles, config comments and docstrings refer to the table numbers of
559
+ the submitted version (e.g. "Table 4" for T2, "Tables 16/17" for the appendix
560
+ ablations) and to review-round row groups (R1-R3, B7, B13, B19, B21); section
561
+ 3 maps every registry table to the camera-ready paper.
562
+
563
+ ## 10. Annotation prompts
564
+
565
+ The action labels were drafted by a vision-language model and then reviewed by a
566
+ human (every released segment was confirmed or corrected). The prompt templates
567
+ for S1-S8 and for the S9 primitive pass, the tables filled into them and a
568
+ script that renders the exact prompt are in `prompts/` (see `prompts/README.md`).
569
+
570
+ ## License
571
+
572
+ Code: MIT (`LICENSE`). Dataset: CC BY-NC 4.0 (see the dataset card). The
573
+ harnet5 weights are loaded from `OxWearables/ssl-wearables` under its own
574
+ licence and are not distributed here.
575
+
576
+ ## Citation
577
+
578
+ ```bibtex
579
+ @inproceedings{liang2026pulse,
580
+ title = {{PULSE}: A Synchronized Five-Modality Dataset for Sensorimotor Coordination in Long-Horizon Daily Activities},
581
+ author = {Liang, Wolin and Gao, Yang and Yan, Peiyu and Hu, Yuexiang and Xiao, Yingjing and Ying, Xiongfeng and Guo, Yingnian and Zhang, Cheng and Jin, Zhanpeng},
582
+ booktitle = {NeurIPS 2026 Evaluations and Datasets Track},
583
+ year = {2026},
584
+ url = {https://openreview.net/forum?id=AoU576r9O2}
585
+ }
586
+ ```
587
+
588
+ Dataset DOI: [10.57967/hf/11314](https://doi.org/10.57967/hf/11314).
configs/datacheck/emg.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Infrastructure smoke test: EMG only (every pool volunteer).
2
+ _base_: emg_pressure.yaml
3
+ modalities: [emg]
configs/datacheck/emg_gate_pressure.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Infrastructure smoke test: EMG restricted to the recordings that have pressure (gating).
2
+ _base_: emg_pressure.yaml
3
+ modalities: [emg]
4
+ gate_modalities: [pressure]
configs/datacheck/emg_pressure.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # Infrastructure smoke test: EMG + pressure (small files, fast on CPU).
2
+ task: datacheck
3
+ modalities: [emg, pressure]
4
+ gate_modalities: []
5
+ require_annotation: false
6
+ downsample: 5
configs/datacheck/mocap.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Infrastructure smoke test: skeleton MoCap (exercises the raw/ and recording-root TSV locations).
2
+ _base_: emg_pressure.yaml
3
+ modalities: [mocap]
configs/f0/README.md ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # F0 case study: onset of EMG and hand motion before contact
2
+
3
+ `python -m pulse.tasks.f0` is a deterministic CPU analysis of the public data
4
+ (no split, no seed); algorithm and parameters in the module docstring and
5
+ `configs/f0/grasp_timing.yaml`.
6
+
7
+ ```bash
8
+ python -m pulse.tasks.f0 --root $PULSE_ROOT --config configs/f0/grasp_timing.yaml --out runs/f0 --workers 16
9
+ python figures/f0/plot_f0.py --in runs/f0 --out figures/out --tables tables
10
+ # -> figures/out/f0_grasp_phase_timing.pdf, f0_delay_distributions.pdf, tables/numbers_f0.tex
11
+ ```
12
+
13
+ * Recordings: every S1-S8 recording whose EMG, pressure and skeleton MoCap pass
14
+ the strict loaders (198 recordings of 26 volunteers in v1.1); the loaded list
15
+ is checked against the availability table.
16
+ * The three streams are placed on the alignment timeline with the shared T2
17
+ rule (`recording_timeline`): the re-trimmed EMG / pressure files of v2s6,
18
+ v13s2 and v13s3 start at their release offset; the analysis uses the frames
19
+ all three cover.
20
+ * Right hand only: the Myo armband is on the right forearm of every volunteer
21
+ (the earlier analysis also paired left-hand contacts with the right-forearm
22
+ EMG, and used an 80-recording pre-release snapshot).
23
+ * Events whose window overlaps a MoCap edge-filled stretch
24
+ (`alignment_metadata.json` warning; v25 s1-s8) or an untracked right-hand
25
+ frame (non-finite or exactly zero position; 330 frames in v3s1, v3s6, v3s7,
26
+ v23s4 in v1.1) are excluded; untracked frames are linearly interpolated before
27
+ the hand speed is computed, so they do not inflate the recording's speed range
28
+ used by the clean rule (the earlier analysis set missing positions to 0).
29
+ * Statistics: n, mean, SD (ddof=1), median, quartiles, IQR per signal; the
30
+ paired motion - EMG difference per event (t-based CI, paired t, Wilcoxon,
31
+ share of events with motion first) and per volunteer (Wilcoxon / t over
32
+ volunteers; volunteer-bootstrap 95 % CIs of the median of the per-volunteer
33
+ median differences and of the pooled event median, each with its own point
34
+ estimate).
35
+ * Caveat (stored in the output): EMG stream start times have whole-second
36
+ resolution in `alignment_metadata.json`.
37
+
38
+ Macros (`tables/numbers_f0.tex`): `\pulsenum{f0/events}`, `f0/onsets`,
39
+ `f0/recordings`, `f0/volunteers`, `f0/emg/{mean,sd,median,p25,p75,iqr}`,
40
+ `f0/motion/...`, `f0/diff/{mean,median,sd,cilo,cihi,medcilo,medcihi,pt,pw,motionfirst,r}`
41
+ (`cilo/cihi`: t-based CI of the mean difference; `medcilo/medcihi`: volunteer-cluster
42
+ bootstrap CI of the event median), `f0/vol/{n,median,pw,pt,cilo,cihi}` (`median`:
43
+ median of the per-volunteer median differences, `cilo/cihi`: its volunteer-bootstrap CI).
configs/f0/grasp_timing.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # F0 case study: onset of right-forearm EMG and right-hand motion before right-hand contact.
2
+ # Algorithm and parameters: pulse/tasks/f0.py (same rules as the original analysis,
3
+ # right hand only because the Myo armband is on the right forearm).
4
+ task: f0
5
+ modalities: [emg, pressure, mocap] # all three must pass the strict loaders
6
+ scenarios: [s1, s2, s3, s4, s5, s6, s7, s8]
7
+ volunteers: null # null = every volunteer with qualifying recordings
8
+ hands: [right]
9
+ pressure_threshold_g: 5.0 # contact = right-hand 25-cell sum above this
10
+ context_sec: 2.0 # look-back window before contact
11
+ post_sec: 0.5 # trace window after contact
12
+ exclude_edge_filled: true # skip events whose window overlaps an edge-filled MoCap stretch
13
+ detect: {min_duration: 10, min_gap: 50, persist_frac: 0.7, release_window: 5, release_frac: 0.3}
14
+ emg_envelope: {window: 20} # 200 ms moving average of the rectified EMG
15
+ velocity: {smooth_window: 3}
16
+ onset: {threshold_ratio: 0.3, baseline_frac: 0.3, baseline_percentile: 25}
17
+ clean: {look_back: 150, rest_window: 50, min_rel_increase: 0.1}
18
+ delay_range_ms: [-1500, 0]
19
+ n_bootstrap: 10000 # volunteer-cluster bootstrap of the median difference
20
+ bootstrap_seed: 0
configs/f5/README.md ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # F5: S9 motion-primitive pre-training -> T2 fine-tuning
2
+
3
+ Paper: Table 9 and Fig. 4 (registry table `5`; the registry and the code
4
+ comments use the submission numbering "Table 5", see the main README). Code: `pulse/tasks/f5.py`,
5
+ `pulse/models/f5/` (S9 labels, pre-training + encoder cache, label budget).
6
+ Model, samples, training loop and metrics are T2's (`pulse/tasks/t2.py`,
7
+ `pulse/tasks/t2_data.py`, `pulse/models/t2`); `configs/f5/base.yaml` inherits
8
+ `configs/t2/daf.yaml`, so F5 and T2 (registry table `4`, paper Table 6) share protocols, evaluation spans,
9
+ normalisation, optimiser, schedule and early stopping.
10
+
11
+ ## Design
12
+
13
+ | | |
14
+ |---|---|
15
+ | Model | T2 DailyActFormer-v2 on EMG + IMU (per-modality conv stems, per-batch whole-modality dropout 0.3, concat + 1x1 fusion, 3-layer Transformer, query pooling, joint verb/noun head), `max_T` 256 |
16
+ | Conditions | `init: scratch` (random) and `init: s9` (every non-head parameter from S9 pre-training; backbone LR x 0.1, head LR x 1) |
17
+ | Label budget | `fraction` 0.05 / 0.10 / 0.25 / 0.50 / 1.00 (the paper's sweep) of the training volunteers' kept T2 segments that overlap the evaluation span; nested random subsets from (seed, fold); a training sample is kept iff the segment labelling it is in the subset; identical for both conditions and both protocols; inner-val and test never subsampled |
18
+ | Protocol | Table `5`: T2 sliding windows (4 s, stride 1 s, centre label) -- the T2 primary protocol. Table `5seg`: T2 segment-aligned samples (secondary) |
19
+ | Pool / folds | `splits/pool27.json`, 5 folds, inner-val volunteers from the split file |
20
+ | Training | T2 `train_model`: AdamW 3e-4, wd 1e-4, batch 32, <= 40 epochs, 5-epoch warm-up + cosine, label smoothing 0.05, loss weights 1 / 0.5 / 1 / 0.5 |
21
+ | Selection | inner-val `verb_composite@1` after every epoch (earliest best), early stopping patience 25 (inherited from T2); test fold evaluated once |
22
+ | Seeds | 42, 43, 44; the random state is reset to the seed before the model is built, so both conditions share the head initialisation and the batch order |
23
+ | Reported | T2 metric set; paper Table 9 columns `verb_composite@1` (primary), `verb_fine@1`, `act_vn@5` |
24
+ | Comparison | each `s9pre_fXXX` row is paired with `scratch_fXXX` (registry `reference:`) on the common test set: paired t over fold means (n=5) and fold x seed (n=15), positive folds |
25
+
26
+ Recordings (EMG + IMU + labels, frozen availability table): v5 is excluded
27
+ (IMU file has 72 columns) and v29 s4-s8 (IMU |x| > 1e6). Per fold train /
28
+ inner-val / test recordings: 134/24/40, 126/24/48, 134/24/40, 144/24/30,
29
+ 139/19/40 (198 test recordings). The re-trimmed EMG files of v2s6, v13s2,
30
+ v13s3 are placed at their release offset (T2 timeline).
31
+
32
+ S9 pre-training (one encoder per seed, shared by every fold, fraction and
33
+ both tables): volunteers v9, v16-v21 (none is in pool27; checked at run time);
34
+ v20s9's IMU fails the |x| > 1e6 rule, so 6 recordings are used. The 16 S9
35
+ primitives map to 12 verbs x 10 nouns (`pulse/models/f5/s9.py`; any other
36
+ name is an error). Network = the T2 DAF with a 12-verb / 10-noun head
37
+ (modality dropout 0), samples = S9 segments cut like the T2 segment protocol,
38
+ z-scored over the S9 frames; 80 epochs, batch 8, AdamW 1e-3, cosine to 0,
39
+ loss verb 1 + noun 0.5; 20 % of the segments (seeded) held out for epoch
40
+ selection. Cached at `runs/_cache/f5_s9_pretrain/<key>/` (`key` = md5 of the
41
+ config, seed, S9 label files, manifests and the source of the `pulse` package;
42
+ the record names the commit that built it); concurrent runs
43
+ wait on a directory lock, so an array builds it once per seed.
44
+
45
+ Differences from the pre-camera-ready runs (deliberate): the 5/10/25/50/100 %
46
+ label fractions are drawn over labelled segments (was over samples); epoch selection on
47
+ inner-val (was the test fold); 3 seeds (was 1); S9 encoder with the fine-tuned
48
+ architecture (was a separate 2-layer encoder; third layer random); no T2-head
49
+ initialisation from S9 (the earlier row copy never happened: 256 vs 128 input
50
+ shapes); v5 and v20s9 IMU excluded instead of zero-padded / used; v1.1
51
+ human-reviewed T2 labels; T2 evaluation spans and trim offsets.
52
+
53
+ ## S9 labels
54
+
55
+ The reviewed S9 labels are published in the dataset (main branch, not in tag
56
+ `v1.1`): `annotations/vX/s9.json` for 30 volunteers; F5 pre-trains on v9 and
57
+ v16-v21 only (`pretrain.volunteers` in `configs/f5/base.yaml`). These are the
58
+ label files the paper's F5 runs used (the per-file md5s recorded in their
59
+ `results.json` match). Download them next to the v1.1 copy and pass their
60
+ directory with `--s9-annotations DIR` or `export PULSE_S9_ANNOTATIONS=DIR`:
61
+
62
+ ```bash
63
+ hf download velvet-pine-22/PULSE --repo-type dataset --revision main \
64
+ --include "annotations/*/s9.json" --local-dir PULSE_S9
65
+ export PULSE_S9_ANNOTATIONS=$PWD/PULSE_S9/annotations
66
+ ```
67
+
68
+ Each run records the md5 of every label file and of the set
69
+ (`extra.s9_pretrain.label_set_md5`; `f4b16ff2c9ebdbd7b1c2548df4155bed` for the
70
+ published labels), never the path. A missing file raises. The encoder cache
71
+ key changes with the label files.
72
+
73
+ ## Commands
74
+
75
+ ```bash
76
+ export PULSE_ROOT=/path/to/PULSE PULSE_S9_ANNOTATIONS=/path/to/PULSE_S9/annotations
77
+ python -m pulse.tasks.f5 --root $PULSE_ROOT --split splits/pool27.json --fold 0 --seed 42 \
78
+ --config configs/f5/sliding/s9pre_f010.yaml # one run
79
+ python -m pulse.tasks.f5 ... --config configs/f5/sliding/s9pre_f100.yaml --pretrain-only # encoder only
80
+ python scripts/run_table.py --table 5 --slurm # 10 rows x 5 folds x 3 seeds
81
+ python scripts/run_table.py --table 5seg --slurm # secondary protocol
82
+ python scripts/aggregate.py --table 5 --table 5seg # tables/tab5*.tex, numbers.tex
83
+ python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out
84
+ # CPU smoke test (fold 0, seed 42, 1 epoch x 20 batches, 2 pre-training epochs)
85
+ python scripts/run_table.py --table 5 --rows s9pre_f010,scratch_f010 --folds 0 --seeds 42 --slurm --cpu \
86
+ --set train.epochs=1 --set train.max_batches_per_epoch=20 --set pretrain.train.epochs=2
87
+ ```
88
+
89
+ Outputs: `runs/5/<row>/fold<k>_seed<s>/results.json` (provenance includes the
90
+ main and the S9 manifest md5s; `extra.label_budget` has the subset md5 and
91
+ segment / sample counts, `extra.s9_pretrain` the encoder record),
92
+ `tables/tab5_fraction.tex`, `figures/out/f5_data_fraction_tab5.pdf`. Macros:
93
+ `\pulsenum{5/s9pre_f010/verb_composite@1/delta}`, `.../p15`, `.../pos`,
94
+ `\pulsenum{5/scratch_f010/verb_composite@1/cpm}`.
configs/f5/base.yaml ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # F5: S9 motion-primitive pre-training -> T2 fine-tuning.
2
+ # Everything not set here comes from the T2 DAF configuration (configs/t2/daf.yaml ->
3
+ # configs/t2/base.yaml): protocol definitions, 20 Hz decimation, DAF-v2 architecture
4
+ # (max_T 256), AdamW 3e-4, batch 32, <= 40 epochs, warm-up + cosine, label smoothing,
5
+ # loss weights, early-stopping patience. Rows set `protocol`, `init` and `fraction`.
6
+ _base_: ../t2/daf.yaml
7
+ task: f5
8
+ modalities: [emg, imu] # the S9 encoder and the fine-tuned model use this order
9
+ init: scratch # scratch | s9
10
+ fraction: 1.0 # share of the training volunteers' labelled segments
11
+ load_workers: 8
12
+ train:
13
+ backbone_lr_scale: 0.1 # LR factor of the S9-initialised backbone (init: s9 only)
14
+ selection:
15
+ metric: verb_composite@1 # Table 5 metric, on the inner-validation volunteers
16
+ mode: max
17
+
18
+ pretrain: # one S9 encoder per seed (pulse/models/f5/pretrain.py)
19
+ volunteers: [v9, v16, v17, v18, v19, v20, v21] # none in pool27; v20s9 IMU fails |x| > 1e6
20
+ scenario: s9
21
+ min_frames: 10 # 0.5 s at 20 Hz
22
+ max_frames: 256 # = model.max_T; longer S9 segments are centre-cropped
23
+ min_segments: 10
24
+ val_fraction: 0.2 # held-out S9 segments for epoch selection
25
+ mod_dropout_p: 0.0
26
+ train:
27
+ epochs: 80
28
+ batch_size: 8
29
+ eval_batch_size: 64
30
+ lr: 1.0e-3
31
+ weight_decay: 1.0e-4
32
+ warmup_epochs: 0
33
+ min_lr_factor: 0.0 # cosine to 0 over the 80 epochs
34
+ grad_clip: 0.0
35
+ label_smoothing: 0.0
36
+ loss_weights: {verb_fine: 1.0, verb_composite: 0.0, noun: 0.5, hand: 0.0} # S9 verb, S9 noun
37
+ max_batches_per_epoch: null
38
+ selection:
39
+ metric: verb_fine@1 # held-out S9 verb accuracy
40
+ mode: max
41
+ patience: null
42
+ cache_dir: runs/_cache/f5_s9_pretrain
43
+ lock_timeout_sec: 7200
configs/f5/segment.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # F5, T2 segment-aligned protocol (secondary): one sample per kept segment (<= 256 frames).
2
+ _base_: base.yaml
3
+ protocol: segment
configs/f5/segment/s9pre_f005.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: S9-pretrained backbone, 5% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: s9
4
+ fraction: 0.05
configs/f5/segment/s9pre_f010.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: S9-pretrained backbone, 10% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: s9
4
+ fraction: 0.10
configs/f5/segment/s9pre_f025.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: S9-pretrained backbone, 25% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: s9
4
+ fraction: 0.25
configs/f5/segment/s9pre_f050.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: S9-pretrained backbone, 50% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: s9
4
+ fraction: 0.50
configs/f5/segment/s9pre_f100.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: S9-pretrained backbone, 100% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: s9
4
+ fraction: 1.00
configs/f5/segment/scratch_f005.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: random init, 5% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: scratch
4
+ fraction: 0.05
configs/f5/segment/scratch_f010.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: random init, 10% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: scratch
4
+ fraction: 0.10
configs/f5/segment/scratch_f025.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: random init, 25% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: scratch
4
+ fraction: 0.25
configs/f5/segment/scratch_f050.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: random init, 50% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: scratch
4
+ fraction: 0.50
configs/f5/segment/scratch_f100.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 segment: random init, 100% of the training segments.
2
+ _base_: ../segment.yaml
3
+ init: scratch
4
+ fraction: 1.00
configs/f5/sliding.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # F5, T2 primary protocol: 4 s windows every 1 s, label = kept segment at the window centre.
2
+ _base_: base.yaml
3
+ protocol: sliding
configs/f5/sliding/s9pre_f005.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: S9-pretrained backbone, 5% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: s9
4
+ fraction: 0.05
configs/f5/sliding/s9pre_f010.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: S9-pretrained backbone, 10% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: s9
4
+ fraction: 0.10
configs/f5/sliding/s9pre_f025.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: S9-pretrained backbone, 25% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: s9
4
+ fraction: 0.25
configs/f5/sliding/s9pre_f050.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: S9-pretrained backbone, 50% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: s9
4
+ fraction: 0.50
configs/f5/sliding/s9pre_f100.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: S9-pretrained backbone, 100% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: s9
4
+ fraction: 1.00
configs/f5/sliding/scratch_f005.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: random init, 5% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: scratch
4
+ fraction: 0.05
configs/f5/sliding/scratch_f010.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: random init, 10% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: scratch
4
+ fraction: 0.10
configs/f5/sliding/scratch_f025.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: random init, 25% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: scratch
4
+ fraction: 0.25
configs/f5/sliding/scratch_f050.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: random init, 50% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: scratch
4
+ fraction: 0.50
configs/f5/sliding/scratch_f100.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # F5 sliding: random init, 100% of the training segments.
2
+ _base_: ../sliding.yaml
3
+ init: scratch
4
+ fraction: 1.00
configs/manifests.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Reference configurations whose expected recordings (per pool / fold /
2
+ # partition) are frozen in manifests/expected_v1.1.json by
3
+ # scripts/build_manifests.py. Tasks compute the same manifest at run time
4
+ # from their own config; this file is the reviewable record. gate = streams a
5
+ # recording must have without being model inputs; labelled = needs v1.1
6
+ # dense annotations.
7
+ configs:
8
+ - {name: pool33_emg_mocap, split: splits/pool33.json, modalities: [emg, mocap]}
9
+ - {name: pool33_m4, split: splits/pool33.json, modalities: [mocap, emg, eyetrack, imu]}
10
+ - {name: pool33_m4_gate_pressure, split: splits/pool33.json, modalities: [mocap, emg, eyetrack, imu], gate: [pressure]}
11
+ - {name: pool33_m5, split: splits/pool33.json, modalities: [mocap, emg, eyetrack, imu, pressure]}
12
+ - {name: pool33_imu, split: splits/pool33.json, modalities: [imu]}
13
+ - {name: pool33_emg, split: splits/pool33.json, modalities: [emg]}
14
+ - {name: pool27_emg_pressure, split: splits/pool27.json, modalities: [emg, pressure]}
15
+ - {name: pool27_emg_gate_pressure, split: splits/pool27.json, modalities: [emg], gate: [pressure]}
16
+ - {name: pool27_m4_gate_pressure, split: splits/pool27.json, modalities: [mocap, emg, eyetrack, imu], gate: [pressure]}
17
+ - {name: pool27_mocap_gate_pressure, split: splits/pool27.json, modalities: [mocap], gate: [pressure]}
18
+ - {name: pool27_m5_labelled, split: splits/pool27.json, modalities: [mocap, emg, eyetrack, imu, pressure], labelled: true}
19
+ - {name: pool27_imu_labelled, split: splits/pool27.json, modalities: [imu], labelled: true}
20
+ - {name: pool27_emg_pressure_labelled, split: splits/pool27.json, modalities: [emg, pressure], labelled: true}
configs/slurm.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Cluster settings used by scripts/run_table.py --slurm.
2
+ # GPU jobs: partition ex03A800 only (hard-coded in slurm/gpu_array.sbatch.in and
3
+ # pulse/slurm.py). Cap on the user's running+pending GPUs there: 4 by default,
4
+ # override with the MAXGPU environment variable (1..8).
5
+ # account: null -> $PULSE_SLURM_ACCOUNT if set, else the user's default account.
6
+ account: null
7
+ gpu:
8
+ gpus_per_task: 1
9
+ cpus_per_task: 8
10
+ mem: 60G # 8 tasks x 60G fit the 503 GiB of one 8-GPU node (8 x 64G do not); peak RSS seen 34 GB (T4)
11
+ time: "08:00:00"
12
+ cpu:
13
+ partition: cpuXeon6458
14
+ cpus_per_task: 8
15
+ mem: 64G # T1/T4 CPU smoke runs peak at 34 GB (32G ran out of memory)
16
+ time: "04:00:00"
17
+ max_concurrent: 16
configs/smoke.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CPU smoke suite (scripts/smoke_suite.py): every run row of every table in
2
+ # configs/tables.yaml, fold 0, seed 42, one short epoch, on the CPU partition.
3
+ # Checks that each row loads its recordings, trains, selects on inner-val,
4
+ # evaluates the test fold and writes a valid results.json, and that the
5
+ # aggregation / reports / figures build from such runs. The numbers mean
6
+ # nothing. Keep the smoke runs in their own --runs directory: the aggregator
7
+ # refuses to mix them with real runs (different --set overrides and device).
8
+ folds: [0]
9
+ seeds: [42]
10
+
11
+ # --set overrides per task (keys the tasks read with a default; the suite passes
12
+ # --allow-new-keys)
13
+ tasks:
14
+ datacheck: []
15
+ t1: [train.epochs=1, train.max_train_batches=2, train.eval_batch_size=4]
16
+ t4: [train.epochs=1, train.max_train_batches=2, train.eval_batch_size=4]
17
+ t2: [train.epochs=1, train.max_batches_per_epoch=20]
18
+ f5: [train.epochs=1, train.max_batches_per_epoch=20, pretrain.train.epochs=2]
19
+ t3: [train.epochs=1, train.max_steps_per_epoch=50]
20
+ t5: [train.epochs=1, train.max_steps_per_epoch=50]
21
+ t6: [train.epochs=1]
22
+ t6ens: []
23
+ t7: [train.epochs=1, train.batch_size=4]
24
+
25
+ # per-table exceptions: set (replaces the task's list), seeds, after (tables whose
26
+ # smoke array must have ended first)
27
+ tables:
28
+ 3harnet: {set: [train.epochs=1, train.max_train_batches=2, train.eval_batch_size=128]}
29
+ # the within-fold ensemble of the paper averages all three seeds
30
+ 11a: {seeds: [42, 43, 44]}
31
+ 11b: {seeds: [42, 43, 44]}
32
+ 11a-ens: {after: [11a]}
33
+ 11b-ens: {after: [11b]}
34
+
35
+ # GPU smoke (W0 in the launch plan): every GPU table on the A800s, fold 0, seed 42,
36
+ # 2 full epochs -- GPU memory and per-epoch time before the full waves. Groups
37
+ # are chained one after another (each may use the whole GPU cap).
38
+ gpu:
39
+ folds: [0]
40
+ seeds: [42]
41
+ tasks:
42
+ t1: [train.epochs=2]
43
+ t4: [train.epochs=2]
44
+ t2: [train.epochs=2]
45
+ f5: [train.epochs=2, pretrain.train.epochs=2]
46
+ t3: [train.epochs=2]
47
+ t5: [train.epochs=2]
48
+ t6: [train.epochs=2]
49
+ t7: [train.epochs=2]
configs/t1/actionsense.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # ActionSense LSTM (DelPreto et al. 2022); 4 modalities.
2
+ _base_: base.yaml
3
+ modalities: [mocap, emg, eyetrack, imu]
4
+ model: {name: actionsense, hidden: 128}
configs/t1/actionsense_m4_gate_pressure.yaml ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # R1 control: ActionSense, 4 modalities, on the 5-modality recordings (pressure gate).
2
+ _base_: base.yaml
3
+ modalities: [mocap, emg, eyetrack, imu]
4
+ model: {name: actionsense, hidden: 128}
5
+ gate_modalities: [pressure]
configs/t1/actionsense_m5.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # R1: ActionSense with pressure as a 5th modality.
2
+ _base_: base.yaml
3
+ modalities: [mocap, emg, eyetrack, imu, pressure]
4
+ model: {name: actionsense, hidden: 128}
configs/t1/base.yaml ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # T1 activity-scenario recognition, whole-recording ("sequence") protocol.
2
+ # Hyper-parameters of the submission's Table 3 launcher (run_t1_cv.sh);
3
+ # model selection now uses the fold's inner-validation volunteers only.
4
+ task: t1
5
+ family: sequence
6
+ modalities: [mocap, emg, eyetrack, imu, pressure]
7
+ gate_modalities: []
8
+ data:
9
+ downsample: 5 # 100 Hz -> 20 Hz
10
+ align_trimmed: true # align the 3 release-trimmed recordings (v2s6, v13s2, v13s3)
11
+ load_workers: 4 # reader threads (does not change the data)
12
+ train:
13
+ epochs: 80
14
+ batch_size: 16
15
+ eval_batch_size: null # null: same as batch_size
16
+ optimizer: adam
17
+ lr: 1.0e-3
18
+ weight_decay: 1.0e-4
19
+ label_smoothing: 0.1
20
+ class_weights: true # inverse frequency of the training labels
21
+ patience: 15 # early stopping on the selection metric
22
+ grad_clip: 1.0
23
+ max_train_batches: null
24
+ scheduler: {name: plateau, factor: 0.5, patience: 7, min_lr: 1.0e-6} # steps on inner-val loss
25
+ selection: {metric: macro_f1, mode: max} # inner-val macro F1
configs/t1/ctrgcn.yaml ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # CTR-GCN (Chen et al. 2021); MoCap skeleton. Batch 4 (memory), as submitted.
2
+ _base_: base.yaml
3
+ modalities: [mocap]
4
+ model: {name: ctrgcn, hidden: 128, n_joints: 52}
5
+ train: {batch_size: 4}
configs/t1/emg_cnn.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # EMG-CNN (sEMG literature); EMG only.
2
+ _base_: base.yaml
3
+ modalities: [emg]
4
+ model: {name: emg_cnn, hidden: 64}
configs/t1/exp1/base.yaml ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ # Appendix Tables 16/17 (backbone and fusion ablations), now under pool33
2
+ # 5-fold CV with inner-validation selection. Hyper-parameters of the
3
+ # submission's exp1 v7/v8 runs: 100 epochs, no label smoothing, the epoch
4
+ # with the lowest inner-validation loss.
5
+ _base_: ../base.yaml
6
+ train:
7
+ epochs: 100
8
+ label_smoothing: 0.0
9
+ selection: {metric: loss, mode: min}
configs/t1/exp1/bb_cnn_emg.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Table 16: cnn backbone on emg alone.
2
+ _base_: base.yaml
3
+ modalities: [emg]
4
+ model: {name: fusion, backbone: cnn, fusion: early, hidden: 128, proj_dim: 0}
configs/t1/exp1/bb_cnn_imu.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Table 16: cnn backbone on imu alone.
2
+ _base_: base.yaml
3
+ modalities: [imu]
4
+ model: {name: fusion, backbone: cnn, fusion: early, hidden: 128, proj_dim: 0}
configs/t1/exp1/bb_cnn_mocap.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Table 16: cnn backbone on mocap alone.
2
+ _base_: base.yaml
3
+ modalities: [mocap]
4
+ model: {name: fusion, backbone: cnn, fusion: early, hidden: 128, proj_dim: 0}
configs/t1/exp1/bb_lstm_emg.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Table 16: lstm backbone on emg alone.
2
+ _base_: base.yaml
3
+ modalities: [emg]
4
+ model: {name: fusion, backbone: lstm, fusion: early, hidden: 128, proj_dim: 0}
configs/t1/exp1/bb_lstm_imu.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Table 16: lstm backbone on imu alone.
2
+ _base_: base.yaml
3
+ modalities: [imu]
4
+ model: {name: fusion, backbone: lstm, fusion: early, hidden: 128, proj_dim: 0}
configs/t1/exp1/bb_lstm_mocap.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Table 16: lstm backbone on mocap alone.
2
+ _base_: base.yaml
3
+ modalities: [mocap]
4
+ model: {name: fusion, backbone: lstm, fusion: early, hidden: 128, proj_dim: 0}