PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +5 -35
- .gitignore +8 -0
- LICENSE +21 -0
- README.md +588 -0
- configs/datacheck/emg.yaml +3 -0
- configs/datacheck/emg_gate_pressure.yaml +4 -0
- configs/datacheck/emg_pressure.yaml +6 -0
- configs/datacheck/mocap.yaml +3 -0
- configs/f0/README.md +43 -0
- configs/f0/grasp_timing.yaml +20 -0
- configs/f5/README.md +94 -0
- configs/f5/base.yaml +43 -0
- configs/f5/segment.yaml +3 -0
- configs/f5/segment/s9pre_f005.yaml +4 -0
- configs/f5/segment/s9pre_f010.yaml +4 -0
- configs/f5/segment/s9pre_f025.yaml +4 -0
- configs/f5/segment/s9pre_f050.yaml +4 -0
- configs/f5/segment/s9pre_f100.yaml +4 -0
- configs/f5/segment/scratch_f005.yaml +4 -0
- configs/f5/segment/scratch_f010.yaml +4 -0
- configs/f5/segment/scratch_f025.yaml +4 -0
- configs/f5/segment/scratch_f050.yaml +4 -0
- configs/f5/segment/scratch_f100.yaml +4 -0
- configs/f5/sliding.yaml +3 -0
- configs/f5/sliding/s9pre_f005.yaml +4 -0
- configs/f5/sliding/s9pre_f010.yaml +4 -0
- configs/f5/sliding/s9pre_f025.yaml +4 -0
- configs/f5/sliding/s9pre_f050.yaml +4 -0
- configs/f5/sliding/s9pre_f100.yaml +4 -0
- configs/f5/sliding/scratch_f005.yaml +4 -0
- configs/f5/sliding/scratch_f010.yaml +4 -0
- configs/f5/sliding/scratch_f025.yaml +4 -0
- configs/f5/sliding/scratch_f050.yaml +4 -0
- configs/f5/sliding/scratch_f100.yaml +4 -0
- configs/manifests.yaml +20 -0
- configs/slurm.yaml +17 -0
- configs/smoke.yaml +49 -0
- configs/t1/actionsense.yaml +4 -0
- configs/t1/actionsense_m4_gate_pressure.yaml +5 -0
- configs/t1/actionsense_m5.yaml +4 -0
- configs/t1/base.yaml +25 -0
- configs/t1/ctrgcn.yaml +5 -0
- configs/t1/emg_cnn.yaml +4 -0
- configs/t1/exp1/base.yaml +9 -0
- configs/t1/exp1/bb_cnn_emg.yaml +4 -0
- configs/t1/exp1/bb_cnn_imu.yaml +4 -0
- configs/t1/exp1/bb_cnn_mocap.yaml +4 -0
- configs/t1/exp1/bb_lstm_emg.yaml +4 -0
- configs/t1/exp1/bb_lstm_imu.yaml +4 -0
- configs/t1/exp1/bb_lstm_mocap.yaml +4 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,5 @@
|
|
| 1 |
-
*
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.
|
| 5 |
-
*.
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
* text=auto eol=lf
|
| 2 |
+
*.png binary
|
| 3 |
+
*.pdf binary
|
| 4 |
+
*.npz binary
|
| 5 |
+
*.pt binary
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
*.egg-info/
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
.cache/
|
| 6 |
+
runs/
|
| 7 |
+
*.tmp
|
| 8 |
+
.DS_Store
|
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 The PULSE authors (Wolin Liang, Yang Gao, Peiyu Yan, Yuexiang Hu, Yingjing Xiao, Xiongfeng Ying, Yingnian Guo, Cheng Zhang, Zhanpeng Jin)
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
README.md
ADDED
|
@@ -0,0 +1,588 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
tags:
|
| 4 |
+
- multimodal
|
| 5 |
+
- human-activity-recognition
|
| 6 |
+
- emg
|
| 7 |
+
- motion-capture
|
| 8 |
+
- eye-tracking
|
| 9 |
+
- tactile
|
| 10 |
+
- benchmark
|
| 11 |
+
datasets:
|
| 12 |
+
- velvet-pine-22/PULSE
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# PULSE benchmark code
|
| 16 |
+
|
| 17 |
+
Reference code of the paper **"PULSE: A Synchronized Five-Modality Dataset for
|
| 18 |
+
Sensorimotor Coordination in Long-Horizon Daily Activities"** (NeurIPS 2026
|
| 19 |
+
Evaluations and Datasets Track, spotlight). It loads the public PULSE dataset,
|
| 20 |
+
defines the volunteer-stratified 5-fold cross-validation splits, trains every
|
| 21 |
+
baseline of the benchmark (T1-T7, F5), runs the F0 case study, and regenerates the
|
| 22 |
+
benchmark tables, figures and numbers of the paper from the run results.
|
| 23 |
+
|
| 24 |
+
| | |
|
| 25 |
+
|---|---|
|
| 26 |
+
| Paper | https://openreview.net/forum?id=AoU576r9O2 |
|
| 27 |
+
| Dataset | https://huggingface.co/datasets/velvet-pine-22/PULSE (CC BY-NC 4.0), DOI [10.57967/hf/11314](https://doi.org/10.57967/hf/11314) |
|
| 28 |
+
| Small sample of the dataset | https://huggingface.co/datasets/velvet-pine-22/PULSE-sample |
|
| 29 |
+
| Code (this repository) | https://huggingface.co/velvet-pine-22/PULSE-code (MIT, see `LICENSE`) |
|
| 30 |
+
| Contact | Yang Gao (corresponding author, gaoyang2025@scut.edu.cn); maintainer Wolin Liang (202610196903@mail.scut.edu.cn) |
|
| 31 |
+
|
| 32 |
+
The benchmark data are the dataset's tag **`v1.1`** (commit
|
| 33 |
+
`e27b1bb6bb35f714705ae8685507f71773c065eb`, 3,001 files, 22.0 GB: sensor streams,
|
| 34 |
+
annotations, metadata). All numbers in the paper were computed on that tag with
|
| 35 |
+
the commands in section 3.
|
| 36 |
+
|
| 37 |
+
## Contents
|
| 38 |
+
|
| 39 |
+
```
|
| 40 |
+
pulse/ Python package: paths, data layer, splits, taxonomy, metrics, statistics,
|
| 41 |
+
provenance, aggregation, SLURM launcher, models/, tasks/ (one module per task)
|
| 42 |
+
configs/ tables.yaml (registry of every table and its rows), slurm.yaml, per-task configs
|
| 43 |
+
splits/ pool33.json, pool27.json (volunteer-disjoint 5-fold CV with inner validation)
|
| 44 |
+
taxonomy/ taxonomy_v1.1.json (T2 / F5 label vocabulary)
|
| 45 |
+
manifests/ hf_PULSE_v1.1_files.tsv (file list of dataset tag v1.1), availability_v1.1.json,
|
| 46 |
+
expected_v1.1.json
|
| 47 |
+
scripts/ run_table, aggregate, build_outputs, smoke_suite, verify_data, fetch_hf_manifest,
|
| 48 |
+
make_splits, build_taxonomy, build_availability, build_manifests
|
| 49 |
+
figures/ figure scripts (T1, T2, T5, F0, F5)
|
| 50 |
+
prompts/ VLM annotation prompt templates used to draft the action labels (see prompts/README.md)
|
| 51 |
+
slurm/ sbatch templates
|
| 52 |
+
tests/ unit tests (synthetic mini dataset + optional checks on the real data)
|
| 53 |
+
environment.yml the environment every number was produced with
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
## 1. Setup
|
| 57 |
+
|
| 58 |
+
```bash
|
| 59 |
+
git clone https://huggingface.co/velvet-pine-22/PULSE-code && cd PULSE-code
|
| 60 |
+
conda env create -f environment.yml # Python 3.12.13, torch 2.4.1+cu121
|
| 61 |
+
conda activate pulse
|
| 62 |
+
|
| 63 |
+
# dataset, pinned to the benchmark tag (22.0 GB, no videos)
|
| 64 |
+
hf download velvet-pine-22/PULSE --repo-type dataset --revision v1.1 --local-dir PULSE
|
| 65 |
+
export PULSE_ROOT=$PWD/PULSE
|
| 66 |
+
|
| 67 |
+
# check every file against the tag's file list (sha256 for LFS files, git blob id otherwise)
|
| 68 |
+
python scripts/verify_data.py --root $PULSE_ROOT --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 8
|
| 69 |
+
python -m pytest -q tests # unit tests on CPU; a few also read $PULSE_ROOT
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
`hf` is the Hugging Face CLI (installed with `huggingface_hub`, which the
|
| 73 |
+
environment contains); with older versions use `huggingface-cli download` with the
|
| 74 |
+
same arguments. Tasks read the dataset from `--root` or `$PULSE_ROOT`. Run every
|
| 75 |
+
command from the repository root (or `pip install -e .` to import `pulse` from
|
| 76 |
+
elsewhere; the configs, splits and manifests are still read from the checkout).
|
| 77 |
+
|
| 78 |
+
**Dataset revisions.** The `main` branch of the dataset contains everything in
|
| 79 |
+
`v1.1` plus the first-person videos (622 files, about 63 GB), the S9
|
| 80 |
+
motion-primitive labels (`annotations/vX/s9.json`, 30 files, and
|
| 81 |
+
`annotations_flat/segments_s9.csv`) and updated documentation. The sensor
|
| 82 |
+
streams and the S1-S8 annotations are identical; `metadata/recordings.csv`
|
| 83 |
+
differs only in the line endings and in marking the scene video of v5s8 and
|
| 84 |
+
v18s2 as missing (not a benchmark modality). The benchmark needs only `v1.1`;
|
| 85 |
+
F5 additionally needs the S9 labels (section 3, F5). Against a full download of
|
| 86 |
+
`main`, `verify_data.py` reports those differences and the extra files; it
|
| 87 |
+
passes with:
|
| 88 |
+
|
| 89 |
+
```bash
|
| 90 |
+
python scripts/verify_data.py --root $PULSE_ROOT --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 8 \
|
| 91 |
+
--expect-different README.md --expect-different DATASHEET.md --expect-different LICENSE \
|
| 92 |
+
--expect-different CODE_LICENSE --expect-different croissant.json \
|
| 93 |
+
--expect-different metadata/recordings.csv --expect-different docs/upload_to_huggingface.md \
|
| 94 |
+
--ignore-extra '*.mp4' --ignore-extra 'annotations/*/s9.json' --ignore-extra annotations_flat/segments_s9.csv
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
## 2. Getting started
|
| 98 |
+
|
| 99 |
+
Load one recording and its action segments, and iterate the folds of a split
|
| 100 |
+
(save as a file in the repository root and run it there):
|
| 101 |
+
|
| 102 |
+
```python
|
| 103 |
+
# Run from the root of a PULSE-code checkout, with $PULSE_ROOT = the dataset (revision v1.1).
|
| 104 |
+
from pulse.paths import DataRoot
|
| 105 |
+
from pulse.splits import load_split
|
| 106 |
+
from pulse.data import load_recording
|
| 107 |
+
from pulse.data.annotations import load_segments
|
| 108 |
+
from pulse.data.availability import Availability
|
| 109 |
+
from pulse.data.manifest import expected_manifest
|
| 110 |
+
|
| 111 |
+
dr = DataRoot.from_arg() # reads $PULSE_ROOT
|
| 112 |
+
av = Availability.load("manifests/availability_v1.1.json") # usable streams per recording
|
| 113 |
+
mods = ["mocap", "emg", "eyetrack", "imu", "pressure"]
|
| 114 |
+
split = load_split("splits/pool27.json") # volunteer-disjoint 5-fold CV
|
| 115 |
+
|
| 116 |
+
# (b) iterate the folds: which recordings each partition uses for these modalities
|
| 117 |
+
for k in range(split.n_folds):
|
| 118 |
+
fold = split.fold(k)
|
| 119 |
+
man = expected_manifest(av, fold, mods, require_annotation=True)
|
| 120 |
+
print(f"fold {k}: test volunteers {list(fold.test)}; {man.summary()}")
|
| 121 |
+
|
| 122 |
+
# (a) one test recording of the last fold: aligned 100 Hz arrays + action segments
|
| 123 |
+
rid = man.partitions["test"][0]
|
| 124 |
+
rec = load_recording(dr, rid, mods, align_trimmed=True)
|
| 125 |
+
print(rid, {m: x.shape for m, x in rec.arrays.items()}) # (T, features), common T
|
| 126 |
+
for s in load_segments(dr, rid)[:3]:
|
| 127 |
+
print(f"{s.start_sec:7.2f} {s.end_sec:7.2f} {s.verb} {s.noun} ({s.hand})")
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
Output on `v1.1`:
|
| 131 |
+
|
| 132 |
+
```
|
| 133 |
+
fold 0: test volunteers ['v25', 'v28', 'v11', 'v38', 'v5', 'v35']; pool27 fold 0 [mocap+emg+eyetrack+imu+pressure; s1,s2,s3,s4,s5,s6,s7,s8; labelled]: train 128 / inner_val 24 / test 40 recordings (excluded 11/0/8)
|
| 134 |
+
fold 1: test volunteers ['v3', 'v32', 'v2', 'v13', 'v36', 'v23']; pool27 fold 1 [...]: train 123 / inner_val 24 / test 45 recordings (excluded 16/0/3)
|
| 135 |
+
fold 2: test volunteers ['v26', 'v33', 'v22', 'v41', 'v24']; pool27 fold 2 [...]: train 128 / inner_val 24 / test 40 recordings (excluded 19/0/0)
|
| 136 |
+
fold 3: test volunteers ['v37', 'v29', 'v12', 'v39', 'v27']; pool27 fold 3 [...]: train 141 / inner_val 23 / test 28 recordings (excluded 11/1/7)
|
| 137 |
+
fold 4: test volunteers ['v1', 'v40', 'v31', 'v34', 'v30']; pool27 fold 4 [...]: train 134 / inner_val 19 / test 39 recordings (excluded 13/5/1)
|
| 138 |
+
v1s2 {'mocap': (5645, 620), 'emg': (5645, 8), 'eyetrack': (5645, 24), 'imu': (5645, 180), 'pressure': (5645, 50)}
|
| 139 |
+
0.00 5.00 open box (both)
|
| 140 |
+
11.00 14.00 close box (both)
|
| 141 |
+
14.00 16.00 open box (both)
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
Every benchmark run has the same interface and writes one `results.json`
|
| 145 |
+
(metrics, per-recording statistics, provenance):
|
| 146 |
+
|
| 147 |
+
```bash
|
| 148 |
+
python -m pulse.tasks.<task> --root $PULSE_ROOT --split splits/<pool>.json \
|
| 149 |
+
--fold <k> --seed <s> --config configs/<...>.yaml [--out DIR] [--set key=value] [--device auto|cpu|cuda]
|
| 150 |
+
python -m pulse.tasks.<task> ... --print-manifest # list the recordings a configuration uses
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
A quick CPU check of the whole chain (nearest-centroid scenario classifier, not
|
| 154 |
+
in the paper):
|
| 155 |
+
|
| 156 |
+
```bash
|
| 157 |
+
python -m pulse.tasks.datacheck --root $PULSE_ROOT --split splits/pool33.json \
|
| 158 |
+
--fold 0 --seed 42 --config configs/datacheck/emg_pressure.yaml
|
| 159 |
+
```
|
| 160 |
+
|
| 161 |
+
## 3. Reproducing the paper
|
| 162 |
+
|
| 163 |
+
`configs/tables.yaml` registers every table: its task, split, seeds and rows
|
| 164 |
+
(one config per row). `scripts/run_table.py` turns a table into one run per
|
| 165 |
+
(row, fold, seed) and `scripts/aggregate.py` turns the runs into LaTeX tables and
|
| 166 |
+
number macros:
|
| 167 |
+
|
| 168 |
+
```bash
|
| 169 |
+
python scripts/run_table.py --table 3 # print the commands (135 runs)
|
| 170 |
+
python scripts/run_table.py --table 3 --local # run them here, one after another
|
| 171 |
+
python scripts/run_table.py --table 3 --slurm # or as a SLURM job array (section 7)
|
| 172 |
+
python scripts/aggregate.py --table 3 # -> tables/tab3.tex, tab3_common.tex, tab3_paired.tex, numbers.tex
|
| 173 |
+
python scripts/build_outputs.py --runs runs --out out --f0 --fig7 # every table, report and figure at once
|
| 174 |
+
```
|
| 175 |
+
|
| 176 |
+
Runs whose `results.json` exists are skipped, so an interrupted table can be
|
| 177 |
+
resumed. The registry ids below are internal names; the registry titles, config
|
| 178 |
+
comments and docstrings use the table numbers of the submitted version of the
|
| 179 |
+
paper, and the table below gives the camera-ready numbers.
|
| 180 |
+
|
| 181 |
+
| Camera-ready paper | Registry table(s) | Task | Split | Runs | Outputs |
|
| 182 |
+
|---|---|---|---|---|---|
|
| 183 |
+
| Table 3, Fig. 3 (T1 scenario recognition, each method's own modalities) | `3` | `t1` | pool33 | 135 | `tab3*.tex`; Fig. 3: `python figures/t1/fig3_confusion.py --runs runs --table 3 --row syncfuse` |
|
| 184 |
+
| Table 4 (T1 matched inputs: IMU only, 4 modalities, 5 modalities, 4 -> 5 modalities on the same recordings) | `3imu`, `3m4`, `3m5`, `3gain`, `3gain-as` | `t1` | pool33 | 120 | `tab3imu*.tex`, ... (rows marked `reuse:` read the runs of the identical row of another table) |
|
| 185 |
+
| Table 5 (T1 on the right-wrist accelerometer: harnet5 vs TinyHAR) | `3harnet` | `t1` | pool33 | 45 | `tab3harnet*.tex` |
|
| 186 |
+
| Table 6, Table 27 (T2 action recognition, sliding windows; full per-head metrics; training-majority row) | `4`, `4maj` | `t2` | pool27 | 120 | `tab4*.tex`, `tab4maj*.tex` |
|
| 187 |
+
| Table 7 (T2, segment-aligned protocol) | `4seg` (`4segmaj`: majority rates, not in the paper) | `t2` | pool27 | 120 | `tab4seg*.tex` |
|
| 188 |
+
| Table 8 (T2 matched inputs: all five / IMU only) | `4m5`, `4imu` | `t2` | pool27 | 240 | `tab4m5*.tex`, `tab4imu*.tex` |
|
| 189 |
+
| Table 9, Fig. 4 (F5 motion-primitive pre-training -> T2) | `5` (`5seg`: segment-aligned variant, not in the paper) | `f5` | pool27 | 150 | `tab5*.tex`; Fig. 4: `python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out` |
|
| 190 |
+
| Table 10 (T3 grasp-onset anticipation) | `6` | `t3` | pool27 | 90 | `tab6*.tex`; per-fold counts: `python -m pulse.tasks.t3_report --table 6` |
|
| 191 |
+
| Table 11 (T4 missing modalities, 4 modalities) | `7` | `t4` | pool33 | 30 | `tab7*.tex` |
|
| 192 |
+
| Table 12 (T4 with five modalities; 5 vs 4 modalities on the same recordings) | `7m5`, `7m5-matched` | `t4` | pool33 | 45 | `tab7m5*.tex`, `tab7m5-matched*.tex` |
|
| 193 |
+
| Tables 13, 14, 15, 28; Figs. 7, 8 (T5 grasp state, EMG vs EMG + pressure) | `8` | `t5` | pool27 | 105 | `python -m pulse.tasks.t5_report --table 8 --table 8full --runs runs --out tables` -> `t5_tab8_lift.tex` (13), `_classes.tex` (14), `_events.tex` (15), `_backbones.tex` (28); Fig. 7: `python figures/t5/fig_label_derivation.py --root $PULSE_ROOT --out figures/t5`; Fig. 8: `python figures/t5/fig_sustained_lift.py --report tables/t5_tab8.json --out figures/t5` |
|
| 194 |
+
| Table 16 (T5 with the full non-tactile stack +- pressure) | `8full` | `t5` | pool27 | 90 | `t5_tab8full_lift.tex` (same report command) |
|
| 195 |
+
| Table 17 (T6a / T6b pressure prediction, single-seed and within-fold 3-seed ensemble columns; Ghorbani GRU with four modalities, marked with a dagger) | `11a`, `11b`, `11a-ens`, `11b-ens`, `11b-b19` | `t6`, `t6ens` | pool27 | 120 GPU + 35 CPU | `tab11a*.tex`, `tab11b*.tex`, `tab11a-ens*.tex`, `tab11b-ens*.tex`, `tab11b-b19*.tex` |
|
| 196 |
+
| Table 18 (T6a budget-matched: EMG vs four modalities on the same recordings) | `11a-b7-pham`, `11a-b7-transformer`, `11a-b7-bao` | `t6` | pool27 | 90 | `tab11a-b7-*.tex` |
|
| 197 |
+
| Table 19 (T7 EMG -> right-hand pose) | `12` | `t7` | pool33 | 75 | `tab12*.tex` |
|
| 198 |
+
| Table 23 (appendix backbone ablation, 1 seed) | `16` | `t1` | pool33 | 45 | `tab16*.tex` |
|
| 199 |
+
| Table 24 (appendix fusion ablation, 1 seed) | `17` | `t1` | pool33 | 45 | `tab17*.tex` |
|
| 200 |
+
| Figs. 5, 6 and the F0 numbers (anticipatory coordination case study, CPU) | -- | `f0` | all recordings with EMG + pressure + MoCap | 1 | `python -m pulse.tasks.f0 --root $PULSE_ROOT --out runs/f0 --workers 16`, then `python figures/f0/plot_f0.py --in runs/f0 --out figures/out --tables tables` (`f0_grasp_phase_timing.pdf` = Fig. 5, `f0_delay_distributions.pdf` = Fig. 6, `numbers_f0.tex`); see `configs/f0/README.md` |
|
| 201 |
+
| Table 25 (verb rollup), Table 26 (test recordings per input configuration) | -- | -- | -- | -- | `taxonomy/taxonomy_v1.1.json` (`scripts/build_taxonomy.py`); `manifests/expected_v1.1.json` (`python scripts/build_manifests.py --check`) |
|
| 202 |
+
|
| 203 |
+
Main tables use 5 folds x 3 seeds (42, 43, 44); the appendix ablations one seed.
|
| 204 |
+
The T2 confusion-matrix script (`figures/t2/confusion.py`) draws a figure that is
|
| 205 |
+
not in the paper.
|
| 206 |
+
|
| 207 |
+
**Result provenance.** This repository is published as a single snapshot; its
|
| 208 |
+
development history is not public. The numbers in the paper come from runs of
|
| 209 |
+
development commits `e792638` (T1-T6 and the appendix ablations), `cb55d98` (T7
|
| 210 |
+
and F0) and `487a753` (F5), all from clean checkouts on data tag `v1.1` (each run
|
| 211 |
+
records its commit id in its `results.json`). Between `e792638`
|
| 212 |
+
and `487a753` the T1-T6 code paths are unchanged except the label of one T6a
|
| 213 |
+
baseline (the multi-branch CNN of Hajian et al. 2021) and documentation. The
|
| 214 |
+
benchmark code in this snapshot is identical to `487a753`; the snapshot only adds
|
| 215 |
+
documentation, packaging metadata, the licence and `prompts/`.
|
| 216 |
+
Every `results.json` records the commit (and whether the checkout had uncommitted
|
| 217 |
+
changes), the data tag and the md5 of the split file, taxonomy, availability table
|
| 218 |
+
and expected manifest, so a rerun can be compared run by run. GPU runs are not
|
| 219 |
+
bit-reproducible across hardware and library versions; expect small differences
|
| 220 |
+
from the paper's numbers within the reported fold-to-fold spread.
|
| 221 |
+
|
| 222 |
+
**F5 needs the S9 labels**, which are not in tag `v1.1`. They are published on
|
| 223 |
+
the dataset's `main` branch (identical to the reviewed labels the paper's F5 runs
|
| 224 |
+
used):
|
| 225 |
+
|
| 226 |
+
```bash
|
| 227 |
+
hf download velvet-pine-22/PULSE --repo-type dataset --revision main \
|
| 228 |
+
--include "annotations/*/s9.json" --local-dir PULSE_S9
|
| 229 |
+
export PULSE_S9_ANNOTATIONS=$PWD/PULSE_S9/annotations
|
| 230 |
+
python scripts/run_table.py --table 5 --local # 10 rows x 5 folds x 3 seeds
|
| 231 |
+
```
|
| 232 |
+
|
| 233 |
+
Pre-training uses volunteers v9 and v16-v21 (none of them is in pool27); v20s9's
|
| 234 |
+
IMU fails the |x| > 1e6 rule, so 6 recordings are used. Details:
|
| 235 |
+
`configs/f5/README.md`.
|
| 236 |
+
|
| 237 |
+
**harnet5 rows** (`3harnet`, and the harnet5 row of `3imu`) load
|
| 238 |
+
`OxWearables/ssl-wearables` at commit `150550ea` with `torch.hub` (University of
|
| 239 |
+
Oxford Academic Use Licence, not vendored; checkpoint SHA-256 verified). On
|
| 240 |
+
machines without internet access fetch it once and point
|
| 241 |
+
`$PULSE_SSL_WEARABLES_DIR` at it:
|
| 242 |
+
|
| 243 |
+
```bash
|
| 244 |
+
python -m pulse.models.t1.harnet --fetch ~/ext/ssl-wearables # once, with internet
|
| 245 |
+
export PULSE_SSL_WEARABLES_DIR=~/ext/ssl-wearables/ssl-wearables-150550ea5d41800229c95e36f88f5bf0d2e7cf04
|
| 246 |
+
```
|
| 247 |
+
|
| 248 |
+
Launch paper runs from a checkout nobody commits to and keep `--runs` outside
|
| 249 |
+
it: every task refuses to write results if the checkout changes while it runs,
|
| 250 |
+
and the aggregator refuses to merge runs from different commits, configurations,
|
| 251 |
+
`--set` overrides, device types or torch versions.
|
| 252 |
+
|
| 253 |
+
## 4. Protocol
|
| 254 |
+
|
| 255 |
+
**Volunteer pools and folds** (`splits/`). Two split files, volunteer-disjoint 5-fold CV:
|
| 256 |
+
|
| 257 |
+
| File | Volunteers | Test folds | Used by |
|
| 258 |
+
|---|---|---|---|
|
| 259 |
+
| `splits/pool33.json` | 33 (all with EMG + skeleton MoCap) | 7/7/7/6/6, round-robin by volunteer number | T1, T4, T7, appendix ablations |
|
| 260 |
+
| `splits/pool27.json` | 27 (all with EMG + pressure) | 6/6/5/5/5, seed-42 random partition | T2, T3, T5, T6, F5 |
|
| 261 |
+
|
| 262 |
+
Within a configuration only recordings that have all its modalities are used
|
| 263 |
+
(e.g. 26 of the 33 pool33 volunteers have pressure; v5 has no MoCap); paper
|
| 264 |
+
Table 26 lists the resulting test recordings per input configuration.
|
| 265 |
+
|
| 266 |
+
**Inner validation.** Each fold also lists inner-validation volunteers:
|
| 267 |
+
`round(0.15 x n_train)` training volunteers (pool33: 4, pool27: 3), chosen
|
| 268 |
+
deterministically (strata by modality coverage, full S1-S8 coverage required,
|
| 269 |
+
each volunteer at most once across folds; the exact rule is stored in the
|
| 270 |
+
split file). Epoch selection, early stopping and LR scheduling use only the
|
| 271 |
+
inner-validation volunteers; the test fold is evaluated once.
|
| 272 |
+
|
| 273 |
+
**Seeds and statistics.** Main tables: seeds 42, 43, 44. Per fold the metric
|
| 274 |
+
is averaged over seeds; tables report mean +- SD (ddof = 1) over the 5 fold
|
| 275 |
+
means. Rows are compared on the *common test set* (per fold, the recordings
|
| 276 |
+
every compared row loaded) with paired t-tests over fold means (n = 5) and over
|
| 277 |
+
fold x seed pairs (n = 15), plus the number of folds where the row is better.
|
| 278 |
+
The T6 within-fold ensemble averages all 3 seeds.
|
| 279 |
+
|
| 280 |
+
**Labels (T2, F5).** v1.1 human-reviewed annotations, frozen in
|
| 281 |
+
`taxonomy/taxonomy_v1.1.json` by `scripts/build_taxonomy.py`: 17 fine verbs
|
| 282 |
+
(the single `support` segment is dropped), 6 composite verbs, 33 nouns with
|
| 283 |
+
>= 50 segments over all 36 annotated volunteers (`cup` as named in v1.1), 3
|
| 284 |
+
hands; 7,076 of 7,356 segments kept. Uniform chance: verb@1 5.9 %, noun@1 3.0 %,
|
| 285 |
+
action@5 (top-5 verbs x top-5 nouns) 25/561 = 4.5 %.
|
| 286 |
+
|
| 287 |
+
**T1 (activity-scenario recognition) and T4 (missing modalities).** One
|
| 288 |
+
sample = one whole S1-S8 recording, label = its scenario (8 classes; no action
|
| 289 |
+
annotations), pool33. The configured modalities are cut to their common
|
| 290 |
+
length (the three re-trimmed recordings v2s6, v13s2, v13s3 aligned first,
|
| 291 |
+
`align_trimmed`), concatenated in config order, decimated to 20 Hz (longest
|
| 292 |
+
sequence 4,844 steps; every positional encoding holds 8,192) and z-scored
|
| 293 |
+
with training-frame statistics; batches are zero-padded with a frame mask.
|
| 294 |
+
Training (`configs/t1/base.yaml`, `configs/t4/base.yaml`): Adam lr 1e-3, weight
|
| 295 |
+
decay 1e-4, batch 16 (ST-GCN / CTR-GCN 4), cross-entropy with inverse-frequency
|
| 296 |
+
class weights and label smoothing 0.1, gradient clipping 1.0, ReduceLROnPlateau
|
| 297 |
+
on the inner-validation loss, early stopping after 15 epochs; the epoch is
|
| 298 |
+
selected on inner-validation macro F1 (T1, at most 80 epochs) or
|
| 299 |
+
inner-validation loss (T4 and the appendix ablations, at most 100 epochs); the
|
| 300 |
+
test fold is evaluated once. SyncFuse is `pulse/models/t1/syncfuse.py`; its
|
| 301 |
+
modality dropout (p = 0.3) is drawn **per sample**, and its parameter count
|
| 302 |
+
(about 3.1M with five modalities) is stored as `n_params` in every results.json.
|
| 303 |
+
"Transformer+Late" (per-modality Transformers, mean of logits) is trained from
|
| 304 |
+
scratch. LIMU-BERT is the supervised-only variant. ST-GCN and CTR-GCN
|
| 305 |
+
average over real frames only (the frame mask follows the stride-2 blocks), so a
|
| 306 |
+
recording's prediction does not depend on the other recordings in its
|
| 307 |
+
evaluation batch. InceptionTime has no frame mask, so it is evaluated one
|
| 308 |
+
recording at a time (`train.eval_batch_size: 1`). T1/T4 refuse a split file
|
| 309 |
+
whose `used_by` does not list them. T4 trains the late-fusion Transformer with
|
| 310 |
+
per-sample input dropout (p = 0 / 0.3) and evaluates it on full, leave-one-out
|
| 311 |
+
and singleton inputs (zeroed z-scored inputs), all on the same test recordings;
|
| 312 |
+
`gate_modalities: [pressure]` restricts a 4-modality run to the 5-modality
|
| 313 |
+
recordings (tables `3gain*`, `7m5-matched`). harnet5 (`3harnet`): right-wrist
|
| 314 |
+
(WT8) acceleration in g resampled to 30 Hz, 5 s windows every 5 s, recording
|
| 315 |
+
prediction = mean of window logits; AdamW lr 1e-3, batch 64, 80 epochs,
|
| 316 |
+
patience 15.
|
| 317 |
+
|
| 318 |
+
**T2 (fine-grained action recognition).** One config per model
|
| 319 |
+
(`configs/t2/<model>.yaml`) serves every T2 table. Primary protocol (paper
|
| 320 |
+
Table 6, registry `4`, `protocol: sliding`): 4 s windows at 1 s stride over each
|
| 321 |
+
recording; a window takes the labels of the kept segment that contains its
|
| 322 |
+
centre and is skipped otherwise; models are trained, inner-validated and tested
|
| 323 |
+
on such windows. Secondary (paper Table 7, registry `4seg`,
|
| 324 |
+
`--set protocol=segment`): one sample per kept segment, cut at its annotated
|
| 325 |
+
[start, end) (the 63 of 5,312 pool27 segments longer than 12.8 s are
|
| 326 |
+
centre-cropped to 256 frames). Signals are decimated to 20 Hz and z-scored with
|
| 327 |
+
training-sample statistics. Windows and segments lie in the recording's
|
| 328 |
+
*evaluation span* -- the frames covered by every usable benchmark modality,
|
| 329 |
+
computed from the availability table -- so two rows that load a recording score
|
| 330 |
+
identical samples; the re-trimmed EMG / eye-tracking / pressure files of v2s6,
|
| 331 |
+
v13s2 and v13s3 are placed at their release offset
|
| 332 |
+
(`mocap_padding.front_samples`) on the annotation timeline. Training: AdamW
|
| 333 |
+
(lr 3e-4, weight decay 1e-4), batch 32, at most 40 epochs, 5-epoch linear
|
| 334 |
+
warm-up then cosine decay (fixed schedule), label smoothing 0.05, no class
|
| 335 |
+
weighting, loss weights verb 1 / composite 0.5 / noun 1 / hand 0.5; the epoch
|
| 336 |
+
is selected and training stopped (patience 25) on inner-validation `act_vn@5`;
|
| 337 |
+
the test fold is evaluated once. Every T2 input set contains IMU or MoCap, so
|
| 338 |
+
v5 (72-column IMU, no MoCap) never enters T2: the effective pool is 26
|
| 339 |
+
volunteers (of the 211 annotated S1-S8 pool27 recordings, 198 have IMU, 198
|
| 340 |
+
MoCap, 192 all five modalities). Metrics: `act_vn@k` = verb in the top-k verbs
|
| 341 |
+
and noun in the top-k nouns (chance at k = 5: 25/561 = 4.5 %), verb / noun top-1
|
| 342 |
+
and top-5, composite-verb and hand top-1, per-head macro-F1, and the accuracy of
|
| 343 |
+
the training-majority class (`4maj`, `4segmaj`). DAF = DailyActFormer:
|
| 344 |
+
per-modality multi-scale conv stems, whole-modality dropout p = 0.3 drawn once
|
| 345 |
+
per batch, concat + 1x1 fusion, 3-layer Transformer, learnable-query pooling,
|
| 346 |
+
joint verb-noun head (17 x 33). "ActionLLM" is a sensor surrogate trained from
|
| 347 |
+
scratch (conv stem + 2-layer Transformer; no language model). The matched-input
|
| 348 |
+
tables (paper Table 8) are `4m5` (all five modalities) and `4imu` (IMU only);
|
| 349 |
+
rows marked `reuse:` in `configs/tables.yaml` read the runs of the identical
|
| 350 |
+
row of table `4`.
|
| 351 |
+
|
| 352 |
+
**T3 (grasp-onset anticipation).** Protocol in the docstring of
|
| 353 |
+
`pulse/tasks/t3.py` and `configs/t3/base.yaml`; per-fold sample counts and the
|
| 354 |
+
positive fraction: `python -m pulse.tasks.t3_report --table 6`.
|
| 355 |
+
|
| 356 |
+
**T5 (tactile-driven grasp-state recognition).** At anchors every 0.25 s the
|
| 357 |
+
past 1 s of the inputs (20 Hz) predicts the state of the next 0.5 s: NoGrasp
|
| 358 |
+
unless >= 50 % of it lies in v1.1 segments with a grasp verb (grasp, hold,
|
| 359 |
+
pick_up, move, place, put_down, pull, rotate, insert, remove); otherwise
|
| 360 |
+
SustainedGrasp if its longest contact run is >= 0.3 s (contact = >= 3 of the 50
|
| 361 |
+
pressure cells > 10 g), else AttemptedGrasp. Anchors are also stratified by
|
| 362 |
+
past / future contact (non-contact, pre-contact, steady-grip, release; paper
|
| 363 |
+
Table 15). All rows of a table load the same streams -- table `8`: EMG +
|
| 364 |
+
pressure (pressure defines the labels; the EMG-only rows do not see it), table
|
| 365 |
+
`8full`: all five -- so all rows score identical anchors (checked by the
|
| 366 |
+
report); test recordings over the 5 folds: 210 (`8`; v5s8 has no pressure) and
|
| 367 |
+
192 (`8full`). Backbones DAF, FUTR, DeepConvLSTM (`pulse/models/t5`).
|
| 368 |
+
Training: each class capped at 10,000 anchors, class-weighted cross-entropy,
|
| 369 |
+
AdamW (wd 1e-3), fixed cosine schedule, d_model 64, dropout 0.3; DAF / FUTR lr
|
| 370 |
+
3e-4, 30 epochs, patience 6; DeepConvLSTM lr 1e-4, 50 epochs, patience 12; the
|
| 371 |
+
epoch is chosen on inner-validation macro F1 and the test fold is evaluated
|
| 372 |
+
once. Lift = with - without pressure, paired per (fold, seed); per backbone
|
| 373 |
+
paired t over the 5 fold means and the 15 fold x seed runs; 3-backbone aggregate
|
| 374 |
+
over fold means (n = 5) and the 15 fold x backbone pairs. In v2s6, v13s2 and
|
| 375 |
+
v13s3 the labels are shifted by the re-trim offset
|
| 376 |
+
(`mocap_padding.mocap_sq.front_samples` = 1048 / 393 / 336 frames): their
|
| 377 |
+
annotations are on the full timeline (the last segments end 1.5 / 3.8 / 0.3 s
|
| 378 |
+
before the end of the re-trimmed streams on that timeline, but 9.0 / 0.1 /
|
| 379 |
+
3.1 s after it without the shift). Details: docstrings of `pulse/tasks/t5.py`
|
| 380 |
+
and `pulse/tasks/t5_data.py`.
|
| 381 |
+
|
| 382 |
+
**T6 (cross-modal pressure prediction).** T6a regresses the right-hand
|
| 383 |
+
pressure sum (`R1..R25`, 1-D; the Myo side), T6b the 50 cells; metrics are
|
| 384 |
+
pooled over all scored test frames of a fold, in grams (T6a `pearson`, T6b
|
| 385 |
+
`pearson_max` = r_max, the best of the 50 per-cell r, and `pearson_mean`).
|
| 386 |
+
Only active segments are trained on and scored: pressure total > 100 g,
|
| 387 |
+
dilated by 1.0 s, 1.0 s context on each side, overlapping segments merged.
|
| 388 |
+
Inputs are z-scored per subject from a calibration window only -- the first 30 s
|
| 389 |
+
of the subject's recordings, S1 first (std floored at 0.1 x the pooled training
|
| 390 |
+
std, clipped at +-10); calibration frames are not scored; no statistic of the
|
| 391 |
+
rest of a test recording is used. EMG is a 200 ms RMS envelope; T6b MoCap is
|
| 392 |
+
the 460-column hand set (both hands and forearms, wrist-relative, + velocity),
|
| 393 |
+
the T6a 4-modality rows use the full skeleton. Reported model = SWA average of
|
| 394 |
+
epochs 30-60 (ReduceLROnPlateau on the inner-validation Huber loss before
|
| 395 |
+
that); the test fold is evaluated once. The ensemble tables
|
| 396 |
+
(`pulse.tasks.t6ens`, CPU) average the per-frame test predictions of seeds 42,
|
| 397 |
+
43 and 44 of a row. The budget-matched rows (paper Table 18) are trained and
|
| 398 |
+
tested on identical recordings (the EMG-only rows are gated on the 4-modality
|
| 399 |
+
recordings); `11b-b19` is the 4-modality Ghorbani GRU row of paper Table 17. A
|
| 400 |
+
T6 run refuses to write results if the checkout changes while it runs;
|
| 401 |
+
`t6ens` refuses a subset of the table's seeds and a `predictions.npz` that does
|
| 402 |
+
not reproduce its member's `results.json`. Protocol details: docstring of
|
| 403 |
+
`pulse/tasks/t6.py`.
|
| 404 |
+
|
| 405 |
+
**T7 (EMG -> right-hand pose).** Input: the 8 Myo channels, pooled to 20 Hz
|
| 406 |
+
by block RMS (5 frames per block); target: the right hand as 21 joints x 3 =
|
| 407 |
+
63-D in the wrist frame (mm) from the skeleton TSV -- wrist + per finger
|
| 408 |
+
`RightHand{f}1` (MCP), `{f}2`, `{f}3` and `Right{f}Tip` (the usual 21-keypoint
|
| 409 |
+
layout). MPJPE is over the 21 joints, fingertip MPJPE over the 5 tips, and
|
| 410 |
+
the angle error is the bone-direction error (`atan2(|u x v|, u . v)`) over the
|
| 411 |
+
20 hand-skeleton bones; `finger_angle_err_deg` (15 finger bones, without the 5
|
| 412 |
+
palm bones that are fixed in the wrist frame) is also written to
|
| 413 |
+
`results.json`. MoCap dropout frames (zero wrist quaternion, NaN) are masked.
|
| 414 |
+
Definitions: `pulse/tasks/t7_pose.py`. Loss (`train.loss`): `mpjpe` (default;
|
| 415 |
+
masked MPJPE in mm, fingertips weighted `train.tip_weight` = 2), or `huber` /
|
| 416 |
+
`mse` on the z-scored target. Rows: "MS-CNN+Transformer" (multi-scale CNN +
|
| 417 |
+
Transformer; named "EMG2Pose" in the submitted version, but it is not the TDS +
|
| 418 |
+
LSTM vemg2pose of the emg2pose paper), NeuroPose, Transformer, TDS (the emg2pose
|
| 419 |
+
featurizer) and the training-set mean pose (reference row). MS-CNN+Transformer
|
| 420 |
+
uses its position head: the velocity-integration head drifts over whole
|
| 421 |
+
recordings (fold-0 inner-validation MPJPE 138.5 mm after 12 epochs vs 21.6 mm;
|
| 422 |
+
`configs/t7/mscnn_transformer_velocity.yaml`, not in the table). Training
|
| 423 |
+
batches are whole recordings padded to the longest one; the BatchNorm layers of
|
| 424 |
+
MS-CNN+Transformer and NeuroPose take their statistics from real frames only,
|
| 425 |
+
and NeuroPose resets padded positions to 0 after every block. RMS pooling is
|
| 426 |
+
used instead of decimation (`emg[::5]`): on fold-0 inner-validation volunteers a
|
| 427 |
+
ridge probe on decimated EMG equals the mean pose (21.95 vs 21.95 mm) while RMS
|
| 428 |
+
blocks give 20.99 mm. Both choices (position head, RMS pooling) were made once,
|
| 429 |
+
on the fold-0 inner-validation volunteers v16/v25/v33/v36, who are test
|
| 430 |
+
volunteers in folds 1-4; they are fixed for all folds.
|
| 431 |
+
|
| 432 |
+
```bash
|
| 433 |
+
python -m pulse.tasks.t7 --root $PULSE_ROOT --split splits/pool33.json \
|
| 434 |
+
--fold 0 --seed 42 --config configs/t7/mscnn_transformer.yaml # one run
|
| 435 |
+
```
|
| 436 |
+
|
| 437 |
+
## 5. Data layer (`pulse/paths.py`, `pulse/data/`)
|
| 438 |
+
|
| 439 |
+
* Modalities: `emg` (Myo, 8 ch, **right forearm for every volunteer**), `imu`
|
| 440 |
+
(10 IMUs x 18 = 180 columns; subsets `imu_wrist`, `imu_finger`), `eyetrack`
|
| 441 |
+
(24 Dikablis core columns; subsets `eyetrack_gaze`, `eyetrack_ocular`),
|
| 442 |
+
`pressure` (25 cells per hand), `mocap` (skeleton TSV: hip-relative positions
|
| 443 |
+
+ position velocity, 620 columns), `mocap_markers`, `myo_quat`. All streams
|
| 444 |
+
are on the dataset's common 100 Hz timebase.
|
| 445 |
+
* The skeleton TSV `aligned_vXsY_s_Q.tsv` is in `data/vX/sY/raw/` for 237
|
| 446 |
+
recordings and at the recording root for 10 (v1s5, v2s2, v2s6, v3s1, v3s2,
|
| 447 |
+
v3s5, v3s6, v11s8, v13s2, v13s3); both are supported.
|
| 448 |
+
* Nothing is skipped silently. A missing file raises `MissingFileError`; a file
|
| 449 |
+
that does not provide the modality raises `ModalityUnavailableError` (v5's IMU
|
| 450 |
+
file has 72 columns -> treated as a missing modality; one-eye eye-tracking
|
| 451 |
+
exports lack core columns); a corrupted stream raises `DataQualityError`.
|
| 452 |
+
Select recordings through the availability table (as in section 2) rather
|
| 453 |
+
than loading every recording id.
|
| 454 |
+
* `manifests/availability_v1.1.json` (built by `scripts/build_availability.py`)
|
| 455 |
+
records, per recording and modality, whether it is usable. For each task
|
| 456 |
+
configuration and fold, `pulse.data.manifest.expected_manifest` lists exactly
|
| 457 |
+
which recordings train / inner-val / test must load; every task checks what it
|
| 458 |
+
loaded against it before writing results. `manifests/expected_v1.1.json`
|
| 459 |
+
(`scripts/build_manifests.py`) freezes these lists for reference
|
| 460 |
+
configurations, e.g. pool33 MoCap+EMG+eye+IMU: 235 test recordings over the
|
| 461 |
+
5 folds; with pressure (or gated on pressure): 192.
|
| 462 |
+
* Documented exclusions found by the scan of v1.1 (no data errors):
|
| 463 |
+
|
| 464 |
+
| Recording(s) | Effect |
|
| 465 |
+
|---|---|
|
| 466 |
+
| v5 (s1-s9) | IMU file has 72 columns -> no `imu`; no MoCap |
|
| 467 |
+
| v14s1, v20s3-s9, v29s4-s9 | IMU values with abs > 1e6 -> no `imu` (`quality:extreme_values`) |
|
| 468 |
+
| v1s1, v14s8 | one-eye eye-tracking export -> no `eyetrack` |
|
| 469 |
+
| v23s3 | scene-camera gaze columns > 90 % zero -> no `eyetrack_gaze` (full `eyetrack` is fine) |
|
| 470 |
+
| v11s6 | marker CSV corrupted -> no `mocap_markers` (skeleton `mocap` is fine) |
|
| 471 |
+
| v4s1, v4s4 | listed in recordings.csv but ship no files |
|
| 472 |
+
|
| 473 |
+
Notes: v17s6 has an aligned skeleton TSV although recordings.csv lists no
|
| 474 |
+
`mocap` (that column tracks the marker CSV), so it is used; for 7 of the 10
|
| 475 |
+
root-level TSVs `alignment_metadata.json` does not list MoCap but the streams
|
| 476 |
+
have exactly the recording's sample count; in v2s6, v13s2 and v13s3 the EMG,
|
| 477 |
+
eye-tracking and pressure streams are shorter than the others (re-trimmed at
|
| 478 |
+
release) and multi-modal inputs are cut to the shortest stream. The short
|
| 479 |
+
files cover frames `[front, front + n)` of the IMU / MoCap timeline
|
| 480 |
+
(`mocap_padding.mocap_sq.front_samples` / `n_frames` in
|
| 481 |
+
`alignment_metadata.json`), so cutting from the start shifts IMU / MoCap
|
| 482 |
+
against EMG / eye / pressure by 10.5 s, 3.9 s and 3.4 s.
|
| 483 |
+
`load_recording(..., align_trimmed=True)` slices the full-timeline streams to
|
| 484 |
+
that window first (off by default in the loader; T1 / T4 turn it on through
|
| 485 |
+
`data.align_trimmed`, T3 always uses it). T6 and T7 use the same window
|
| 486 |
+
(`release_trim_window`); T2, T5, F5 and F0 place the short streams at their
|
| 487 |
+
release offset on the annotation timeline (`recording_timeline`).
|
| 488 |
+
|
| 489 |
+
## 6. Results files
|
| 490 |
+
|
| 491 |
+
`results.json` (schema `pulse-results/1`, see `pulse/provenance.py`) holds the
|
| 492 |
+
test metrics, per-recording sufficient statistics (confusion matrices, top-k
|
| 493 |
+
hits, Pearson sums, ...), from which every metric can be recomputed on any
|
| 494 |
+
recording subset (the aggregator does this for the common test set; per-volunteer
|
| 495 |
+
metrics can be computed the same way), and provenance: git commit (+ dirty
|
| 496 |
+
flag), data tag `v1.1`, split file md5, taxonomy md5, availability md5,
|
| 497 |
+
manifest md5, seed, device, train / inner-val / test volunteers, loaded
|
| 498 |
+
recording ids and the full config. `scripts/aggregate.py` writes
|
| 499 |
+
`tables/tab<N>.tex`, `tab<N>_common.tex`, `tab<N>_paired.tex`, `tab<N>.json` and
|
| 500 |
+
`tables/numbers.tex` (macros `\pulsenum{<table>/<row>/<metric>/<stat>}`).
|
| 501 |
+
|
| 502 |
+
## 7. Running on a SLURM cluster
|
| 503 |
+
|
| 504 |
+
`scripts/run_table.py --slurm` writes `<runs>/_jobs/<name>-<time>/{commands.txt,job.sbatch,job.json}`
|
| 505 |
+
and submits one job array (one GPU, 8 CPUs and 60 GB per task, 8 h limit;
|
| 506 |
+
`configs/slurm.yaml`); repeat `--table` to put several tables into one array.
|
| 507 |
+
The launcher was written for the authors' cluster: the GPU partition name is
|
| 508 |
+
hard-coded in `pulse/slurm.py` (`GPU_PARTITION`) and
|
| 509 |
+
`slurm/gpu_array.sbatch.in`, and the CPU partition is set in
|
| 510 |
+
`configs/slurm.yaml`; change these for another cluster, or use `--local` /
|
| 511 |
+
`--dry-run` (writes the job files without submitting). The user's running and
|
| 512 |
+
pending GPUs on that partition are counted first and the array is throttled to
|
| 513 |
+
stay within a cap (default 4, `MAXGPU=1..8`). `--after JOBID[,JOBID]` adds
|
| 514 |
+
`--dependency=afterany:...`. GPU arrays are refused from a checkout with
|
| 515 |
+
uncommitted changes, and every `--set` key (CLI and the rows' `set:`) must exist
|
| 516 |
+
in the row's config (`--allow-new-keys` for smoke-test keys such as
|
| 517 |
+
`train.max_train_batches`). The variables the rows need are checked before
|
| 518 |
+
submission and exported in the job script: `PULSE_ROOT`,
|
| 519 |
+
`PULSE_SSL_WEARABLES_DIR` (harnet rows) and `PULSE_S9_ANNOTATIONS` (F5 rows with
|
| 520 |
+
S9 pre-training). The account is `$PULSE_SLURM_ACCOUNT`, else `account` in
|
| 521 |
+
`configs/slurm.yaml`, else the user's default. `--cpu` runs on the CPU
|
| 522 |
+
partition. Set `PULSE_PYTHON` (the environment's python) before submitting.
|
| 523 |
+
|
| 524 |
+
```bash
|
| 525 |
+
# CPU smoke suite: every run row of every table, fold 0, seed 42, 1 short epoch (configs/smoke.yaml)
|
| 526 |
+
python scripts/smoke_suite.py --runs SMOKE/runs --submit
|
| 527 |
+
# GPU smoke before the full runs: GPU tables only, 2 full epochs
|
| 528 |
+
python scripts/smoke_suite.py --runs GPUSMOKE/runs --gpu --submit
|
| 529 |
+
# every table, numbers.tex, report and figure from one runs directory (with a LaTeX build check)
|
| 530 |
+
python scripts/build_outputs.py --runs SMOKE/runs --out SMOKE/out --allow-incomplete --latex
|
| 531 |
+
```
|
| 532 |
+
|
| 533 |
+
## 8. Compute
|
| 534 |
+
|
| 535 |
+
The 1,515 GPU runs behind the T1-T7 tables of the paper (including the appendix
|
| 536 |
+
ablations and the matched-input comparisons) took about 53 GPU-hours on NVIDIA
|
| 537 |
+
A800 80 GB GPUs, one GPU per run (median about 2 minutes, at most 22 minutes per
|
| 538 |
+
run); the F5 runs (150 for paper Table 9) are not included in that figure. The
|
| 539 |
+
T6 seed ensembles and F0 run on CPU. Peak host memory of a run was about 34 GB
|
| 540 |
+
(T4). On CPU, MulT at batch 16 needs more than 64 GB (no fused attention over
|
| 541 |
+
4,844 steps); the CPU smoke suite runs it at batch 4.
|
| 542 |
+
|
| 543 |
+
## 9. Known issues and naming
|
| 544 |
+
|
| 545 |
+
* v5: IMU has 4 sensors (72 columns) and is treated as missing; no MoCap.
|
| 546 |
+
* 10 skeleton TSVs sit at the recording root (handled, see section 5).
|
| 547 |
+
* F5 reads the S9 labels from `--s9-annotations` / `$PULSE_S9_ANNOTATIONS`
|
| 548 |
+
(`vX/s9.json`, published on the dataset's `main` branch); a missing label file
|
| 549 |
+
raises `MissingFileError`. v20s9's IMU fails the |x| > 1e6 rule, so
|
| 550 |
+
pre-training uses the other 6 recordings.
|
| 551 |
+
* "Transformer+Late" (T1) is trained from scratch; the earlier name
|
| 552 |
+
"Transformer+Late+Pretrain" is dropped because no pretraining is performed.
|
| 553 |
+
* `metadata/recordings.csv` column `split` is a legacy fixed hold-out and is
|
| 554 |
+
not used; all results use the 5-fold split files in `splits/`.
|
| 555 |
+
* T7: the TSV fingertip points `Right{f}Tip` were added in post-processing
|
| 556 |
+
(`alignment_metadata.json`: `tips_added`, `tip_scale` 0.75) and the MCP
|
| 557 |
+
joints are rigid in the wrist frame (fixed bone lengths per volunteer).
|
| 558 |
+
* Registry titles, config comments and docstrings refer to the table numbers of
|
| 559 |
+
the submitted version (e.g. "Table 4" for T2, "Tables 16/17" for the appendix
|
| 560 |
+
ablations) and to review-round row groups (R1-R3, B7, B13, B19, B21); section
|
| 561 |
+
3 maps every registry table to the camera-ready paper.
|
| 562 |
+
|
| 563 |
+
## 10. Annotation prompts
|
| 564 |
+
|
| 565 |
+
The action labels were drafted by a vision-language model and then reviewed by a
|
| 566 |
+
human (every released segment was confirmed or corrected). The prompt templates
|
| 567 |
+
for S1-S8 and for the S9 primitive pass, the tables filled into them and a
|
| 568 |
+
script that renders the exact prompt are in `prompts/` (see `prompts/README.md`).
|
| 569 |
+
|
| 570 |
+
## License
|
| 571 |
+
|
| 572 |
+
Code: MIT (`LICENSE`). Dataset: CC BY-NC 4.0 (see the dataset card). The
|
| 573 |
+
harnet5 weights are loaded from `OxWearables/ssl-wearables` under its own
|
| 574 |
+
licence and are not distributed here.
|
| 575 |
+
|
| 576 |
+
## Citation
|
| 577 |
+
|
| 578 |
+
```bibtex
|
| 579 |
+
@inproceedings{liang2026pulse,
|
| 580 |
+
title = {{PULSE}: A Synchronized Five-Modality Dataset for Sensorimotor Coordination in Long-Horizon Daily Activities},
|
| 581 |
+
author = {Liang, Wolin and Gao, Yang and Yan, Peiyu and Hu, Yuexiang and Xiao, Yingjing and Ying, Xiongfeng and Guo, Yingnian and Zhang, Cheng and Jin, Zhanpeng},
|
| 582 |
+
booktitle = {NeurIPS 2026 Evaluations and Datasets Track},
|
| 583 |
+
year = {2026},
|
| 584 |
+
url = {https://openreview.net/forum?id=AoU576r9O2}
|
| 585 |
+
}
|
| 586 |
+
```
|
| 587 |
+
|
| 588 |
+
Dataset DOI: [10.57967/hf/11314](https://doi.org/10.57967/hf/11314).
|
configs/datacheck/emg.yaml
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Infrastructure smoke test: EMG only (every pool volunteer).
|
| 2 |
+
_base_: emg_pressure.yaml
|
| 3 |
+
modalities: [emg]
|
configs/datacheck/emg_gate_pressure.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Infrastructure smoke test: EMG restricted to the recordings that have pressure (gating).
|
| 2 |
+
_base_: emg_pressure.yaml
|
| 3 |
+
modalities: [emg]
|
| 4 |
+
gate_modalities: [pressure]
|
configs/datacheck/emg_pressure.yaml
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Infrastructure smoke test: EMG + pressure (small files, fast on CPU).
|
| 2 |
+
task: datacheck
|
| 3 |
+
modalities: [emg, pressure]
|
| 4 |
+
gate_modalities: []
|
| 5 |
+
require_annotation: false
|
| 6 |
+
downsample: 5
|
configs/datacheck/mocap.yaml
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Infrastructure smoke test: skeleton MoCap (exercises the raw/ and recording-root TSV locations).
|
| 2 |
+
_base_: emg_pressure.yaml
|
| 3 |
+
modalities: [mocap]
|
configs/f0/README.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F0 case study: onset of EMG and hand motion before contact
|
| 2 |
+
|
| 3 |
+
`python -m pulse.tasks.f0` is a deterministic CPU analysis of the public data
|
| 4 |
+
(no split, no seed); algorithm and parameters in the module docstring and
|
| 5 |
+
`configs/f0/grasp_timing.yaml`.
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
python -m pulse.tasks.f0 --root $PULSE_ROOT --config configs/f0/grasp_timing.yaml --out runs/f0 --workers 16
|
| 9 |
+
python figures/f0/plot_f0.py --in runs/f0 --out figures/out --tables tables
|
| 10 |
+
# -> figures/out/f0_grasp_phase_timing.pdf, f0_delay_distributions.pdf, tables/numbers_f0.tex
|
| 11 |
+
```
|
| 12 |
+
|
| 13 |
+
* Recordings: every S1-S8 recording whose EMG, pressure and skeleton MoCap pass
|
| 14 |
+
the strict loaders (198 recordings of 26 volunteers in v1.1); the loaded list
|
| 15 |
+
is checked against the availability table.
|
| 16 |
+
* The three streams are placed on the alignment timeline with the shared T2
|
| 17 |
+
rule (`recording_timeline`): the re-trimmed EMG / pressure files of v2s6,
|
| 18 |
+
v13s2 and v13s3 start at their release offset; the analysis uses the frames
|
| 19 |
+
all three cover.
|
| 20 |
+
* Right hand only: the Myo armband is on the right forearm of every volunteer
|
| 21 |
+
(the earlier analysis also paired left-hand contacts with the right-forearm
|
| 22 |
+
EMG, and used an 80-recording pre-release snapshot).
|
| 23 |
+
* Events whose window overlaps a MoCap edge-filled stretch
|
| 24 |
+
(`alignment_metadata.json` warning; v25 s1-s8) or an untracked right-hand
|
| 25 |
+
frame (non-finite or exactly zero position; 330 frames in v3s1, v3s6, v3s7,
|
| 26 |
+
v23s4 in v1.1) are excluded; untracked frames are linearly interpolated before
|
| 27 |
+
the hand speed is computed, so they do not inflate the recording's speed range
|
| 28 |
+
used by the clean rule (the earlier analysis set missing positions to 0).
|
| 29 |
+
* Statistics: n, mean, SD (ddof=1), median, quartiles, IQR per signal; the
|
| 30 |
+
paired motion - EMG difference per event (t-based CI, paired t, Wilcoxon,
|
| 31 |
+
share of events with motion first) and per volunteer (Wilcoxon / t over
|
| 32 |
+
volunteers; volunteer-bootstrap 95 % CIs of the median of the per-volunteer
|
| 33 |
+
median differences and of the pooled event median, each with its own point
|
| 34 |
+
estimate).
|
| 35 |
+
* Caveat (stored in the output): EMG stream start times have whole-second
|
| 36 |
+
resolution in `alignment_metadata.json`.
|
| 37 |
+
|
| 38 |
+
Macros (`tables/numbers_f0.tex`): `\pulsenum{f0/events}`, `f0/onsets`,
|
| 39 |
+
`f0/recordings`, `f0/volunteers`, `f0/emg/{mean,sd,median,p25,p75,iqr}`,
|
| 40 |
+
`f0/motion/...`, `f0/diff/{mean,median,sd,cilo,cihi,medcilo,medcihi,pt,pw,motionfirst,r}`
|
| 41 |
+
(`cilo/cihi`: t-based CI of the mean difference; `medcilo/medcihi`: volunteer-cluster
|
| 42 |
+
bootstrap CI of the event median), `f0/vol/{n,median,pw,pt,cilo,cihi}` (`median`:
|
| 43 |
+
median of the per-volunteer median differences, `cilo/cihi`: its volunteer-bootstrap CI).
|
configs/f0/grasp_timing.yaml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F0 case study: onset of right-forearm EMG and right-hand motion before right-hand contact.
|
| 2 |
+
# Algorithm and parameters: pulse/tasks/f0.py (same rules as the original analysis,
|
| 3 |
+
# right hand only because the Myo armband is on the right forearm).
|
| 4 |
+
task: f0
|
| 5 |
+
modalities: [emg, pressure, mocap] # all three must pass the strict loaders
|
| 6 |
+
scenarios: [s1, s2, s3, s4, s5, s6, s7, s8]
|
| 7 |
+
volunteers: null # null = every volunteer with qualifying recordings
|
| 8 |
+
hands: [right]
|
| 9 |
+
pressure_threshold_g: 5.0 # contact = right-hand 25-cell sum above this
|
| 10 |
+
context_sec: 2.0 # look-back window before contact
|
| 11 |
+
post_sec: 0.5 # trace window after contact
|
| 12 |
+
exclude_edge_filled: true # skip events whose window overlaps an edge-filled MoCap stretch
|
| 13 |
+
detect: {min_duration: 10, min_gap: 50, persist_frac: 0.7, release_window: 5, release_frac: 0.3}
|
| 14 |
+
emg_envelope: {window: 20} # 200 ms moving average of the rectified EMG
|
| 15 |
+
velocity: {smooth_window: 3}
|
| 16 |
+
onset: {threshold_ratio: 0.3, baseline_frac: 0.3, baseline_percentile: 25}
|
| 17 |
+
clean: {look_back: 150, rest_window: 50, min_rel_increase: 0.1}
|
| 18 |
+
delay_range_ms: [-1500, 0]
|
| 19 |
+
n_bootstrap: 10000 # volunteer-cluster bootstrap of the median difference
|
| 20 |
+
bootstrap_seed: 0
|
configs/f5/README.md
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5: S9 motion-primitive pre-training -> T2 fine-tuning
|
| 2 |
+
|
| 3 |
+
Paper: Table 9 and Fig. 4 (registry table `5`; the registry and the code
|
| 4 |
+
comments use the submission numbering "Table 5", see the main README). Code: `pulse/tasks/f5.py`,
|
| 5 |
+
`pulse/models/f5/` (S9 labels, pre-training + encoder cache, label budget).
|
| 6 |
+
Model, samples, training loop and metrics are T2's (`pulse/tasks/t2.py`,
|
| 7 |
+
`pulse/tasks/t2_data.py`, `pulse/models/t2`); `configs/f5/base.yaml` inherits
|
| 8 |
+
`configs/t2/daf.yaml`, so F5 and T2 (registry table `4`, paper Table 6) share protocols, evaluation spans,
|
| 9 |
+
normalisation, optimiser, schedule and early stopping.
|
| 10 |
+
|
| 11 |
+
## Design
|
| 12 |
+
|
| 13 |
+
| | |
|
| 14 |
+
|---|---|
|
| 15 |
+
| Model | T2 DailyActFormer-v2 on EMG + IMU (per-modality conv stems, per-batch whole-modality dropout 0.3, concat + 1x1 fusion, 3-layer Transformer, query pooling, joint verb/noun head), `max_T` 256 |
|
| 16 |
+
| Conditions | `init: scratch` (random) and `init: s9` (every non-head parameter from S9 pre-training; backbone LR x 0.1, head LR x 1) |
|
| 17 |
+
| Label budget | `fraction` 0.05 / 0.10 / 0.25 / 0.50 / 1.00 (the paper's sweep) of the training volunteers' kept T2 segments that overlap the evaluation span; nested random subsets from (seed, fold); a training sample is kept iff the segment labelling it is in the subset; identical for both conditions and both protocols; inner-val and test never subsampled |
|
| 18 |
+
| Protocol | Table `5`: T2 sliding windows (4 s, stride 1 s, centre label) -- the T2 primary protocol. Table `5seg`: T2 segment-aligned samples (secondary) |
|
| 19 |
+
| Pool / folds | `splits/pool27.json`, 5 folds, inner-val volunteers from the split file |
|
| 20 |
+
| Training | T2 `train_model`: AdamW 3e-4, wd 1e-4, batch 32, <= 40 epochs, 5-epoch warm-up + cosine, label smoothing 0.05, loss weights 1 / 0.5 / 1 / 0.5 |
|
| 21 |
+
| Selection | inner-val `verb_composite@1` after every epoch (earliest best), early stopping patience 25 (inherited from T2); test fold evaluated once |
|
| 22 |
+
| Seeds | 42, 43, 44; the random state is reset to the seed before the model is built, so both conditions share the head initialisation and the batch order |
|
| 23 |
+
| Reported | T2 metric set; paper Table 9 columns `verb_composite@1` (primary), `verb_fine@1`, `act_vn@5` |
|
| 24 |
+
| Comparison | each `s9pre_fXXX` row is paired with `scratch_fXXX` (registry `reference:`) on the common test set: paired t over fold means (n=5) and fold x seed (n=15), positive folds |
|
| 25 |
+
|
| 26 |
+
Recordings (EMG + IMU + labels, frozen availability table): v5 is excluded
|
| 27 |
+
(IMU file has 72 columns) and v29 s4-s8 (IMU |x| > 1e6). Per fold train /
|
| 28 |
+
inner-val / test recordings: 134/24/40, 126/24/48, 134/24/40, 144/24/30,
|
| 29 |
+
139/19/40 (198 test recordings). The re-trimmed EMG files of v2s6, v13s2,
|
| 30 |
+
v13s3 are placed at their release offset (T2 timeline).
|
| 31 |
+
|
| 32 |
+
S9 pre-training (one encoder per seed, shared by every fold, fraction and
|
| 33 |
+
both tables): volunteers v9, v16-v21 (none is in pool27; checked at run time);
|
| 34 |
+
v20s9's IMU fails the |x| > 1e6 rule, so 6 recordings are used. The 16 S9
|
| 35 |
+
primitives map to 12 verbs x 10 nouns (`pulse/models/f5/s9.py`; any other
|
| 36 |
+
name is an error). Network = the T2 DAF with a 12-verb / 10-noun head
|
| 37 |
+
(modality dropout 0), samples = S9 segments cut like the T2 segment protocol,
|
| 38 |
+
z-scored over the S9 frames; 80 epochs, batch 8, AdamW 1e-3, cosine to 0,
|
| 39 |
+
loss verb 1 + noun 0.5; 20 % of the segments (seeded) held out for epoch
|
| 40 |
+
selection. Cached at `runs/_cache/f5_s9_pretrain/<key>/` (`key` = md5 of the
|
| 41 |
+
config, seed, S9 label files, manifests and the source of the `pulse` package;
|
| 42 |
+
the record names the commit that built it); concurrent runs
|
| 43 |
+
wait on a directory lock, so an array builds it once per seed.
|
| 44 |
+
|
| 45 |
+
Differences from the pre-camera-ready runs (deliberate): the 5/10/25/50/100 %
|
| 46 |
+
label fractions are drawn over labelled segments (was over samples); epoch selection on
|
| 47 |
+
inner-val (was the test fold); 3 seeds (was 1); S9 encoder with the fine-tuned
|
| 48 |
+
architecture (was a separate 2-layer encoder; third layer random); no T2-head
|
| 49 |
+
initialisation from S9 (the earlier row copy never happened: 256 vs 128 input
|
| 50 |
+
shapes); v5 and v20s9 IMU excluded instead of zero-padded / used; v1.1
|
| 51 |
+
human-reviewed T2 labels; T2 evaluation spans and trim offsets.
|
| 52 |
+
|
| 53 |
+
## S9 labels
|
| 54 |
+
|
| 55 |
+
The reviewed S9 labels are published in the dataset (main branch, not in tag
|
| 56 |
+
`v1.1`): `annotations/vX/s9.json` for 30 volunteers; F5 pre-trains on v9 and
|
| 57 |
+
v16-v21 only (`pretrain.volunteers` in `configs/f5/base.yaml`). These are the
|
| 58 |
+
label files the paper's F5 runs used (the per-file md5s recorded in their
|
| 59 |
+
`results.json` match). Download them next to the v1.1 copy and pass their
|
| 60 |
+
directory with `--s9-annotations DIR` or `export PULSE_S9_ANNOTATIONS=DIR`:
|
| 61 |
+
|
| 62 |
+
```bash
|
| 63 |
+
hf download velvet-pine-22/PULSE --repo-type dataset --revision main \
|
| 64 |
+
--include "annotations/*/s9.json" --local-dir PULSE_S9
|
| 65 |
+
export PULSE_S9_ANNOTATIONS=$PWD/PULSE_S9/annotations
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
Each run records the md5 of every label file and of the set
|
| 69 |
+
(`extra.s9_pretrain.label_set_md5`; `f4b16ff2c9ebdbd7b1c2548df4155bed` for the
|
| 70 |
+
published labels), never the path. A missing file raises. The encoder cache
|
| 71 |
+
key changes with the label files.
|
| 72 |
+
|
| 73 |
+
## Commands
|
| 74 |
+
|
| 75 |
+
```bash
|
| 76 |
+
export PULSE_ROOT=/path/to/PULSE PULSE_S9_ANNOTATIONS=/path/to/PULSE_S9/annotations
|
| 77 |
+
python -m pulse.tasks.f5 --root $PULSE_ROOT --split splits/pool27.json --fold 0 --seed 42 \
|
| 78 |
+
--config configs/f5/sliding/s9pre_f010.yaml # one run
|
| 79 |
+
python -m pulse.tasks.f5 ... --config configs/f5/sliding/s9pre_f100.yaml --pretrain-only # encoder only
|
| 80 |
+
python scripts/run_table.py --table 5 --slurm # 10 rows x 5 folds x 3 seeds
|
| 81 |
+
python scripts/run_table.py --table 5seg --slurm # secondary protocol
|
| 82 |
+
python scripts/aggregate.py --table 5 --table 5seg # tables/tab5*.tex, numbers.tex
|
| 83 |
+
python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out
|
| 84 |
+
# CPU smoke test (fold 0, seed 42, 1 epoch x 20 batches, 2 pre-training epochs)
|
| 85 |
+
python scripts/run_table.py --table 5 --rows s9pre_f010,scratch_f010 --folds 0 --seeds 42 --slurm --cpu \
|
| 86 |
+
--set train.epochs=1 --set train.max_batches_per_epoch=20 --set pretrain.train.epochs=2
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
Outputs: `runs/5/<row>/fold<k>_seed<s>/results.json` (provenance includes the
|
| 90 |
+
main and the S9 manifest md5s; `extra.label_budget` has the subset md5 and
|
| 91 |
+
segment / sample counts, `extra.s9_pretrain` the encoder record),
|
| 92 |
+
`tables/tab5_fraction.tex`, `figures/out/f5_data_fraction_tab5.pdf`. Macros:
|
| 93 |
+
`\pulsenum{5/s9pre_f010/verb_composite@1/delta}`, `.../p15`, `.../pos`,
|
| 94 |
+
`\pulsenum{5/scratch_f010/verb_composite@1/cpm}`.
|
configs/f5/base.yaml
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5: S9 motion-primitive pre-training -> T2 fine-tuning.
|
| 2 |
+
# Everything not set here comes from the T2 DAF configuration (configs/t2/daf.yaml ->
|
| 3 |
+
# configs/t2/base.yaml): protocol definitions, 20 Hz decimation, DAF-v2 architecture
|
| 4 |
+
# (max_T 256), AdamW 3e-4, batch 32, <= 40 epochs, warm-up + cosine, label smoothing,
|
| 5 |
+
# loss weights, early-stopping patience. Rows set `protocol`, `init` and `fraction`.
|
| 6 |
+
_base_: ../t2/daf.yaml
|
| 7 |
+
task: f5
|
| 8 |
+
modalities: [emg, imu] # the S9 encoder and the fine-tuned model use this order
|
| 9 |
+
init: scratch # scratch | s9
|
| 10 |
+
fraction: 1.0 # share of the training volunteers' labelled segments
|
| 11 |
+
load_workers: 8
|
| 12 |
+
train:
|
| 13 |
+
backbone_lr_scale: 0.1 # LR factor of the S9-initialised backbone (init: s9 only)
|
| 14 |
+
selection:
|
| 15 |
+
metric: verb_composite@1 # Table 5 metric, on the inner-validation volunteers
|
| 16 |
+
mode: max
|
| 17 |
+
|
| 18 |
+
pretrain: # one S9 encoder per seed (pulse/models/f5/pretrain.py)
|
| 19 |
+
volunteers: [v9, v16, v17, v18, v19, v20, v21] # none in pool27; v20s9 IMU fails |x| > 1e6
|
| 20 |
+
scenario: s9
|
| 21 |
+
min_frames: 10 # 0.5 s at 20 Hz
|
| 22 |
+
max_frames: 256 # = model.max_T; longer S9 segments are centre-cropped
|
| 23 |
+
min_segments: 10
|
| 24 |
+
val_fraction: 0.2 # held-out S9 segments for epoch selection
|
| 25 |
+
mod_dropout_p: 0.0
|
| 26 |
+
train:
|
| 27 |
+
epochs: 80
|
| 28 |
+
batch_size: 8
|
| 29 |
+
eval_batch_size: 64
|
| 30 |
+
lr: 1.0e-3
|
| 31 |
+
weight_decay: 1.0e-4
|
| 32 |
+
warmup_epochs: 0
|
| 33 |
+
min_lr_factor: 0.0 # cosine to 0 over the 80 epochs
|
| 34 |
+
grad_clip: 0.0
|
| 35 |
+
label_smoothing: 0.0
|
| 36 |
+
loss_weights: {verb_fine: 1.0, verb_composite: 0.0, noun: 0.5, hand: 0.0} # S9 verb, S9 noun
|
| 37 |
+
max_batches_per_epoch: null
|
| 38 |
+
selection:
|
| 39 |
+
metric: verb_fine@1 # held-out S9 verb accuracy
|
| 40 |
+
mode: max
|
| 41 |
+
patience: null
|
| 42 |
+
cache_dir: runs/_cache/f5_s9_pretrain
|
| 43 |
+
lock_timeout_sec: 7200
|
configs/f5/segment.yaml
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5, T2 segment-aligned protocol (secondary): one sample per kept segment (<= 256 frames).
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
protocol: segment
|
configs/f5/segment/s9pre_f005.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: S9-pretrained backbone, 5% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.05
|
configs/f5/segment/s9pre_f010.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: S9-pretrained backbone, 10% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.10
|
configs/f5/segment/s9pre_f025.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: S9-pretrained backbone, 25% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.25
|
configs/f5/segment/s9pre_f050.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: S9-pretrained backbone, 50% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.50
|
configs/f5/segment/s9pre_f100.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: S9-pretrained backbone, 100% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 1.00
|
configs/f5/segment/scratch_f005.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: random init, 5% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.05
|
configs/f5/segment/scratch_f010.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: random init, 10% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.10
|
configs/f5/segment/scratch_f025.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: random init, 25% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.25
|
configs/f5/segment/scratch_f050.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: random init, 50% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.50
|
configs/f5/segment/scratch_f100.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 segment: random init, 100% of the training segments.
|
| 2 |
+
_base_: ../segment.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 1.00
|
configs/f5/sliding.yaml
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5, T2 primary protocol: 4 s windows every 1 s, label = kept segment at the window centre.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
protocol: sliding
|
configs/f5/sliding/s9pre_f005.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: S9-pretrained backbone, 5% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.05
|
configs/f5/sliding/s9pre_f010.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: S9-pretrained backbone, 10% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.10
|
configs/f5/sliding/s9pre_f025.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: S9-pretrained backbone, 25% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.25
|
configs/f5/sliding/s9pre_f050.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: S9-pretrained backbone, 50% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 0.50
|
configs/f5/sliding/s9pre_f100.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: S9-pretrained backbone, 100% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: s9
|
| 4 |
+
fraction: 1.00
|
configs/f5/sliding/scratch_f005.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: random init, 5% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.05
|
configs/f5/sliding/scratch_f010.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: random init, 10% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.10
|
configs/f5/sliding/scratch_f025.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: random init, 25% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.25
|
configs/f5/sliding/scratch_f050.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: random init, 50% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 0.50
|
configs/f5/sliding/scratch_f100.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# F5 sliding: random init, 100% of the training segments.
|
| 2 |
+
_base_: ../sliding.yaml
|
| 3 |
+
init: scratch
|
| 4 |
+
fraction: 1.00
|
configs/manifests.yaml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reference configurations whose expected recordings (per pool / fold /
|
| 2 |
+
# partition) are frozen in manifests/expected_v1.1.json by
|
| 3 |
+
# scripts/build_manifests.py. Tasks compute the same manifest at run time
|
| 4 |
+
# from their own config; this file is the reviewable record. gate = streams a
|
| 5 |
+
# recording must have without being model inputs; labelled = needs v1.1
|
| 6 |
+
# dense annotations.
|
| 7 |
+
configs:
|
| 8 |
+
- {name: pool33_emg_mocap, split: splits/pool33.json, modalities: [emg, mocap]}
|
| 9 |
+
- {name: pool33_m4, split: splits/pool33.json, modalities: [mocap, emg, eyetrack, imu]}
|
| 10 |
+
- {name: pool33_m4_gate_pressure, split: splits/pool33.json, modalities: [mocap, emg, eyetrack, imu], gate: [pressure]}
|
| 11 |
+
- {name: pool33_m5, split: splits/pool33.json, modalities: [mocap, emg, eyetrack, imu, pressure]}
|
| 12 |
+
- {name: pool33_imu, split: splits/pool33.json, modalities: [imu]}
|
| 13 |
+
- {name: pool33_emg, split: splits/pool33.json, modalities: [emg]}
|
| 14 |
+
- {name: pool27_emg_pressure, split: splits/pool27.json, modalities: [emg, pressure]}
|
| 15 |
+
- {name: pool27_emg_gate_pressure, split: splits/pool27.json, modalities: [emg], gate: [pressure]}
|
| 16 |
+
- {name: pool27_m4_gate_pressure, split: splits/pool27.json, modalities: [mocap, emg, eyetrack, imu], gate: [pressure]}
|
| 17 |
+
- {name: pool27_mocap_gate_pressure, split: splits/pool27.json, modalities: [mocap], gate: [pressure]}
|
| 18 |
+
- {name: pool27_m5_labelled, split: splits/pool27.json, modalities: [mocap, emg, eyetrack, imu, pressure], labelled: true}
|
| 19 |
+
- {name: pool27_imu_labelled, split: splits/pool27.json, modalities: [imu], labelled: true}
|
| 20 |
+
- {name: pool27_emg_pressure_labelled, split: splits/pool27.json, modalities: [emg, pressure], labelled: true}
|
configs/slurm.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Cluster settings used by scripts/run_table.py --slurm.
|
| 2 |
+
# GPU jobs: partition ex03A800 only (hard-coded in slurm/gpu_array.sbatch.in and
|
| 3 |
+
# pulse/slurm.py). Cap on the user's running+pending GPUs there: 4 by default,
|
| 4 |
+
# override with the MAXGPU environment variable (1..8).
|
| 5 |
+
# account: null -> $PULSE_SLURM_ACCOUNT if set, else the user's default account.
|
| 6 |
+
account: null
|
| 7 |
+
gpu:
|
| 8 |
+
gpus_per_task: 1
|
| 9 |
+
cpus_per_task: 8
|
| 10 |
+
mem: 60G # 8 tasks x 60G fit the 503 GiB of one 8-GPU node (8 x 64G do not); peak RSS seen 34 GB (T4)
|
| 11 |
+
time: "08:00:00"
|
| 12 |
+
cpu:
|
| 13 |
+
partition: cpuXeon6458
|
| 14 |
+
cpus_per_task: 8
|
| 15 |
+
mem: 64G # T1/T4 CPU smoke runs peak at 34 GB (32G ran out of memory)
|
| 16 |
+
time: "04:00:00"
|
| 17 |
+
max_concurrent: 16
|
configs/smoke.yaml
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CPU smoke suite (scripts/smoke_suite.py): every run row of every table in
|
| 2 |
+
# configs/tables.yaml, fold 0, seed 42, one short epoch, on the CPU partition.
|
| 3 |
+
# Checks that each row loads its recordings, trains, selects on inner-val,
|
| 4 |
+
# evaluates the test fold and writes a valid results.json, and that the
|
| 5 |
+
# aggregation / reports / figures build from such runs. The numbers mean
|
| 6 |
+
# nothing. Keep the smoke runs in their own --runs directory: the aggregator
|
| 7 |
+
# refuses to mix them with real runs (different --set overrides and device).
|
| 8 |
+
folds: [0]
|
| 9 |
+
seeds: [42]
|
| 10 |
+
|
| 11 |
+
# --set overrides per task (keys the tasks read with a default; the suite passes
|
| 12 |
+
# --allow-new-keys)
|
| 13 |
+
tasks:
|
| 14 |
+
datacheck: []
|
| 15 |
+
t1: [train.epochs=1, train.max_train_batches=2, train.eval_batch_size=4]
|
| 16 |
+
t4: [train.epochs=1, train.max_train_batches=2, train.eval_batch_size=4]
|
| 17 |
+
t2: [train.epochs=1, train.max_batches_per_epoch=20]
|
| 18 |
+
f5: [train.epochs=1, train.max_batches_per_epoch=20, pretrain.train.epochs=2]
|
| 19 |
+
t3: [train.epochs=1, train.max_steps_per_epoch=50]
|
| 20 |
+
t5: [train.epochs=1, train.max_steps_per_epoch=50]
|
| 21 |
+
t6: [train.epochs=1]
|
| 22 |
+
t6ens: []
|
| 23 |
+
t7: [train.epochs=1, train.batch_size=4]
|
| 24 |
+
|
| 25 |
+
# per-table exceptions: set (replaces the task's list), seeds, after (tables whose
|
| 26 |
+
# smoke array must have ended first)
|
| 27 |
+
tables:
|
| 28 |
+
3harnet: {set: [train.epochs=1, train.max_train_batches=2, train.eval_batch_size=128]}
|
| 29 |
+
# the within-fold ensemble of the paper averages all three seeds
|
| 30 |
+
11a: {seeds: [42, 43, 44]}
|
| 31 |
+
11b: {seeds: [42, 43, 44]}
|
| 32 |
+
11a-ens: {after: [11a]}
|
| 33 |
+
11b-ens: {after: [11b]}
|
| 34 |
+
|
| 35 |
+
# GPU smoke (W0 in the launch plan): every GPU table on the A800s, fold 0, seed 42,
|
| 36 |
+
# 2 full epochs -- GPU memory and per-epoch time before the full waves. Groups
|
| 37 |
+
# are chained one after another (each may use the whole GPU cap).
|
| 38 |
+
gpu:
|
| 39 |
+
folds: [0]
|
| 40 |
+
seeds: [42]
|
| 41 |
+
tasks:
|
| 42 |
+
t1: [train.epochs=2]
|
| 43 |
+
t4: [train.epochs=2]
|
| 44 |
+
t2: [train.epochs=2]
|
| 45 |
+
f5: [train.epochs=2, pretrain.train.epochs=2]
|
| 46 |
+
t3: [train.epochs=2]
|
| 47 |
+
t5: [train.epochs=2]
|
| 48 |
+
t6: [train.epochs=2]
|
| 49 |
+
t7: [train.epochs=2]
|
configs/t1/actionsense.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ActionSense LSTM (DelPreto et al. 2022); 4 modalities.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [mocap, emg, eyetrack, imu]
|
| 4 |
+
model: {name: actionsense, hidden: 128}
|
configs/t1/actionsense_m4_gate_pressure.yaml
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# R1 control: ActionSense, 4 modalities, on the 5-modality recordings (pressure gate).
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [mocap, emg, eyetrack, imu]
|
| 4 |
+
model: {name: actionsense, hidden: 128}
|
| 5 |
+
gate_modalities: [pressure]
|
configs/t1/actionsense_m5.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# R1: ActionSense with pressure as a 5th modality.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [mocap, emg, eyetrack, imu, pressure]
|
| 4 |
+
model: {name: actionsense, hidden: 128}
|
configs/t1/base.yaml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# T1 activity-scenario recognition, whole-recording ("sequence") protocol.
|
| 2 |
+
# Hyper-parameters of the submission's Table 3 launcher (run_t1_cv.sh);
|
| 3 |
+
# model selection now uses the fold's inner-validation volunteers only.
|
| 4 |
+
task: t1
|
| 5 |
+
family: sequence
|
| 6 |
+
modalities: [mocap, emg, eyetrack, imu, pressure]
|
| 7 |
+
gate_modalities: []
|
| 8 |
+
data:
|
| 9 |
+
downsample: 5 # 100 Hz -> 20 Hz
|
| 10 |
+
align_trimmed: true # align the 3 release-trimmed recordings (v2s6, v13s2, v13s3)
|
| 11 |
+
load_workers: 4 # reader threads (does not change the data)
|
| 12 |
+
train:
|
| 13 |
+
epochs: 80
|
| 14 |
+
batch_size: 16
|
| 15 |
+
eval_batch_size: null # null: same as batch_size
|
| 16 |
+
optimizer: adam
|
| 17 |
+
lr: 1.0e-3
|
| 18 |
+
weight_decay: 1.0e-4
|
| 19 |
+
label_smoothing: 0.1
|
| 20 |
+
class_weights: true # inverse frequency of the training labels
|
| 21 |
+
patience: 15 # early stopping on the selection metric
|
| 22 |
+
grad_clip: 1.0
|
| 23 |
+
max_train_batches: null
|
| 24 |
+
scheduler: {name: plateau, factor: 0.5, patience: 7, min_lr: 1.0e-6} # steps on inner-val loss
|
| 25 |
+
selection: {metric: macro_f1, mode: max} # inner-val macro F1
|
configs/t1/ctrgcn.yaml
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CTR-GCN (Chen et al. 2021); MoCap skeleton. Batch 4 (memory), as submitted.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [mocap]
|
| 4 |
+
model: {name: ctrgcn, hidden: 128, n_joints: 52}
|
| 5 |
+
train: {batch_size: 4}
|
configs/t1/emg_cnn.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# EMG-CNN (sEMG literature); EMG only.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [emg]
|
| 4 |
+
model: {name: emg_cnn, hidden: 64}
|
configs/t1/exp1/base.yaml
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Appendix Tables 16/17 (backbone and fusion ablations), now under pool33
|
| 2 |
+
# 5-fold CV with inner-validation selection. Hyper-parameters of the
|
| 3 |
+
# submission's exp1 v7/v8 runs: 100 epochs, no label smoothing, the epoch
|
| 4 |
+
# with the lowest inner-validation loss.
|
| 5 |
+
_base_: ../base.yaml
|
| 6 |
+
train:
|
| 7 |
+
epochs: 100
|
| 8 |
+
label_smoothing: 0.0
|
| 9 |
+
selection: {metric: loss, mode: min}
|
configs/t1/exp1/bb_cnn_emg.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Table 16: cnn backbone on emg alone.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [emg]
|
| 4 |
+
model: {name: fusion, backbone: cnn, fusion: early, hidden: 128, proj_dim: 0}
|
configs/t1/exp1/bb_cnn_imu.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Table 16: cnn backbone on imu alone.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [imu]
|
| 4 |
+
model: {name: fusion, backbone: cnn, fusion: early, hidden: 128, proj_dim: 0}
|
configs/t1/exp1/bb_cnn_mocap.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Table 16: cnn backbone on mocap alone.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [mocap]
|
| 4 |
+
model: {name: fusion, backbone: cnn, fusion: early, hidden: 128, proj_dim: 0}
|
configs/t1/exp1/bb_lstm_emg.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Table 16: lstm backbone on emg alone.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [emg]
|
| 4 |
+
model: {name: fusion, backbone: lstm, fusion: early, hidden: 128, proj_dim: 0}
|
configs/t1/exp1/bb_lstm_imu.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Table 16: lstm backbone on imu alone.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [imu]
|
| 4 |
+
model: {name: fusion, backbone: lstm, fusion: early, hidden: 128, proj_dim: 0}
|
configs/t1/exp1/bb_lstm_mocap.yaml
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Table 16: lstm backbone on mocap alone.
|
| 2 |
+
_base_: base.yaml
|
| 3 |
+
modalities: [mocap]
|
| 4 |
+
model: {name: fusion, backbone: lstm, fusion: early, hidden: 128, proj_dim: 0}
|