velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
10 kB
"""F5: S9 motion-primitive pre-training -> T2 fine-tuning (Table 5, data-fraction figure).
Model, samples, training and metrics are those of T2 (``pulse.tasks.t2``,
``pulse.tasks.t2_data``, ``pulse.models.t2``): DailyActFormer-v2 on EMG + IMU,
the T2 evaluation spans, protocols and normalisation, inner-validation epoch
selection and early stopping, test fold evaluated once. F5 adds two factors:
``init`` ``scratch`` (random) or ``s9`` (backbone from S9 pre-training on
v9, v16-v21, none of whom is in pool27; backbone LR x
``train.backbone_lr_scale``);
``fraction`` share of the training volunteers' labelled segments used for
training (nested random subsets, identical for both conditions;
``pulse.models.f5.subset``).
The random state is reset to the run seed right before the T2 model is built,
so both conditions start from the same head initialisation and see the same
batch order; only the backbone initialisation differs.
python -m pulse.tasks.f5 --root $PULSE_ROOT --split splits/pool27.json --fold 0 \
--seed 42 --config configs/f5/sliding/s9pre_f010.yaml \
[--s9-annotations DIR] # DIR/vX/s9.json (or $PULSE_S9_ANNOTATIONS)
``--pretrain-only`` builds (or checks) the cached S9 encoder of this seed and exits.
"""
from __future__ import annotations
import os
import sys
import time
from pathlib import Path
import numpy as np
from .. import DATA_TAG
from ..data.manifest import expected_manifest, verify_loaded
from ..data.recordings import SCENARIOS_TASK
from ..errors import ProvenanceError
from ..models.f5 import pretrain as P
from ..models.f5 import s9 as S9
from ..models.f5 import subset as SUB
from ..provenance import git_state, json_md5
from ..splits import FoldSplit
from . import t2 as T2
from . import t2_data as D
from .common import REPO_ROOT, TaskRun, build_parser, set_seed
def log(msg: str) -> None:
print(msg, flush=True)
def check_config(cfg: dict) -> None:
T2.check_config(cfg)
if cfg["init"] not in ("scratch", "s9"):
raise ValueError(f"init must be 'scratch' or 's9', got {cfg['init']!r}")
if not 0 < float(cfg["fraction"]) <= 1:
raise ValueError("fraction must be in (0, 1]")
if float(cfg["train"].get("subsample", 1.0)) != 1.0:
raise ValueError("F5 sets the label budget with `fraction`; train.subsample must stay 1.0")
if cfg["model"]["name"] != "dailyactformer_v2":
raise ValueError("F5 transfers the DailyActFormer-v2 backbone; model.name must be dailyactformer_v2")
def s9_manifest(run: TaskRun, cfg: dict):
vols = tuple(cfg["pretrain"]["volunteers"])
S9.check_disjoint(vols, run.split.volunteers)
pseudo = FoldSplit(pool="s9_pretrain", fold=0, train=vols, inner_val=(), test=())
man = expected_manifest(run.availability, pseudo, cfg["modalities"],
scenarios=(cfg["pretrain"].get("scenario", "s9"),))
run.manifests["s9_pretrain"] = man
log(f"[manifest:s9_pretrain] {man.summary()}")
if not man.partitions["train"]:
raise ProvenanceError("no S9 recording qualifies for pre-training")
return man
def get_encoder(run: TaskRun, cfg: dict, workers: int):
man = s9_manifest(run, cfg)
rids = man.partitions["train"]
md5s = {rid: S9.load_s9_segments(run.dr, rid)[1] for rid in rids} # checks every label file
pc = cfg["pretrain"]
payload = P.key_payload(cfg, run.seed, md5s, man.md5, run.availability.md5, DATA_TAG)
key = json_md5(payload)
cache = pc.get("cache_dir", "runs/_cache/f5_s9_pretrain")
cache = cache if os.path.isabs(cache) else str(REPO_ROOT / cache)
def build():
return P.pretrain_s9(run.dr, run.availability, rids, cfg, run.seed, run.device, workers, log)
state, record, built = P.ensure_encoder(cache, key, payload, build,
float(pc.get("lock_timeout_sec", 7200)), log=log)
s = record["summary"]
verify_loaded(man, {"train": s["recordings"], "inner_val": [], "test": []})
info = {"key": key, "encoder_md5": record["encoder_md5"], "built_by_this_run": built,
"label_set_md5": S9.label_set_md5(md5s), "label_files_md5": md5s,
"s9_manifest_md5": man.md5, "s9_excluded": man.excluded["train"],
"summary": {k: v for k, v in s.items() if k != "history"}, "built_at": record.get("built_at"),
"built_by": record.get("built_by")}
log(f"[s9 pretrain] encoder {key[:12]} ({'built' if built else 'cached'}): {len(s['recordings'])} "
f"recordings, {s['n_samples']} segments, best held-out verb acc {s['best_val_verb_acc']:.3f} "
f"(majority {s['val_verb_majority_rate']:.3f})")
return state, info
def main(argv=None) -> int:
ap = build_parser("f5", __doc__)
ap.add_argument("--pretrain-only", action="store_true",
help="build / check the S9 encoder of this seed and exit")
args = ap.parse_args(argv)
run = TaskRun("f5", args, needs_taxonomy=True)
cfg = run.config
check_config(cfg)
mods = list(cfg["modalities"])
man = run.manifest(mods, (), SCENARIOS_TASK, require_annotation=True)
if args.print_manifest:
if cfg["init"] == "s9":
s9_manifest(run, cfg)
return 0
import torch
from ..models.t2 import build_model, n_params
# as T2: results.json records the commit at finish, valid only if the checkout
# did not change while the run executed
git_start = git_state()
if str(run.device) == "cpu" and os.environ.get("SLURM_CPUS_PER_TASK"):
torch.set_num_threads(int(os.environ["SLURM_CPUS_PER_TASK"]))
workers = args.num_workers or int(cfg.get("load_workers", 4))
t0 = time.time()
pre_state, pre_info = None, None
if cfg["init"] == "s9" or args.pretrain_only:
pre_state, pre_info = get_encoder(run, cfg, workers)
if args.pretrain_only:
return 0
tax = run.taxonomy
sizes = tax.sizes
recs = {p: D.load_partition(run.dr, run.availability, tax, man.partitions[p], mods, cfg,
workers, log, p) for p in ("train", "inner_val", "test")}
stats = D.normalisation_stats(recs["train"], mods) # all training samples, every fraction
for p in recs:
D.apply_normalisation(recs[p], stats)
sets = {p: D.SampleSet(recs[p], mods) for p in recs}
pool, sample_keys = SUB.segment_pool(run.dr, tax, recs["train"], float(cfg["segment"]["min_sec"]))
keep = SUB.subset_segments(pool, float(cfg["fraction"]), run.seed, run.fold.fold)
idx = SUB.subset_index(sample_keys, keep)
n_all = len(sets["train"])
sets["train"] = D.SampleSet(recs["train"], mods, index=idx)
sub = SUB.subset_summary(pool, keep, n_all, len(sets["train"]))
log(f"[f5] {cfg['protocol']}, init {cfg['init']}, fraction {cfg['fraction']}: "
f"{sub['segments_used']}/{sub['segments_total']} segments -> {sub['samples_used']}/"
f"{sub['samples_total']} training samples; inner_val {len(sets['inner_val'])}, "
f"test {len(sets['test'])}")
train_labels = sets["train"].labels
majority = {h: int(np.bincount(train_labels[:, i], minlength=sizes[h]).argmax())
for i, h in enumerate(D.HEADS)}
set_seed(run.seed, bool(cfg.get("deterministic", True)))
mk = {k: v for k, v in cfg["model"].items() if k != "name"}
model = build_model(cfg["model"]["name"], sets["train"].modality_dims(), sizes, **mk)
groups = None
if pre_state is not None:
P.load_backbone(model, pre_state)
groups = P.backbone_param_groups(model, float(cfg["train"]["lr"]),
float(cfg["train"]["backbone_lr_scale"]))
model.to(run.device)
npar = n_params(model)
log(f"[f5] DAF-v2 {sets['train'].modality_dims()} params {npar:,} device {run.device}")
selection = T2.train_model(model, sets["train"], sets["inner_val"], cfg, run.device, run.seed,
sizes, majority, param_groups=groups, log=log,
num_workers=args.num_workers)
ebs = int(cfg["train"]["eval_batch_size"])
iv_metrics = T2.evaluate(model, sets["inner_val"], run.device, ebs, sizes, majority)
lg = T2.predict(model, sets["test"], run.device, ebs)
y, ri = sets["test"].labels, sets["test"].rec_index
test_metrics = T2.metrics_from_stats(T2.sample_stats(lg, y, sizes, majority))
per_rec = {}
for i, r in enumerate(recs["test"]):
sel = ri == i
per_rec[r.rid] = T2.sample_stats({h: lg[h][sel] if len(y) else lg[h] for h in D.HEADS},
y[sel], sizes, majority)
log("[f5] test " + " ".join(f"{k} {test_metrics[k]:.4f}" for k in
("verb_composite@1", "verb_fine@1", "noun@1", "act_vn@5")))
loaded = {p: [r.rid for r in recs[p]] for p in recs}
if git_state() != git_start:
raise ProvenanceError("the repository changed while this run was executing; results.json "
"would record the wrong code version. Run from a checkout nobody "
"commits to or edits.")
run.finish(
loaded=loaded, metric_defs=T2.metric_defs(), per_recording={"test": per_rec},
metrics={"test": test_metrics, "inner_val": iv_metrics}, manifest=man, selection=selection,
n_params=npar, run_name=Path(args.config).stem,
extra={"protocol": cfg["protocol"], "init": cfg["init"], "fraction": float(cfg["fraction"]),
"label_budget": sub, "modalities": mods, "head_sizes": sizes,
"data": T2.summarise(recs, sets, sizes),
"normalisation": {"basis": "all training samples of the fold (before the label "
"subset)", "md5": D.stats_md5(stats)},
"s9_pretrain": pre_info, "seconds": round(time.time() - t0, 1)})
return 0
if __name__ == "__main__":
sys.exit(main())