"""F5: S9 motion-primitive pre-training -> T2 fine-tuning (Table 5, data-fraction figure). Model, samples, training and metrics are those of T2 (``pulse.tasks.t2``, ``pulse.tasks.t2_data``, ``pulse.models.t2``): DailyActFormer-v2 on EMG + IMU, the T2 evaluation spans, protocols and normalisation, inner-validation epoch selection and early stopping, test fold evaluated once. F5 adds two factors: ``init`` ``scratch`` (random) or ``s9`` (backbone from S9 pre-training on v9, v16-v21, none of whom is in pool27; backbone LR x ``train.backbone_lr_scale``); ``fraction`` share of the training volunteers' labelled segments used for training (nested random subsets, identical for both conditions; ``pulse.models.f5.subset``). The random state is reset to the run seed right before the T2 model is built, so both conditions start from the same head initialisation and see the same batch order; only the backbone initialisation differs. python -m pulse.tasks.f5 --root $PULSE_ROOT --split splits/pool27.json --fold 0 \ --seed 42 --config configs/f5/sliding/s9pre_f010.yaml \ [--s9-annotations DIR] # DIR/vX/s9.json (or $PULSE_S9_ANNOTATIONS) ``--pretrain-only`` builds (or checks) the cached S9 encoder of this seed and exits. """ from __future__ import annotations import os import sys import time from pathlib import Path import numpy as np from .. import DATA_TAG from ..data.manifest import expected_manifest, verify_loaded from ..data.recordings import SCENARIOS_TASK from ..errors import ProvenanceError from ..models.f5 import pretrain as P from ..models.f5 import s9 as S9 from ..models.f5 import subset as SUB from ..provenance import git_state, json_md5 from ..splits import FoldSplit from . import t2 as T2 from . import t2_data as D from .common import REPO_ROOT, TaskRun, build_parser, set_seed def log(msg: str) -> None: print(msg, flush=True) def check_config(cfg: dict) -> None: T2.check_config(cfg) if cfg["init"] not in ("scratch", "s9"): raise ValueError(f"init must be 'scratch' or 's9', got {cfg['init']!r}") if not 0 < float(cfg["fraction"]) <= 1: raise ValueError("fraction must be in (0, 1]") if float(cfg["train"].get("subsample", 1.0)) != 1.0: raise ValueError("F5 sets the label budget with `fraction`; train.subsample must stay 1.0") if cfg["model"]["name"] != "dailyactformer_v2": raise ValueError("F5 transfers the DailyActFormer-v2 backbone; model.name must be dailyactformer_v2") def s9_manifest(run: TaskRun, cfg: dict): vols = tuple(cfg["pretrain"]["volunteers"]) S9.check_disjoint(vols, run.split.volunteers) pseudo = FoldSplit(pool="s9_pretrain", fold=0, train=vols, inner_val=(), test=()) man = expected_manifest(run.availability, pseudo, cfg["modalities"], scenarios=(cfg["pretrain"].get("scenario", "s9"),)) run.manifests["s9_pretrain"] = man log(f"[manifest:s9_pretrain] {man.summary()}") if not man.partitions["train"]: raise ProvenanceError("no S9 recording qualifies for pre-training") return man def get_encoder(run: TaskRun, cfg: dict, workers: int): man = s9_manifest(run, cfg) rids = man.partitions["train"] md5s = {rid: S9.load_s9_segments(run.dr, rid)[1] for rid in rids} # checks every label file pc = cfg["pretrain"] payload = P.key_payload(cfg, run.seed, md5s, man.md5, run.availability.md5, DATA_TAG) key = json_md5(payload) cache = pc.get("cache_dir", "runs/_cache/f5_s9_pretrain") cache = cache if os.path.isabs(cache) else str(REPO_ROOT / cache) def build(): return P.pretrain_s9(run.dr, run.availability, rids, cfg, run.seed, run.device, workers, log) state, record, built = P.ensure_encoder(cache, key, payload, build, float(pc.get("lock_timeout_sec", 7200)), log=log) s = record["summary"] verify_loaded(man, {"train": s["recordings"], "inner_val": [], "test": []}) info = {"key": key, "encoder_md5": record["encoder_md5"], "built_by_this_run": built, "label_set_md5": S9.label_set_md5(md5s), "label_files_md5": md5s, "s9_manifest_md5": man.md5, "s9_excluded": man.excluded["train"], "summary": {k: v for k, v in s.items() if k != "history"}, "built_at": record.get("built_at"), "built_by": record.get("built_by")} log(f"[s9 pretrain] encoder {key[:12]} ({'built' if built else 'cached'}): {len(s['recordings'])} " f"recordings, {s['n_samples']} segments, best held-out verb acc {s['best_val_verb_acc']:.3f} " f"(majority {s['val_verb_majority_rate']:.3f})") return state, info def main(argv=None) -> int: ap = build_parser("f5", __doc__) ap.add_argument("--pretrain-only", action="store_true", help="build / check the S9 encoder of this seed and exit") args = ap.parse_args(argv) run = TaskRun("f5", args, needs_taxonomy=True) cfg = run.config check_config(cfg) mods = list(cfg["modalities"]) man = run.manifest(mods, (), SCENARIOS_TASK, require_annotation=True) if args.print_manifest: if cfg["init"] == "s9": s9_manifest(run, cfg) return 0 import torch from ..models.t2 import build_model, n_params # as T2: results.json records the commit at finish, valid only if the checkout # did not change while the run executed git_start = git_state() if str(run.device) == "cpu" and os.environ.get("SLURM_CPUS_PER_TASK"): torch.set_num_threads(int(os.environ["SLURM_CPUS_PER_TASK"])) workers = args.num_workers or int(cfg.get("load_workers", 4)) t0 = time.time() pre_state, pre_info = None, None if cfg["init"] == "s9" or args.pretrain_only: pre_state, pre_info = get_encoder(run, cfg, workers) if args.pretrain_only: return 0 tax = run.taxonomy sizes = tax.sizes recs = {p: D.load_partition(run.dr, run.availability, tax, man.partitions[p], mods, cfg, workers, log, p) for p in ("train", "inner_val", "test")} stats = D.normalisation_stats(recs["train"], mods) # all training samples, every fraction for p in recs: D.apply_normalisation(recs[p], stats) sets = {p: D.SampleSet(recs[p], mods) for p in recs} pool, sample_keys = SUB.segment_pool(run.dr, tax, recs["train"], float(cfg["segment"]["min_sec"])) keep = SUB.subset_segments(pool, float(cfg["fraction"]), run.seed, run.fold.fold) idx = SUB.subset_index(sample_keys, keep) n_all = len(sets["train"]) sets["train"] = D.SampleSet(recs["train"], mods, index=idx) sub = SUB.subset_summary(pool, keep, n_all, len(sets["train"])) log(f"[f5] {cfg['protocol']}, init {cfg['init']}, fraction {cfg['fraction']}: " f"{sub['segments_used']}/{sub['segments_total']} segments -> {sub['samples_used']}/" f"{sub['samples_total']} training samples; inner_val {len(sets['inner_val'])}, " f"test {len(sets['test'])}") train_labels = sets["train"].labels majority = {h: int(np.bincount(train_labels[:, i], minlength=sizes[h]).argmax()) for i, h in enumerate(D.HEADS)} set_seed(run.seed, bool(cfg.get("deterministic", True))) mk = {k: v for k, v in cfg["model"].items() if k != "name"} model = build_model(cfg["model"]["name"], sets["train"].modality_dims(), sizes, **mk) groups = None if pre_state is not None: P.load_backbone(model, pre_state) groups = P.backbone_param_groups(model, float(cfg["train"]["lr"]), float(cfg["train"]["backbone_lr_scale"])) model.to(run.device) npar = n_params(model) log(f"[f5] DAF-v2 {sets['train'].modality_dims()} params {npar:,} device {run.device}") selection = T2.train_model(model, sets["train"], sets["inner_val"], cfg, run.device, run.seed, sizes, majority, param_groups=groups, log=log, num_workers=args.num_workers) ebs = int(cfg["train"]["eval_batch_size"]) iv_metrics = T2.evaluate(model, sets["inner_val"], run.device, ebs, sizes, majority) lg = T2.predict(model, sets["test"], run.device, ebs) y, ri = sets["test"].labels, sets["test"].rec_index test_metrics = T2.metrics_from_stats(T2.sample_stats(lg, y, sizes, majority)) per_rec = {} for i, r in enumerate(recs["test"]): sel = ri == i per_rec[r.rid] = T2.sample_stats({h: lg[h][sel] if len(y) else lg[h] for h in D.HEADS}, y[sel], sizes, majority) log("[f5] test " + " ".join(f"{k} {test_metrics[k]:.4f}" for k in ("verb_composite@1", "verb_fine@1", "noun@1", "act_vn@5"))) loaded = {p: [r.rid for r in recs[p]] for p in recs} if git_state() != git_start: raise ProvenanceError("the repository changed while this run was executing; results.json " "would record the wrong code version. Run from a checkout nobody " "commits to or edits.") run.finish( loaded=loaded, metric_defs=T2.metric_defs(), per_recording={"test": per_rec}, metrics={"test": test_metrics, "inner_val": iv_metrics}, manifest=man, selection=selection, n_params=npar, run_name=Path(args.config).stem, extra={"protocol": cfg["protocol"], "init": cfg["init"], "fraction": float(cfg["fraction"]), "label_budget": sub, "modalities": mods, "head_sizes": sizes, "data": T2.summarise(recs, sets, sizes), "normalisation": {"basis": "all training samples of the fold (before the label " "subset)", "md5": D.stats_md5(stats)}, "s9_pretrain": pre_info, "seconds": round(time.time() - t0, 1)}) return 0 if __name__ == "__main__": sys.exit(main())