PULSE-code / tests /test_pipeline.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
21.8 kB
import json
import subprocess
import sys
import numpy as np
import pytest
import yaml
from conftest import REPO, mini_split
from pulse import DATA_TAG
from pulse import aggregate as A
from pulse import slurm as S
from pulse.data.availability import build_availability
from pulse.errors import ProvenanceError
from pulse.metrics import confusion_stat
from pulse.paths import DataRoot
from pulse.provenance import REQUIRED_PROVENANCE, load_results, write_results
SQUEUE_SAMPLE = """\
466145 |cpuXeon6458 |PENDING |1 |cpu=4,mem=16G,node=1,billing=4 |N/A |N/A |
466323 |ex03A800 |RUNNING |1 |cpu=16,mem=64G,node=1,billing=16,gres/gpu=1 |gpu:1,gpu:1 |N/A |
470000_3 |ex03A800 |RUNNING |1 |cpu=8,mem=64G,node=1,gres/gpu=1 |gpu:1 |N/A |
470000_[4-20%2] |ex03A800 |PENDING |1 |cpu=8,mem=64G,node=1,gres/gpu=1 |gpu:1 |N/A |
470100 |ex03A800,gpuA800 |PENDING |1 |cpu=8,mem=64G,node=1 |gpu:2 |N/A |
"""
def test_squeue_accounting():
e = S.parse_squeue(SQUEUE_SAMPLE)
assert len(e) == 5
grp = next(x for x in e if x.job_id.startswith("470000_["))
assert grp.pending_tasks == 17 and grp.throttle == 2 and grp.array_id == "470000"
# running: 1 + 1; pending array throttled to 2 - 1 running = 1; pending job 2 GPUs
assert S.gpus_in_use(e) == 1 + 1 + 1 + 2
assert S.gpus_in_use(e, "gpuA800") == 2
def test_gpu_cap():
assert S.gpu_cap_from_env({}) == 4
assert S.gpu_cap_from_env({"MAXGPU": "8"}) == 8
for bad in ("9", "0", "x"):
with pytest.raises(S.GpuBudgetError):
S.gpu_cap_from_env({"MAXGPU": bad})
assert S.plan_throttle(135, 1, 4, 1) == 3
assert S.plan_throttle(2, 1, 8, 0) == 2
with pytest.raises(S.GpuBudgetError):
S.plan_throttle(10, 1, 4, 4)
with pytest.raises(S.GpuBudgetError):
S.plan_throttle(10, 2, 4, 3)
def test_gpu_template_targets_ex03_only():
vals = dict(JOB_NAME="x", ACCOUNT="acct", GPUS_PER_TASK=1, CPUS=8, MEM="64G", TIME="1:00:00",
LAST_INDEX=3, THROTTLE=2, LOG_DIR="/l", REPO_DIR="/r", PULSE_ROOT="/d",
COMMANDS="/c")
text = S.render("gpu_array.sbatch.in", vals)
parts = [l for l in text.splitlines() if l.startswith("#SBATCH --partition")]
assert parts == ["#SBATCH --partition=ex03A800"]
assert "--array=0-3%2" in text and "ulimit -n 51200" in text
with pytest.raises(ValueError):
S.render("gpu_array.sbatch.in", {k: v for k, v in vals.items() if k != "THROTTLE"})
def _fake(task, fold, seed, rids, rng, n_classes=4, split_md5="abc", sha="deadbeef", acc=0.6):
per = {}
for r in rids:
y = rng.integers(0, n_classes, 20)
p = np.where(rng.random(20) < acc, y, rng.integers(0, n_classes, 20))
per[r] = {"confusion": confusion_stat(y, p, n_classes)}
prov = {k: None for k in REQUIRED_PROVENANCE}
prov.update(git_sha=sha, git_dirty=False, code_version="t", data_tag=DATA_TAG,
split_md5=split_md5, taxonomy_md5=None, availability_md5="av", fold=fold,
seed=seed, loaded_recordings={"train": [], "inner_val": [], "test": list(rids)})
d = {"schema": "pulse-results/1", "task": task, "metric_defs": {
"macro_f1": {"fn": "macro_f1", "stat": "confusion"},
"accuracy": {"fn": "accuracy", "stat": "confusion"}},
"per_recording": {"test": per}, "metrics": {"test": {}}, "provenance": prov}
from pulse.metrics import compute_metrics
d["metrics"]["test"] = compute_metrics(d)
return d
def _spec(tmp_path, ref="a"):
return {"id": "99", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)),
"seeds": [42, 43, 44], "metrics": [
{"key": "macro_f1", "label": "Macro-F1", "digits": 3, "better": "higher"},
{"key": "accuracy", "label": "Acc", "digits": 3, "better": "higher"}],
"primary_metric": "macro_f1", "reference_row": ref,
"rows": [{"key": "a", "label": "Row_A"}, {"key": "b", "label": "Row B & co"}]}
def _write_runs(tmp_path, spec, **kw):
rng = np.random.default_rng(0)
for row, acc in (("a", 0.5), ("b", 0.8)):
for k in range(5):
rids = [f"v{k + 1}s{i}" for i in range(1, 9)]
if row == "b":
rids = rids[:6] # row b lost two recordings per fold
for s in spec["seeds"]:
write_results(A.run_dir(tmp_path / "runs", spec["id"], row, k, s),
_fake("tx", k, s, rids, rng, acc=acc, **kw.get(row, {})))
def test_aggregate_common_test_set_and_outputs(tmp_path):
spec = _spec(tmp_path)
_write_runs(tmp_path, spec)
runs, missing = A.collect(spec, tmp_path / "runs")
assert not missing
summ = A.aggregate(spec, runs)
assert all(len(v) == 6 for v in summ["common_test_recordings"].values())
b = summ["rows"]["b"]
assert b["n_test_recordings"] == {str(k): 6 for k in range(5)}
assert summ["rows"]["a"]["n_test_recordings"] == {str(k): 8 for k in range(5)}
c = b["vs_reference"]["macro_f1"]
assert c["positive_folds"] == 5 and c["t5"]["n"] == 5 and c["t15"]["n"] == 15
fm = [b["own"]["macro_f1"]["fold_means"][str(k)] for k in range(5)]
assert b["own"]["macro_f1"]["sd"] == pytest.approx(np.std(fm, ddof=1))
files = A.write_outputs(summ, tmp_path / "tables")
names = {p.name for p in files}
assert {"tab99.tex", "tab99_common.tex", "tab99_paired.tex", "numbers.tex"} <= names
tex = (tmp_path / "tables" / "tab99.tex").read_text(encoding="utf-8")
assert r"Row\_A" in tex and r"Row B \& co" in tex and r"$\pm$" in tex
nums = (tmp_path / "tables" / "numbers.tex").read_text(encoding="utf-8")
assert r"\csname pulse@99/b/macro_f1/pm\endcsname" in nums
assert r"\csname pulse@99/b/macro_f1/p15\endcsname" in nums
def test_aggregate_refuses_inconsistent_provenance(tmp_path):
spec = _spec(tmp_path)
_write_runs(tmp_path, spec, b={"split_md5": "other"})
runs, _ = A.collect(spec, tmp_path / "runs")
with pytest.raises(ProvenanceError, match="split_md5"):
A.aggregate(spec, runs)
def test_aggregate_refuses_missing_and_tampered(tmp_path):
spec = _spec(tmp_path)
_write_runs(tmp_path, spec)
p = A.run_dir(tmp_path / "runs", "99", "a", 0, 42) / "results.json"
d = json.loads(p.read_text(encoding="utf-8"))
d["metrics"]["test"]["macro_f1"] += 0.01
p.write_text(json.dumps(d), encoding="utf-8")
runs, _ = A.collect(spec, tmp_path / "runs")
with pytest.raises(ProvenanceError, match="recomputed"):
A.aggregate(spec, runs)
p.unlink()
runs, missing = A.collect(spec, tmp_path / "runs")
assert missing == ["a/fold0_seed42"]
with pytest.raises(ProvenanceError, match="missing"):
A.aggregate(spec, runs, missing=missing)
summ = A.aggregate(spec, runs, missing=missing, allow_incomplete=True)
assert not summ["complete"]
def test_datacheck_end_to_end(mini_root, tmp_path):
dr = DataRoot.from_arg(mini_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
split = mini_split(tmp_path / "mini_split.json")
cfg = REPO / "configs" / "datacheck" / "emg_gate_pressure.yaml"
out = tmp_path / "run"
from pulse.tasks import datacheck
rc = datacheck.main(["--root", str(mini_root), "--split", str(split), "--fold", "2",
"--seed", "42", "--config", str(cfg), "--availability", str(av),
"--out", str(out), "--device", "cpu"])
assert rc == 0
r = load_results(out / "results.json")
prov = r["provenance"]
assert prov["loaded_recordings"]["test"] == ["v6s1", "v6s2"] # v3 has no pressure
assert prov["inner_val_volunteers"] == ["v1"] and prov["data_tag"] == DATA_TAG
assert set(r["per_recording"]["test"]) == {"v6s1", "v6s2"}
assert str(mini_root) not in json.dumps(r) # no absolute data path
def test_run_table_prints_commands():
res = subprocess.run([sys.executable, "scripts/run_table.py", "--table", "datacheck",
"--folds", "0", "--seeds", "42", "--runs", "runs_test_tmp",
"--rows", "emg", "--set", "downsample=10"],
cwd=str(REPO), capture_output=True, text=True)
assert res.returncode == 0, res.stderr
lines = res.stdout.strip().splitlines()
assert len(lines) == 1
assert '--root "$PULSE_ROOT"' in lines[0] and "--fold 0 --seed 42" in lines[0]
assert "--set downsample=10" in lines[0]
assert "runs_test_tmp/datacheck/emg/fold0_seed42" in lines[0]
def test_registry_is_consistent():
reg = A.load_registry()
for tid, t in reg.items():
spec = A.table_spec(reg, tid)
assert spec["task"] and spec["split"].startswith("splits/")
assert spec["seeds"] and all(m["key"] for m in spec["metrics"])
for row in spec["rows"]:
assert (REPO / row["config"]).is_file(), row
if spec["rows"] and spec.get("reference_row"):
assert spec["reference_row"] in {r["key"] for r in spec["rows"]}
cfg = yaml.safe_load((REPO / "configs" / "slurm.yaml").read_text(encoding="utf-8"))
assert "partition" not in cfg.get("gpu", {}) # GPU partition is fixed in code/template
def test_committed_manifests_up_to_date():
res = subprocess.run([sys.executable, "scripts/build_manifests.py", "--check"], cwd=str(REPO),
capture_output=True, text=True)
assert res.returncode == 0, res.stdout + res.stderr
def test_committed_availability_table():
from pulse.data.availability import Availability, default_availability_path
av = Availability.load(default_availability_path())
assert av.data_tag == DATA_TAG and av.errors() == []
s = av.table["summary"]
assert s["mocap_tsv_locations"] == {"raw": 237, "recording_root": 10}
assert s["recordings_without_data"] == ["v4s1", "v4s4"]
assert all(av.status(f"v5s{i}", "imu") == "unavailable:imu_columns_72" for i in range(1, 10))
def test_verify_data_detects_changes(mini_root, tmp_path):
import hashlib
rows = []
for p in sorted(mini_root.rglob("*")):
if p.is_file():
b = p.read_bytes()
rel = p.relative_to(mini_root).as_posix()
if rel.endswith(".csv") and "aligned_" in rel: # pretend these are LFS files
rows.append(f"{rel}\t{len(b)}\t\t{hashlib.sha256(b).hexdigest()}")
else:
oid = hashlib.sha1(f"blob {len(b)}\0".encode() + b).hexdigest()
rows.append(f"{rel}\t{len(b)}\t{oid}\t")
man = tmp_path / "files.tsv"
man.write_text("# repo=x\trevision=t\tcommit=c\n" + "path\tsize\tgit_oid\tlfs_sha256\n"
+ "\n".join(rows) + "\n", encoding="utf-8")
cmd = [sys.executable, "scripts/verify_data.py", "--root", str(mini_root), "--manifest",
str(man), "--workers", "2"]
assert subprocess.run(cmd, cwd=str(REPO), capture_output=True).returncode == 0
f = mini_root / "data" / "v1" / "s1" / "aligned_emg_100hz.csv"
b = bytearray(f.read_bytes())
b[-2] = ord("7") if b[-2] != ord("7") else ord("8")
f.write_bytes(bytes(b))
(mini_root / "stray.txt").write_text("x", encoding="utf-8")
res = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True)
assert res.returncode == 1 and "sha256_mismatch" in res.stdout and "stray.txt" in res.stdout
res = subprocess.run(cmd + ["--expect-different", "data/v1/s1/aligned_emg_100hz.csv",
"--ignore-extra", "stray.txt"], cwd=str(REPO), capture_output=True)
assert res.returncode == 0
# ---------------------------------------------------------------------------
# launcher: dependencies, combined arrays, override keys, environment, smoke plan
def _load_script(name):
import importlib.util
spec = importlib.util.spec_from_file_location(f"_script_{name}", REPO / "scripts" / f"{name}.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def test_dependency_chain_not_counted():
assert S.parse_dependency("(null)") == ()
assert S.parse_dependency("afterok:467240(unfulfilled),afterany:470000_*(unfulfilled)") == \
("467240", "470000")
assert S.parse_dependency("afterany:1:2_3?afterok:4+30") == ("1", "2", "4")
assert S.normalize_job_ids("470000, 470001:470002_[0-3%2]") == ("470000", "470001", "470002")
with pytest.raises(S.GpuBudgetError):
S.normalize_job_ids("afterany:1")
q = ("467240|ex03A800|RUNNING|1|cpu=16,gres/gpu=8|gpu:8|N/A|(null)|\n"
"500_[0-99%8]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:467240(unfulfilled)|\n"
"501_[0-50%8]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:500_*(unfulfilled)|\n"
"777|ex03A800|RUNNING|1|cpu=8,gres/gpu=1|gpu:1|N/A|(null)|\n")
e = S.parse_squeue(q)
assert S.gpus_in_use(e) == 8 + 8 + 8 + 1
# a new array after 501 starts once 501, 500 and 467240 have ended; job 777 may still run
chain = S.dependency_chain(e, ("501",))
assert chain == {"501", "500", "467240"}
assert S.gpus_in_use(e, exclude=chain) == 1
assert S.plan_throttle(300, 1, 8, S.gpus_in_use(e, exclude=chain)) == 7
def test_render_optional_lines_and_account():
vals = dict(JOB_NAME="x", ACCOUNT="", GPUS_PER_TASK=1, CPUS=8, MEM="60G", TIME="1:00:00",
LAST_INDEX=3, THROTTLE=2, LOG_DIR="/l", REPO_DIR="/r", PULSE_ROOT="/d", COMMANDS="/c")
text = S.render("gpu_array.sbatch.in", vals)
assert "--account" not in text and "@@" not in text and "--dependency" not in text
text = S.render("gpu_array.sbatch.in", dict(vals, ACCOUNT="acct",
SBATCH_EXTRA="#SBATCH --dependency=afterany:5:6",
EXTRA_ENV="export A=b"))
assert "#SBATCH --account=acct" in text and "#SBATCH --dependency=afterany:5:6" in text
sb = [l for l in text.splitlines() if l.startswith("#SBATCH")]
assert sb[-1].startswith("#SBATCH --error") # dependency line inside the header
assert text.index("export A=b") > text.index("export PULSE_ROOT")
def _run_table(*a, env=None):
return subprocess.run([sys.executable, "scripts/run_table.py", *a], cwd=str(REPO),
capture_output=True, text=True, env=env)
def test_run_table_override_keys_checked():
base = ["--table", "datacheck", "--folds", "0", "--seeds", "42", "--rows", "emg", "--runs", "runs_tmp"]
res = _run_table(*base, "--set", "downsampel=10")
assert res.returncode != 0 and "downsampel" in res.stderr and "did you mean downsample" in res.stderr
assert _run_table(*base, "--set", "nonexistent.key=1", "--allow-new-keys").returncode == 0
res = _run_table("--table", "3", "--table", "7", "--folds", "0", "--seeds", "42", "--runs", "runs_tmp")
assert res.returncode == 0, res.stderr
lines = res.stdout.strip().splitlines()
assert len(lines) == 9 + 2 and sum("pulse.tasks.t4" in l for l in lines) == 2
assert _run_table("--table", "3", "--table", "7", "--rows", "stgcn").returncode != 0
res = _run_table("--table", "3", "--table", "datacheck")
assert res.returncode != 0 and "mix resources" in res.stderr
def test_run_table_cpu_dry_run_dependency_and_env(tmp_path):
import os
env = dict(os.environ, PULSE_ROOT=str(tmp_path))
env.pop("PULSE_SLURM_ACCOUNT", None)
env.pop("PULSE_SSL_WEARABLES_DIR", None)
runs = tmp_path / "runs"
res = _run_table("--table", "datacheck", "--folds", "0", "--seeds", "42",
"--runs", str(runs), "--slurm", "--cpu", "--dry-run", "--name", "W9",
"--after", "123,456", env=env)
assert res.returncode == 0, res.stderr
job = next((runs / "_jobs").glob("W9-*/job.sbatch")).read_text(encoding="utf-8")
assert "#SBATCH --dependency=afterany:123:456" in job and "--account" not in job
assert "#SBATCH --job-name=W9" in job and "--array=0-3%4" in job
info = json.loads(next((runs / "_jobs").glob("W9-*/job.json")).read_text(encoding="utf-8"))
assert info["after"] == ["123", "456"] and info["n_tasks"] == 4
# a second submission with the same name in the same second gets its own directory
res2 = _run_table("--table", "datacheck", "--folds", "0", "--seeds", "42", "--runs", str(runs),
"--slurm", "--cpu", "--dry-run", "--name", "W9", env=env)
assert res2.returncode == 0, res2.stderr
rt0 = _load_script("run_table")
d1, d2 = rt0.new_job_dir(runs / "_jobs", "Z"), rt0.new_job_dir(runs / "_jobs", "Z")
assert d1 != d2 and d1.is_dir() and d2.is_dir()
assert len({p.read_text(encoding="utf-8") for p in (runs / "_jobs").glob("W9-*/commands.txt")}) == 1
assert len(list((runs / "_jobs").glob("W9-*/commands.txt"))) == 2
# harnet rows need the offline ssl-wearables checkout
res = _run_table("--table", "3harnet", "--folds", "0", "--seeds", "42", "--runs", str(runs),
"--slurm", "--cpu", "--dry-run", env=env)
assert res.returncode != 0 and "PULSE_SSL_WEARABLES_DIR" in res.stderr
rt = _load_script("run_table")
with pytest.raises(SystemExit, match="no vX/s9.json"):
rt.check_env({rt.S9_ENV: "x"}, {rt.S9_ENV: str(tmp_path)})
(tmp_path / "v9").mkdir()
(tmp_path / "v9" / "s9.json").write_text("{}", encoding="utf-8")
assert rt.check_env({rt.S9_ENV: "x"}, {rt.S9_ENV: str(tmp_path)}) == \
{rt.S9_ENV: tmp_path.resolve().as_posix()}
cfg = {"init": "s9", "model": {"name": "dailyactformer_v2"}}
assert set(rt.env_requirements("f5", cfg)) == {rt.S9_ENV}
assert rt.env_requirements("f5", dict(cfg, init="scratch")) == {}
def test_smoke_plan_covers_every_run_row():
sm = _load_script("smoke_suite")
reg = A.load_registry()
smoke = yaml.safe_load((REPO / "configs" / "smoke.yaml").read_text(encoding="utf-8"))
groups = sm.plan(reg, smoke)
planned = [t for g in groups for t in g["tables"]]
expect = [t for t, s in reg.items() if any(not r.get("reuse") for r in s.get("rows", []))]
assert sorted(planned) == sorted(expect) and len(planned) == len(set(planned))
order = {t: i for i, g in enumerate(groups) for t in g["tables"]}
for g in groups:
for t in g["after"]:
assert order[t] < order[g["tables"][0]] # members are submitted first
ens = next(g for g in groups if "11a-ens" in g["tables"])
assert ens["seeds"] == ("ens",)
assert next(g for g in groups if "11a" in g["tables"])["seeds"] == (42, 43, 44)
def test_numbers_ignore_report_json(tmp_path):
spec = _spec(tmp_path)
_write_runs(tmp_path, spec)
runs, _ = A.collect(spec, tmp_path / "runs")
out = tmp_path / "tables"
out.mkdir()
(out / "tab99_folds.json").write_text(json.dumps({"table": "99", "rows": {}}), encoding="utf-8")
A.write_outputs(A.aggregate(spec, runs), out)
nums = (out / "numbers.tex").read_text(encoding="utf-8")
assert r"pulse@99/b/macro_f1/pm" in nums
def test_every_task_refuses_a_changed_checkout(mini_root, tmp_path, monkeypatch):
from pulse.tasks import common, datacheck
states = iter([{"git_sha": "a" * 40, "git_dirty": False, "git_diff_md5": None},
{"git_sha": "b" * 40, "git_dirty": False, "git_diff_md5": None}])
monkeypatch.setattr(common, "git_state", lambda: next(states))
dr = DataRoot.from_arg(mini_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
split = mini_split(tmp_path / "mini_split.json")
with pytest.raises(ProvenanceError, match="changed while this run"):
datacheck.main(["--root", str(mini_root), "--split", str(split), "--fold", "2", "--seed", "42",
"--config", str(REPO / "configs" / "datacheck" / "emg_gate_pressure.yaml"),
"--availability", str(av), "--out", str(tmp_path / "run"), "--device", "cpu"])
assert not (tmp_path / "run" / "results.json").exists()
def test_gpu_smoke_plan_only_gpu_tables():
sm = _load_script("smoke_suite")
reg = A.load_registry()
smoke = yaml.safe_load((REPO / "configs" / "smoke.yaml").read_text(encoding="utf-8"))
groups = sm.plan(reg, smoke, gpu=True)
planned = [t for g in groups for t in g["tables"]]
expect = [t for t, s in reg.items() if s.get("resources", "gpu") == "gpu"
and any(not r.get("reuse") for r in s.get("rows", []))]
assert sorted(planned) == sorted(expect) and all(g["seeds"] == (42,) for g in groups)
assert all(any(kv.startswith("train.epochs=") for kv in g["set"]) for g in groups)
def test_squeue_format_ties_array_tasks_to_their_array():
# squeue -O JobID prints a running array task under its raw id (467656) and a pending
# array without range / throttle; JobArrayID prints 467654_0 and 467654_[1-3%1]
assert S.SQUEUE_FORMAT.startswith("JobArrayID:")
q = ("467654_[1-3%1]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:467641(unfulfilled)|\n"
"467654_0|ex03A800|RUNNING|1|cpu=8,gres/gpu=1|gpu:1|N/A|(null)|\n"
"467641|ex03A800|RUNNING|1|cpu=16,gres/gpu=1|gpu:1,gpu:1|N/A|(null)|\n")
e = S.parse_squeue(q)
assert [x.base_id for x in e] == ["467654", "467654", "467641"]
assert S.gpus_in_use(e) == 1 + 1 + 0 # pending tasks capped by throttle 1 - 1 running
assert S.gpus_in_use(e, exclude=S.dependency_chain(e, ("467654",))) == 0