import json import subprocess import sys import numpy as np import pytest import yaml from conftest import REPO, mini_split from pulse import DATA_TAG from pulse import aggregate as A from pulse import slurm as S from pulse.data.availability import build_availability from pulse.errors import ProvenanceError from pulse.metrics import confusion_stat from pulse.paths import DataRoot from pulse.provenance import REQUIRED_PROVENANCE, load_results, write_results SQUEUE_SAMPLE = """\ 466145 |cpuXeon6458 |PENDING |1 |cpu=4,mem=16G,node=1,billing=4 |N/A |N/A | 466323 |ex03A800 |RUNNING |1 |cpu=16,mem=64G,node=1,billing=16,gres/gpu=1 |gpu:1,gpu:1 |N/A | 470000_3 |ex03A800 |RUNNING |1 |cpu=8,mem=64G,node=1,gres/gpu=1 |gpu:1 |N/A | 470000_[4-20%2] |ex03A800 |PENDING |1 |cpu=8,mem=64G,node=1,gres/gpu=1 |gpu:1 |N/A | 470100 |ex03A800,gpuA800 |PENDING |1 |cpu=8,mem=64G,node=1 |gpu:2 |N/A | """ def test_squeue_accounting(): e = S.parse_squeue(SQUEUE_SAMPLE) assert len(e) == 5 grp = next(x for x in e if x.job_id.startswith("470000_[")) assert grp.pending_tasks == 17 and grp.throttle == 2 and grp.array_id == "470000" # running: 1 + 1; pending array throttled to 2 - 1 running = 1; pending job 2 GPUs assert S.gpus_in_use(e) == 1 + 1 + 1 + 2 assert S.gpus_in_use(e, "gpuA800") == 2 def test_gpu_cap(): assert S.gpu_cap_from_env({}) == 4 assert S.gpu_cap_from_env({"MAXGPU": "8"}) == 8 for bad in ("9", "0", "x"): with pytest.raises(S.GpuBudgetError): S.gpu_cap_from_env({"MAXGPU": bad}) assert S.plan_throttle(135, 1, 4, 1) == 3 assert S.plan_throttle(2, 1, 8, 0) == 2 with pytest.raises(S.GpuBudgetError): S.plan_throttle(10, 1, 4, 4) with pytest.raises(S.GpuBudgetError): S.plan_throttle(10, 2, 4, 3) def test_gpu_template_targets_ex03_only(): vals = dict(JOB_NAME="x", ACCOUNT="acct", GPUS_PER_TASK=1, CPUS=8, MEM="64G", TIME="1:00:00", LAST_INDEX=3, THROTTLE=2, LOG_DIR="/l", REPO_DIR="/r", PULSE_ROOT="/d", COMMANDS="/c") text = S.render("gpu_array.sbatch.in", vals) parts = [l for l in text.splitlines() if l.startswith("#SBATCH --partition")] assert parts == ["#SBATCH --partition=ex03A800"] assert "--array=0-3%2" in text and "ulimit -n 51200" in text with pytest.raises(ValueError): S.render("gpu_array.sbatch.in", {k: v for k, v in vals.items() if k != "THROTTLE"}) def _fake(task, fold, seed, rids, rng, n_classes=4, split_md5="abc", sha="deadbeef", acc=0.6): per = {} for r in rids: y = rng.integers(0, n_classes, 20) p = np.where(rng.random(20) < acc, y, rng.integers(0, n_classes, 20)) per[r] = {"confusion": confusion_stat(y, p, n_classes)} prov = {k: None for k in REQUIRED_PROVENANCE} prov.update(git_sha=sha, git_dirty=False, code_version="t", data_tag=DATA_TAG, split_md5=split_md5, taxonomy_md5=None, availability_md5="av", fold=fold, seed=seed, loaded_recordings={"train": [], "inner_val": [], "test": list(rids)}) d = {"schema": "pulse-results/1", "task": task, "metric_defs": { "macro_f1": {"fn": "macro_f1", "stat": "confusion"}, "accuracy": {"fn": "accuracy", "stat": "confusion"}}, "per_recording": {"test": per}, "metrics": {"test": {}}, "provenance": prov} from pulse.metrics import compute_metrics d["metrics"]["test"] = compute_metrics(d) return d def _spec(tmp_path, ref="a"): return {"id": "99", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)), "seeds": [42, 43, 44], "metrics": [ {"key": "macro_f1", "label": "Macro-F1", "digits": 3, "better": "higher"}, {"key": "accuracy", "label": "Acc", "digits": 3, "better": "higher"}], "primary_metric": "macro_f1", "reference_row": ref, "rows": [{"key": "a", "label": "Row_A"}, {"key": "b", "label": "Row B & co"}]} def _write_runs(tmp_path, spec, **kw): rng = np.random.default_rng(0) for row, acc in (("a", 0.5), ("b", 0.8)): for k in range(5): rids = [f"v{k + 1}s{i}" for i in range(1, 9)] if row == "b": rids = rids[:6] # row b lost two recordings per fold for s in spec["seeds"]: write_results(A.run_dir(tmp_path / "runs", spec["id"], row, k, s), _fake("tx", k, s, rids, rng, acc=acc, **kw.get(row, {}))) def test_aggregate_common_test_set_and_outputs(tmp_path): spec = _spec(tmp_path) _write_runs(tmp_path, spec) runs, missing = A.collect(spec, tmp_path / "runs") assert not missing summ = A.aggregate(spec, runs) assert all(len(v) == 6 for v in summ["common_test_recordings"].values()) b = summ["rows"]["b"] assert b["n_test_recordings"] == {str(k): 6 for k in range(5)} assert summ["rows"]["a"]["n_test_recordings"] == {str(k): 8 for k in range(5)} c = b["vs_reference"]["macro_f1"] assert c["positive_folds"] == 5 and c["t5"]["n"] == 5 and c["t15"]["n"] == 15 fm = [b["own"]["macro_f1"]["fold_means"][str(k)] for k in range(5)] assert b["own"]["macro_f1"]["sd"] == pytest.approx(np.std(fm, ddof=1)) files = A.write_outputs(summ, tmp_path / "tables") names = {p.name for p in files} assert {"tab99.tex", "tab99_common.tex", "tab99_paired.tex", "numbers.tex"} <= names tex = (tmp_path / "tables" / "tab99.tex").read_text(encoding="utf-8") assert r"Row\_A" in tex and r"Row B \& co" in tex and r"$\pm$" in tex nums = (tmp_path / "tables" / "numbers.tex").read_text(encoding="utf-8") assert r"\csname pulse@99/b/macro_f1/pm\endcsname" in nums assert r"\csname pulse@99/b/macro_f1/p15\endcsname" in nums def test_aggregate_refuses_inconsistent_provenance(tmp_path): spec = _spec(tmp_path) _write_runs(tmp_path, spec, b={"split_md5": "other"}) runs, _ = A.collect(spec, tmp_path / "runs") with pytest.raises(ProvenanceError, match="split_md5"): A.aggregate(spec, runs) def test_aggregate_refuses_missing_and_tampered(tmp_path): spec = _spec(tmp_path) _write_runs(tmp_path, spec) p = A.run_dir(tmp_path / "runs", "99", "a", 0, 42) / "results.json" d = json.loads(p.read_text(encoding="utf-8")) d["metrics"]["test"]["macro_f1"] += 0.01 p.write_text(json.dumps(d), encoding="utf-8") runs, _ = A.collect(spec, tmp_path / "runs") with pytest.raises(ProvenanceError, match="recomputed"): A.aggregate(spec, runs) p.unlink() runs, missing = A.collect(spec, tmp_path / "runs") assert missing == ["a/fold0_seed42"] with pytest.raises(ProvenanceError, match="missing"): A.aggregate(spec, runs, missing=missing) summ = A.aggregate(spec, runs, missing=missing, allow_incomplete=True) assert not summ["complete"] def test_datacheck_end_to_end(mini_root, tmp_path): dr = DataRoot.from_arg(mini_root) av = tmp_path / "availability.json" av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") split = mini_split(tmp_path / "mini_split.json") cfg = REPO / "configs" / "datacheck" / "emg_gate_pressure.yaml" out = tmp_path / "run" from pulse.tasks import datacheck rc = datacheck.main(["--root", str(mini_root), "--split", str(split), "--fold", "2", "--seed", "42", "--config", str(cfg), "--availability", str(av), "--out", str(out), "--device", "cpu"]) assert rc == 0 r = load_results(out / "results.json") prov = r["provenance"] assert prov["loaded_recordings"]["test"] == ["v6s1", "v6s2"] # v3 has no pressure assert prov["inner_val_volunteers"] == ["v1"] and prov["data_tag"] == DATA_TAG assert set(r["per_recording"]["test"]) == {"v6s1", "v6s2"} assert str(mini_root) not in json.dumps(r) # no absolute data path def test_run_table_prints_commands(): res = subprocess.run([sys.executable, "scripts/run_table.py", "--table", "datacheck", "--folds", "0", "--seeds", "42", "--runs", "runs_test_tmp", "--rows", "emg", "--set", "downsample=10"], cwd=str(REPO), capture_output=True, text=True) assert res.returncode == 0, res.stderr lines = res.stdout.strip().splitlines() assert len(lines) == 1 assert '--root "$PULSE_ROOT"' in lines[0] and "--fold 0 --seed 42" in lines[0] assert "--set downsample=10" in lines[0] assert "runs_test_tmp/datacheck/emg/fold0_seed42" in lines[0] def test_registry_is_consistent(): reg = A.load_registry() for tid, t in reg.items(): spec = A.table_spec(reg, tid) assert spec["task"] and spec["split"].startswith("splits/") assert spec["seeds"] and all(m["key"] for m in spec["metrics"]) for row in spec["rows"]: assert (REPO / row["config"]).is_file(), row if spec["rows"] and spec.get("reference_row"): assert spec["reference_row"] in {r["key"] for r in spec["rows"]} cfg = yaml.safe_load((REPO / "configs" / "slurm.yaml").read_text(encoding="utf-8")) assert "partition" not in cfg.get("gpu", {}) # GPU partition is fixed in code/template def test_committed_manifests_up_to_date(): res = subprocess.run([sys.executable, "scripts/build_manifests.py", "--check"], cwd=str(REPO), capture_output=True, text=True) assert res.returncode == 0, res.stdout + res.stderr def test_committed_availability_table(): from pulse.data.availability import Availability, default_availability_path av = Availability.load(default_availability_path()) assert av.data_tag == DATA_TAG and av.errors() == [] s = av.table["summary"] assert s["mocap_tsv_locations"] == {"raw": 237, "recording_root": 10} assert s["recordings_without_data"] == ["v4s1", "v4s4"] assert all(av.status(f"v5s{i}", "imu") == "unavailable:imu_columns_72" for i in range(1, 10)) def test_verify_data_detects_changes(mini_root, tmp_path): import hashlib rows = [] for p in sorted(mini_root.rglob("*")): if p.is_file(): b = p.read_bytes() rel = p.relative_to(mini_root).as_posix() if rel.endswith(".csv") and "aligned_" in rel: # pretend these are LFS files rows.append(f"{rel}\t{len(b)}\t\t{hashlib.sha256(b).hexdigest()}") else: oid = hashlib.sha1(f"blob {len(b)}\0".encode() + b).hexdigest() rows.append(f"{rel}\t{len(b)}\t{oid}\t") man = tmp_path / "files.tsv" man.write_text("# repo=x\trevision=t\tcommit=c\n" + "path\tsize\tgit_oid\tlfs_sha256\n" + "\n".join(rows) + "\n", encoding="utf-8") cmd = [sys.executable, "scripts/verify_data.py", "--root", str(mini_root), "--manifest", str(man), "--workers", "2"] assert subprocess.run(cmd, cwd=str(REPO), capture_output=True).returncode == 0 f = mini_root / "data" / "v1" / "s1" / "aligned_emg_100hz.csv" b = bytearray(f.read_bytes()) b[-2] = ord("7") if b[-2] != ord("7") else ord("8") f.write_bytes(bytes(b)) (mini_root / "stray.txt").write_text("x", encoding="utf-8") res = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True) assert res.returncode == 1 and "sha256_mismatch" in res.stdout and "stray.txt" in res.stdout res = subprocess.run(cmd + ["--expect-different", "data/v1/s1/aligned_emg_100hz.csv", "--ignore-extra", "stray.txt"], cwd=str(REPO), capture_output=True) assert res.returncode == 0 # --------------------------------------------------------------------------- # launcher: dependencies, combined arrays, override keys, environment, smoke plan def _load_script(name): import importlib.util spec = importlib.util.spec_from_file_location(f"_script_{name}", REPO / "scripts" / f"{name}.py") mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def test_dependency_chain_not_counted(): assert S.parse_dependency("(null)") == () assert S.parse_dependency("afterok:467240(unfulfilled),afterany:470000_*(unfulfilled)") == \ ("467240", "470000") assert S.parse_dependency("afterany:1:2_3?afterok:4+30") == ("1", "2", "4") assert S.normalize_job_ids("470000, 470001:470002_[0-3%2]") == ("470000", "470001", "470002") with pytest.raises(S.GpuBudgetError): S.normalize_job_ids("afterany:1") q = ("467240|ex03A800|RUNNING|1|cpu=16,gres/gpu=8|gpu:8|N/A|(null)|\n" "500_[0-99%8]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:467240(unfulfilled)|\n" "501_[0-50%8]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:500_*(unfulfilled)|\n" "777|ex03A800|RUNNING|1|cpu=8,gres/gpu=1|gpu:1|N/A|(null)|\n") e = S.parse_squeue(q) assert S.gpus_in_use(e) == 8 + 8 + 8 + 1 # a new array after 501 starts once 501, 500 and 467240 have ended; job 777 may still run chain = S.dependency_chain(e, ("501",)) assert chain == {"501", "500", "467240"} assert S.gpus_in_use(e, exclude=chain) == 1 assert S.plan_throttle(300, 1, 8, S.gpus_in_use(e, exclude=chain)) == 7 def test_render_optional_lines_and_account(): vals = dict(JOB_NAME="x", ACCOUNT="", GPUS_PER_TASK=1, CPUS=8, MEM="60G", TIME="1:00:00", LAST_INDEX=3, THROTTLE=2, LOG_DIR="/l", REPO_DIR="/r", PULSE_ROOT="/d", COMMANDS="/c") text = S.render("gpu_array.sbatch.in", vals) assert "--account" not in text and "@@" not in text and "--dependency" not in text text = S.render("gpu_array.sbatch.in", dict(vals, ACCOUNT="acct", SBATCH_EXTRA="#SBATCH --dependency=afterany:5:6", EXTRA_ENV="export A=b")) assert "#SBATCH --account=acct" in text and "#SBATCH --dependency=afterany:5:6" in text sb = [l for l in text.splitlines() if l.startswith("#SBATCH")] assert sb[-1].startswith("#SBATCH --error") # dependency line inside the header assert text.index("export A=b") > text.index("export PULSE_ROOT") def _run_table(*a, env=None): return subprocess.run([sys.executable, "scripts/run_table.py", *a], cwd=str(REPO), capture_output=True, text=True, env=env) def test_run_table_override_keys_checked(): base = ["--table", "datacheck", "--folds", "0", "--seeds", "42", "--rows", "emg", "--runs", "runs_tmp"] res = _run_table(*base, "--set", "downsampel=10") assert res.returncode != 0 and "downsampel" in res.stderr and "did you mean downsample" in res.stderr assert _run_table(*base, "--set", "nonexistent.key=1", "--allow-new-keys").returncode == 0 res = _run_table("--table", "3", "--table", "7", "--folds", "0", "--seeds", "42", "--runs", "runs_tmp") assert res.returncode == 0, res.stderr lines = res.stdout.strip().splitlines() assert len(lines) == 9 + 2 and sum("pulse.tasks.t4" in l for l in lines) == 2 assert _run_table("--table", "3", "--table", "7", "--rows", "stgcn").returncode != 0 res = _run_table("--table", "3", "--table", "datacheck") assert res.returncode != 0 and "mix resources" in res.stderr def test_run_table_cpu_dry_run_dependency_and_env(tmp_path): import os env = dict(os.environ, PULSE_ROOT=str(tmp_path)) env.pop("PULSE_SLURM_ACCOUNT", None) env.pop("PULSE_SSL_WEARABLES_DIR", None) runs = tmp_path / "runs" res = _run_table("--table", "datacheck", "--folds", "0", "--seeds", "42", "--runs", str(runs), "--slurm", "--cpu", "--dry-run", "--name", "W9", "--after", "123,456", env=env) assert res.returncode == 0, res.stderr job = next((runs / "_jobs").glob("W9-*/job.sbatch")).read_text(encoding="utf-8") assert "#SBATCH --dependency=afterany:123:456" in job and "--account" not in job assert "#SBATCH --job-name=W9" in job and "--array=0-3%4" in job info = json.loads(next((runs / "_jobs").glob("W9-*/job.json")).read_text(encoding="utf-8")) assert info["after"] == ["123", "456"] and info["n_tasks"] == 4 # a second submission with the same name in the same second gets its own directory res2 = _run_table("--table", "datacheck", "--folds", "0", "--seeds", "42", "--runs", str(runs), "--slurm", "--cpu", "--dry-run", "--name", "W9", env=env) assert res2.returncode == 0, res2.stderr rt0 = _load_script("run_table") d1, d2 = rt0.new_job_dir(runs / "_jobs", "Z"), rt0.new_job_dir(runs / "_jobs", "Z") assert d1 != d2 and d1.is_dir() and d2.is_dir() assert len({p.read_text(encoding="utf-8") for p in (runs / "_jobs").glob("W9-*/commands.txt")}) == 1 assert len(list((runs / "_jobs").glob("W9-*/commands.txt"))) == 2 # harnet rows need the offline ssl-wearables checkout res = _run_table("--table", "3harnet", "--folds", "0", "--seeds", "42", "--runs", str(runs), "--slurm", "--cpu", "--dry-run", env=env) assert res.returncode != 0 and "PULSE_SSL_WEARABLES_DIR" in res.stderr rt = _load_script("run_table") with pytest.raises(SystemExit, match="no vX/s9.json"): rt.check_env({rt.S9_ENV: "x"}, {rt.S9_ENV: str(tmp_path)}) (tmp_path / "v9").mkdir() (tmp_path / "v9" / "s9.json").write_text("{}", encoding="utf-8") assert rt.check_env({rt.S9_ENV: "x"}, {rt.S9_ENV: str(tmp_path)}) == \ {rt.S9_ENV: tmp_path.resolve().as_posix()} cfg = {"init": "s9", "model": {"name": "dailyactformer_v2"}} assert set(rt.env_requirements("f5", cfg)) == {rt.S9_ENV} assert rt.env_requirements("f5", dict(cfg, init="scratch")) == {} def test_smoke_plan_covers_every_run_row(): sm = _load_script("smoke_suite") reg = A.load_registry() smoke = yaml.safe_load((REPO / "configs" / "smoke.yaml").read_text(encoding="utf-8")) groups = sm.plan(reg, smoke) planned = [t for g in groups for t in g["tables"]] expect = [t for t, s in reg.items() if any(not r.get("reuse") for r in s.get("rows", []))] assert sorted(planned) == sorted(expect) and len(planned) == len(set(planned)) order = {t: i for i, g in enumerate(groups) for t in g["tables"]} for g in groups: for t in g["after"]: assert order[t] < order[g["tables"][0]] # members are submitted first ens = next(g for g in groups if "11a-ens" in g["tables"]) assert ens["seeds"] == ("ens",) assert next(g for g in groups if "11a" in g["tables"])["seeds"] == (42, 43, 44) def test_numbers_ignore_report_json(tmp_path): spec = _spec(tmp_path) _write_runs(tmp_path, spec) runs, _ = A.collect(spec, tmp_path / "runs") out = tmp_path / "tables" out.mkdir() (out / "tab99_folds.json").write_text(json.dumps({"table": "99", "rows": {}}), encoding="utf-8") A.write_outputs(A.aggregate(spec, runs), out) nums = (out / "numbers.tex").read_text(encoding="utf-8") assert r"pulse@99/b/macro_f1/pm" in nums def test_every_task_refuses_a_changed_checkout(mini_root, tmp_path, monkeypatch): from pulse.tasks import common, datacheck states = iter([{"git_sha": "a" * 40, "git_dirty": False, "git_diff_md5": None}, {"git_sha": "b" * 40, "git_dirty": False, "git_diff_md5": None}]) monkeypatch.setattr(common, "git_state", lambda: next(states)) dr = DataRoot.from_arg(mini_root) av = tmp_path / "availability.json" av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") split = mini_split(tmp_path / "mini_split.json") with pytest.raises(ProvenanceError, match="changed while this run"): datacheck.main(["--root", str(mini_root), "--split", str(split), "--fold", "2", "--seed", "42", "--config", str(REPO / "configs" / "datacheck" / "emg_gate_pressure.yaml"), "--availability", str(av), "--out", str(tmp_path / "run"), "--device", "cpu"]) assert not (tmp_path / "run" / "results.json").exists() def test_gpu_smoke_plan_only_gpu_tables(): sm = _load_script("smoke_suite") reg = A.load_registry() smoke = yaml.safe_load((REPO / "configs" / "smoke.yaml").read_text(encoding="utf-8")) groups = sm.plan(reg, smoke, gpu=True) planned = [t for g in groups for t in g["tables"]] expect = [t for t, s in reg.items() if s.get("resources", "gpu") == "gpu" and any(not r.get("reuse") for r in s.get("rows", []))] assert sorted(planned) == sorted(expect) and all(g["seeds"] == (42,) for g in groups) assert all(any(kv.startswith("train.epochs=") for kv in g["set"]) for g in groups) def test_squeue_format_ties_array_tasks_to_their_array(): # squeue -O JobID prints a running array task under its raw id (467656) and a pending # array without range / throttle; JobArrayID prints 467654_0 and 467654_[1-3%1] assert S.SQUEUE_FORMAT.startswith("JobArrayID:") q = ("467654_[1-3%1]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:467641(unfulfilled)|\n" "467654_0|ex03A800|RUNNING|1|cpu=8,gres/gpu=1|gpu:1|N/A|(null)|\n" "467641|ex03A800|RUNNING|1|cpu=16,gres/gpu=1|gpu:1,gpu:1|N/A|(null)|\n") e = S.parse_squeue(q) assert [x.base_id for x in e] == ["467654", "467654", "467641"] assert S.gpus_in_use(e) == 1 + 1 + 0 # pending tasks capped by throttle 1 - 1 running assert S.gpus_in_use(e, exclude=S.dependency_chain(e, ("467654",))) == 0