Download tests/test_pipeline.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 21.8 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/tests/test_pipeline.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/tests/test_pipeline.py
-
curl -L -o test_pipeline.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/tests/test_pipeline.py
21.8 kB
| import json | |
| import subprocess | |
| import sys | |
| import numpy as np | |
| import pytest | |
| import yaml | |
| from conftest import REPO, mini_split | |
| from pulse import DATA_TAG | |
| from pulse import aggregate as A | |
| from pulse import slurm as S | |
| from pulse.data.availability import build_availability | |
| from pulse.errors import ProvenanceError | |
| from pulse.metrics import confusion_stat | |
| from pulse.paths import DataRoot | |
| from pulse.provenance import REQUIRED_PROVENANCE, load_results, write_results | |
| SQUEUE_SAMPLE = """\ | |
| 466145 |cpuXeon6458 |PENDING |1 |cpu=4,mem=16G,node=1,billing=4 |N/A |N/A | | |
| 466323 |ex03A800 |RUNNING |1 |cpu=16,mem=64G,node=1,billing=16,gres/gpu=1 |gpu:1,gpu:1 |N/A | | |
| 470000_3 |ex03A800 |RUNNING |1 |cpu=8,mem=64G,node=1,gres/gpu=1 |gpu:1 |N/A | | |
| 470000_[4-20%2] |ex03A800 |PENDING |1 |cpu=8,mem=64G,node=1,gres/gpu=1 |gpu:1 |N/A | | |
| 470100 |ex03A800,gpuA800 |PENDING |1 |cpu=8,mem=64G,node=1 |gpu:2 |N/A | | |
| """ | |
| def test_squeue_accounting(): | |
| e = S.parse_squeue(SQUEUE_SAMPLE) | |
| assert len(e) == 5 | |
| grp = next(x for x in e if x.job_id.startswith("470000_[")) | |
| assert grp.pending_tasks == 17 and grp.throttle == 2 and grp.array_id == "470000" | |
| # running: 1 + 1; pending array throttled to 2 - 1 running = 1; pending job 2 GPUs | |
| assert S.gpus_in_use(e) == 1 + 1 + 1 + 2 | |
| assert S.gpus_in_use(e, "gpuA800") == 2 | |
| def test_gpu_cap(): | |
| assert S.gpu_cap_from_env({}) == 4 | |
| assert S.gpu_cap_from_env({"MAXGPU": "8"}) == 8 | |
| for bad in ("9", "0", "x"): | |
| with pytest.raises(S.GpuBudgetError): | |
| S.gpu_cap_from_env({"MAXGPU": bad}) | |
| assert S.plan_throttle(135, 1, 4, 1) == 3 | |
| assert S.plan_throttle(2, 1, 8, 0) == 2 | |
| with pytest.raises(S.GpuBudgetError): | |
| S.plan_throttle(10, 1, 4, 4) | |
| with pytest.raises(S.GpuBudgetError): | |
| S.plan_throttle(10, 2, 4, 3) | |
| def test_gpu_template_targets_ex03_only(): | |
| vals = dict(JOB_NAME="x", ACCOUNT="acct", GPUS_PER_TASK=1, CPUS=8, MEM="64G", TIME="1:00:00", | |
| LAST_INDEX=3, THROTTLE=2, LOG_DIR="/l", REPO_DIR="/r", PULSE_ROOT="/d", | |
| COMMANDS="/c") | |
| text = S.render("gpu_array.sbatch.in", vals) | |
| parts = [l for l in text.splitlines() if l.startswith("#SBATCH --partition")] | |
| assert parts == ["#SBATCH --partition=ex03A800"] | |
| assert "--array=0-3%2" in text and "ulimit -n 51200" in text | |
| with pytest.raises(ValueError): | |
| S.render("gpu_array.sbatch.in", {k: v for k, v in vals.items() if k != "THROTTLE"}) | |
| def _fake(task, fold, seed, rids, rng, n_classes=4, split_md5="abc", sha="deadbeef", acc=0.6): | |
| per = {} | |
| for r in rids: | |
| y = rng.integers(0, n_classes, 20) | |
| p = np.where(rng.random(20) < acc, y, rng.integers(0, n_classes, 20)) | |
| per[r] = {"confusion": confusion_stat(y, p, n_classes)} | |
| prov = {k: None for k in REQUIRED_PROVENANCE} | |
| prov.update(git_sha=sha, git_dirty=False, code_version="t", data_tag=DATA_TAG, | |
| split_md5=split_md5, taxonomy_md5=None, availability_md5="av", fold=fold, | |
| seed=seed, loaded_recordings={"train": [], "inner_val": [], "test": list(rids)}) | |
| d = {"schema": "pulse-results/1", "task": task, "metric_defs": { | |
| "macro_f1": {"fn": "macro_f1", "stat": "confusion"}, | |
| "accuracy": {"fn": "accuracy", "stat": "confusion"}}, | |
| "per_recording": {"test": per}, "metrics": {"test": {}}, "provenance": prov} | |
| from pulse.metrics import compute_metrics | |
| d["metrics"]["test"] = compute_metrics(d) | |
| return d | |
| def _spec(tmp_path, ref="a"): | |
| return {"id": "99", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)), | |
| "seeds": [42, 43, 44], "metrics": [ | |
| {"key": "macro_f1", "label": "Macro-F1", "digits": 3, "better": "higher"}, | |
| {"key": "accuracy", "label": "Acc", "digits": 3, "better": "higher"}], | |
| "primary_metric": "macro_f1", "reference_row": ref, | |
| "rows": [{"key": "a", "label": "Row_A"}, {"key": "b", "label": "Row B & co"}]} | |
| def _write_runs(tmp_path, spec, **kw): | |
| rng = np.random.default_rng(0) | |
| for row, acc in (("a", 0.5), ("b", 0.8)): | |
| for k in range(5): | |
| rids = [f"v{k + 1}s{i}" for i in range(1, 9)] | |
| if row == "b": | |
| rids = rids[:6] # row b lost two recordings per fold | |
| for s in spec["seeds"]: | |
| write_results(A.run_dir(tmp_path / "runs", spec["id"], row, k, s), | |
| _fake("tx", k, s, rids, rng, acc=acc, **kw.get(row, {}))) | |
| def test_aggregate_common_test_set_and_outputs(tmp_path): | |
| spec = _spec(tmp_path) | |
| _write_runs(tmp_path, spec) | |
| runs, missing = A.collect(spec, tmp_path / "runs") | |
| assert not missing | |
| summ = A.aggregate(spec, runs) | |
| assert all(len(v) == 6 for v in summ["common_test_recordings"].values()) | |
| b = summ["rows"]["b"] | |
| assert b["n_test_recordings"] == {str(k): 6 for k in range(5)} | |
| assert summ["rows"]["a"]["n_test_recordings"] == {str(k): 8 for k in range(5)} | |
| c = b["vs_reference"]["macro_f1"] | |
| assert c["positive_folds"] == 5 and c["t5"]["n"] == 5 and c["t15"]["n"] == 15 | |
| fm = [b["own"]["macro_f1"]["fold_means"][str(k)] for k in range(5)] | |
| assert b["own"]["macro_f1"]["sd"] == pytest.approx(np.std(fm, ddof=1)) | |
| files = A.write_outputs(summ, tmp_path / "tables") | |
| names = {p.name for p in files} | |
| assert {"tab99.tex", "tab99_common.tex", "tab99_paired.tex", "numbers.tex"} <= names | |
| tex = (tmp_path / "tables" / "tab99.tex").read_text(encoding="utf-8") | |
| assert r"Row\_A" in tex and r"Row B \& co" in tex and r"$\pm$" in tex | |
| nums = (tmp_path / "tables" / "numbers.tex").read_text(encoding="utf-8") | |
| assert r"\csname pulse@99/b/macro_f1/pm\endcsname" in nums | |
| assert r"\csname pulse@99/b/macro_f1/p15\endcsname" in nums | |
| def test_aggregate_refuses_inconsistent_provenance(tmp_path): | |
| spec = _spec(tmp_path) | |
| _write_runs(tmp_path, spec, b={"split_md5": "other"}) | |
| runs, _ = A.collect(spec, tmp_path / "runs") | |
| with pytest.raises(ProvenanceError, match="split_md5"): | |
| A.aggregate(spec, runs) | |
| def test_aggregate_refuses_missing_and_tampered(tmp_path): | |
| spec = _spec(tmp_path) | |
| _write_runs(tmp_path, spec) | |
| p = A.run_dir(tmp_path / "runs", "99", "a", 0, 42) / "results.json" | |
| d = json.loads(p.read_text(encoding="utf-8")) | |
| d["metrics"]["test"]["macro_f1"] += 0.01 | |
| p.write_text(json.dumps(d), encoding="utf-8") | |
| runs, _ = A.collect(spec, tmp_path / "runs") | |
| with pytest.raises(ProvenanceError, match="recomputed"): | |
| A.aggregate(spec, runs) | |
| p.unlink() | |
| runs, missing = A.collect(spec, tmp_path / "runs") | |
| assert missing == ["a/fold0_seed42"] | |
| with pytest.raises(ProvenanceError, match="missing"): | |
| A.aggregate(spec, runs, missing=missing) | |
| summ = A.aggregate(spec, runs, missing=missing, allow_incomplete=True) | |
| assert not summ["complete"] | |
| def test_datacheck_end_to_end(mini_root, tmp_path): | |
| dr = DataRoot.from_arg(mini_root) | |
| av = tmp_path / "availability.json" | |
| av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") | |
| split = mini_split(tmp_path / "mini_split.json") | |
| cfg = REPO / "configs" / "datacheck" / "emg_gate_pressure.yaml" | |
| out = tmp_path / "run" | |
| from pulse.tasks import datacheck | |
| rc = datacheck.main(["--root", str(mini_root), "--split", str(split), "--fold", "2", | |
| "--seed", "42", "--config", str(cfg), "--availability", str(av), | |
| "--out", str(out), "--device", "cpu"]) | |
| assert rc == 0 | |
| r = load_results(out / "results.json") | |
| prov = r["provenance"] | |
| assert prov["loaded_recordings"]["test"] == ["v6s1", "v6s2"] # v3 has no pressure | |
| assert prov["inner_val_volunteers"] == ["v1"] and prov["data_tag"] == DATA_TAG | |
| assert set(r["per_recording"]["test"]) == {"v6s1", "v6s2"} | |
| assert str(mini_root) not in json.dumps(r) # no absolute data path | |
| def test_run_table_prints_commands(): | |
| res = subprocess.run([sys.executable, "scripts/run_table.py", "--table", "datacheck", | |
| "--folds", "0", "--seeds", "42", "--runs", "runs_test_tmp", | |
| "--rows", "emg", "--set", "downsample=10"], | |
| cwd=str(REPO), capture_output=True, text=True) | |
| assert res.returncode == 0, res.stderr | |
| lines = res.stdout.strip().splitlines() | |
| assert len(lines) == 1 | |
| assert '--root "$PULSE_ROOT"' in lines[0] and "--fold 0 --seed 42" in lines[0] | |
| assert "--set downsample=10" in lines[0] | |
| assert "runs_test_tmp/datacheck/emg/fold0_seed42" in lines[0] | |
| def test_registry_is_consistent(): | |
| reg = A.load_registry() | |
| for tid, t in reg.items(): | |
| spec = A.table_spec(reg, tid) | |
| assert spec["task"] and spec["split"].startswith("splits/") | |
| assert spec["seeds"] and all(m["key"] for m in spec["metrics"]) | |
| for row in spec["rows"]: | |
| assert (REPO / row["config"]).is_file(), row | |
| if spec["rows"] and spec.get("reference_row"): | |
| assert spec["reference_row"] in {r["key"] for r in spec["rows"]} | |
| cfg = yaml.safe_load((REPO / "configs" / "slurm.yaml").read_text(encoding="utf-8")) | |
| assert "partition" not in cfg.get("gpu", {}) # GPU partition is fixed in code/template | |
| def test_committed_manifests_up_to_date(): | |
| res = subprocess.run([sys.executable, "scripts/build_manifests.py", "--check"], cwd=str(REPO), | |
| capture_output=True, text=True) | |
| assert res.returncode == 0, res.stdout + res.stderr | |
| def test_committed_availability_table(): | |
| from pulse.data.availability import Availability, default_availability_path | |
| av = Availability.load(default_availability_path()) | |
| assert av.data_tag == DATA_TAG and av.errors() == [] | |
| s = av.table["summary"] | |
| assert s["mocap_tsv_locations"] == {"raw": 237, "recording_root": 10} | |
| assert s["recordings_without_data"] == ["v4s1", "v4s4"] | |
| assert all(av.status(f"v5s{i}", "imu") == "unavailable:imu_columns_72" for i in range(1, 10)) | |
| def test_verify_data_detects_changes(mini_root, tmp_path): | |
| import hashlib | |
| rows = [] | |
| for p in sorted(mini_root.rglob("*")): | |
| if p.is_file(): | |
| b = p.read_bytes() | |
| rel = p.relative_to(mini_root).as_posix() | |
| if rel.endswith(".csv") and "aligned_" in rel: # pretend these are LFS files | |
| rows.append(f"{rel}\t{len(b)}\t\t{hashlib.sha256(b).hexdigest()}") | |
| else: | |
| oid = hashlib.sha1(f"blob {len(b)}\0".encode() + b).hexdigest() | |
| rows.append(f"{rel}\t{len(b)}\t{oid}\t") | |
| man = tmp_path / "files.tsv" | |
| man.write_text("# repo=x\trevision=t\tcommit=c\n" + "path\tsize\tgit_oid\tlfs_sha256\n" | |
| + "\n".join(rows) + "\n", encoding="utf-8") | |
| cmd = [sys.executable, "scripts/verify_data.py", "--root", str(mini_root), "--manifest", | |
| str(man), "--workers", "2"] | |
| assert subprocess.run(cmd, cwd=str(REPO), capture_output=True).returncode == 0 | |
| f = mini_root / "data" / "v1" / "s1" / "aligned_emg_100hz.csv" | |
| b = bytearray(f.read_bytes()) | |
| b[-2] = ord("7") if b[-2] != ord("7") else ord("8") | |
| f.write_bytes(bytes(b)) | |
| (mini_root / "stray.txt").write_text("x", encoding="utf-8") | |
| res = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True) | |
| assert res.returncode == 1 and "sha256_mismatch" in res.stdout and "stray.txt" in res.stdout | |
| res = subprocess.run(cmd + ["--expect-different", "data/v1/s1/aligned_emg_100hz.csv", | |
| "--ignore-extra", "stray.txt"], cwd=str(REPO), capture_output=True) | |
| assert res.returncode == 0 | |
| # --------------------------------------------------------------------------- | |
| # launcher: dependencies, combined arrays, override keys, environment, smoke plan | |
| def _load_script(name): | |
| import importlib.util | |
| spec = importlib.util.spec_from_file_location(f"_script_{name}", REPO / "scripts" / f"{name}.py") | |
| mod = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(mod) | |
| return mod | |
| def test_dependency_chain_not_counted(): | |
| assert S.parse_dependency("(null)") == () | |
| assert S.parse_dependency("afterok:467240(unfulfilled),afterany:470000_*(unfulfilled)") == \ | |
| ("467240", "470000") | |
| assert S.parse_dependency("afterany:1:2_3?afterok:4+30") == ("1", "2", "4") | |
| assert S.normalize_job_ids("470000, 470001:470002_[0-3%2]") == ("470000", "470001", "470002") | |
| with pytest.raises(S.GpuBudgetError): | |
| S.normalize_job_ids("afterany:1") | |
| q = ("467240|ex03A800|RUNNING|1|cpu=16,gres/gpu=8|gpu:8|N/A|(null)|\n" | |
| "500_[0-99%8]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:467240(unfulfilled)|\n" | |
| "501_[0-50%8]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:500_*(unfulfilled)|\n" | |
| "777|ex03A800|RUNNING|1|cpu=8,gres/gpu=1|gpu:1|N/A|(null)|\n") | |
| e = S.parse_squeue(q) | |
| assert S.gpus_in_use(e) == 8 + 8 + 8 + 1 | |
| # a new array after 501 starts once 501, 500 and 467240 have ended; job 777 may still run | |
| chain = S.dependency_chain(e, ("501",)) | |
| assert chain == {"501", "500", "467240"} | |
| assert S.gpus_in_use(e, exclude=chain) == 1 | |
| assert S.plan_throttle(300, 1, 8, S.gpus_in_use(e, exclude=chain)) == 7 | |
| def test_render_optional_lines_and_account(): | |
| vals = dict(JOB_NAME="x", ACCOUNT="", GPUS_PER_TASK=1, CPUS=8, MEM="60G", TIME="1:00:00", | |
| LAST_INDEX=3, THROTTLE=2, LOG_DIR="/l", REPO_DIR="/r", PULSE_ROOT="/d", COMMANDS="/c") | |
| text = S.render("gpu_array.sbatch.in", vals) | |
| assert "--account" not in text and "@@" not in text and "--dependency" not in text | |
| text = S.render("gpu_array.sbatch.in", dict(vals, ACCOUNT="acct", | |
| SBATCH_EXTRA="#SBATCH --dependency=afterany:5:6", | |
| EXTRA_ENV="export A=b")) | |
| assert "#SBATCH --account=acct" in text and "#SBATCH --dependency=afterany:5:6" in text | |
| sb = [l for l in text.splitlines() if l.startswith("#SBATCH")] | |
| assert sb[-1].startswith("#SBATCH --error") # dependency line inside the header | |
| assert text.index("export A=b") > text.index("export PULSE_ROOT") | |
| def _run_table(*a, env=None): | |
| return subprocess.run([sys.executable, "scripts/run_table.py", *a], cwd=str(REPO), | |
| capture_output=True, text=True, env=env) | |
| def test_run_table_override_keys_checked(): | |
| base = ["--table", "datacheck", "--folds", "0", "--seeds", "42", "--rows", "emg", "--runs", "runs_tmp"] | |
| res = _run_table(*base, "--set", "downsampel=10") | |
| assert res.returncode != 0 and "downsampel" in res.stderr and "did you mean downsample" in res.stderr | |
| assert _run_table(*base, "--set", "nonexistent.key=1", "--allow-new-keys").returncode == 0 | |
| res = _run_table("--table", "3", "--table", "7", "--folds", "0", "--seeds", "42", "--runs", "runs_tmp") | |
| assert res.returncode == 0, res.stderr | |
| lines = res.stdout.strip().splitlines() | |
| assert len(lines) == 9 + 2 and sum("pulse.tasks.t4" in l for l in lines) == 2 | |
| assert _run_table("--table", "3", "--table", "7", "--rows", "stgcn").returncode != 0 | |
| res = _run_table("--table", "3", "--table", "datacheck") | |
| assert res.returncode != 0 and "mix resources" in res.stderr | |
| def test_run_table_cpu_dry_run_dependency_and_env(tmp_path): | |
| import os | |
| env = dict(os.environ, PULSE_ROOT=str(tmp_path)) | |
| env.pop("PULSE_SLURM_ACCOUNT", None) | |
| env.pop("PULSE_SSL_WEARABLES_DIR", None) | |
| runs = tmp_path / "runs" | |
| res = _run_table("--table", "datacheck", "--folds", "0", "--seeds", "42", | |
| "--runs", str(runs), "--slurm", "--cpu", "--dry-run", "--name", "W9", | |
| "--after", "123,456", env=env) | |
| assert res.returncode == 0, res.stderr | |
| job = next((runs / "_jobs").glob("W9-*/job.sbatch")).read_text(encoding="utf-8") | |
| assert "#SBATCH --dependency=afterany:123:456" in job and "--account" not in job | |
| assert "#SBATCH --job-name=W9" in job and "--array=0-3%4" in job | |
| info = json.loads(next((runs / "_jobs").glob("W9-*/job.json")).read_text(encoding="utf-8")) | |
| assert info["after"] == ["123", "456"] and info["n_tasks"] == 4 | |
| # a second submission with the same name in the same second gets its own directory | |
| res2 = _run_table("--table", "datacheck", "--folds", "0", "--seeds", "42", "--runs", str(runs), | |
| "--slurm", "--cpu", "--dry-run", "--name", "W9", env=env) | |
| assert res2.returncode == 0, res2.stderr | |
| rt0 = _load_script("run_table") | |
| d1, d2 = rt0.new_job_dir(runs / "_jobs", "Z"), rt0.new_job_dir(runs / "_jobs", "Z") | |
| assert d1 != d2 and d1.is_dir() and d2.is_dir() | |
| assert len({p.read_text(encoding="utf-8") for p in (runs / "_jobs").glob("W9-*/commands.txt")}) == 1 | |
| assert len(list((runs / "_jobs").glob("W9-*/commands.txt"))) == 2 | |
| # harnet rows need the offline ssl-wearables checkout | |
| res = _run_table("--table", "3harnet", "--folds", "0", "--seeds", "42", "--runs", str(runs), | |
| "--slurm", "--cpu", "--dry-run", env=env) | |
| assert res.returncode != 0 and "PULSE_SSL_WEARABLES_DIR" in res.stderr | |
| rt = _load_script("run_table") | |
| with pytest.raises(SystemExit, match="no vX/s9.json"): | |
| rt.check_env({rt.S9_ENV: "x"}, {rt.S9_ENV: str(tmp_path)}) | |
| (tmp_path / "v9").mkdir() | |
| (tmp_path / "v9" / "s9.json").write_text("{}", encoding="utf-8") | |
| assert rt.check_env({rt.S9_ENV: "x"}, {rt.S9_ENV: str(tmp_path)}) == \ | |
| {rt.S9_ENV: tmp_path.resolve().as_posix()} | |
| cfg = {"init": "s9", "model": {"name": "dailyactformer_v2"}} | |
| assert set(rt.env_requirements("f5", cfg)) == {rt.S9_ENV} | |
| assert rt.env_requirements("f5", dict(cfg, init="scratch")) == {} | |
| def test_smoke_plan_covers_every_run_row(): | |
| sm = _load_script("smoke_suite") | |
| reg = A.load_registry() | |
| smoke = yaml.safe_load((REPO / "configs" / "smoke.yaml").read_text(encoding="utf-8")) | |
| groups = sm.plan(reg, smoke) | |
| planned = [t for g in groups for t in g["tables"]] | |
| expect = [t for t, s in reg.items() if any(not r.get("reuse") for r in s.get("rows", []))] | |
| assert sorted(planned) == sorted(expect) and len(planned) == len(set(planned)) | |
| order = {t: i for i, g in enumerate(groups) for t in g["tables"]} | |
| for g in groups: | |
| for t in g["after"]: | |
| assert order[t] < order[g["tables"][0]] # members are submitted first | |
| ens = next(g for g in groups if "11a-ens" in g["tables"]) | |
| assert ens["seeds"] == ("ens",) | |
| assert next(g for g in groups if "11a" in g["tables"])["seeds"] == (42, 43, 44) | |
| def test_numbers_ignore_report_json(tmp_path): | |
| spec = _spec(tmp_path) | |
| _write_runs(tmp_path, spec) | |
| runs, _ = A.collect(spec, tmp_path / "runs") | |
| out = tmp_path / "tables" | |
| out.mkdir() | |
| (out / "tab99_folds.json").write_text(json.dumps({"table": "99", "rows": {}}), encoding="utf-8") | |
| A.write_outputs(A.aggregate(spec, runs), out) | |
| nums = (out / "numbers.tex").read_text(encoding="utf-8") | |
| assert r"pulse@99/b/macro_f1/pm" in nums | |
| def test_every_task_refuses_a_changed_checkout(mini_root, tmp_path, monkeypatch): | |
| from pulse.tasks import common, datacheck | |
| states = iter([{"git_sha": "a" * 40, "git_dirty": False, "git_diff_md5": None}, | |
| {"git_sha": "b" * 40, "git_dirty": False, "git_diff_md5": None}]) | |
| monkeypatch.setattr(common, "git_state", lambda: next(states)) | |
| dr = DataRoot.from_arg(mini_root) | |
| av = tmp_path / "availability.json" | |
| av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") | |
| split = mini_split(tmp_path / "mini_split.json") | |
| with pytest.raises(ProvenanceError, match="changed while this run"): | |
| datacheck.main(["--root", str(mini_root), "--split", str(split), "--fold", "2", "--seed", "42", | |
| "--config", str(REPO / "configs" / "datacheck" / "emg_gate_pressure.yaml"), | |
| "--availability", str(av), "--out", str(tmp_path / "run"), "--device", "cpu"]) | |
| assert not (tmp_path / "run" / "results.json").exists() | |
| def test_gpu_smoke_plan_only_gpu_tables(): | |
| sm = _load_script("smoke_suite") | |
| reg = A.load_registry() | |
| smoke = yaml.safe_load((REPO / "configs" / "smoke.yaml").read_text(encoding="utf-8")) | |
| groups = sm.plan(reg, smoke, gpu=True) | |
| planned = [t for g in groups for t in g["tables"]] | |
| expect = [t for t, s in reg.items() if s.get("resources", "gpu") == "gpu" | |
| and any(not r.get("reuse") for r in s.get("rows", []))] | |
| assert sorted(planned) == sorted(expect) and all(g["seeds"] == (42,) for g in groups) | |
| assert all(any(kv.startswith("train.epochs=") for kv in g["set"]) for g in groups) | |
| def test_squeue_format_ties_array_tasks_to_their_array(): | |
| # squeue -O JobID prints a running array task under its raw id (467656) and a pending | |
| # array without range / throttle; JobArrayID prints 467654_0 and 467654_[1-3%1] | |
| assert S.SQUEUE_FORMAT.startswith("JobArrayID:") | |
| q = ("467654_[1-3%1]|ex03A800|PENDING|1|cpu=8|gpu:1|N/A|afterany:467641(unfulfilled)|\n" | |
| "467654_0|ex03A800|RUNNING|1|cpu=8,gres/gpu=1|gpu:1|N/A|(null)|\n" | |
| "467641|ex03A800|RUNNING|1|cpu=16,gres/gpu=1|gpu:1,gpu:1|N/A|(null)|\n") | |
| e = S.parse_squeue(q) | |
| assert [x.base_id for x in e] == ["467654", "467654", "467641"] | |
| assert S.gpus_in_use(e) == 1 + 1 + 0 # pending tasks capped by throttle 1 - 1 running | |
| assert S.gpus_in_use(e, exclude=S.dependency_chain(e, ("467654",))) == 0 | |