PULSE-code / scripts /smoke_suite.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
5.97 kB
#!/usr/bin/env python
"""CPU smoke suite: every run row of every table, fold 0, seed 42, one short epoch.
export PULSE_ROOT=... PULSE_PYTHON=... # + PULSE_SSL_WEARABLES_DIR, PULSE_S9_ANNOTATIONS
python scripts/smoke_suite.py --runs ~/smoke/runs # print the run_table calls
python scripts/smoke_suite.py --runs ~/smoke/runs --submit # submit CPU job arrays
python scripts/build_outputs.py --runs ~/smoke/runs --out ~/smoke/out --allow-incomplete
python scripts/smoke_suite.py --runs ~/gpusmoke/runs --gpu --after JOBID --submit
# GPU variant (W0): GPU tables only, 2 full epochs,
# groups chained one after another
Overrides and exceptions: configs/smoke.yaml. Tables of one task with the same
overrides and seeds share one CPU array; tables listed with ``after`` wait for
the arrays of those tables (``--after``). Tables whose rows all reuse other
tables' runs (``4maj``, ``4segmaj``) have nothing to run. Keep ``--runs``
separate from the paper runs.
"""
from __future__ import annotations
import argparse
import os
import shlex
import subprocess
import sys
from collections import OrderedDict
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
import yaml # noqa: E402
from pulse.aggregate import load_registry # noqa: E402
def plan(registry, smoke, only=None, gpu=False):
"""``[{tables, task, set, seeds, after}]`` in registry order (``gpu``: the GPU tables with
the ``gpu:`` section of configs/smoke.yaml)."""
groups: "OrderedDict[tuple, dict]" = OrderedDict()
sec = smoke["gpu"] if gpu else smoke
for tid, t in registry.items():
if only and tid not in only:
continue
rows = [r for r in t.get("rows", []) if not r.get("reuse")]
if not rows or (gpu and t.get("resources", "gpu") != "gpu"):
continue
ex = (sec.get("tables") or {}).get(tid, {})
task = t["task"]
if task not in sec["tasks"]:
raise SystemExit(f"configs/smoke.yaml has no overrides for task {task!r} (table {tid})")
sets = tuple(ex.get("set", sec["tasks"][task]))
seeds = t["seeds"] if any(not isinstance(s, int) for s in t["seeds"]) \
else tuple(ex.get("seeds", sec["seeds"]))
after = tuple(ex.get("after", ()))
key = (task, sets, tuple(seeds), after, t.get("resources", "gpu"))
g = groups.setdefault(key, {"tables": [], "task": task, "set": sets, "seeds": tuple(seeds),
"after": after})
g["tables"].append(tid)
return list(groups.values())
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--runs", required=True, help="smoke runs directory (not the paper runs)")
ap.add_argument("--smoke-config", default=str(REPO / "configs" / "smoke.yaml"))
ap.add_argument("--registry", default=None)
ap.add_argument("--tables", default=None, help="comma-separated subset of tables")
ap.add_argument("--submit", action="store_true", help="submit the CPU arrays (default: print)")
ap.add_argument("--dry-run", action="store_true", help="write job files, do not submit")
ap.add_argument("--gpu", action="store_true",
help="GPU smoke: GPU tables on the GPU partition, groups chained one after another")
ap.add_argument("--after", default=None, help="job id(s) the first group waits for (--gpu)")
args = ap.parse_args(argv)
smoke = yaml.safe_load(Path(args.smoke_config).read_text(encoding="utf-8"))
registry = load_registry(args.registry)
only = set(args.tables.split(",")) if args.tables else None
groups = plan(registry, smoke, only, gpu=args.gpu)
python = os.environ.get("PULSE_PYTHON") or sys.executable
folds = ",".join(str(f) for f in (smoke["gpu"] if args.gpu else smoke)["folds"])
job_of_table = {}
prev = args.after or ""
for g in groups:
cmd = [python, str(REPO / "scripts" / "run_table.py")]
for t in g["tables"]:
cmd += ["--table", t]
cmd += ["--runs", args.runs, "--folds", folds, "--allow-new-keys",
"--name", f"{'gpusmoke' if args.gpu else 'smoke'}-{g['task']}-{g['tables'][0]}"]
if not args.gpu:
cmd += ["--cpu"]
if all(isinstance(s, int) for s in g["seeds"]):
cmd += ["--seeds", ",".join(str(s) for s in g["seeds"])]
for kv in g["set"]:
cmd += ["--set", kv]
after = sorted({job_of_table[t] for t in g["after"] if job_of_table.get(t)})
missing = [t for t in g["after"] if t not in job_of_table and (not only or t in only)]
if missing and args.submit:
raise SystemExit(f"{g['tables']}: wait-for tables {missing} were not submitted first")
if args.gpu and prev:
after = [prev] # GPU groups one after another: each may use the whole cap
if after:
cmd += ["--after", ",".join(after)]
if args.submit or args.dry_run:
cmd += ["--slurm"] + (["--dry-run"] if args.dry_run else [])
print("+ " + " ".join(shlex.quote(c) for c in cmd), file=sys.stderr, flush=True)
if not (args.submit or args.dry_run):
continue
res = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True)
sys.stderr.write(res.stderr)
if res.returncode != 0:
raise SystemExit(f"run_table failed for {g['tables']} (exit {res.returncode})")
jid = res.stdout.strip().splitlines()[-1] if res.stdout.strip() else ""
for t in g["tables"]:
job_of_table[t] = jid
prev = jid or prev
print(f"{'+'.join(g['tables'])}\t{jid or '(nothing to run / dry run)'}", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())