PULSE-code / scripts /run_table.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
18.3 kB
#!/usr/bin/env python
"""Generate (and optionally submit) every run of one or more tables.
python scripts/run_table.py --table 3 # print the commands
python scripts/run_table.py --table 3 --local # run them here, one by one
python scripts/run_table.py --table 3 --slurm # GPU job array on ex03A800
python scripts/run_table.py --table 3 --table 7 --slurm --name W1 --after 467240
# one array for both tables, starts
# after job 467240 has ended
python scripts/run_table.py --table 3 --slurm --cpu --folds 0 --seeds 42 \
--set train.epochs=1 --allow-new-keys # CPU smoke test
One run = (row, fold, seed) -> runs/<table>/<row>/fold<k>_seed<s>/results.json.
Runs whose results.json exists are skipped unless --force. Several --table
flags put all their runs into one job array (all tables must have the same
``resources``).
Overrides (``--set`` and the rows' ``set:``) must name keys that exist in the
row's configuration (after ``_base_`` inheritance); a misspelled key is
refused. ``--allow-new-keys`` accepts keys the task reads with a default but
the YAML does not list (e.g. ``train.max_train_batches`` in smoke tests).
GPU submission: partition ex03A800 only. The user's running + pending GPUs on
that partition are counted first; the array is throttled so the total stays
within the cap (default 4; MAXGPU=1..8), and nothing is submitted if the cap
is already reached. ``--after JOBID[,JOBID...]`` adds
``--dependency=afterany:...``; those jobs and every job they wait for are not
counted, because the new array starts only after they have ended. GPU arrays
are refused from a checkout with uncommitted changes (the runs would record a
dirty tree). --dry-run writes the job files without submitting. After a
submission the job id is printed on stdout (``W1=$(python scripts/run_table.py ...)``).
Environment the runs need is checked before anything is written and exported
in the job script: ``PULSE_ROOT``; ``PULSE_SSL_WEARABLES_DIR`` for harnet rows;
``PULSE_S9_ANNOTATIONS`` for F5 rows with S9 pre-training. The SLURM account
is ``$PULSE_SLURM_ACCOUNT`` or ``account`` in configs/slurm.yaml (none: the
user's default account).
"""
from __future__ import annotations
import argparse
import datetime as dt
import difflib
import importlib.util
import json
import os
import re
import shlex
import subprocess
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
import yaml # noqa: E402
from pulse.aggregate import load_registry, row_source, run_dir, table_spec # noqa: E402
from pulse.config import apply_overrides, load_yaml # noqa: E402
from pulse.slurm import (GPU_PARTITION, GpuBudgetError, dependency_chain, # noqa: E402
gpu_cap_from_env, gpus_in_use, normalize_job_ids, parse_squeue,
plan_throttle, query_squeue, render)
ROOT_TOKEN = '"$PULSE_ROOT"'
SSL_ENV = "PULSE_SSL_WEARABLES_DIR"
S9_ENV = "PULSE_S9_ANNOTATIONS"
ACCOUNT_ENV = "PULSE_SLURM_ACCOUNT"
def _ints(s):
return [int(x) for x in s.split(",") if x != ""] if s else None
def check_reuse(spec, row, registry):
"""A ``reuse: "<table>/<row>"`` row is produced by that table: same config and overrides."""
src_t, src_r = row_source(spec, row)
if registry is None or src_t not in registry:
raise SystemExit(f"row {row['key']}: reuse source table {src_t!r} not in the registry")
src = next((r for r in registry[src_t].get("rows", []) if r["key"] == src_r), None)
if src is None or src.get("reuse"):
raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} is not a run row")
if (src["config"], list(src.get("set", []))) != (row["config"], list(row.get("set", []))) \
or registry[src_t]["task"] != spec["task"] or registry[src_t]["split"] != spec["split"]:
raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} has a different "
"task/split/config/set")
def _key_paths(d, prefix=""):
for k, v in d.items():
p = f"{prefix}{k}"
yield p
if isinstance(v, dict):
yield from _key_paths(v, p + ".")
def unknown_override_keys(config_path, overrides):
"""``[(key, suggestion or None)]`` for overrides whose key is not in the merged config."""
cfg = load_yaml(config_path)
known = list(_key_paths(cfg))
bad = []
for item in overrides:
if "=" not in item:
raise SystemExit(f"override must be key=value, got {item!r}")
key = item.split("=", 1)[0].strip()
node, ok = cfg, True
for part in key.split("."):
if not isinstance(node, dict) or part not in node:
ok = False
break
node = node[part]
if not ok:
near = difflib.get_close_matches(key, known, n=1, cutoff=0.6)
bad.append((key, near[0] if near else None))
return bad
def env_requirements(task, cfg):
"""Environment variables a run of this (resolved) config needs: ``{name: why}``."""
need = {}
model = cfg.get("model") or {}
if isinstance(model, dict) and model.get("name") == "harnet" and not model.get("repo_dir"):
need[SSL_ENV] = "harnet rows load the pinned ssl-wearables checkout (compute nodes are offline)"
if task == "f5" and cfg.get("init") == "s9":
need[S9_ENV] = "F5 S9 pre-training reads <dir>/vX/s9.json"
return need
def check_env(need, environ=None):
"""Validate the needed variables; return ``{name: absolute value}`` to export in the job."""
environ = os.environ if environ is None else environ
out, errors = {}, []
for name, why in sorted(need.items()):
val = environ.get(name)
if not val:
errors.append(f"set {name} ({why})")
continue
p = Path(os.path.expanduser(val)).resolve()
if name == SSL_ENV and not (p / "hubconf.py").is_file():
errors.append(f"{name}={val}: no hubconf.py there (fetch it with "
"python -m pulse.models.t1.harnet --fetch DIR)")
elif name == S9_ENV and not (p.is_dir() and any(p.glob("v*/s9.json"))):
errors.append(f"{name}={val}: no vX/s9.json label files there")
out[name] = p.as_posix()
if errors:
raise SystemExit("missing environment for these runs:\n " + "\n ".join(errors))
return out
def build_commands(spec, args, python, registry=None, rows_filter=None):
"""``(commands, n_skipped, unknown_keys, env_needed)`` for one table."""
if not spec["rows"]:
raise SystemExit(f"table {spec['id']} has no rows yet (status: {spec.get('status')}); "
"add them to configs/tables.yaml")
if importlib.util.find_spec(f"pulse.tasks.{spec['task']}") is None:
raise SystemExit(f"task module pulse.tasks.{spec['task']} does not exist yet")
rows = spec["rows"]
if rows_filter:
want = set(rows_filter.split(","))
unknown = want - {r["key"] for r in rows}
if unknown:
raise SystemExit(f"unknown rows {sorted(unknown)}")
rows = [r for r in rows if r["key"] in want]
folds = _ints(args.folds) if args.folds else spec["folds"]
seeds = _ints(args.seeds) if args.seeds else spec["seeds"]
cmds, skipped, bad_keys, env_need = [], 0, [], {}
for row in rows:
cfg = REPO / row["config"]
if not cfg.is_file():
raise SystemExit(f"row {row['key']}: config {row['config']} does not exist")
if row.get("reuse"):
check_reuse(spec, row, registry)
print(f"row {spec['id']}/{row['key']}: reuses the runs of {row['reuse']} (run that table)",
file=sys.stderr)
continue
overrides = list(row.get("set", [])) + list(args.set)
for key, near in unknown_override_keys(cfg, overrides):
bad_keys.append((f"{spec['id']}/{row['key']}", key, near))
row_cmds = 0
for k in folds:
for s in seeds:
out = run_dir(args.runs, spec["id"], row["key"], k, s)
if (Path(out) / "results.json").is_file() and not args.force:
skipped += 1
continue
parts = [python, "-m", f"pulse.tasks.{spec['task']}", "--root", ROOT_TOKEN,
"--split", spec["split"], "--fold", str(k), "--seed", str(s),
"--config", row["config"], "--out", Path(out).as_posix()]
for kv in overrides:
parts += ["--set", kv]
if args.cpu:
parts += ["--device", "cpu"]
# the dataset root stays a variable: it is exported by the job script
cmds.append(" ".join(p if p is ROOT_TOKEN else shlex.quote(p) for p in parts))
row_cmds += 1
if row_cmds:
env_need.update(env_requirements(spec["task"], apply_overrides(load_yaml(cfg), overrides)))
return cmds, skipped, bad_keys, env_need
def _git(*a):
try:
r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True, timeout=60)
except OSError:
return None
return r.stdout.strip() if r.returncode == 0 else None
def new_job_dir(jobs_root: Path, name: str) -> Path:
"""A fresh ``<jobs_root>/<name>-<time>[-k]`` directory. Never reused: a pending array reads
its commands.txt when each task starts, so two submissions must not share a directory."""
base = f"{re.sub(r'[^A-Za-z0-9_.+-]', '_', name)}-{dt.datetime.now().strftime('%Y%m%d-%H%M%S')}"
jobs_root.mkdir(parents=True, exist_ok=True)
for k in range(1000):
d = jobs_root / (base if k == 0 else f"{base}-{k}")
try:
d.mkdir() # atomic: fails if another submission created it first
return d
except FileExistsError:
continue
raise SystemExit(f"cannot create a fresh job directory under {jobs_root}")
def _budget(after_ids, dry_run):
"""GPUs on the partition that may run alongside the new array."""
try:
entries = parse_squeue(query_squeue())
except (OSError, GpuBudgetError) as e:
if dry_run:
print(f"warning: cannot read squeue ({e}); dry run assumes 0 GPUs in use", file=sys.stderr)
return 0, set()
raise SystemExit(f"cannot check the GPU budget ({e}); refusing to submit")
chain = dependency_chain(entries, after_ids) if after_ids else set()
return gpus_in_use(entries, GPU_PARTITION, exclude=chain), chain
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--table", required=True, action="append",
help="table id; repeat to put several tables into one job array")
ap.add_argument("--registry", default=None)
ap.add_argument("--runs", default="runs")
ap.add_argument("--rows", default=None, help="comma-separated subset of row keys (one --table only)")
ap.add_argument("--folds", default=None, help="e.g. 0 or 0,1,2")
ap.add_argument("--seeds", default=None, help="e.g. 42 or 42,43")
ap.add_argument("--set", action="append", default=[], help="override passed to every run")
ap.add_argument("--allow-new-keys", action="store_true",
help="accept overrides of keys the config does not list (smoke tests)")
ap.add_argument("--force", action="store_true", help="rerun runs that already have results")
ap.add_argument("--python", default=None, help="python executable (default $PULSE_PYTHON or this one)")
ap.add_argument("--local", action="store_true", help="run sequentially in this shell")
ap.add_argument("--slurm", action="store_true", help="write and submit a SLURM job array")
ap.add_argument("--cpu", action="store_true", help="CPU partition and --device cpu (smoke tests)")
ap.add_argument("--name", default=None, help="job name (default pulse-tab<ids>)")
ap.add_argument("--after", default=None,
help="job id(s) the array waits for (sbatch --dependency=afterany:...)")
ap.add_argument("--allow-dirty", action="store_true",
help="submit GPU runs from a checkout with uncommitted changes (not for the paper)")
ap.add_argument("--dry-run", action="store_true", help="write job files but do not submit")
ap.add_argument("--slurm-config", default=str(REPO / "configs" / "slurm.yaml"))
args = ap.parse_args(argv)
registry = load_registry(args.registry)
specs = [table_spec(registry, t) for t in dict.fromkeys(args.table)]
if len(specs) > 1 and args.rows:
raise SystemExit("--rows needs a single --table")
kinds = {s.get("resources", "gpu") for s in specs}
if len(kinds) > 1:
raise SystemExit(f"tables {args.table} mix resources {sorted(kinds)}; submit them separately")
python = args.python or os.environ.get("PULSE_PYTHON") or sys.executable
cmds, bad_keys, env_need, per_table = [], [], {}, []
for spec in specs:
c, skipped, bad, need = build_commands(spec, args, python, registry, args.rows)
print(f"table {spec['id']}: {len(c)} run(s) to do, {skipped} already done", file=sys.stderr)
cmds += c
bad_keys += bad
env_need.update(need)
per_table.append({"table": spec["id"], "runs": len(c), "skipped": skipped})
if bad_keys and not args.allow_new_keys:
lines = [f"{where}: {key}" + (f" (did you mean {near}?)" if near else "")
for where, key, near in bad_keys]
raise SystemExit("override key(s) not in the row's config (misspelled?); pass "
"--allow-new-keys if the task reads them with a default:\n "
+ "\n ".join(dict.fromkeys(lines)))
if not cmds:
return 0
if not (args.slurm or args.local):
print("\n".join(cmds))
return 0
root = os.environ.get("PULSE_ROOT")
if not root:
raise SystemExit("set PULSE_ROOT to the dataset root first")
env_export = check_env(env_need)
if args.local:
env = dict(os.environ)
for c in cmds:
print(f"+ {c}", file=sys.stderr, flush=True)
subprocess.run(c, shell=True, check=True, cwd=str(REPO), env=env)
return 0
with open(args.slurm_config, encoding="utf-8") as f:
sc = yaml.safe_load(f)
use_gpu = kinds == {"gpu"} and not args.cpu
after_ids = normalize_job_ids(args.after) if args.after else ()
sha = _git("rev-parse", "HEAD")
dirty = _git("status", "--porcelain", "--untracked-files=no")
if use_gpu and dirty and not args.allow_dirty:
raise SystemExit(f"checkout {REPO} has uncommitted changes; GPU runs must come from a "
"clean commit (launch from a frozen clone; --allow-dirty to override)")
name = args.name or "pulse-tab" + "+".join(s["id"] for s in specs)
job_dir = new_job_dir(Path(args.runs) / "_jobs", name)
(job_dir / "logs").mkdir()
cmd_file = job_dir / "commands.txt"
cmd_file.write_text("\n".join(cmds) + "\n", encoding="utf-8", newline="\n")
account = os.environ.get(ACCOUNT_ENV) or sc.get("account") or ""
values = {"JOB_NAME": name, "ACCOUNT": account,
"LAST_INDEX": len(cmds) - 1, "LOG_DIR": (job_dir / "logs").resolve().as_posix(),
"REPO_DIR": REPO.as_posix(), "PULSE_ROOT": root,
"COMMANDS": cmd_file.resolve().as_posix(),
"SBATCH_EXTRA": (f"#SBATCH --dependency=afterany:{':'.join(after_ids)}"
if after_ids else ""),
"EXTRA_ENV": "\n".join(f"export {k}={shlex.quote(v)}" for k, v in env_export.items())}
info = {"name": name, "tables": per_table, "n_tasks": len(cmds), "after": list(after_ids),
"git_sha": sha, "git_dirty": bool(dirty), "resources": "gpu" if use_gpu else "cpu"}
if use_gpu:
g = sc["gpu"]
cap = gpu_cap_from_env()
in_use, chain = _budget(after_ids, args.dry_run)
try:
throttle = plan_throttle(len(cmds), int(g["gpus_per_task"]), cap, in_use)
except GpuBudgetError as e:
raise SystemExit(str(e))
not_counted = sorted(chain)
print(f"GPU budget: {in_use} in use on {GPU_PARTITION} besides the dependency chain "
f"{not_counted or '(none)'}, cap {cap} -> array throttle {throttle} x "
f"{g['gpus_per_task']} GPU", file=sys.stderr)
values.update(GPUS_PER_TASK=g["gpus_per_task"], CPUS=g["cpus_per_task"], MEM=g["mem"],
TIME=g["time"], THROTTLE=throttle)
text = render("gpu_array.sbatch.in", values)
info.update(throttle=throttle, gpu_cap=cap, gpus_in_use=in_use, not_counted=not_counted)
else:
c = sc["cpu"]
throttle = min(len(cmds), int(c.get("max_concurrent", 8)))
values.update(PARTITION=c["partition"], CPUS=c["cpus_per_task"], MEM=c["mem"],
TIME=c["time"], THROTTLE=throttle)
text = render("cpu_array.sbatch.in", values)
info.update(throttle=throttle)
job_file = job_dir / "job.sbatch"
job_file.write_text(text, encoding="utf-8", newline="\n")
print(f"wrote {job_file} ({len(cmds)} tasks, throttle {throttle}"
+ (f", after {':'.join(after_ids)}" if after_ids else "") + ")", file=sys.stderr)
if args.dry_run:
(job_dir / "job.json").write_text(json.dumps(info, indent=1) + "\n", encoding="utf-8",
newline="\n")
return 0
res = subprocess.run(["sbatch", "--parsable", str(job_file)], capture_output=True, text=True)
sys.stderr.write(res.stderr)
if res.returncode != 0:
return res.returncode
job_id = res.stdout.strip().split(";")[0]
info["job_id"] = job_id
(job_dir / "job.json").write_text(json.dumps(info, indent=1) + "\n", encoding="utf-8", newline="\n")
print(f"submitted {name}: job {job_id}", file=sys.stderr)
print(job_id)
return 0
if __name__ == "__main__":
sys.exit(main())