#!/usr/bin/env python """CPU smoke suite: every run row of every table, fold 0, seed 42, one short epoch. export PULSE_ROOT=... PULSE_PYTHON=... # + PULSE_SSL_WEARABLES_DIR, PULSE_S9_ANNOTATIONS python scripts/smoke_suite.py --runs ~/smoke/runs # print the run_table calls python scripts/smoke_suite.py --runs ~/smoke/runs --submit # submit CPU job arrays python scripts/build_outputs.py --runs ~/smoke/runs --out ~/smoke/out --allow-incomplete python scripts/smoke_suite.py --runs ~/gpusmoke/runs --gpu --after JOBID --submit # GPU variant (W0): GPU tables only, 2 full epochs, # groups chained one after another Overrides and exceptions: configs/smoke.yaml. Tables of one task with the same overrides and seeds share one CPU array; tables listed with ``after`` wait for the arrays of those tables (``--after``). Tables whose rows all reuse other tables' runs (``4maj``, ``4segmaj``) have nothing to run. Keep ``--runs`` separate from the paper runs. """ from __future__ import annotations import argparse import os import shlex import subprocess import sys from collections import OrderedDict from pathlib import Path REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO)) import yaml # noqa: E402 from pulse.aggregate import load_registry # noqa: E402 def plan(registry, smoke, only=None, gpu=False): """``[{tables, task, set, seeds, after}]`` in registry order (``gpu``: the GPU tables with the ``gpu:`` section of configs/smoke.yaml).""" groups: "OrderedDict[tuple, dict]" = OrderedDict() sec = smoke["gpu"] if gpu else smoke for tid, t in registry.items(): if only and tid not in only: continue rows = [r for r in t.get("rows", []) if not r.get("reuse")] if not rows or (gpu and t.get("resources", "gpu") != "gpu"): continue ex = (sec.get("tables") or {}).get(tid, {}) task = t["task"] if task not in sec["tasks"]: raise SystemExit(f"configs/smoke.yaml has no overrides for task {task!r} (table {tid})") sets = tuple(ex.get("set", sec["tasks"][task])) seeds = t["seeds"] if any(not isinstance(s, int) for s in t["seeds"]) \ else tuple(ex.get("seeds", sec["seeds"])) after = tuple(ex.get("after", ())) key = (task, sets, tuple(seeds), after, t.get("resources", "gpu")) g = groups.setdefault(key, {"tables": [], "task": task, "set": sets, "seeds": tuple(seeds), "after": after}) g["tables"].append(tid) return list(groups.values()) def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--runs", required=True, help="smoke runs directory (not the paper runs)") ap.add_argument("--smoke-config", default=str(REPO / "configs" / "smoke.yaml")) ap.add_argument("--registry", default=None) ap.add_argument("--tables", default=None, help="comma-separated subset of tables") ap.add_argument("--submit", action="store_true", help="submit the CPU arrays (default: print)") ap.add_argument("--dry-run", action="store_true", help="write job files, do not submit") ap.add_argument("--gpu", action="store_true", help="GPU smoke: GPU tables on the GPU partition, groups chained one after another") ap.add_argument("--after", default=None, help="job id(s) the first group waits for (--gpu)") args = ap.parse_args(argv) smoke = yaml.safe_load(Path(args.smoke_config).read_text(encoding="utf-8")) registry = load_registry(args.registry) only = set(args.tables.split(",")) if args.tables else None groups = plan(registry, smoke, only, gpu=args.gpu) python = os.environ.get("PULSE_PYTHON") or sys.executable folds = ",".join(str(f) for f in (smoke["gpu"] if args.gpu else smoke)["folds"]) job_of_table = {} prev = args.after or "" for g in groups: cmd = [python, str(REPO / "scripts" / "run_table.py")] for t in g["tables"]: cmd += ["--table", t] cmd += ["--runs", args.runs, "--folds", folds, "--allow-new-keys", "--name", f"{'gpusmoke' if args.gpu else 'smoke'}-{g['task']}-{g['tables'][0]}"] if not args.gpu: cmd += ["--cpu"] if all(isinstance(s, int) for s in g["seeds"]): cmd += ["--seeds", ",".join(str(s) for s in g["seeds"])] for kv in g["set"]: cmd += ["--set", kv] after = sorted({job_of_table[t] for t in g["after"] if job_of_table.get(t)}) missing = [t for t in g["after"] if t not in job_of_table and (not only or t in only)] if missing and args.submit: raise SystemExit(f"{g['tables']}: wait-for tables {missing} were not submitted first") if args.gpu and prev: after = [prev] # GPU groups one after another: each may use the whole cap if after: cmd += ["--after", ",".join(after)] if args.submit or args.dry_run: cmd += ["--slurm"] + (["--dry-run"] if args.dry_run else []) print("+ " + " ".join(shlex.quote(c) for c in cmd), file=sys.stderr, flush=True) if not (args.submit or args.dry_run): continue res = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True) sys.stderr.write(res.stderr) if res.returncode != 0: raise SystemExit(f"run_table failed for {g['tables']} (exit {res.returncode})") jid = res.stdout.strip().splitlines()[-1] if res.stdout.strip() else "" for t in g["tables"]: job_of_table[t] = jid prev = jid or prev print(f"{'+'.join(g['tables'])}\t{jid or '(nothing to run / dry run)'}", flush=True) return 0 if __name__ == "__main__": sys.exit(main())