Download scripts/run_table.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 18.3 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/run_table.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/scripts/run_table.py
-
curl -L -o run_table.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/run_table.py
18.3 kB
| #!/usr/bin/env python | |
| """Generate (and optionally submit) every run of one or more tables. | |
| python scripts/run_table.py --table 3 # print the commands | |
| python scripts/run_table.py --table 3 --local # run them here, one by one | |
| python scripts/run_table.py --table 3 --slurm # GPU job array on ex03A800 | |
| python scripts/run_table.py --table 3 --table 7 --slurm --name W1 --after 467240 | |
| # one array for both tables, starts | |
| # after job 467240 has ended | |
| python scripts/run_table.py --table 3 --slurm --cpu --folds 0 --seeds 42 \ | |
| --set train.epochs=1 --allow-new-keys # CPU smoke test | |
| One run = (row, fold, seed) -> runs/<table>/<row>/fold<k>_seed<s>/results.json. | |
| Runs whose results.json exists are skipped unless --force. Several --table | |
| flags put all their runs into one job array (all tables must have the same | |
| ``resources``). | |
| Overrides (``--set`` and the rows' ``set:``) must name keys that exist in the | |
| row's configuration (after ``_base_`` inheritance); a misspelled key is | |
| refused. ``--allow-new-keys`` accepts keys the task reads with a default but | |
| the YAML does not list (e.g. ``train.max_train_batches`` in smoke tests). | |
| GPU submission: partition ex03A800 only. The user's running + pending GPUs on | |
| that partition are counted first; the array is throttled so the total stays | |
| within the cap (default 4; MAXGPU=1..8), and nothing is submitted if the cap | |
| is already reached. ``--after JOBID[,JOBID...]`` adds | |
| ``--dependency=afterany:...``; those jobs and every job they wait for are not | |
| counted, because the new array starts only after they have ended. GPU arrays | |
| are refused from a checkout with uncommitted changes (the runs would record a | |
| dirty tree). --dry-run writes the job files without submitting. After a | |
| submission the job id is printed on stdout (``W1=$(python scripts/run_table.py ...)``). | |
| Environment the runs need is checked before anything is written and exported | |
| in the job script: ``PULSE_ROOT``; ``PULSE_SSL_WEARABLES_DIR`` for harnet rows; | |
| ``PULSE_S9_ANNOTATIONS`` for F5 rows with S9 pre-training. The SLURM account | |
| is ``$PULSE_SLURM_ACCOUNT`` or ``account`` in configs/slurm.yaml (none: the | |
| user's default account). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import datetime as dt | |
| import difflib | |
| import importlib.util | |
| import json | |
| import os | |
| import re | |
| import shlex | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| REPO = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(REPO)) | |
| import yaml # noqa: E402 | |
| from pulse.aggregate import load_registry, row_source, run_dir, table_spec # noqa: E402 | |
| from pulse.config import apply_overrides, load_yaml # noqa: E402 | |
| from pulse.slurm import (GPU_PARTITION, GpuBudgetError, dependency_chain, # noqa: E402 | |
| gpu_cap_from_env, gpus_in_use, normalize_job_ids, parse_squeue, | |
| plan_throttle, query_squeue, render) | |
| ROOT_TOKEN = '"$PULSE_ROOT"' | |
| SSL_ENV = "PULSE_SSL_WEARABLES_DIR" | |
| S9_ENV = "PULSE_S9_ANNOTATIONS" | |
| ACCOUNT_ENV = "PULSE_SLURM_ACCOUNT" | |
| def _ints(s): | |
| return [int(x) for x in s.split(",") if x != ""] if s else None | |
| def check_reuse(spec, row, registry): | |
| """A ``reuse: "<table>/<row>"`` row is produced by that table: same config and overrides.""" | |
| src_t, src_r = row_source(spec, row) | |
| if registry is None or src_t not in registry: | |
| raise SystemExit(f"row {row['key']}: reuse source table {src_t!r} not in the registry") | |
| src = next((r for r in registry[src_t].get("rows", []) if r["key"] == src_r), None) | |
| if src is None or src.get("reuse"): | |
| raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} is not a run row") | |
| if (src["config"], list(src.get("set", []))) != (row["config"], list(row.get("set", []))) \ | |
| or registry[src_t]["task"] != spec["task"] or registry[src_t]["split"] != spec["split"]: | |
| raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} has a different " | |
| "task/split/config/set") | |
| def _key_paths(d, prefix=""): | |
| for k, v in d.items(): | |
| p = f"{prefix}{k}" | |
| yield p | |
| if isinstance(v, dict): | |
| yield from _key_paths(v, p + ".") | |
| def unknown_override_keys(config_path, overrides): | |
| """``[(key, suggestion or None)]`` for overrides whose key is not in the merged config.""" | |
| cfg = load_yaml(config_path) | |
| known = list(_key_paths(cfg)) | |
| bad = [] | |
| for item in overrides: | |
| if "=" not in item: | |
| raise SystemExit(f"override must be key=value, got {item!r}") | |
| key = item.split("=", 1)[0].strip() | |
| node, ok = cfg, True | |
| for part in key.split("."): | |
| if not isinstance(node, dict) or part not in node: | |
| ok = False | |
| break | |
| node = node[part] | |
| if not ok: | |
| near = difflib.get_close_matches(key, known, n=1, cutoff=0.6) | |
| bad.append((key, near[0] if near else None)) | |
| return bad | |
| def env_requirements(task, cfg): | |
| """Environment variables a run of this (resolved) config needs: ``{name: why}``.""" | |
| need = {} | |
| model = cfg.get("model") or {} | |
| if isinstance(model, dict) and model.get("name") == "harnet" and not model.get("repo_dir"): | |
| need[SSL_ENV] = "harnet rows load the pinned ssl-wearables checkout (compute nodes are offline)" | |
| if task == "f5" and cfg.get("init") == "s9": | |
| need[S9_ENV] = "F5 S9 pre-training reads <dir>/vX/s9.json" | |
| return need | |
| def check_env(need, environ=None): | |
| """Validate the needed variables; return ``{name: absolute value}`` to export in the job.""" | |
| environ = os.environ if environ is None else environ | |
| out, errors = {}, [] | |
| for name, why in sorted(need.items()): | |
| val = environ.get(name) | |
| if not val: | |
| errors.append(f"set {name} ({why})") | |
| continue | |
| p = Path(os.path.expanduser(val)).resolve() | |
| if name == SSL_ENV and not (p / "hubconf.py").is_file(): | |
| errors.append(f"{name}={val}: no hubconf.py there (fetch it with " | |
| "python -m pulse.models.t1.harnet --fetch DIR)") | |
| elif name == S9_ENV and not (p.is_dir() and any(p.glob("v*/s9.json"))): | |
| errors.append(f"{name}={val}: no vX/s9.json label files there") | |
| out[name] = p.as_posix() | |
| if errors: | |
| raise SystemExit("missing environment for these runs:\n " + "\n ".join(errors)) | |
| return out | |
| def build_commands(spec, args, python, registry=None, rows_filter=None): | |
| """``(commands, n_skipped, unknown_keys, env_needed)`` for one table.""" | |
| if not spec["rows"]: | |
| raise SystemExit(f"table {spec['id']} has no rows yet (status: {spec.get('status')}); " | |
| "add them to configs/tables.yaml") | |
| if importlib.util.find_spec(f"pulse.tasks.{spec['task']}") is None: | |
| raise SystemExit(f"task module pulse.tasks.{spec['task']} does not exist yet") | |
| rows = spec["rows"] | |
| if rows_filter: | |
| want = set(rows_filter.split(",")) | |
| unknown = want - {r["key"] for r in rows} | |
| if unknown: | |
| raise SystemExit(f"unknown rows {sorted(unknown)}") | |
| rows = [r for r in rows if r["key"] in want] | |
| folds = _ints(args.folds) if args.folds else spec["folds"] | |
| seeds = _ints(args.seeds) if args.seeds else spec["seeds"] | |
| cmds, skipped, bad_keys, env_need = [], 0, [], {} | |
| for row in rows: | |
| cfg = REPO / row["config"] | |
| if not cfg.is_file(): | |
| raise SystemExit(f"row {row['key']}: config {row['config']} does not exist") | |
| if row.get("reuse"): | |
| check_reuse(spec, row, registry) | |
| print(f"row {spec['id']}/{row['key']}: reuses the runs of {row['reuse']} (run that table)", | |
| file=sys.stderr) | |
| continue | |
| overrides = list(row.get("set", [])) + list(args.set) | |
| for key, near in unknown_override_keys(cfg, overrides): | |
| bad_keys.append((f"{spec['id']}/{row['key']}", key, near)) | |
| row_cmds = 0 | |
| for k in folds: | |
| for s in seeds: | |
| out = run_dir(args.runs, spec["id"], row["key"], k, s) | |
| if (Path(out) / "results.json").is_file() and not args.force: | |
| skipped += 1 | |
| continue | |
| parts = [python, "-m", f"pulse.tasks.{spec['task']}", "--root", ROOT_TOKEN, | |
| "--split", spec["split"], "--fold", str(k), "--seed", str(s), | |
| "--config", row["config"], "--out", Path(out).as_posix()] | |
| for kv in overrides: | |
| parts += ["--set", kv] | |
| if args.cpu: | |
| parts += ["--device", "cpu"] | |
| # the dataset root stays a variable: it is exported by the job script | |
| cmds.append(" ".join(p if p is ROOT_TOKEN else shlex.quote(p) for p in parts)) | |
| row_cmds += 1 | |
| if row_cmds: | |
| env_need.update(env_requirements(spec["task"], apply_overrides(load_yaml(cfg), overrides))) | |
| return cmds, skipped, bad_keys, env_need | |
| def _git(*a): | |
| try: | |
| r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True, timeout=60) | |
| except OSError: | |
| return None | |
| return r.stdout.strip() if r.returncode == 0 else None | |
| def new_job_dir(jobs_root: Path, name: str) -> Path: | |
| """A fresh ``<jobs_root>/<name>-<time>[-k]`` directory. Never reused: a pending array reads | |
| its commands.txt when each task starts, so two submissions must not share a directory.""" | |
| base = f"{re.sub(r'[^A-Za-z0-9_.+-]', '_', name)}-{dt.datetime.now().strftime('%Y%m%d-%H%M%S')}" | |
| jobs_root.mkdir(parents=True, exist_ok=True) | |
| for k in range(1000): | |
| d = jobs_root / (base if k == 0 else f"{base}-{k}") | |
| try: | |
| d.mkdir() # atomic: fails if another submission created it first | |
| return d | |
| except FileExistsError: | |
| continue | |
| raise SystemExit(f"cannot create a fresh job directory under {jobs_root}") | |
| def _budget(after_ids, dry_run): | |
| """GPUs on the partition that may run alongside the new array.""" | |
| try: | |
| entries = parse_squeue(query_squeue()) | |
| except (OSError, GpuBudgetError) as e: | |
| if dry_run: | |
| print(f"warning: cannot read squeue ({e}); dry run assumes 0 GPUs in use", file=sys.stderr) | |
| return 0, set() | |
| raise SystemExit(f"cannot check the GPU budget ({e}); refusing to submit") | |
| chain = dependency_chain(entries, after_ids) if after_ids else set() | |
| return gpus_in_use(entries, GPU_PARTITION, exclude=chain), chain | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--table", required=True, action="append", | |
| help="table id; repeat to put several tables into one job array") | |
| ap.add_argument("--registry", default=None) | |
| ap.add_argument("--runs", default="runs") | |
| ap.add_argument("--rows", default=None, help="comma-separated subset of row keys (one --table only)") | |
| ap.add_argument("--folds", default=None, help="e.g. 0 or 0,1,2") | |
| ap.add_argument("--seeds", default=None, help="e.g. 42 or 42,43") | |
| ap.add_argument("--set", action="append", default=[], help="override passed to every run") | |
| ap.add_argument("--allow-new-keys", action="store_true", | |
| help="accept overrides of keys the config does not list (smoke tests)") | |
| ap.add_argument("--force", action="store_true", help="rerun runs that already have results") | |
| ap.add_argument("--python", default=None, help="python executable (default $PULSE_PYTHON or this one)") | |
| ap.add_argument("--local", action="store_true", help="run sequentially in this shell") | |
| ap.add_argument("--slurm", action="store_true", help="write and submit a SLURM job array") | |
| ap.add_argument("--cpu", action="store_true", help="CPU partition and --device cpu (smoke tests)") | |
| ap.add_argument("--name", default=None, help="job name (default pulse-tab<ids>)") | |
| ap.add_argument("--after", default=None, | |
| help="job id(s) the array waits for (sbatch --dependency=afterany:...)") | |
| ap.add_argument("--allow-dirty", action="store_true", | |
| help="submit GPU runs from a checkout with uncommitted changes (not for the paper)") | |
| ap.add_argument("--dry-run", action="store_true", help="write job files but do not submit") | |
| ap.add_argument("--slurm-config", default=str(REPO / "configs" / "slurm.yaml")) | |
| args = ap.parse_args(argv) | |
| registry = load_registry(args.registry) | |
| specs = [table_spec(registry, t) for t in dict.fromkeys(args.table)] | |
| if len(specs) > 1 and args.rows: | |
| raise SystemExit("--rows needs a single --table") | |
| kinds = {s.get("resources", "gpu") for s in specs} | |
| if len(kinds) > 1: | |
| raise SystemExit(f"tables {args.table} mix resources {sorted(kinds)}; submit them separately") | |
| python = args.python or os.environ.get("PULSE_PYTHON") or sys.executable | |
| cmds, bad_keys, env_need, per_table = [], [], {}, [] | |
| for spec in specs: | |
| c, skipped, bad, need = build_commands(spec, args, python, registry, args.rows) | |
| print(f"table {spec['id']}: {len(c)} run(s) to do, {skipped} already done", file=sys.stderr) | |
| cmds += c | |
| bad_keys += bad | |
| env_need.update(need) | |
| per_table.append({"table": spec["id"], "runs": len(c), "skipped": skipped}) | |
| if bad_keys and not args.allow_new_keys: | |
| lines = [f"{where}: {key}" + (f" (did you mean {near}?)" if near else "") | |
| for where, key, near in bad_keys] | |
| raise SystemExit("override key(s) not in the row's config (misspelled?); pass " | |
| "--allow-new-keys if the task reads them with a default:\n " | |
| + "\n ".join(dict.fromkeys(lines))) | |
| if not cmds: | |
| return 0 | |
| if not (args.slurm or args.local): | |
| print("\n".join(cmds)) | |
| return 0 | |
| root = os.environ.get("PULSE_ROOT") | |
| if not root: | |
| raise SystemExit("set PULSE_ROOT to the dataset root first") | |
| env_export = check_env(env_need) | |
| if args.local: | |
| env = dict(os.environ) | |
| for c in cmds: | |
| print(f"+ {c}", file=sys.stderr, flush=True) | |
| subprocess.run(c, shell=True, check=True, cwd=str(REPO), env=env) | |
| return 0 | |
| with open(args.slurm_config, encoding="utf-8") as f: | |
| sc = yaml.safe_load(f) | |
| use_gpu = kinds == {"gpu"} and not args.cpu | |
| after_ids = normalize_job_ids(args.after) if args.after else () | |
| sha = _git("rev-parse", "HEAD") | |
| dirty = _git("status", "--porcelain", "--untracked-files=no") | |
| if use_gpu and dirty and not args.allow_dirty: | |
| raise SystemExit(f"checkout {REPO} has uncommitted changes; GPU runs must come from a " | |
| "clean commit (launch from a frozen clone; --allow-dirty to override)") | |
| name = args.name or "pulse-tab" + "+".join(s["id"] for s in specs) | |
| job_dir = new_job_dir(Path(args.runs) / "_jobs", name) | |
| (job_dir / "logs").mkdir() | |
| cmd_file = job_dir / "commands.txt" | |
| cmd_file.write_text("\n".join(cmds) + "\n", encoding="utf-8", newline="\n") | |
| account = os.environ.get(ACCOUNT_ENV) or sc.get("account") or "" | |
| values = {"JOB_NAME": name, "ACCOUNT": account, | |
| "LAST_INDEX": len(cmds) - 1, "LOG_DIR": (job_dir / "logs").resolve().as_posix(), | |
| "REPO_DIR": REPO.as_posix(), "PULSE_ROOT": root, | |
| "COMMANDS": cmd_file.resolve().as_posix(), | |
| "SBATCH_EXTRA": (f"#SBATCH --dependency=afterany:{':'.join(after_ids)}" | |
| if after_ids else ""), | |
| "EXTRA_ENV": "\n".join(f"export {k}={shlex.quote(v)}" for k, v in env_export.items())} | |
| info = {"name": name, "tables": per_table, "n_tasks": len(cmds), "after": list(after_ids), | |
| "git_sha": sha, "git_dirty": bool(dirty), "resources": "gpu" if use_gpu else "cpu"} | |
| if use_gpu: | |
| g = sc["gpu"] | |
| cap = gpu_cap_from_env() | |
| in_use, chain = _budget(after_ids, args.dry_run) | |
| try: | |
| throttle = plan_throttle(len(cmds), int(g["gpus_per_task"]), cap, in_use) | |
| except GpuBudgetError as e: | |
| raise SystemExit(str(e)) | |
| not_counted = sorted(chain) | |
| print(f"GPU budget: {in_use} in use on {GPU_PARTITION} besides the dependency chain " | |
| f"{not_counted or '(none)'}, cap {cap} -> array throttle {throttle} x " | |
| f"{g['gpus_per_task']} GPU", file=sys.stderr) | |
| values.update(GPUS_PER_TASK=g["gpus_per_task"], CPUS=g["cpus_per_task"], MEM=g["mem"], | |
| TIME=g["time"], THROTTLE=throttle) | |
| text = render("gpu_array.sbatch.in", values) | |
| info.update(throttle=throttle, gpu_cap=cap, gpus_in_use=in_use, not_counted=not_counted) | |
| else: | |
| c = sc["cpu"] | |
| throttle = min(len(cmds), int(c.get("max_concurrent", 8))) | |
| values.update(PARTITION=c["partition"], CPUS=c["cpus_per_task"], MEM=c["mem"], | |
| TIME=c["time"], THROTTLE=throttle) | |
| text = render("cpu_array.sbatch.in", values) | |
| info.update(throttle=throttle) | |
| job_file = job_dir / "job.sbatch" | |
| job_file.write_text(text, encoding="utf-8", newline="\n") | |
| print(f"wrote {job_file} ({len(cmds)} tasks, throttle {throttle}" | |
| + (f", after {':'.join(after_ids)}" if after_ids else "") + ")", file=sys.stderr) | |
| if args.dry_run: | |
| (job_dir / "job.json").write_text(json.dumps(info, indent=1) + "\n", encoding="utf-8", | |
| newline="\n") | |
| return 0 | |
| res = subprocess.run(["sbatch", "--parsable", str(job_file)], capture_output=True, text=True) | |
| sys.stderr.write(res.stderr) | |
| if res.returncode != 0: | |
| return res.returncode | |
| job_id = res.stdout.strip().split(";")[0] | |
| info["job_id"] = job_id | |
| (job_dir / "job.json").write_text(json.dumps(info, indent=1) + "\n", encoding="utf-8", newline="\n") | |
| print(f"submitted {name}: job {job_id}", file=sys.stderr) | |
| print(job_id) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |