#!/usr/bin/env python
"""Generate (and optionally submit) every run of one or more tables.
python scripts/run_table.py --table 3 # print the commands
python scripts/run_table.py --table 3 --local # run them here, one by one
python scripts/run_table.py --table 3 --slurm # GPU job array on ex03A800
python scripts/run_table.py --table 3 --table 7 --slurm --name W1 --after 467240
# one array for both tables, starts
# after job 467240 has ended
python scripts/run_table.py --table 3 --slurm --cpu --folds 0 --seeds 42 \
--set train.epochs=1 --allow-new-keys # CPU smoke test
One run = (row, fold, seed) -> runs/
//fold_seed/results.json.
Runs whose results.json exists are skipped unless --force. Several --table
flags put all their runs into one job array (all tables must have the same
``resources``).
Overrides (``--set`` and the rows' ``set:``) must name keys that exist in the
row's configuration (after ``_base_`` inheritance); a misspelled key is
refused. ``--allow-new-keys`` accepts keys the task reads with a default but
the YAML does not list (e.g. ``train.max_train_batches`` in smoke tests).
GPU submission: partition ex03A800 only. The user's running + pending GPUs on
that partition are counted first; the array is throttled so the total stays
within the cap (default 4; MAXGPU=1..8), and nothing is submitted if the cap
is already reached. ``--after JOBID[,JOBID...]`` adds
``--dependency=afterany:...``; those jobs and every job they wait for are not
counted, because the new array starts only after they have ended. GPU arrays
are refused from a checkout with uncommitted changes (the runs would record a
dirty tree). --dry-run writes the job files without submitting. After a
submission the job id is printed on stdout (``W1=$(python scripts/run_table.py ...)``).
Environment the runs need is checked before anything is written and exported
in the job script: ``PULSE_ROOT``; ``PULSE_SSL_WEARABLES_DIR`` for harnet rows;
``PULSE_S9_ANNOTATIONS`` for F5 rows with S9 pre-training. The SLURM account
is ``$PULSE_SLURM_ACCOUNT`` or ``account`` in configs/slurm.yaml (none: the
user's default account).
"""
from __future__ import annotations
import argparse
import datetime as dt
import difflib
import importlib.util
import json
import os
import re
import shlex
import subprocess
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
import yaml # noqa: E402
from pulse.aggregate import load_registry, row_source, run_dir, table_spec # noqa: E402
from pulse.config import apply_overrides, load_yaml # noqa: E402
from pulse.slurm import (GPU_PARTITION, GpuBudgetError, dependency_chain, # noqa: E402
gpu_cap_from_env, gpus_in_use, normalize_job_ids, parse_squeue,
plan_throttle, query_squeue, render)
ROOT_TOKEN = '"$PULSE_ROOT"'
SSL_ENV = "PULSE_SSL_WEARABLES_DIR"
S9_ENV = "PULSE_S9_ANNOTATIONS"
ACCOUNT_ENV = "PULSE_SLURM_ACCOUNT"
def _ints(s):
return [int(x) for x in s.split(",") if x != ""] if s else None
def check_reuse(spec, row, registry):
"""A ``reuse: "
/"`` row is produced by that table: same config and overrides."""
src_t, src_r = row_source(spec, row)
if registry is None or src_t not in registry:
raise SystemExit(f"row {row['key']}: reuse source table {src_t!r} not in the registry")
src = next((r for r in registry[src_t].get("rows", []) if r["key"] == src_r), None)
if src is None or src.get("reuse"):
raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} is not a run row")
if (src["config"], list(src.get("set", []))) != (row["config"], list(row.get("set", []))) \
or registry[src_t]["task"] != spec["task"] or registry[src_t]["split"] != spec["split"]:
raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} has a different "
"task/split/config/set")
def _key_paths(d, prefix=""):
for k, v in d.items():
p = f"{prefix}{k}"
yield p
if isinstance(v, dict):
yield from _key_paths(v, p + ".")
def unknown_override_keys(config_path, overrides):
"""``[(key, suggestion or None)]`` for overrides whose key is not in the merged config."""
cfg = load_yaml(config_path)
known = list(_key_paths(cfg))
bad = []
for item in overrides:
if "=" not in item:
raise SystemExit(f"override must be key=value, got {item!r}")
key = item.split("=", 1)[0].strip()
node, ok = cfg, True
for part in key.split("."):
if not isinstance(node, dict) or part not in node:
ok = False
break
node = node[part]
if not ok:
near = difflib.get_close_matches(key, known, n=1, cutoff=0.6)
bad.append((key, near[0] if near else None))
return bad
def env_requirements(task, cfg):
"""Environment variables a run of this (resolved) config needs: ``{name: why}``."""
need = {}
model = cfg.get("model") or {}
if isinstance(model, dict) and model.get("name") == "harnet" and not model.get("repo_dir"):
need[SSL_ENV] = "harnet rows load the pinned ssl-wearables checkout (compute nodes are offline)"
if task == "f5" and cfg.get("init") == "s9":
need[S9_ENV] = "F5 S9 pre-training reads /vX/s9.json"
return need
def check_env(need, environ=None):
"""Validate the needed variables; return ``{name: absolute value}`` to export in the job."""
environ = os.environ if environ is None else environ
out, errors = {}, []
for name, why in sorted(need.items()):
val = environ.get(name)
if not val:
errors.append(f"set {name} ({why})")
continue
p = Path(os.path.expanduser(val)).resolve()
if name == SSL_ENV and not (p / "hubconf.py").is_file():
errors.append(f"{name}={val}: no hubconf.py there (fetch it with "
"python -m pulse.models.t1.harnet --fetch DIR)")
elif name == S9_ENV and not (p.is_dir() and any(p.glob("v*/s9.json"))):
errors.append(f"{name}={val}: no vX/s9.json label files there")
out[name] = p.as_posix()
if errors:
raise SystemExit("missing environment for these runs:\n " + "\n ".join(errors))
return out
def build_commands(spec, args, python, registry=None, rows_filter=None):
"""``(commands, n_skipped, unknown_keys, env_needed)`` for one table."""
if not spec["rows"]:
raise SystemExit(f"table {spec['id']} has no rows yet (status: {spec.get('status')}); "
"add them to configs/tables.yaml")
if importlib.util.find_spec(f"pulse.tasks.{spec['task']}") is None:
raise SystemExit(f"task module pulse.tasks.{spec['task']} does not exist yet")
rows = spec["rows"]
if rows_filter:
want = set(rows_filter.split(","))
unknown = want - {r["key"] for r in rows}
if unknown:
raise SystemExit(f"unknown rows {sorted(unknown)}")
rows = [r for r in rows if r["key"] in want]
folds = _ints(args.folds) if args.folds else spec["folds"]
seeds = _ints(args.seeds) if args.seeds else spec["seeds"]
cmds, skipped, bad_keys, env_need = [], 0, [], {}
for row in rows:
cfg = REPO / row["config"]
if not cfg.is_file():
raise SystemExit(f"row {row['key']}: config {row['config']} does not exist")
if row.get("reuse"):
check_reuse(spec, row, registry)
print(f"row {spec['id']}/{row['key']}: reuses the runs of {row['reuse']} (run that table)",
file=sys.stderr)
continue
overrides = list(row.get("set", [])) + list(args.set)
for key, near in unknown_override_keys(cfg, overrides):
bad_keys.append((f"{spec['id']}/{row['key']}", key, near))
row_cmds = 0
for k in folds:
for s in seeds:
out = run_dir(args.runs, spec["id"], row["key"], k, s)
if (Path(out) / "results.json").is_file() and not args.force:
skipped += 1
continue
parts = [python, "-m", f"pulse.tasks.{spec['task']}", "--root", ROOT_TOKEN,
"--split", spec["split"], "--fold", str(k), "--seed", str(s),
"--config", row["config"], "--out", Path(out).as_posix()]
for kv in overrides:
parts += ["--set", kv]
if args.cpu:
parts += ["--device", "cpu"]
# the dataset root stays a variable: it is exported by the job script
cmds.append(" ".join(p if p is ROOT_TOKEN else shlex.quote(p) for p in parts))
row_cmds += 1
if row_cmds:
env_need.update(env_requirements(spec["task"], apply_overrides(load_yaml(cfg), overrides)))
return cmds, skipped, bad_keys, env_need
def _git(*a):
try:
r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True, timeout=60)
except OSError:
return None
return r.stdout.strip() if r.returncode == 0 else None
def new_job_dir(jobs_root: Path, name: str) -> Path:
"""A fresh ``/-