File size: 18,312 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
#!/usr/bin/env python
"""Generate (and optionally submit) every run of one or more tables.

    python scripts/run_table.py --table 3                  # print the commands
    python scripts/run_table.py --table 3 --local          # run them here, one by one
    python scripts/run_table.py --table 3 --slurm          # GPU job array on ex03A800
    python scripts/run_table.py --table 3 --table 7 --slurm --name W1 --after 467240
                                                           # one array for both tables, starts
                                                           # after job 467240 has ended
    python scripts/run_table.py --table 3 --slurm --cpu --folds 0 --seeds 42 \
        --set train.epochs=1 --allow-new-keys              # CPU smoke test

One run = (row, fold, seed) -> runs/<table>/<row>/fold<k>_seed<s>/results.json.
Runs whose results.json exists are skipped unless --force.  Several --table
flags put all their runs into one job array (all tables must have the same
``resources``).

Overrides (``--set`` and the rows' ``set:``) must name keys that exist in the
row's configuration (after ``_base_`` inheritance); a misspelled key is
refused.  ``--allow-new-keys`` accepts keys the task reads with a default but
the YAML does not list (e.g. ``train.max_train_batches`` in smoke tests).

GPU submission: partition ex03A800 only.  The user's running + pending GPUs on
that partition are counted first; the array is throttled so the total stays
within the cap (default 4; MAXGPU=1..8), and nothing is submitted if the cap
is already reached.  ``--after JOBID[,JOBID...]`` adds
``--dependency=afterany:...``; those jobs and every job they wait for are not
counted, because the new array starts only after they have ended.  GPU arrays
are refused from a checkout with uncommitted changes (the runs would record a
dirty tree).  --dry-run writes the job files without submitting.  After a
submission the job id is printed on stdout (``W1=$(python scripts/run_table.py ...)``).

Environment the runs need is checked before anything is written and exported
in the job script: ``PULSE_ROOT``; ``PULSE_SSL_WEARABLES_DIR`` for harnet rows;
``PULSE_S9_ANNOTATIONS`` for F5 rows with S9 pre-training.  The SLURM account
is ``$PULSE_SLURM_ACCOUNT`` or ``account`` in configs/slurm.yaml (none: the
user's default account).
"""
from __future__ import annotations

import argparse
import datetime as dt
import difflib
import importlib.util
import json
import os
import re
import shlex
import subprocess
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))

import yaml  # noqa: E402

from pulse.aggregate import load_registry, row_source, run_dir, table_spec  # noqa: E402
from pulse.config import apply_overrides, load_yaml  # noqa: E402
from pulse.slurm import (GPU_PARTITION, GpuBudgetError, dependency_chain,  # noqa: E402
                         gpu_cap_from_env, gpus_in_use, normalize_job_ids, parse_squeue,
                         plan_throttle, query_squeue, render)


ROOT_TOKEN = '"$PULSE_ROOT"'
SSL_ENV = "PULSE_SSL_WEARABLES_DIR"
S9_ENV = "PULSE_S9_ANNOTATIONS"
ACCOUNT_ENV = "PULSE_SLURM_ACCOUNT"


def _ints(s):
    return [int(x) for x in s.split(",") if x != ""] if s else None


def check_reuse(spec, row, registry):
    """A ``reuse: "<table>/<row>"`` row is produced by that table: same config and overrides."""
    src_t, src_r = row_source(spec, row)
    if registry is None or src_t not in registry:
        raise SystemExit(f"row {row['key']}: reuse source table {src_t!r} not in the registry")
    src = next((r for r in registry[src_t].get("rows", []) if r["key"] == src_r), None)
    if src is None or src.get("reuse"):
        raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} is not a run row")
    if (src["config"], list(src.get("set", []))) != (row["config"], list(row.get("set", []))) \
            or registry[src_t]["task"] != spec["task"] or registry[src_t]["split"] != spec["split"]:
        raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} has a different "
                         "task/split/config/set")


def _key_paths(d, prefix=""):
    for k, v in d.items():
        p = f"{prefix}{k}"
        yield p
        if isinstance(v, dict):
            yield from _key_paths(v, p + ".")


def unknown_override_keys(config_path, overrides):
    """``[(key, suggestion or None)]`` for overrides whose key is not in the merged config."""
    cfg = load_yaml(config_path)
    known = list(_key_paths(cfg))
    bad = []
    for item in overrides:
        if "=" not in item:
            raise SystemExit(f"override must be key=value, got {item!r}")
        key = item.split("=", 1)[0].strip()
        node, ok = cfg, True
        for part in key.split("."):
            if not isinstance(node, dict) or part not in node:
                ok = False
                break
            node = node[part]
        if not ok:
            near = difflib.get_close_matches(key, known, n=1, cutoff=0.6)
            bad.append((key, near[0] if near else None))
    return bad


def env_requirements(task, cfg):
    """Environment variables a run of this (resolved) config needs: ``{name: why}``."""
    need = {}
    model = cfg.get("model") or {}
    if isinstance(model, dict) and model.get("name") == "harnet" and not model.get("repo_dir"):
        need[SSL_ENV] = "harnet rows load the pinned ssl-wearables checkout (compute nodes are offline)"
    if task == "f5" and cfg.get("init") == "s9":
        need[S9_ENV] = "F5 S9 pre-training reads <dir>/vX/s9.json"
    return need


def check_env(need, environ=None):
    """Validate the needed variables; return ``{name: absolute value}`` to export in the job."""
    environ = os.environ if environ is None else environ
    out, errors = {}, []
    for name, why in sorted(need.items()):
        val = environ.get(name)
        if not val:
            errors.append(f"set {name} ({why})")
            continue
        p = Path(os.path.expanduser(val)).resolve()
        if name == SSL_ENV and not (p / "hubconf.py").is_file():
            errors.append(f"{name}={val}: no hubconf.py there (fetch it with "
                          "python -m pulse.models.t1.harnet --fetch DIR)")
        elif name == S9_ENV and not (p.is_dir() and any(p.glob("v*/s9.json"))):
            errors.append(f"{name}={val}: no vX/s9.json label files there")
        out[name] = p.as_posix()
    if errors:
        raise SystemExit("missing environment for these runs:\n  " + "\n  ".join(errors))
    return out


def build_commands(spec, args, python, registry=None, rows_filter=None):
    """``(commands, n_skipped, unknown_keys, env_needed)`` for one table."""
    if not spec["rows"]:
        raise SystemExit(f"table {spec['id']} has no rows yet (status: {spec.get('status')}); "
                         "add them to configs/tables.yaml")
    if importlib.util.find_spec(f"pulse.tasks.{spec['task']}") is None:
        raise SystemExit(f"task module pulse.tasks.{spec['task']} does not exist yet")
    rows = spec["rows"]
    if rows_filter:
        want = set(rows_filter.split(","))
        unknown = want - {r["key"] for r in rows}
        if unknown:
            raise SystemExit(f"unknown rows {sorted(unknown)}")
        rows = [r for r in rows if r["key"] in want]
    folds = _ints(args.folds) if args.folds else spec["folds"]
    seeds = _ints(args.seeds) if args.seeds else spec["seeds"]
    cmds, skipped, bad_keys, env_need = [], 0, [], {}
    for row in rows:
        cfg = REPO / row["config"]
        if not cfg.is_file():
            raise SystemExit(f"row {row['key']}: config {row['config']} does not exist")
        if row.get("reuse"):
            check_reuse(spec, row, registry)
            print(f"row {spec['id']}/{row['key']}: reuses the runs of {row['reuse']} (run that table)",
                  file=sys.stderr)
            continue
        overrides = list(row.get("set", [])) + list(args.set)
        for key, near in unknown_override_keys(cfg, overrides):
            bad_keys.append((f"{spec['id']}/{row['key']}", key, near))
        row_cmds = 0
        for k in folds:
            for s in seeds:
                out = run_dir(args.runs, spec["id"], row["key"], k, s)
                if (Path(out) / "results.json").is_file() and not args.force:
                    skipped += 1
                    continue
                parts = [python, "-m", f"pulse.tasks.{spec['task']}", "--root", ROOT_TOKEN,
                         "--split", spec["split"], "--fold", str(k), "--seed", str(s),
                         "--config", row["config"], "--out", Path(out).as_posix()]
                for kv in overrides:
                    parts += ["--set", kv]
                if args.cpu:
                    parts += ["--device", "cpu"]
                # the dataset root stays a variable: it is exported by the job script
                cmds.append(" ".join(p if p is ROOT_TOKEN else shlex.quote(p) for p in parts))
                row_cmds += 1
        if row_cmds:
            env_need.update(env_requirements(spec["task"], apply_overrides(load_yaml(cfg), overrides)))
    return cmds, skipped, bad_keys, env_need


def _git(*a):
    try:
        r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True, timeout=60)
    except OSError:
        return None
    return r.stdout.strip() if r.returncode == 0 else None


def new_job_dir(jobs_root: Path, name: str) -> Path:
    """A fresh ``<jobs_root>/<name>-<time>[-k]`` directory.  Never reused: a pending array reads
    its commands.txt when each task starts, so two submissions must not share a directory."""
    base = f"{re.sub(r'[^A-Za-z0-9_.+-]', '_', name)}-{dt.datetime.now().strftime('%Y%m%d-%H%M%S')}"
    jobs_root.mkdir(parents=True, exist_ok=True)
    for k in range(1000):
        d = jobs_root / (base if k == 0 else f"{base}-{k}")
        try:
            d.mkdir()          # atomic: fails if another submission created it first
            return d
        except FileExistsError:
            continue
    raise SystemExit(f"cannot create a fresh job directory under {jobs_root}")


def _budget(after_ids, dry_run):
    """GPUs on the partition that may run alongside the new array."""
    try:
        entries = parse_squeue(query_squeue())
    except (OSError, GpuBudgetError) as e:
        if dry_run:
            print(f"warning: cannot read squeue ({e}); dry run assumes 0 GPUs in use", file=sys.stderr)
            return 0, set()
        raise SystemExit(f"cannot check the GPU budget ({e}); refusing to submit")
    chain = dependency_chain(entries, after_ids) if after_ids else set()
    return gpus_in_use(entries, GPU_PARTITION, exclude=chain), chain


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--table", required=True, action="append",
                    help="table id; repeat to put several tables into one job array")
    ap.add_argument("--registry", default=None)
    ap.add_argument("--runs", default="runs")
    ap.add_argument("--rows", default=None, help="comma-separated subset of row keys (one --table only)")
    ap.add_argument("--folds", default=None, help="e.g. 0 or 0,1,2")
    ap.add_argument("--seeds", default=None, help="e.g. 42 or 42,43")
    ap.add_argument("--set", action="append", default=[], help="override passed to every run")
    ap.add_argument("--allow-new-keys", action="store_true",
                    help="accept overrides of keys the config does not list (smoke tests)")
    ap.add_argument("--force", action="store_true", help="rerun runs that already have results")
    ap.add_argument("--python", default=None, help="python executable (default $PULSE_PYTHON or this one)")
    ap.add_argument("--local", action="store_true", help="run sequentially in this shell")
    ap.add_argument("--slurm", action="store_true", help="write and submit a SLURM job array")
    ap.add_argument("--cpu", action="store_true", help="CPU partition and --device cpu (smoke tests)")
    ap.add_argument("--name", default=None, help="job name (default pulse-tab<ids>)")
    ap.add_argument("--after", default=None,
                    help="job id(s) the array waits for (sbatch --dependency=afterany:...)")
    ap.add_argument("--allow-dirty", action="store_true",
                    help="submit GPU runs from a checkout with uncommitted changes (not for the paper)")
    ap.add_argument("--dry-run", action="store_true", help="write job files but do not submit")
    ap.add_argument("--slurm-config", default=str(REPO / "configs" / "slurm.yaml"))
    args = ap.parse_args(argv)

    registry = load_registry(args.registry)
    specs = [table_spec(registry, t) for t in dict.fromkeys(args.table)]
    if len(specs) > 1 and args.rows:
        raise SystemExit("--rows needs a single --table")
    kinds = {s.get("resources", "gpu") for s in specs}
    if len(kinds) > 1:
        raise SystemExit(f"tables {args.table} mix resources {sorted(kinds)}; submit them separately")
    python = args.python or os.environ.get("PULSE_PYTHON") or sys.executable
    cmds, bad_keys, env_need, per_table = [], [], {}, []
    for spec in specs:
        c, skipped, bad, need = build_commands(spec, args, python, registry, args.rows)
        print(f"table {spec['id']}: {len(c)} run(s) to do, {skipped} already done", file=sys.stderr)
        cmds += c
        bad_keys += bad
        env_need.update(need)
        per_table.append({"table": spec["id"], "runs": len(c), "skipped": skipped})
    if bad_keys and not args.allow_new_keys:
        lines = [f"{where}: {key}" + (f"  (did you mean {near}?)" if near else "")
                 for where, key, near in bad_keys]
        raise SystemExit("override key(s) not in the row's config (misspelled?); pass "
                         "--allow-new-keys if the task reads them with a default:\n  "
                         + "\n  ".join(dict.fromkeys(lines)))
    if not cmds:
        return 0
    if not (args.slurm or args.local):
        print("\n".join(cmds))
        return 0
    root = os.environ.get("PULSE_ROOT")
    if not root:
        raise SystemExit("set PULSE_ROOT to the dataset root first")
    env_export = check_env(env_need)
    if args.local:
        env = dict(os.environ)
        for c in cmds:
            print(f"+ {c}", file=sys.stderr, flush=True)
            subprocess.run(c, shell=True, check=True, cwd=str(REPO), env=env)
        return 0

    with open(args.slurm_config, encoding="utf-8") as f:
        sc = yaml.safe_load(f)
    use_gpu = kinds == {"gpu"} and not args.cpu
    after_ids = normalize_job_ids(args.after) if args.after else ()
    sha = _git("rev-parse", "HEAD")
    dirty = _git("status", "--porcelain", "--untracked-files=no")
    if use_gpu and dirty and not args.allow_dirty:
        raise SystemExit(f"checkout {REPO} has uncommitted changes; GPU runs must come from a "
                         "clean commit (launch from a frozen clone; --allow-dirty to override)")
    name = args.name or "pulse-tab" + "+".join(s["id"] for s in specs)
    job_dir = new_job_dir(Path(args.runs) / "_jobs", name)
    (job_dir / "logs").mkdir()
    cmd_file = job_dir / "commands.txt"
    cmd_file.write_text("\n".join(cmds) + "\n", encoding="utf-8", newline="\n")
    account = os.environ.get(ACCOUNT_ENV) or sc.get("account") or ""
    values = {"JOB_NAME": name, "ACCOUNT": account,
              "LAST_INDEX": len(cmds) - 1, "LOG_DIR": (job_dir / "logs").resolve().as_posix(),
              "REPO_DIR": REPO.as_posix(), "PULSE_ROOT": root,
              "COMMANDS": cmd_file.resolve().as_posix(),
              "SBATCH_EXTRA": (f"#SBATCH --dependency=afterany:{':'.join(after_ids)}"
                               if after_ids else ""),
              "EXTRA_ENV": "\n".join(f"export {k}={shlex.quote(v)}" for k, v in env_export.items())}
    info = {"name": name, "tables": per_table, "n_tasks": len(cmds), "after": list(after_ids),
            "git_sha": sha, "git_dirty": bool(dirty), "resources": "gpu" if use_gpu else "cpu"}
    if use_gpu:
        g = sc["gpu"]
        cap = gpu_cap_from_env()
        in_use, chain = _budget(after_ids, args.dry_run)
        try:
            throttle = plan_throttle(len(cmds), int(g["gpus_per_task"]), cap, in_use)
        except GpuBudgetError as e:
            raise SystemExit(str(e))
        not_counted = sorted(chain)
        print(f"GPU budget: {in_use} in use on {GPU_PARTITION} besides the dependency chain "
              f"{not_counted or '(none)'}, cap {cap} -> array throttle {throttle} x "
              f"{g['gpus_per_task']} GPU", file=sys.stderr)
        values.update(GPUS_PER_TASK=g["gpus_per_task"], CPUS=g["cpus_per_task"], MEM=g["mem"],
                      TIME=g["time"], THROTTLE=throttle)
        text = render("gpu_array.sbatch.in", values)
        info.update(throttle=throttle, gpu_cap=cap, gpus_in_use=in_use, not_counted=not_counted)
    else:
        c = sc["cpu"]
        throttle = min(len(cmds), int(c.get("max_concurrent", 8)))
        values.update(PARTITION=c["partition"], CPUS=c["cpus_per_task"], MEM=c["mem"],
                      TIME=c["time"], THROTTLE=throttle)
        text = render("cpu_array.sbatch.in", values)
        info.update(throttle=throttle)
    job_file = job_dir / "job.sbatch"
    job_file.write_text(text, encoding="utf-8", newline="\n")
    print(f"wrote {job_file} ({len(cmds)} tasks, throttle {throttle}"
          + (f", after {':'.join(after_ids)}" if after_ids else "") + ")", file=sys.stderr)
    if args.dry_run:
        (job_dir / "job.json").write_text(json.dumps(info, indent=1) + "\n", encoding="utf-8",
                                          newline="\n")
        return 0
    res = subprocess.run(["sbatch", "--parsable", str(job_file)], capture_output=True, text=True)
    sys.stderr.write(res.stderr)
    if res.returncode != 0:
        return res.returncode
    job_id = res.stdout.strip().split(";")[0]
    info["job_id"] = job_id
    (job_dir / "job.json").write_text(json.dumps(info, indent=1) + "\n", encoding="utf-8", newline="\n")
    print(f"submitted {name}: job {job_id}", file=sys.stderr)
    print(job_id)
    return 0


if __name__ == "__main__":
    sys.exit(main())