File size: 5,965 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
#!/usr/bin/env python
"""CPU smoke suite: every run row of every table, fold 0, seed 42, one short epoch.

    export PULSE_ROOT=... PULSE_PYTHON=...                  # + PULSE_SSL_WEARABLES_DIR, PULSE_S9_ANNOTATIONS
    python scripts/smoke_suite.py --runs ~/smoke/runs              # print the run_table calls
    python scripts/smoke_suite.py --runs ~/smoke/runs --submit     # submit CPU job arrays
    python scripts/build_outputs.py --runs ~/smoke/runs --out ~/smoke/out --allow-incomplete
    python scripts/smoke_suite.py --runs ~/gpusmoke/runs --gpu --after JOBID --submit
                                  # GPU variant (W0): GPU tables only, 2 full epochs,
                                  # groups chained one after another

Overrides and exceptions: configs/smoke.yaml.  Tables of one task with the same
overrides and seeds share one CPU array; tables listed with ``after`` wait for
the arrays of those tables (``--after``).  Tables whose rows all reuse other
tables' runs (``4maj``, ``4segmaj``) have nothing to run.  Keep ``--runs``
separate from the paper runs.
"""
from __future__ import annotations

import argparse
import os
import shlex
import subprocess
import sys
from collections import OrderedDict
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))

import yaml  # noqa: E402

from pulse.aggregate import load_registry  # noqa: E402


def plan(registry, smoke, only=None, gpu=False):
    """``[{tables, task, set, seeds, after}]`` in registry order (``gpu``: the GPU tables with
    the ``gpu:`` section of configs/smoke.yaml)."""
    groups: "OrderedDict[tuple, dict]" = OrderedDict()
    sec = smoke["gpu"] if gpu else smoke
    for tid, t in registry.items():
        if only and tid not in only:
            continue
        rows = [r for r in t.get("rows", []) if not r.get("reuse")]
        if not rows or (gpu and t.get("resources", "gpu") != "gpu"):
            continue
        ex = (sec.get("tables") or {}).get(tid, {})
        task = t["task"]
        if task not in sec["tasks"]:
            raise SystemExit(f"configs/smoke.yaml has no overrides for task {task!r} (table {tid})")
        sets = tuple(ex.get("set", sec["tasks"][task]))
        seeds = t["seeds"] if any(not isinstance(s, int) for s in t["seeds"]) \
            else tuple(ex.get("seeds", sec["seeds"]))
        after = tuple(ex.get("after", ()))
        key = (task, sets, tuple(seeds), after, t.get("resources", "gpu"))
        g = groups.setdefault(key, {"tables": [], "task": task, "set": sets, "seeds": tuple(seeds),
                                    "after": after})
        g["tables"].append(tid)
    return list(groups.values())


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--runs", required=True, help="smoke runs directory (not the paper runs)")
    ap.add_argument("--smoke-config", default=str(REPO / "configs" / "smoke.yaml"))
    ap.add_argument("--registry", default=None)
    ap.add_argument("--tables", default=None, help="comma-separated subset of tables")
    ap.add_argument("--submit", action="store_true", help="submit the CPU arrays (default: print)")
    ap.add_argument("--dry-run", action="store_true", help="write job files, do not submit")
    ap.add_argument("--gpu", action="store_true",
                    help="GPU smoke: GPU tables on the GPU partition, groups chained one after another")
    ap.add_argument("--after", default=None, help="job id(s) the first group waits for (--gpu)")
    args = ap.parse_args(argv)
    smoke = yaml.safe_load(Path(args.smoke_config).read_text(encoding="utf-8"))
    registry = load_registry(args.registry)
    only = set(args.tables.split(",")) if args.tables else None
    groups = plan(registry, smoke, only, gpu=args.gpu)
    python = os.environ.get("PULSE_PYTHON") or sys.executable
    folds = ",".join(str(f) for f in (smoke["gpu"] if args.gpu else smoke)["folds"])
    job_of_table = {}
    prev = args.after or ""
    for g in groups:
        cmd = [python, str(REPO / "scripts" / "run_table.py")]
        for t in g["tables"]:
            cmd += ["--table", t]
        cmd += ["--runs", args.runs, "--folds", folds, "--allow-new-keys",
                "--name", f"{'gpusmoke' if args.gpu else 'smoke'}-{g['task']}-{g['tables'][0]}"]
        if not args.gpu:
            cmd += ["--cpu"]
        if all(isinstance(s, int) for s in g["seeds"]):
            cmd += ["--seeds", ",".join(str(s) for s in g["seeds"])]
        for kv in g["set"]:
            cmd += ["--set", kv]
        after = sorted({job_of_table[t] for t in g["after"] if job_of_table.get(t)})
        missing = [t for t in g["after"] if t not in job_of_table and (not only or t in only)]
        if missing and args.submit:
            raise SystemExit(f"{g['tables']}: wait-for tables {missing} were not submitted first")
        if args.gpu and prev:
            after = [prev]          # GPU groups one after another: each may use the whole cap
        if after:
            cmd += ["--after", ",".join(after)]
        if args.submit or args.dry_run:
            cmd += ["--slurm"] + (["--dry-run"] if args.dry_run else [])
        print("+ " + " ".join(shlex.quote(c) for c in cmd), file=sys.stderr, flush=True)
        if not (args.submit or args.dry_run):
            continue
        res = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True)
        sys.stderr.write(res.stderr)
        if res.returncode != 0:
            raise SystemExit(f"run_table failed for {g['tables']} (exit {res.returncode})")
        jid = res.stdout.strip().splitlines()[-1] if res.stdout.strip() else ""
        for t in g["tables"]:
            job_of_table[t] = jid
        prev = jid or prev
        print(f"{'+'.join(g['tables'])}\t{jid or '(nothing to run / dry run)'}", flush=True)
    return 0


if __name__ == "__main__":
    sys.exit(main())