#!/usr/bin/env python """Generate (and optionally submit) every run of one or more tables. python scripts/run_table.py --table 3 # print the commands python scripts/run_table.py --table 3 --local # run them here, one by one python scripts/run_table.py --table 3 --slurm # GPU job array on ex03A800 python scripts/run_table.py --table 3 --table 7 --slurm --name W1 --after 467240 # one array for both tables, starts # after job 467240 has ended python scripts/run_table.py --table 3 --slurm --cpu --folds 0 --seeds 42 \ --set train.epochs=1 --allow-new-keys # CPU smoke test One run = (row, fold, seed) -> runs///fold_seed/results.json. Runs whose results.json exists are skipped unless --force. Several --table flags put all their runs into one job array (all tables must have the same ``resources``). Overrides (``--set`` and the rows' ``set:``) must name keys that exist in the row's configuration (after ``_base_`` inheritance); a misspelled key is refused. ``--allow-new-keys`` accepts keys the task reads with a default but the YAML does not list (e.g. ``train.max_train_batches`` in smoke tests). GPU submission: partition ex03A800 only. The user's running + pending GPUs on that partition are counted first; the array is throttled so the total stays within the cap (default 4; MAXGPU=1..8), and nothing is submitted if the cap is already reached. ``--after JOBID[,JOBID...]`` adds ``--dependency=afterany:...``; those jobs and every job they wait for are not counted, because the new array starts only after they have ended. GPU arrays are refused from a checkout with uncommitted changes (the runs would record a dirty tree). --dry-run writes the job files without submitting. After a submission the job id is printed on stdout (``W1=$(python scripts/run_table.py ...)``). Environment the runs need is checked before anything is written and exported in the job script: ``PULSE_ROOT``; ``PULSE_SSL_WEARABLES_DIR`` for harnet rows; ``PULSE_S9_ANNOTATIONS`` for F5 rows with S9 pre-training. The SLURM account is ``$PULSE_SLURM_ACCOUNT`` or ``account`` in configs/slurm.yaml (none: the user's default account). """ from __future__ import annotations import argparse import datetime as dt import difflib import importlib.util import json import os import re import shlex import subprocess import sys from pathlib import Path REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO)) import yaml # noqa: E402 from pulse.aggregate import load_registry, row_source, run_dir, table_spec # noqa: E402 from pulse.config import apply_overrides, load_yaml # noqa: E402 from pulse.slurm import (GPU_PARTITION, GpuBudgetError, dependency_chain, # noqa: E402 gpu_cap_from_env, gpus_in_use, normalize_job_ids, parse_squeue, plan_throttle, query_squeue, render) ROOT_TOKEN = '"$PULSE_ROOT"' SSL_ENV = "PULSE_SSL_WEARABLES_DIR" S9_ENV = "PULSE_S9_ANNOTATIONS" ACCOUNT_ENV = "PULSE_SLURM_ACCOUNT" def _ints(s): return [int(x) for x in s.split(",") if x != ""] if s else None def check_reuse(spec, row, registry): """A ``reuse: "
/"`` row is produced by that table: same config and overrides.""" src_t, src_r = row_source(spec, row) if registry is None or src_t not in registry: raise SystemExit(f"row {row['key']}: reuse source table {src_t!r} not in the registry") src = next((r for r in registry[src_t].get("rows", []) if r["key"] == src_r), None) if src is None or src.get("reuse"): raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} is not a run row") if (src["config"], list(src.get("set", []))) != (row["config"], list(row.get("set", []))) \ or registry[src_t]["task"] != spec["task"] or registry[src_t]["split"] != spec["split"]: raise SystemExit(f"row {row['key']}: reuse source {src_t}/{src_r} has a different " "task/split/config/set") def _key_paths(d, prefix=""): for k, v in d.items(): p = f"{prefix}{k}" yield p if isinstance(v, dict): yield from _key_paths(v, p + ".") def unknown_override_keys(config_path, overrides): """``[(key, suggestion or None)]`` for overrides whose key is not in the merged config.""" cfg = load_yaml(config_path) known = list(_key_paths(cfg)) bad = [] for item in overrides: if "=" not in item: raise SystemExit(f"override must be key=value, got {item!r}") key = item.split("=", 1)[0].strip() node, ok = cfg, True for part in key.split("."): if not isinstance(node, dict) or part not in node: ok = False break node = node[part] if not ok: near = difflib.get_close_matches(key, known, n=1, cutoff=0.6) bad.append((key, near[0] if near else None)) return bad def env_requirements(task, cfg): """Environment variables a run of this (resolved) config needs: ``{name: why}``.""" need = {} model = cfg.get("model") or {} if isinstance(model, dict) and model.get("name") == "harnet" and not model.get("repo_dir"): need[SSL_ENV] = "harnet rows load the pinned ssl-wearables checkout (compute nodes are offline)" if task == "f5" and cfg.get("init") == "s9": need[S9_ENV] = "F5 S9 pre-training reads /vX/s9.json" return need def check_env(need, environ=None): """Validate the needed variables; return ``{name: absolute value}`` to export in the job.""" environ = os.environ if environ is None else environ out, errors = {}, [] for name, why in sorted(need.items()): val = environ.get(name) if not val: errors.append(f"set {name} ({why})") continue p = Path(os.path.expanduser(val)).resolve() if name == SSL_ENV and not (p / "hubconf.py").is_file(): errors.append(f"{name}={val}: no hubconf.py there (fetch it with " "python -m pulse.models.t1.harnet --fetch DIR)") elif name == S9_ENV and not (p.is_dir() and any(p.glob("v*/s9.json"))): errors.append(f"{name}={val}: no vX/s9.json label files there") out[name] = p.as_posix() if errors: raise SystemExit("missing environment for these runs:\n " + "\n ".join(errors)) return out def build_commands(spec, args, python, registry=None, rows_filter=None): """``(commands, n_skipped, unknown_keys, env_needed)`` for one table.""" if not spec["rows"]: raise SystemExit(f"table {spec['id']} has no rows yet (status: {spec.get('status')}); " "add them to configs/tables.yaml") if importlib.util.find_spec(f"pulse.tasks.{spec['task']}") is None: raise SystemExit(f"task module pulse.tasks.{spec['task']} does not exist yet") rows = spec["rows"] if rows_filter: want = set(rows_filter.split(",")) unknown = want - {r["key"] for r in rows} if unknown: raise SystemExit(f"unknown rows {sorted(unknown)}") rows = [r for r in rows if r["key"] in want] folds = _ints(args.folds) if args.folds else spec["folds"] seeds = _ints(args.seeds) if args.seeds else spec["seeds"] cmds, skipped, bad_keys, env_need = [], 0, [], {} for row in rows: cfg = REPO / row["config"] if not cfg.is_file(): raise SystemExit(f"row {row['key']}: config {row['config']} does not exist") if row.get("reuse"): check_reuse(spec, row, registry) print(f"row {spec['id']}/{row['key']}: reuses the runs of {row['reuse']} (run that table)", file=sys.stderr) continue overrides = list(row.get("set", [])) + list(args.set) for key, near in unknown_override_keys(cfg, overrides): bad_keys.append((f"{spec['id']}/{row['key']}", key, near)) row_cmds = 0 for k in folds: for s in seeds: out = run_dir(args.runs, spec["id"], row["key"], k, s) if (Path(out) / "results.json").is_file() and not args.force: skipped += 1 continue parts = [python, "-m", f"pulse.tasks.{spec['task']}", "--root", ROOT_TOKEN, "--split", spec["split"], "--fold", str(k), "--seed", str(s), "--config", row["config"], "--out", Path(out).as_posix()] for kv in overrides: parts += ["--set", kv] if args.cpu: parts += ["--device", "cpu"] # the dataset root stays a variable: it is exported by the job script cmds.append(" ".join(p if p is ROOT_TOKEN else shlex.quote(p) for p in parts)) row_cmds += 1 if row_cmds: env_need.update(env_requirements(spec["task"], apply_overrides(load_yaml(cfg), overrides))) return cmds, skipped, bad_keys, env_need def _git(*a): try: r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True, timeout=60) except OSError: return None return r.stdout.strip() if r.returncode == 0 else None def new_job_dir(jobs_root: Path, name: str) -> Path: """A fresh ``/-