Buckets:
| """cwm3/train/run_queue.py — run a list of (stage, variant, seed) trainer invocations sequentially (each waits for the GPU lock itself). | |
| python run_queue.py --jobs U0:kin_can:0 U0:kin_can_vc:0 V0:can4s:0 ... [--extra "--minutes 18"] | |
| Skips jobs whose run dir already has PRED_DONE; stops on the first failure unless --keep-going. Log: results/runs/queue_<utc>.log. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| PY = sys.executable | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--jobs", nargs="+", required=True, help="stage:variant:seed") | |
| ap.add_argument("--extra", default="", help="extra trainer args") | |
| ap.add_argument("--keep-going", action="store_true") | |
| a = ap.parse_args() | |
| logp = CWM3 / "results" / "runs" / f"queue_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.log" | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(logp, "a") as f: | |
| f.write(line + "\n") | |
| for job in a.jobs: | |
| stage, variant, seed = job.split(":") | |
| out = CWM3 / "results" / "runs" / f"{stage}_{variant}_s{seed}" | |
| if (out / "PRED_DONE").exists(): | |
| log(f"skip {job} (PRED_DONE)") | |
| continue | |
| cmd = [PY, str(CWM3 / "train" / "train_cwm3.py"), "--stage", stage, "--variant", variant, "--seed", seed] + a.extra.split() | |
| log(f"start {job}: {' '.join(cmd)}") | |
| t0 = time.time() | |
| with open(out.parent / f"{stage}_{variant}_s{seed}.stdout.log", "a") as f: | |
| code = subprocess.call(cmd, stdout=f, stderr=subprocess.STDOUT, cwd=str(CWM3)) | |
| log(f"end {job}: exit {code} in {(time.time() - t0) / 60:.1f} min") | |
| if code != 0 and not a.keep_going: | |
| log("stopping on failure") | |
| sys.exit(code) | |
| log("QUEUE DONE") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 1.98 kB
- Xet hash:
- 732da597d06ffe82f989eb3325c2d4c4b9650a1667f1cd7222514cba8a4292f7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.