Buckets:
| """Roadmap supplement B §B2 (AIDE SOTA hardening) GPU queue -- concurrent, no lock, launch checks. | |
| python aide/train/run_hardening.py [--max-parallel 3] [--only a,b] [--skip c,d] [--dry-run] [--max-minutes 45] | |
| Launch checks before every job (re-checked every 30 s): free VRAM >= need + 4 GB, MemAvailable >= 14 GB (coordinator rule), | |
| `pgrep -c -f "train_cwm3|train_kp|pretrain"` + 1 < 4 (coordinator cap 2026-09-12: fewer than four trainers alive across | |
| ALL agents), OMP/MKL threads 4. Runs whose train log was touched in the last 15 min without a run json are treated as | |
| in progress (another driver instance) and skipped; --skip lists names explicitly. | |
| Every job is `train_kp.py <argv> --no-lock` with evaluation inside the job (val + test harness, run json). | |
| Queue (all on the locked protocol; selection on val only; test evaluated once per run): | |
| gate cwm3d_vis_sem_gate_s{123,456}_v2 completes the traffic-gate variant (iii) to 3 seeds (P0-C gate_override rows) | |
| D motionbert_kp_s*_v2, stgcn_kp_s*_v2 fair keypoint baselines, shared recipe + augmentation (P1-D) | |
| hybrid cwm3d_kp_spatial_s*_v2 CWM3-D-kp + spatial attention block before the part tokenizer (kp ref seed-matched); | |
| adopt only if paired Δ HM vs CWM3-D-kp CI excludes 0 on >= 2/3 seeds with All not worse | |
| B1 cwm3d_vis_sem_mw5hw_s*_v2 --motion-weight 5 --h-weights 1,1,1,2,3 ; cwm3d_vis_sem_mos2_s*_v2 --motion-oversample 2 | |
| B2 cwm3d_vis_sem_dct_s*_v2 --dct head | |
| B3 cwm3d_vis_sem_rl_s*_v2 --refine-late (zero-init late-horizon refinement head) | |
| A1 cwm3d_vis_sema_vgp_s*_v2 view-gate heads + --patience-f1 40 (dual selection -> __f1sel) | |
| A2 cwm3d_vis_sema_vge_s*_v2 view-gate heads + --egomotion (dual selection -> __f1sel) | |
| Kill rules are applied afterwards by aide/eval/report_hardening.py from the runs' val metrics (never from test). | |
| Log: results/runs/hardening_driver.log. Failures are logged and do not stop the queue. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS"): | |
| os.environ.setdefault(_v, "4") | |
| import argparse # noqa: E402 | |
| import re # noqa: E402 | |
| import subprocess # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.train import run_concurrent as RC # noqa: E402 | |
| from aide.train import run_stage_bc as Q # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| PY = sys.executable | |
| RUNS = P.RESULTS / "runs" | |
| LOG = RUNS / "hardening_driver.log" | |
| SEEDS = (42, 123, 456) | |
| REFS_KP = "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s42_v2" | |
| MAX_TRAINERS = 4 # rule (coordinator 2026-09-12): pgrep -c -f "train_cwm3|train_kp|pretrain" must stay < 4 | |
| TRAINER_PATTERN = "train_cwm3|train_kp|pretrain" | |
| IN_PROGRESS_MIN = 15 | |
| def cfg(name, model, seed, family, extra=(), refs=Q.REFS_B2, notes="", minutes=None): | |
| argv = ["--model", model, "--seed", str(seed), "--name", name, *Q.RECIPE, *extra, "--refs", refs, "--no-lock", | |
| "--notes", notes] | |
| return {"name": name, "family": family, "argv": argv, "minutes": minutes} | |
| QUEUE = ( | |
| [c for c in Q.QUEUE if c["name"] in ("cwm3d_vis_sem_gate_s123_v2", "cwm3d_vis_sem_gate_s456_v2")] | |
| # coordinator 2026-09-12: CWM3-D-kp + DSTformer-style spatial attention before the part tokenizer; kp reference seed-matched | |
| + [cfg(f"cwm3d_kp_spatial_s{s}_v2", "cwm3d_kp_spatial", s, "cwm3d", refs=f"zvshrink=zv_shrink_val,simlpe=simlpe_kp_s{s}_v2,kp=cwm3d_kp_s{s}_v2", | |
| notes="hardening;hybrid:spatial_attention_before_part_tokenizer;kp_ref_seed_matched") for s in SEEDS] | |
| + [cfg(f"motionbert_kp_s{s}_v2", "motionbert_kp", s, "cwm3d", ["--loss", "euc"], refs=REFS_KP, | |
| notes="hardening;D:motionbert_kp_shared_recipe") for s in SEEDS] | |
| + [cfg(f"stgcn_kp_s{s}_v2", "stgcn_kp", s, "cwm3d", ["--loss", "euc"], refs=REFS_KP, | |
| notes="hardening;D:stgcn_kp_shared_recipe") for s in SEEDS] | |
| + [cfg(f"cwm3d_vis_sema_vgp_s{42}_v2", "cwm3d", 42, "cwm3d_vis", Q.VIS + Q.SEMA + ["--sem-view-gate", "--patience-f1", "40"], | |
| notes="hardening;A1:view_gate+patience_f1_40", minutes=90)] | |
| + [cfg(f"cwm3d_vis_sem_mw5hw_s{s}_v2", "cwm3d", s, "cwm3d_vis", Q.VIS + Q.SEM + ["--motion-weight", "5", "--h-weights", "1,1,1,2,3"], | |
| notes="hardening;B1:motion_weight5+h_weights_1,1,1,2,3") for s in SEEDS] | |
| + [cfg(f"cwm3d_vis_sem_mos2_s{s}_v2", "cwm3d", s, "cwm3d_vis", Q.VIS + Q.SEM + ["--motion-oversample", "2"], | |
| notes="hardening;B1:motion_oversample2") for s in SEEDS] | |
| + [cfg(f"cwm3d_vis_sema_vgp_s{123}_v2", "cwm3d", 123, "cwm3d_vis", Q.VIS + Q.SEMA + ["--sem-view-gate", "--patience-f1", "40"], | |
| notes="hardening;A1:view_gate+patience_f1_40", minutes=90)] | |
| + [cfg(f"cwm3d_vis_sem_dct_s{s}_v2", "cwm3d", s, "cwm3d_vis", Q.VIS + Q.SEM + ["--dct"], | |
| notes="hardening;B2:dct_head") for s in SEEDS] | |
| + [cfg(f"cwm3d_vis_sem_rl_s{s}_v2", "cwm3d", s, "cwm3d_vis", Q.VIS + Q.SEM + ["--refine-late"], | |
| notes="hardening;B3:late_horizon_refinement_head") for s in SEEDS] | |
| + [cfg(f"cwm3d_vis_sema_vgp_s{456}_v2", "cwm3d", 456, "cwm3d_vis", Q.VIS + Q.SEMA + ["--sem-view-gate", "--patience-f1", "40"], | |
| notes="hardening;A1:view_gate+patience_f1_40", minutes=90)] | |
| + [cfg(f"cwm3d_vis_sema_vge_s{s}_v2", "cwm3d", s, "cwm3d_vis", Q.VIS + Q.SEMA + ["--sem-view-gate", "--egomotion"], | |
| notes="hardening;A2:view_gate+egomotion", minutes=60) for s in SEEDS] | |
| ) | |
| def log(msg): | |
| line = f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {msg}" | |
| print(line, flush=True) | |
| with open(LOG, "a") as f: | |
| f.write(line + "\n") | |
| def trainers_alive(): | |
| """Distinct trainer PROCESSES alive across all agents: python processes whose command line matches TRAINER_PATTERN and | |
| whose parent does not match itself (DataLoader / evaluation forks share the parent's command line). `tail -F`, grep and | |
| shell wrappers whose argument strings happen to contain the pattern are not trainers and are excluded (2026-09-12: a | |
| `tail -F ...pretrain...log` of another agent was counted by the plain `pgrep -c -f` and stalled the queue).""" | |
| out = subprocess.run(["ps", "-eo", "pid=,ppid=,args="], capture_output=True, text=True).stdout | |
| rows = [] | |
| for line in out.splitlines(): | |
| parts = line.split(None, 2) | |
| if len(parts) < 3: | |
| continue | |
| pid, ppid, args = int(parts[0]), int(parts[1]), parts[2] | |
| if re.search(TRAINER_PATTERN, args) and "python" in args.split()[0]: | |
| rows.append((pid, ppid)) | |
| pids = {p_ for p_, _ in rows} | |
| return sum(1 for p_, pp in rows if pp not in pids) | |
| def in_progress(name): | |
| """Another driver instance is training this run: train log touched recently and no run json yet.""" | |
| lg = RUNS / f"{name}_train.log" | |
| return lg.exists() and (time.time() - lg.stat().st_mtime) < IN_PROGRESS_MIN * 60 and not Q.is_done(name) | |
| RAM_MIN_GB = 14.0 # coordinator rule 2026-09-12: host MemAvailable >= 14 GB (the shared host runs other agents' trainers) | |
| def checks_ok(family): | |
| g, r = RC.gpu_free_gb(), RC.ram_avail_gb() | |
| n = trainers_alive() | |
| ok = g >= RC.NEED_GB.get(family, 2.0) + 4.0 and r >= RAM_MIN_GB and (n + 1 < MAX_TRAINERS) | |
| return ok, f"vram_free {g:.1f} GB, ram_avail {r:.1f} GB, trainers_alive {n}" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--max-parallel", type=int, default=3) | |
| ap.add_argument("--only", default="") | |
| ap.add_argument("--skip", default="", help="comma list of run names to leave alone (in flight elsewhere)") | |
| ap.add_argument("--dry-run", action="store_true") | |
| ap.add_argument("--max-minutes", type=float, default=30.0) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| queue = [c for c in QUEUE if not a.only or c["name"] in a.only.split(",")] | |
| skip = set(x for x in a.skip.split(",") if x) | |
| todo = [c for c in queue if not Q.is_done(c["name"]) and c["name"] not in skip and not in_progress(c["name"])] | |
| log(f"hardening runner start (max {a.max_parallel} parallel, trainers < {MAX_TRAINERS}): {len(todo)} to run: {[c['name'] for c in todo]}") | |
| if a.dry_run: | |
| for c in todo: | |
| print(c["name"], " ".join(c["argv"]), "--max-minutes", c.get("minutes") or a.max_minutes) | |
| return 0 | |
| env = {**os.environ, "OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4", "OPENBLAS_NUM_THREADS": "4"} | |
| running, results, retried = {}, [], set() | |
| while todo or running: | |
| for name, (proc, t_s, logf) in list(running.items()): | |
| rc = proc.poll() | |
| if rc is not None: | |
| logf.close() | |
| dur = (time.time() - t_s) / 60 | |
| results.append({"name": name, "rc": rc, "minutes": round(dur, 2)}) | |
| log(f"{'end' if rc == 0 else 'FAILED'} {name}: rc {rc}, {dur:.2f} min") | |
| del running[name] | |
| if rc != 0 and "CUDA out of memory" in (RUNS / f"{name}_train.log").read_text()[-4000:] and name not in retried: | |
| retried.add(name) | |
| todo.append(next(c for c in queue if c["name"] == name)) | |
| log(f"requeue {name} once (CUDA OOM caused by concurrent GPU use)") | |
| while todo and len(running) < a.max_parallel: | |
| c = todo[0] | |
| ok, msg = checks_ok(c["family"]) | |
| if not ok: | |
| log(f"launch check failed for {c['name']} ({msg}); retry in 30 s") | |
| break | |
| todo.pop(0) | |
| if Q.is_done(c["name"]) or in_progress(c["name"]): | |
| log(f"skip {c['name']}: finished or in progress elsewhere") | |
| continue | |
| argv = [x for x in c["argv"] if x != "--defer-eval"] + ["--max-minutes", str(c.get("minutes") or a.max_minutes)] | |
| logf = open(RUNS / f"{c['name']}_train.log", "a") | |
| proc = subprocess.Popen([PY, str(ROOT / "aide/train/train_kp.py")] + argv, cwd=str(ROOT), env=env, | |
| stdout=logf, stderr=subprocess.STDOUT) | |
| running[c["name"]] = (proc, time.time(), logf) | |
| log(f"start {c['name']} (pid {proc.pid}; {msg}; {len(running)} running)") | |
| time.sleep(20) # let the new trainer register on the GPU before the next check | |
| time.sleep(5 if running else 30) | |
| log(f"queue finished: {len(results)} jobs, failures {[r['name'] for r in results if r['rc'] != 0]}") | |
| write_run_json(P.RESULTS / "tables", "hardening_driver", [__file__, Q.__file__, P.__file__], [P.CACHE_NPZ], seed=None, | |
| t_start=t0, input_listing=False, extra={"results": results, "max_parallel": a.max_parallel}) | |
| log("driver exit 0") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 11 kB
- Xet hash:
- 6564ae7f3904d84875c209ffb8af9ff3093398d7c92b591d112d4b42d70da4b9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.