Buckets:
| # Pre-registered HM h=5 arms — docs/decisions.md row "PRE-REG (2026-09-17T07:00:22Z) — three owner-requested experiments", part (A) — detached queue. | |
| # | |
| # setsid nohup bash aide/train/run_hm_h5_arms.sh > aide/results/hm_h5_arms/queue.log 2>&1 & | |
| # | |
| # 5 arms x 3 seeds (42 / 123 / 456) = 15 fine-tunes of the paper's adopted row (CockpitWM `cwm3d` + BATON pretraining): the argv of | |
| # cwm3d_kp_batonpre_trunkp123_s<k>_v2 VERBATIM (run json `command`) + the arm flag(s); same pretraining init | |
| # aide/results/runs/cwm3d_kp_batonpre_pretrain_s123/ckpt_best.pt (sha256 d8200062a1e539bd…, trunk only); selection = val All-MPJPE only (inside | |
| # train_kp, EMA weights); the official test split is evaluated once per run by the harness. | |
| # a1 hw2 --hand-weight 2 (42 hand keypoints x2 in the Euclidean loss, other joints 1) | |
| # a2 hw4 --hand-weight 4 | |
| # b hgain --horizon-gain learned (5 scalars g_h = 1 + δ_h on the anchored residual) | |
| # c hgainp --horizon-gain parts (5 x 2 scalars: hands / other joints) | |
| # d hw2hgain --hand-weight 2 --horizon-gain learned | |
| # Run names: cwm3d_kp_hm5_<tag>_batonpre_trunkp123_s<k>_v2 (new run dirs only; a run whose run json exists is skipped; nothing is ever killed). | |
| # Launch gate before EVERY trainer (task rule): free VRAM >= NEED + 4 GB (NEED = 3 GB = the measured cwm3d fine-tune footprint, rounded up), | |
| # MemAvailable >= 28 GB, trainers alive (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"`) < 4, OMP / MKL threads 4. The holdout V-JEPA2 | |
| # extractor (~28 GB VRAM in ~25-min chunks) and the ECAN control runs share the GPU: this queue only interleaves (waits for the gate), never kills. | |
| # PIDs -> aide/results/hm_h5_arms/pids.txt; per-run train logs -> aide/results/runs/<name>_train.log (as every other lane). | |
| set -u | |
| ROOT=/home/USER/cwm3 | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| OUTD=$ROOT/aide/results/hm_h5_arms | |
| PIDS=$OUTD/pids.txt | |
| NEED_GB=${NEED_GB:-3} | |
| RAM_MIN_GB=${RAM_MIN_GB:-28} | |
| MAXTR=${MAXTR:-4} | |
| POLL_S=${POLL_S:-60} | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| mkdir -p "$OUTD" | |
| cd "$ROOT" || exit 1 | |
| # adopted recipe verbatim (run json of cwm3d_kp_batonpre_trunkp123_s42_v2; --loss euc is the default and all four loss terms stay active) | |
| RECIPE="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0" | |
| INIT="aide/results/runs/cwm3d_kp_batonpre_pretrain_s123/ckpt_best.pt" | |
| SEEDS="42 123 456" | |
| ARMS="hw2 hw4 hgain hgainp hw2hgain" | |
| log() { echo "$(date -u +%FT%TZ) [hm5] $*"; } | |
| arm_flags() { | |
| case "$1" in | |
| hw2) echo "--hand-weight 2" ;; | |
| hw4) echo "--hand-weight 4" ;; | |
| hgain) echo "--horizon-gain learned" ;; | |
| hgainp) echo "--horizon-gain parts" ;; | |
| hw2hgain) echo "--hand-weight 2 --horizon-gain learned" ;; | |
| *) echo "unknown arm $1" >&2; return 1 ;; | |
| esac | |
| } | |
| gate_ok() { # 0 iff the launch gate is satisfied now; GATE_STATE describes it | |
| local free_mib avail_kb ntr ok=1 | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ') | |
| avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain)") | |
| [ "${free_mib:-0}" -lt $(( (NEED_GB + 4) * 1024 )) ] && ok=0 | |
| [ "${avail_kb:-0}" -lt $(( RAM_MIN_GB * 1024 * 1024 )) ] && ok=0 | |
| [ "${ntr:-9}" -ge "$MAXTR" ] && ok=0 | |
| GATE_STATE="free VRAM ${free_mib} MiB (need $((NEED_GB + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= ${RAM_MIN_GB}), trainers ${ntr} (< ${MAXTR})" | |
| [ "$ok" = 1 ] | |
| } | |
| gate() { | |
| local n=0 | |
| until gate_ok; do | |
| [ $((n % 5)) -eq 0 ] && log "gate WAIT [$1]: $GATE_STATE" | |
| n=$((n + 1)); sleep "$POLL_S" | |
| done | |
| log "gate OK [$1]: $GATE_STATE" | |
| } | |
| run() { # run <name> <cmd...>: skip if the run json exists; gate; launch; record pid; wait | |
| local name=$1; shift | |
| local pid rc | |
| if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists; existing runs are never re-run)"; return 0; fi | |
| if pgrep -f -- "python.*--name ${name}( |$)" >/dev/null; then log "skip $name (a trainer with this name is already running)"; return 0; fi | |
| gate "$name" | |
| "$@" > "$RUNS/${name}_train.log" 2>&1 & | |
| pid=$! | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid START" >> "$PIDS" | |
| log "launched $name pid $pid: $*" | |
| wait "$pid"; rc=$? | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid END rc=$rc" >> "$PIDS" | |
| log "finished $name rc=$rc" | |
| sleep 20 # let the harness / file system settle before the next gate check | |
| return $rc | |
| } | |
| [ -f "$INIT" ] || { log "ABORT: pretraining init $INIT missing"; exit 2; } | |
| log "queue start: arms [$ARMS] x seeds [$SEEDS]; gate free VRAM >= $((NEED_GB + 4)) GB, MemAvailable >= ${RAM_MIN_GB} GB, trainers < ${MAXTR}" | |
| for tag in $ARMS; do | |
| for s in $SEEDS; do | |
| name="cwm3d_kp_hm5_${tag}_batonpre_trunkp123_s${s}_v2" | |
| run "$name" "$PY" aide/train/train_kp.py --model cwm3d --seed "$s" --name "$name" $RECIPE $(arm_flags "$tag") \ | |
| --init-ckpt "$INIT" --init-parts trunk \ | |
| --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2" \ | |
| --no-lock --max-minutes 25 \ | |
| --notes "hm_h5_arms;arm=${tag};prereg=2026-09-17T07:00:22Z;init=cwm3d_kp_batonpre_pretrain_s123:trunk;adopted_recipe_verbatim" \ | |
| || log "run $name failed (rc != 0)" | |
| done | |
| done | |
| log "queue done" | |
| echo "$(date -u +%FT%TZ) QUEUE_DONE" >> "$PIDS" | |
Xet Storage Details
- Size:
- 5.59 kB
- Xet hash:
- ff3ed6630e231137b635173359f35e631a4fd058e9c2315c2a7ef77509e7c5d5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.