Buckets:
| # Formulation ablation (docs/decisions.md 2026-09-15 "PRE-REGISTRATION (2026-09-15T02:19:20Z) — Formulation ablation", EXPLANATORY, not for | |
| # adoption) — detached lane runner. | |
| # | |
| # setsid nohup bash aide/train/run_formulation_abs.sh <A|B> > aide/results/formulation_abs/lane_<X>.log 2>&1 & | |
| # | |
| # Lane A motionbert_kp_abs_s{42,123,456}_v2 : --model motionbert_kp --formulation abs, hardening-D recipe verbatim + --loss euc (as motionbert_kp_s*_v2) | |
| # Lane B cwm3d_kp_abs_s{42,123,456}_v2 : --model cwm3d --formulation abs, Stage-A recipe verbatim (K's loss defaults euc + 0.1 NLL + 0.05 bone + 0.01 jerk) | |
| # Every python launch passes the launch gate first (re-checked every 30 s, under a launcher-wide flock so two lanes cannot pass it at once): | |
| # free VRAM >= need + 4 GB, MemAvailable >= 28 GB, fewer than MAX_TRAINERS (= 2) trainer processes alive on the host | |
| # (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"` — this COUNTS the BATON U0 arms of train/run_can_arms.sh), OMP/MKL threads 4. | |
| # Sequential inside a lane -> at most 2 trainers alive in total. Existing runs are never touched: a run whose run json exists is skipped; new | |
| # run dirs only; no pkill. PIDs -> aide/results/formulation_abs/pids.txt. Selection is val-only inside train_kp (EMA All-MPJPE); the official | |
| # test split is evaluated once per run by the existing harness. | |
| set -u | |
| LANE=${1:?lane A|B} | |
| ROOT=/home/USER/cwm3 | |
| # Run from a per-launch snapshot so that editing this file while a lane is alive can never change the running lane (as run_kv2_program.sh). | |
| if [ -z "${FA_SNAPSHOT:-}" ]; then | |
| mkdir -p "$ROOT/aide/results/formulation_abs/launcher_snapshots" | |
| SNAP="$ROOT/aide/results/formulation_abs/launcher_snapshots/run_formulation_abs_${LANE}_$(date -u +%Y%m%dT%H%M%SZ).sh" | |
| cp "${BASH_SOURCE[0]}" "$SNAP" && FA_SNAPSHOT=$SNAP exec bash "$SNAP" "$@" | |
| fi | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| FA=$ROOT/aide/results/formulation_abs | |
| PIDS=$FA/pids.txt | |
| LOCK=$FA/launch.lock | |
| MAX_TRAINERS=${MAX_TRAINERS:-2} | |
| DRY_RUN=${DRY_RUN:-0} # 1: print every assembled command instead of launching | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| mkdir -p "$FA" | |
| cd "$ROOT" || exit 1 | |
| # Stage-A / hardening-D recipe verbatim (aide/train/run_arch_attribution.sh:24 without --loss euc = K's loss defaults); tensor-output models add --loss euc. | |
| RECIPE_K="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0" | |
| RECIPE="$RECIPE_K --loss euc" | |
| COMMON="--no-lock --max-minutes 45" | |
| NOTES="formulation_abs;explanatory" | |
| SEEDS="42 123 456" | |
| log() { echo "$(date -u +%FT%TZ) [lane $LANE] $*"; } | |
| gate() { # gate <need_gb>: block until the launch gate is satisfied (caller holds the launcher flock) | |
| local need=$1 free_mib avail_kb ntr ok | |
| while :; do | |
| ok=1 | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ') | |
| avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain)") | |
| [ "${free_mib:-0}" -lt $(( (need + 4) * 1024 )) ] && ok=0 | |
| [ "${avail_kb:-0}" -lt $(( 28 * 1024 * 1024 )) ] && ok=0 | |
| [ "${ntr:-9}" -ge "$MAX_TRAINERS" ] && ok=0 | |
| if [ "$ok" = 1 ]; then | |
| log "gate OK: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= 28), trainers ${ntr} (< ${MAX_TRAINERS})" | |
| return 0 | |
| fi | |
| log "gate WAIT: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB, trainers ${ntr} (< ${MAX_TRAINERS})" | |
| sleep 30 | |
| done | |
| } | |
| run() { # run <name> <need_gb> <cmd...>: skip if finished, flock -> gate -> launch -> release, record pid, wait | |
| local name=$1 need=$2; shift 2 | |
| if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists; existing runs are never re-run)"; return 0; fi | |
| if [ "$DRY_RUN" = 1 ]; then log "DRY $name (need ${need} GB): $*"; return 0; fi | |
| exec 9>"$LOCK" | |
| flock 9 | |
| gate "$need" | |
| "$@" > "$RUNS/${name}_train.log" 2>&1 & | |
| local pid=$! | |
| echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid START" >> "$PIDS" | |
| log "launched $name pid $pid" | |
| sleep 5 # the new trainer is visible to the other lane's pgrep before the lock is released | |
| flock -u 9 | |
| exec 9>&- | |
| wait "$pid"; local rc=$? | |
| echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid END rc=$rc" >> "$PIDS" | |
| log "finished $name rc=$rc" | |
| return $rc | |
| } | |
| refs_for() { # <seed>: evaluation-only paired references (all exist for seeds 42 / 123 / 456) | |
| echo "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${1}_v2,kp=cwm3d_kp_s${1}_v2,mb=motionbert_kp_s${1}_v2" | |
| } | |
| case "$LANE" in | |
| A) | |
| for s in $SEEDS; do | |
| name="motionbert_kp_abs_s${s}_v2" | |
| # shellcheck disable=SC2086 | |
| run "$name" 3 "$PY" aide/train/train_kp.py --model motionbert_kp --formulation abs --seed "$s" --name "$name" $RECIPE \ | |
| --refs "$(refs_for "$s")" $COMMON --notes "$NOTES" || log "WARN: $name failed (rc $?), continuing" | |
| done ;; | |
| B) | |
| for s in $SEEDS; do | |
| name="cwm3d_kp_abs_s${s}_v2" | |
| # shellcheck disable=SC2086 | |
| run "$name" 3 "$PY" aide/train/train_kp.py --model cwm3d --formulation abs --seed "$s" --name "$name" $RECIPE_K \ | |
| --refs "$(refs_for "$s")" $COMMON --notes "$NOTES" || log "WARN: $name failed (rc $?), continuing" | |
| done ;; | |
| *) echo "unknown lane $LANE"; exit 2 ;; | |
| esac | |
| log "lane $LANE done" | |
| [ "$DRY_RUN" = 1 ] || echo "$(date -u +%FT%TZ) lane=$LANE LANE_DONE" >> "$PIDS" | |
Xet Storage Details
- Size:
- 5.64 kB
- Xet hash:
- 0d85520c0009343761ea2afefc13adac9a33562dc5e0a88e2567d07a1ff8ad2b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.