Buckets:
| # Faithful MotionBERT baseline on AIDE (pre-registration docs/decisions.md 2026-09-16T21:28:48Z) — detached, SEQUENTIAL, GPU-gated queue. | |
| # | |
| # setsid nohup bash aide/train/run_motionbert_faithful.sh > aide/results/motionbert_faithful/queue.log 2>&1 & | |
| # | |
| # Runs (aide/train/train_motionbert_official.py, locked harness; new run dirs only, a run whose run json exists is skipped, never re-run): | |
| # (a) mb_official_res_s{42,123,456}_v2 released MB_release weights, anchored residual read-out (zero-init head) | |
| # (b) mb_official_abs_s{42,123,456}_v2 released MB_release weights, absolute-coordinate read-out (their formulation) | |
| # (c) mb_official_rand_s{42,123,456}_v2 same architecture, official random init, residual read-out (value of the released weights) | |
| # then (a) seeds 789 / 1011 ONLY if the total GPU minutes of the 9 runs <= 60 (pre-registered budget rule). | |
| # Launch gate before EVERY trainer (re-checked every 60 s): free VRAM >= need + 4 GB, MemAvailable >= 28 GiB, fewer than 4 trainer processes | |
| # alive (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain|aide/train)"`), OMP/MKL threads 4. `need` = NEED_GB (default 16) until a run of this | |
| # family has recorded its peak VRAM (`peak_gpu_gib` in its run json) -> need = ceil(max peak) + 1. Never pkill; nothing else is touched. | |
| set -u | |
| ROOT=/home/USER/cwm3 | |
| # per-launch snapshot: editing this file while the queue is alive cannot change the running queue | |
| if [ -z "${MF_SNAPSHOT:-}" ]; then | |
| mkdir -p "$ROOT/aide/results/motionbert_faithful/launcher_snapshots" | |
| SNAP="$ROOT/aide/results/motionbert_faithful/launcher_snapshots/run_motionbert_faithful_$(date -u +%Y%m%dT%H%M%SZ).sh" | |
| cp "${BASH_SOURCE[0]}" "$SNAP" && MF_SNAPSHOT=$SNAP exec bash "$SNAP" "$@" | |
| fi | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| MF=$ROOT/aide/results/motionbert_faithful | |
| PIDS=$MF/pids.txt | |
| NEED_GB=${NEED_GB:-16} | |
| MAX_TRAINERS=4 | |
| DRY_RUN=${DRY_RUN:-0} | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| mkdir -p "$MF" | |
| cd "$ROOT" || exit 1 | |
| # pre-registered optimisation + the harness recipe of motionbert_kp_s*_v2 (augmentation / EMA / clip / loss), written out for the record | |
| RECIPE="--epochs 60 --patience 10 --bs 32 --lr-backbone 1e-4 --lr-head 1e-3 --wd 0.01 --warmup 10 --ema 0.999 --clip 1.0 --loss euc --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0" | |
| COMMON="--no-lock --max-minutes 60" | |
| NOTES="motionbert_faithful;prereg=2026-09-16T21:28:48Z" | |
| log() { echo "$(date -u +%FT%TZ) [mb_faithful] $*"; } | |
| need_gb() { # ceil(max peak_gpu_gib over finished runs of this family) + 1, else NEED_GB | |
| "$PY" - "$RUNS" "$NEED_GB" <<'EOF' | |
| import glob, json, math, sys | |
| runs, default = sys.argv[1], int(sys.argv[2]) | |
| peaks = [] | |
| for f in glob.glob(f"{runs}/mb_official_*_v2/run_mb_official_*_2*.json"): | |
| try: | |
| p = json.load(open(f)).get("peak_gpu_gib") | |
| if p: peaks.append(float(p)) | |
| except Exception: | |
| pass | |
| print(math.ceil(max(peaks)) + 1 if peaks else default) | |
| EOF | |
| } | |
| gate() { # gate <need_gb>: block until the launch gate is satisfied (poll every 60 s) | |
| local need=$1 free_mib avail_kb ntr ok | |
| while :; do | |
| ok=1 | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ') | |
| avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain|aide/train)") | |
| [ "${free_mib:-0}" -lt $(( (need + 4) * 1024 )) ] && ok=0 | |
| [ "${avail_kb:-0}" -lt $(( 28 * 1024 * 1024 )) ] && ok=0 | |
| [ "${ntr:-9}" -ge "$MAX_TRAINERS" ] && ok=0 | |
| if [ "$ok" = 1 ]; then | |
| log "gate OK: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GiB (>= 28), trainers ${ntr} (< ${MAX_TRAINERS})" | |
| return 0 | |
| fi | |
| log "gate WAIT: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GiB, trainers ${ntr}" | |
| sleep 60 | |
| done | |
| } | |
| run() { # run <name> <formulation> <init> <seed>: skip if finished, gate, launch, record pid, wait | |
| local name=$1 form=$2 init=$3 seed=$4 need | |
| if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists)"; return 0; fi | |
| need=$(need_gb) | |
| if [ "$DRY_RUN" = 1 ]; then log "DRY $name (need ${need} GB): --formulation $form --init $init --seed $seed $RECIPE $COMMON"; return 0; fi | |
| gate "$need" | |
| # shellcheck disable=SC2086 | |
| "$PY" aide/train/train_motionbert_official.py --formulation "$form" --init "$init" --seed "$seed" --name "$name" $RECIPE $COMMON --notes "$NOTES" \ | |
| > "$RUNS/${name}_train.log" 2>&1 & | |
| local pid=$! | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid START" >> "$PIDS" | |
| log "launched $name pid $pid (log $RUNS/${name}_train.log)" | |
| wait "$pid"; local rc=$? | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid END rc=$rc" >> "$PIDS" | |
| log "finished $name rc=$rc" | |
| return $rc | |
| } | |
| total_gpu_min() { # sum of gpu_minutes over the 9 pre-registered runs (missing run -> 9999 so the budget rule fails closed) | |
| "$PY" - "$RUNS" <<'EOF' | |
| import glob, json, sys | |
| runs = sys.argv[1]; tot = 0.0 | |
| for arm in ("res", "abs", "rand"): | |
| for s in (42, 123, 456): | |
| fs = sorted(glob.glob(f"{runs}/mb_official_{arm}_s{s}_v2/run_mb_official_{arm}_s{s}_v2_2*.json")) | |
| if not fs: print(9999); sys.exit() | |
| tot += float(json.load(open(fs[-1])).get("gpu_minutes", 9999)) | |
| print(f"{tot:.1f}") | |
| EOF | |
| } | |
| MODE=${MODE:-prereg} | |
| log "queue start (snapshot ${MF_SNAPSHOT:-none}); MODE $MODE; NEED_GB default $NEED_GB" | |
| if [ "$MODE" = "posthoc_noema" ]; then | |
| # POST-HOC SENSITIVITY (not pre-registered; run only after the pre-registered set): (b) with the harness EMA disabled (--ema 0 -> selection and | |
| # prediction on the raw weights). Motivation: under the pre-registered 60-epoch cap the EMA (0.999, ~17-epoch time constant) is still dominated by the | |
| # default-init absolute head (val All 250 -> 59 px over the run, cap hit at 59/60) -> the (b) level and the (a) - (b) "value of anchoring" are confounded | |
| # by the EMA lag. Reported in a separate, labelled block; the pre-registered verdicts use the pre-registered runs only. | |
| NOTES="motionbert_faithful;POSTHOC_noema;not_preregistered" | |
| RECIPE="${RECIPE/--ema 0.999/--ema 0.0}" | |
| for s in 42 123 456; do run "mb_official_abs_noema_s${s}_v2" abs released "$s" || log "WARN: mb_official_abs_noema_s${s}_v2 failed (rc $?)"; done | |
| log "post-hoc queue done"; [ "$DRY_RUN" = 1 ] || echo "$(date -u +%FT%TZ) POSTHOC_DONE" >> "$PIDS"; exit 0 | |
| fi | |
| for s in 42 123 456; do run "mb_official_res_s${s}_v2" residual released "$s" || log "WARN: mb_official_res_s${s}_v2 failed (rc $?)"; done | |
| for s in 42 123 456; do run "mb_official_abs_s${s}_v2" abs released "$s" || log "WARN: mb_official_abs_s${s}_v2 failed (rc $?)"; done | |
| for s in 42 123 456; do run "mb_official_rand_s${s}_v2" residual random "$s" || log "WARN: mb_official_rand_s${s}_v2 failed (rc $?)"; done | |
| TOT=$(total_gpu_min) | |
| log "total GPU minutes of the 9 runs: $TOT (budget rule for (a) seeds 789 / 1011: <= 60)" | |
| if [ "$(echo "$TOT <= 60" | bc -l 2>/dev/null || echo 0)" = 1 ]; then | |
| for s in 789 1011; do run "mb_official_res_s${s}_v2" residual released "$s" || log "WARN: mb_official_res_s${s}_v2 failed (rc $?)"; done | |
| else | |
| log "budget rule NOT met -> (a) seeds 789 / 1011 not run" | |
| fi | |
| log "queue done" | |
| [ "$DRY_RUN" = 1 ] || echo "$(date -u +%FT%TZ) QUEUE_DONE" >> "$PIDS" | |
Xet Storage Details
- Size:
- 7.38 kB
- Xet hash:
- 6b0a2e16ea5cd5264f651b5440f75633c21057e23838bc7731643c2940bfb270
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.