Buckets:
| # F7 lane — docs/decisions.md pre-registration row 2026-09-17 "F7 AIDE-only multi-stride pretraining + ensemble distillation" — detached queue. | |
| # | |
| # setsid nohup bash aide/train/run_f7_pretrain_distil.sh > aide/results/f7/queue.log 2>&1 & (pid -> aide/results/f7/queue.pid) | |
| # | |
| # ISOLATION: this lane reads AIDE data only (aide/results/cache/aide_kp.npz) plus AIDE run checkpoints under aide/results/runs; the BATON-derived | |
| # checkpoints it touches (cwm3d_kp_batonpre_pretrain_s123, the 5 adopted seeds, motionbert_kp_batonpre_*) were pretrained on the BATON pose corpus | |
| # whose 440 routes include NONE of the 60 hold-out devices (checked 2026-09-17, results/holdout/holdout_leaves.txt); assert_aide_only() rejects | |
| # any BATON corpus / route / hold-out path in every argv (phase 0 below). | |
| # Phase 1 (parallel): pretraining (a) cwm3d_kp_aidepre_pretrain_s123 (seed init) and (a2) cwm3d_kp_aidepre_fb_pretrain_s123 (init = adopted BATON ckpt). | |
| # Phase 2 (3 seeds of an arm in parallel, arms sequential): fine-tunes with the adopted recipe VERBATIM (run json `command` of | |
| # cwm3d_kp_batonpre_trunkp123_s<k>_v2 / motionbert_kp_batonpre_trunkp123_s<k>_v2) + the arm's init / distillation flags: | |
| # aidepre cwm3d --init-ckpt <a>/ckpt_best.pt --init-parts trunk | |
| # aidepre_fb cwm3d --init-ckpt <a2>/ckpt_best.pt --init-parts trunk | |
| # distil5 cwm3d adopted init + --distil-teachers <5 adopted seeds> --w-distil 1.0 --w-distil-modes 0.1 --w-distil-logb 0.1 | |
| # mb_distil5 motionbert --init-ckpt motionbert_kp_batonpre_pretrain_s123 trunk + --distil-teachers <5 MB pretrained seeds> --w-distil 1.0 | |
| # aidepre_fb_distil5 cwm3d <a2> trunk init + the distil5 flags | |
| # Phase 3: aide/eval/report_f7_pretrain_distil.py -> aide/results/tables/f7_pretrain_distil.md (+ .json). | |
| # Gate before EVERY launch (task rule): free VRAM >= NEED + 4 GB (NEED 4 GB = measured cwm3d footprint 3.3 GB + 5 frozen teachers, rounded up), | |
| # MemAvailable >= 28 GB, trainers alive (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"`) < 4, OMP/MKL threads 4, disk >= 30 GB free (owner rule 2026-09-17T19:26:58Z; was 60). | |
| # New run dirs only (a run whose run json exists is skipped); nothing is ever killed or deleted. | |
| set -u | |
| ROOT=/home/USER/cwm3 | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| OUTD=$ROOT/aide/results/f7 | |
| PIDS=$OUTD/pids.txt | |
| NEED_GB=${NEED_GB:-4} | |
| RAM_MIN_GB=${RAM_MIN_GB:-28} | |
| DISK_MIN_GB=${DISK_MIN_GB:-30} # 60 -> 30 on 2026-09-17T21:4xZ: owner rule row 2026-09-17T19:26:58Z set the disk floor to >= 30 GB (chain restarted from arm (bc); no other change) | |
| MAXTR=${MAXTR:-4} | |
| POLL_S=${POLL_S:-45} | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| mkdir -p "$OUTD" | |
| cd "$ROOT" || exit 1 | |
| echo $$ > "$OUTD/queue.pid" | |
| RECIPE="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0" | |
| INIT_BATON="aide/results/runs/cwm3d_kp_batonpre_pretrain_s123/ckpt_best.pt" | |
| INIT_MB="aide/results/runs/motionbert_kp_batonpre_pretrain_s123/ckpt_best.pt" | |
| PRE_A="cwm3d_kp_aidepre_pretrain_s123" | |
| PRE_A2="cwm3d_kp_aidepre_fb_pretrain_s123" | |
| TEACH_K="cwm3d_kp_batonpre_trunkp123_s42_v2,cwm3d_kp_batonpre_trunkp123_s123_v2,cwm3d_kp_batonpre_trunkp123_s456_v2,cwm3d_kp_batonpre_trunkp123_s789_v2,cwm3d_kp_batonpre_trunkp123_s1011_v2" | |
| TEACH_MB="motionbert_kp_batonpre_trunkp123_s42_v2,motionbert_kp_batonpre_trunkp123_s123_v2,motionbert_kp_batonpre_trunkp123_s456_v2,motionbert_kp_batonpre_trunkp123_s789_v2,motionbert_kp_batonpre_trunkp123_s1011_v2" | |
| DISTIL_K="--distil-teachers $TEACH_K --w-distil 1.0 --w-distil-modes 0.1 --w-distil-logb 0.1" | |
| DISTIL_MB="--distil-teachers $TEACH_MB --w-distil 1.0" | |
| SEEDS="42 123 456" | |
| PREREG=${PREREG:-2026-09-17T19:37:12Z} | |
| log() { echo "$(date -u +%FT%TZ) [f7] $*"; } | |
| gate_ok() { | |
| local free_mib avail_kb ntr disk_gb ok=1 | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ') | |
| avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain)") | |
| disk_gb=$(df -BG --output=avail "$ROOT" | tail -1 | tr -dc '0-9') | |
| [ "${free_mib:-0}" -lt $(( (NEED_GB + 4) * 1024 )) ] && ok=0 | |
| [ "${avail_kb:-0}" -lt $(( RAM_MIN_GB * 1024 * 1024 )) ] && ok=0 | |
| [ "${ntr:-9}" -ge "$MAXTR" ] && ok=0 | |
| [ "${disk_gb:-0}" -lt "$DISK_MIN_GB" ] && ok=0 | |
| GATE_STATE="free VRAM ${free_mib} MiB (need $((NEED_GB + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= ${RAM_MIN_GB}), trainers ${ntr} (< ${MAXTR}), disk ${disk_gb} GB (>= ${DISK_MIN_GB})" | |
| [ "$ok" = 1 ] | |
| } | |
| gate() { local n=0; until gate_ok; do [ $((n % 4)) -eq 0 ] && log "gate WAIT [$1]: $GATE_STATE"; n=$((n + 1)); sleep "$POLL_S"; done; log "gate OK [$1]: $GATE_STATE"; } | |
| declare -a BG_PIDS=() BG_NAMES=() | |
| launch() { # launch <name> <cmd...>: skip if run json exists; isolation assert on argv; gate; start detached in background; record pid | |
| local name=$1; shift | |
| if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists)"; return 0; fi | |
| if pgrep -f -- "python.*--name ${name}( |$)" >/dev/null; then log "skip $name (already running)"; return 0; fi | |
| "$PY" -c "import sys; sys.path.insert(0,'$ROOT'); from aide.train.pretrain_aide_multistride import assert_aide_only; assert_aide_only(sys.argv[1:]); print('isolation OK')" "$@" \ | |
| || { log "ABORT $name: isolation assert failed"; echo "$(date -u +%FT%TZ) name=$name ISOLATION_FAIL" >> "$PIDS"; return 3; } | |
| gate "$name" | |
| "$@" > "$RUNS/${name}_train.log" 2>&1 & | |
| local pid=$! | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid START" >> "$PIDS" | |
| echo "$pid" > "$OUTD/${name}.pid" | |
| log "launched $name pid $pid: $*" | |
| BG_PIDS+=("$pid"); BG_NAMES+=("$name") | |
| sleep 30 # let the new process claim its VRAM before the next gate check | |
| } | |
| wait_all() { | |
| local i rc | |
| for i in "${!BG_PIDS[@]}"; do | |
| wait "${BG_PIDS[$i]}"; rc=$? | |
| echo "$(date -u +%FT%TZ) name=${BG_NAMES[$i]} pid=${BG_PIDS[$i]} END rc=$rc" >> "$PIDS" | |
| log "finished ${BG_NAMES[$i]} rc=$rc" | |
| done | |
| BG_PIDS=(); BG_NAMES=() | |
| sleep 15 | |
| } | |
| for f in "$INIT_BATON" "$INIT_MB"; do [ -f "$f" ] || { log "ABORT: init $f missing"; exit 2; }; done | |
| log "queue start (prereg $PREREG): phase 1 pretraining x2 | phase 2 5 arms x seeds [$SEEDS] | phase 3 report; gate VRAM >= $((NEED_GB + 4)) GB, RAM >= ${RAM_MIN_GB} GB, trainers < ${MAXTR}, disk >= ${DISK_MIN_GB} GB" | |
| # ---- phase 1: pretraining (parallel) | |
| launch "$PRE_A" "$PY" aide/train/pretrain_aide_multistride.py --seed 123 --name "$PRE_A" --strides 1,2,3,4,mixed --windows-per-clip 8 --epochs 60 --patience 10 --bs 128 --max-minutes 30 \ | |
| --notes "f7;arm=a;aide_only_multistride;prereg=$PREREG" | |
| launch "$PRE_A2" "$PY" aide/train/pretrain_aide_multistride.py --seed 123 --name "$PRE_A2" --strides 1,2,3,4,mixed --windows-per-clip 8 --epochs 60 --patience 10 --bs 128 --max-minutes 30 \ | |
| --init-ckpt "$INIT_BATON" --init-parts all --notes "f7;arm=a2;baton_ckpt_then_aide_multistride;prereg=$PREREG" | |
| wait_all | |
| for p in "$PRE_A" "$PRE_A2"; do [ -f "$RUNS/$p/ckpt_best.pt" ] || log "WARNING: $p has no ckpt_best.pt — its fine-tunes will be skipped"; done | |
| # ---- phase 2: fine-tunes (3 seeds in parallel per arm) | |
| ft_cwm3d() { # ft_cwm3d <arm> <init_ckpt> <extra flags...> | |
| local arm=$1 init=$2; shift 2 | |
| for s in $SEEDS; do | |
| local name="cwm3d_kp_${arm}_s${s}_v2" | |
| launch "$name" "$PY" aide/train/train_kp.py --model cwm3d --seed "$s" --name "$name" $RECIPE --init-ckpt "$init" --init-parts trunk "$@" \ | |
| --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${s}_v2,kp=cwm3d_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2,mbpre=motionbert_kp_batonpre_trunkp123_s${s}_v2" \ | |
| --no-lock --max-minutes 25 --notes "f7;arm=${arm};prereg=$PREREG;adopted_recipe_verbatim;init=$(basename "$(dirname "$init")"):trunk" | |
| done | |
| wait_all | |
| } | |
| [ -f "$RUNS/$PRE_A/ckpt_best.pt" ] && ft_cwm3d "aidepre_trunkp123" "$RUNS/$PRE_A/ckpt_best.pt" | |
| [ -f "$RUNS/$PRE_A2/ckpt_best.pt" ] && ft_cwm3d "aidepre_fb_trunkp123" "$RUNS/$PRE_A2/ckpt_best.pt" | |
| ft_cwm3d "distil5_batonpre_trunkp123" "$INIT_BATON" $DISTIL_K | |
| for s in $SEEDS; do | |
| name="motionbert_kp_distil5_batonpre_trunkp123_s${s}_v2" | |
| launch "$name" "$PY" aide/train/train_kp.py --model motionbert_kp --seed "$s" --name "$name" $RECIPE --loss euc --init-ckpt "$INIT_MB" --init-parts trunk $DISTIL_MB \ | |
| --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${s}_v2,kp=cwm3d_kp_s${s}_v2,seedinit=motionbert_kp_s${s}_v2,mbpre=motionbert_kp_batonpre_trunkp123_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2" \ | |
| --no-lock --max-minutes 45 --notes "f7;arm=mb_distil5;prereg=$PREREG;hardening_D_recipe_verbatim;init=motionbert_kp_batonpre_pretrain_s123:trunk" | |
| done | |
| wait_all | |
| [ -f "$RUNS/$PRE_A2/ckpt_best.pt" ] && ft_cwm3d "aidepre_fb_distil5_trunkp123" "$RUNS/$PRE_A2/ckpt_best.pt" $DISTIL_K | |
| # ---- phase 3: report (CPU) | |
| log "phase 3: report" | |
| "$PY" aide/eval/report_f7_pretrain_distil.py --n-boot 10000 > "$OUTD/report.log" 2>&1 && log "report written" || log "report FAILED (see $OUTD/report.log)" | |
| log "queue done" | |
| echo "$(date -u +%FT%TZ) QUEUE_DONE" >> "$PIDS" | |
Xet Storage Details
- Size:
- 9.22 kB
- Xet hash:
- 398d83987df80aab1bf90447f38907ec1b37c0b87515b54667e7ed51dc71351e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.