Buckets:
| # AIDE architecture-attribution fair experiments (docs/decisions.md 2026-09-14 "AIDE 架构归因公平实验") — detached lane runner. | |
| # | |
| # setsid nohup bash aide/train/run_arch_attribution.sh <A|B|C> > aide/results/arch_attribution/lane_<X>.log 2>&1 & | |
| # | |
| # Lane A (1) matched BATON in-cabin pose pretraining of MotionBERT-kp (seed 123, pretrain_cwm3d_baton.py --model motionbert_kp: same corpus / | |
| # task / loss call / schedule / EMA / selection as cwm3d_kp_batonpre_pretrain_s123) -> trunk-only fine-tunes seeds 42 / 123 / 456 | |
| # with the hardening-D recipe verbatim (run_stage_bc.RECIPE + --loss euc), runs motionbert_kp_batonpre_trunkp123_s<k>_v2 | |
| # Lane B (1) the same for ST-GCN-kp: stgcn_kp_batonpre_pretrain_s123 -> stgcn_kp_batonpre_trunkp123_s<k>_v2 | |
| # Lane C (3) MotionBERT-kp seeds 789 / 1011 with the hardening-D recipe verbatim (motionbert_kp_s789_v2, motionbert_kp_s1011_v2), then | |
| # (2) Driver-WM-kp seeds 42 / 123 / 456 on the shared recipe (driverwm_kp_s<k>_v2) | |
| # Every python launch passes the project GPU gate first (re-checked every 30 s): free VRAM >= need + 4 GB, MemAvailable >= 28 GB, fewer than | |
| # 4 trainer processes (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"`), OMP/MKL threads 4. One trainer per lane -> at most 3 alive. | |
| # Existing runs are never touched: a run whose run json exists is skipped; new run dirs only. PIDs -> aide/results/arch_attribution/pids.txt. | |
| # Selection is val-only inside train_kp / the pretraining script; the official test split is evaluated once per run by the existing harness. | |
| set -u | |
| LANE=${1:?lane A|B|C} | |
| ROOT=/home/USER/cwm3 | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| AA=$ROOT/aide/results/arch_attribution | |
| PIDS=$AA/pids.txt | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| mkdir -p "$AA" | |
| cd "$ROOT" || exit 1 | |
| # hardening-D recipe (aide/train/run_stage_bc.RECIPE) + plain Euclidean loss for tensor-output models — verbatim as motionbert_kp_s*_v2 / stgcn_kp_s*_v2 | |
| RECIPE="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0 --loss euc" | |
| log() { echo "$(date -u +%FT%TZ) [lane $LANE] $*"; } | |
| gate() { # gate <need_gb>: block until the project launch gate is satisfied | |
| local need=$1 free_mib avail_kb ntr ok | |
| while :; do | |
| ok=1 | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ') | |
| avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain)") | |
| [ "${free_mib:-0}" -lt $(( (need + 4) * 1024 )) ] && ok=0 | |
| [ "${avail_kb:-0}" -lt $(( 28 * 1024 * 1024 )) ] && ok=0 | |
| [ "${ntr:-9}" -ge 4 ] && ok=0 | |
| if [ "$ok" = 1 ]; then | |
| log "gate OK: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= 28), trainers ${ntr} (< 4)" | |
| return 0 | |
| fi | |
| log "gate WAIT: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB, trainers ${ntr}" | |
| sleep 30 | |
| done | |
| } | |
| run() { # run <name> <need_gb> <cmd...>: gate, launch, record pid, wait | |
| local name=$1 need=$2; shift 2 | |
| if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists; existing runs are never re-run)"; return 0; fi | |
| gate "$need" | |
| "$@" > "$RUNS/${name}_train.log" 2>&1 & | |
| local pid=$! | |
| echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid START" >> "$PIDS" | |
| log "launched $name pid $pid" | |
| wait "$pid"; local rc=$? | |
| echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid END rc=$rc" >> "$PIDS" | |
| log "finished $name rc=$rc" | |
| return $rc | |
| } | |
| pretrain_then_finetune() { # <model> | |
| local model=$1 pre="${1}_batonpre_pretrain_s123" s | |
| run "$pre" 10 "$PY" aide/train/pretrain_cwm3d_baton.py --model "$model" --seed 123 || { log "ABORT lane: pretraining $pre failed"; return 1; } | |
| [ -f "$RUNS/$pre/ckpt_best.pt" ] && [ -f "$RUNS/$pre/TRAIN_DONE" ] || { log "ABORT lane: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; } | |
| for s in 42 123 456; do | |
| run "${model}_batonpre_trunkp123_s${s}_v2" 3 "$PY" aide/train/train_kp.py --model "$model" --seed "$s" --name "${model}_batonpre_trunkp123_s${s}_v2" \ | |
| $RECIPE --init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \ | |
| --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${s}_v2,kp=cwm3d_kp_s${s}_v2,seedinit=${model}_s${s}_v2" --no-lock --max-minutes 45 \ | |
| --notes "arch_attribution;(1):matched_baton_pretrain;init=${pre}:trunk;hardening_D_recipe_verbatim" | |
| done | |
| } | |
| case "$LANE" in | |
| A) pretrain_then_finetune motionbert_kp ;; | |
| B) pretrain_then_finetune stgcn_kp ;; | |
| C) | |
| for s in 789 1011; do | |
| run "motionbert_kp_s${s}_v2" 3 "$PY" aide/train/train_kp.py --model motionbert_kp --seed "$s" --name "motionbert_kp_s${s}_v2" $RECIPE \ | |
| --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2" --no-lock --max-minutes 30 \ | |
| --notes "arch_attribution;(3):motionbert_kp_5seed_extension;hardening_D_recipe_verbatim" | |
| done | |
| for s in 42 123 456; do | |
| run "driverwm_kp_s${s}_v2" 6 "$PY" aide/train/train_kp.py --model driverwm_kp --seed "$s" --name "driverwm_kp_s${s}_v2" $RECIPE \ | |
| --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${s}_v2,kp=cwm3d_kp_s${s}_v2,mb=motionbert_kp_s${s}_v2" --no-lock --max-minutes 60 \ | |
| --notes "arch_attribution;(2):driverwm_kp_shared_recipe;internal_stream=cwm3d_part_tokens;external_stream=qwen_exterior_as_repro" | |
| done ;; | |
| *) echo "unknown lane $LANE"; exit 2 ;; | |
| esac | |
| log "lane $LANE done" | |
| echo "$(date -u +%FT%TZ) lane=$LANE LANE_DONE" >> "$PIDS" | |
Xet Storage Details
- Size:
- 5.72 kB
- Xet hash:
- d158a9b9b729892d9780a5944f52e88d47abb15ee0f5cc12682e02827c8603d2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.