Buckets:
| # Driver-WM Table-2 / Table-3 mirror completion (docs/decisions.md 2026-09-14 "PRE-REGISTRATION — mirror completion: DESCRIPTIVE, NO ADOPTION"). | |
| # | |
| # setsid nohup bash aide/train/run_mirror_completion.sh > aide/results/mirror_completion/lane.log 2>&1 & | |
| # DRY_RUN=1 bash aide/train/run_mirror_completion.sh # print every assembled command, launch nothing | |
| # | |
| # One lane, sequential: | |
| # T1 cwm3d_recursive_s{123,456}_v2 : cwm3d_kp_s<k>_v2 Stage-A argv verbatim + --rollout recursive (as cwm3d_recursive_s42_v2) | |
| # T2 cwm3d_vis_sem_vp_s{123,456}_v2 : cwm3d_vis_sem_s<k>_v2 B2-main argv verbatim + --vis-key vis_pooled (as cwm3d_vis_sem_vp_s42_v2) | |
| # E1 interventions (eval only, existing harness, --cuda) on cwm3d_vis_sema_vg_s{42,123,456}_v2 | |
| # E2 interventions (eval only, existing harness, --cuda) on cwm3d_vis_sem_batonpre_trunkp123_s{42,123,456}_v2 | |
| # Every python launch passes the launch gate first (re-checked every 30 s, under the launcher flock): free VRAM >= need + 4 GB, | |
| # MemAvailable >= 28 GB, fewer than MAX_TRAINERS (= 2) trainer processes alive on the host (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"`). | |
| # Existing runs are never touched: a run whose run json (or interventions json) exists is skipped; new run dirs only; no pkill. | |
| # Selection is val-only inside train_kp (EMA All-MPJPE); the official test split is evaluated once per run by the existing harness. | |
| set -u | |
| ROOT=/home/USER/cwm3 | |
| # per-launch snapshot so that editing this file while the lane is alive can never change the running lane (as run_formulation_abs.sh) | |
| if [ -z "${MC_SNAPSHOT:-}" ]; then | |
| mkdir -p "$ROOT/aide/results/mirror_completion/launcher_snapshots" | |
| SNAP="$ROOT/aide/results/mirror_completion/launcher_snapshots/run_mirror_completion_$(date -u +%Y%m%dT%H%M%SZ).sh" | |
| cp "${BASH_SOURCE[0]}" "$SNAP" && MC_SNAPSHOT=$SNAP exec bash "$SNAP" "$@" | |
| fi | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| MC=$ROOT/aide/results/mirror_completion | |
| PIDS=$MC/pids.txt | |
| LOCK=$MC/launch.lock | |
| MAX_TRAINERS=${MAX_TRAINERS:-2} | |
| DRY_RUN=${DRY_RUN:-0} | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| mkdir -p "$MC" | |
| cd "$ROOT" || exit 1 | |
| # Stage-A recipe verbatim (aide/train/run_stage_a.py RECIPE == aide/train/run_stage_bc.py RECIPE) | |
| RECIPE="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0" | |
| REFS_A="zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2" # run_stage_a.py REFS_FULL | |
| REFS_BC="zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s42_v2" # run_stage_bc.py REFS | |
| VIS="--visual --modality-dropout 0.3" # run_stage_bc.py VIS | |
| SEM="--semantic --lambda-sem 0.5 --label-smoothing 0.1 --class-balanced 1" # run_stage_bc.py SEM | |
| COMMON="--no-lock --max-minutes 45" | |
| log() { echo "$(date -u +%FT%TZ) [mirror] $*"; } | |
| gate() { # gate <need_gb>: block until the launch gate is satisfied (caller holds the launcher flock) | |
| local need=$1 free_mib avail_kb ntr ok | |
| while :; do | |
| ok=1 | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ') | |
| avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain)") | |
| [ "${free_mib:-0}" -lt $(( (need + 4) * 1024 )) ] && ok=0 | |
| [ "${avail_kb:-0}" -lt $(( 28 * 1024 * 1024 )) ] && ok=0 | |
| [ "${ntr:-9}" -ge "$MAX_TRAINERS" ] && ok=0 | |
| if [ "$ok" = 1 ]; then | |
| log "gate OK: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= 28), trainers ${ntr} (< ${MAX_TRAINERS})" | |
| return 0 | |
| fi | |
| log "gate WAIT: free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB, trainers ${ntr} (< ${MAX_TRAINERS})" | |
| sleep 30 | |
| done | |
| } | |
| launch() { # launch <name> <need_gb> <done-glob> <cmd...>: gate under the flock, run to completion, log rc | |
| local name=$1 need=$2 doneglob=$3; shift 3 | |
| if compgen -G "$doneglob" > /dev/null; then log "skip $name: $(ls $doneglob | head -1) exists"; return 0; fi | |
| if [ "$DRY_RUN" = 1 ]; then echo "DRY $name: $*"; return 0; fi | |
| exec 9>"$LOCK"; flock 9 | |
| gate "$need" | |
| log "launch $name: $*" | |
| "$@" > "$MC/$name.log" 2>&1 & | |
| local pid=$! | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid START" >> "$PIDS" | |
| flock -u 9 | |
| wait "$pid"; local rc=$? | |
| echo "$(date -u +%FT%TZ) name=$name pid=$pid END rc=$rc" >> "$PIDS" | |
| if [ "$rc" = 0 ]; then log "finished $name rc=0"; else log "FAILED $name rc=$rc (see $MC/$name.log)"; fi | |
| return 0 | |
| } | |
| # T1 recursive rollout, keypoint-only (Stage-A ablation recipe, seeds 123 / 456) | |
| for s in 123 456; do | |
| n=cwm3d_recursive_s${s}_v2 | |
| launch "$n" 2 "$RUNS/$n/run_${n}_2*.json" $PY aide/train/train_kp.py --model cwm3d --seed $s --name $n $RECIPE --rollout recursive --refs $REFS_A $COMMON --notes "mirror_completion;stage_a_recipe;ablation:recursive_1step" | |
| done | |
| # T2 vis_pooled (no-prompt) features on the B2 main recipe (seeds 123 / 456) | |
| for s in 123 456; do | |
| n=cwm3d_vis_sem_vp_s${s}_v2 | |
| launch "$n" 4 "$RUNS/$n/run_${n}_2*.json" $PY aide/train/train_kp.py --model cwm3d --seed $s --name $n $RECIPE $VIS $SEM --vis-key vis_pooled --refs $REFS_BC $COMMON --notes "mirror_completion;B2_main_recipe;B2(iv):vis_pooled" | |
| done | |
| # E1 / E2 interventions, evaluation only (existing harness; one invocation per run so a failure cannot take the others down) | |
| for n in cwm3d_vis_sema_vg_s42_v2 cwm3d_vis_sema_vg_s123_v2 cwm3d_vis_sema_vg_s456_v2 \ | |
| cwm3d_vis_sem_batonpre_trunkp123_s42_v2 cwm3d_vis_sem_batonpre_trunkp123_s123_v2 cwm3d_vis_sem_batonpre_trunkp123_s456_v2; do | |
| launch "interv_$n" 4 "$RUNS/$n/interventions_2*.json" $PY aide/eval/interventions.py --cuda --n-boot 2000 --run $n | |
| done | |
| log "LANE_DONE" | |
| echo "$(date -u +%FT%TZ) LANE_DONE" >> "$PIDS" | |
Xet Storage Details
- Size:
- 5.89 kB
- Xet hash:
- c3eac52305bf0cc90714e8afa5d3091d64bfcfee13fd028bd56ec74321e1ba91
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.