Buckets:

AsianPlayer
/

TriDrive

Files

xet

AsianPlayer/TriDrive / code /aide /train /run_pretrain_variants.sh

AsianPlayer

16 days ago

download

raw

16.4 kB

#!/usr/bin/env bash
# Pre-registered pretraining experiments P1 / P1-ext / P2 / P3 (docs/decisions.md 2026-09-15 rows "PRE-REGISTRATION — CockpitWM pretraining
# variants …" and "… Driver-WM architecture on CockpitWM tokens + BATON pretraining") — detached lane runner.
#
#   setsid nohup bash aide/train/run_pretrain_variants.sh <K|MB|ST|P2|F|MBX|STX> > aide/results/pretrain_variants/lane_<X>.log 2>&1 &
#
# Lane K    CockpitWM (cwm3d_kp) pretraining variants on the SAME corpus (sha256 465dac1cf0a8265a…), seed 123, bs 256, lr 3e-4, EMA 0.999,
#           held-out-route selection, --zeroshot 0:  euc (6 ep, Euclidean only) -> conv (<= 36 ep, patience 4, all four loss terms)
#           -> conveuc (conv + Euclidean only) -> scale (6 ep, all terms, --motion-scale-max 2.5) -> convscale (conv + --motion-scale-max 2.5);
#           after each pretraining the 5 trunk-only AIDE fine-tunes cwm3d_kp_batonpre_<tag>_s<k>_v2 (Stage-A recipe verbatim as
#           cwm3d_kp_batonpre_trunkp123_s<k>_v2, refs incl. kppre = that adopted arm).  The scale variants wait for GO_scale (code + tests landed).
# Lane MB   MotionBERT architecture, converged matched recipe: motionbert_kp_batonpre_pretrain_conv_s123 (<= 36 ep, patience 4, --max-minutes 400)
#           -> 5 trunk-only fine-tunes motionbert_kp_batonpre_conv_s<k>_v2 (hardening-D recipe verbatim, --loss euc).  For tensor-output
#           architectures conv == conveuc (the nll / bone / jerk weights are inert), so this run is the matched run for either selection.
# Lane ST   the same for the ST-GCN architecture (stgcn_kp_batonpre_pretrain_conv_s123 -> stgcn_kp_batonpre_conv_s<k>_v2); waits for GO_st.
# Lane P2   Driver-WM architecture on CockpitWM tokens: driverwm_kp_batonpre_pretrain_s123 (6 ep, bs $P2_BS, --max-minutes 150) -> 3 trunk-only
#           fine-tunes driverwm_kp_batonpre_trunkp123_s{42,123,456}_v2 (shared recipe verbatim, --loss euc); waits for GO_p2 (code + tests + row).
# Lane F    lean-read-out CockpitWM (cwm3d_v2 --no-v2-spatial): the best two K variants by the pre-registered val rule (file SELECTED, one tag per
#           line, written by report_pretrain_variants.py --select) -> cwm3d_v2_nospatial_batonpre_pretrain_<tag>_s123 -> 5 trunk-only fine-tunes.
# Lane MBX / STX  only if the selected recipe is convscale: motionbert_kp / stgcn_kp pretraining with --motion-scale-max 2.5 (waits for GO_mbx / GO_stx).
#
# The pretraining script's own resource gate receives the same floor (--min-free-vram-gb need + 4; its default 12 GB aborted the conveuc / scale launches at 06:08Z).
# Launch gate (coordinator 2026-09-15 05:45Z revision): free VRAM >= need + 4 GB (need = the job's measured peak footprint, rounded up) and
# trainers alive (`pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"`) < MAXTR (default 3; project cap < 4) for every job; host RAM:
# pretraining processes (name contains _pretrain_, they load the corpus) keep the project rule MemAvailable >= 28 GB; the small AIDE fine-tunes are
# need-based: MemAvailable >= est. RSS + 10 GB, est. RSS = the largest VmHWM recorded so far for a fine-tune of this lane runner
# (aide/results/pretrain_variants/rss/<run>.txt, sampled every 15 s while the job runs) or 4 GB before any measurement.  The gate is polled WITHOUT
# the launch lock; the lock (aide/results/pretrain_variants/launch.lock) is taken only for re-check + launch + 45 s settle, so a lane whose gate cannot
# be met never blocks the others (the 04:31Z–05:46Z stall).  A lane restarted while its job is still running ATTACHES to that process instead of
# re-launching.  Existing runs are never touched: a run whose run json exists is skipped; new run dirs only.  PIDs -> pids.txt.
# Selection is val-only inside train_kp / held-out-route only inside the pretraining script; the official test split is evaluated once per run.
set -u
LANE=${1:?lane K|MB|ST|P2|F|MBX|STX}
ROOT=/home/USER/cwm3
PY=python
RUNS=$ROOT/aide/results/runs
PV=$ROOT/aide/results/pretrain_variants
PIDS=$PV/pids.txt
MAXTR=${MAXTR:-3}
P2_BS=${P2_BS:-256}
RAM_PRE_GB=28          # project rule for corpus-loading pretraining processes
RAM_FT_MARGIN_GB=10    # fine-tunes: MemAvailable >= est. RSS + this
RSS_DEFAULT_GB=4
export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4
mkdir -p "$PV/rss"
cd "$ROOT" || exit 1
# Stage-A / hardening-D fine-tune recipe (aide/train/run_stage_bc.RECIPE + --loss euc) — verbatim as aide/train/run_arch_attribution.sh
RECIPE="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0 --loss euc"
# pretraining constants shared by every variant (== the adopted run cwm3d_kp_batonpre_pretrain_s123 except the varied flags)
PRE_COMMON="--seed 123 --bs 256 --lr 3e-4 --ema 0.999 --zeroshot 0"
SEEDS5="42 123 456 789 1011"
# measured peak footprints (GiB reserved, rounded up): cwm3d pretraining 3.98 -> 4; cwm3d fine-tune ~1 -> 3 (conservative); driverwm_kp fine-tune
# 2.46 -> 3; motionbert_kp pretraining 12.7 -> 13 (bs 256 + bs-1024 held-out eval cache); stgcn_kp pretraining 14.0 -> 14; cwm3d_v2 pretraining unmeasured -> 8
NEED_PRE_K=4; NEED_FT=3; NEED_FT_DWM=3; NEED_PRE_MB=13; NEED_PRE_ST=14; NEED_PRE_V2=5; NEED_PRE_DWM=7   # V2: estimate (cwm3d footprint + 1 GB; 2.7 M params), measured VRAM is written to rss/<run>.vram

log() { echo "$(date -u +%FT%TZ) [lane $LANE] $*"; }

variant_flags() {   # <tag> -> the flags that define the variant (everything else = the adopted run's defaults)
  case "$1" in
    euc)       echo "--epochs 6 --w-nll 0 --w-bone 0 --w-jerk 0" ;;
    scale)     echo "--epochs 6 --motion-scale-max 2.5" ;;
    conv)      echo "--epochs 36 --patience 4" ;;
    conveuc)   echo "--epochs 36 --patience 4 --w-nll 0 --w-bone 0 --w-jerk 0" ;;
    convscale) echo "--epochs 36 --patience 4 --motion-scale-max 2.5" ;;
    *) echo "unknown variant $1" >&2; return 1 ;;
  esac
}

wait_file() {   # block until the file exists (code / pre-registration landed, selection written)
  while [ ! -f "$1" ]; do log "waiting for $1"; sleep 60; done
  log "found $1: $(tr '\n' ' ' < "$1")"
}

ref_if_exists() {   # <key> <run> -> ",key=run" if the run has a test prediction, else ""
  [ -f "$RUNS/$2/pred_test.npz" ] && echo ",$1=$2" || echo ""
}

est_rss_gb() {   # <glob> -> largest recorded VmHWM (GB, rounded up) among the matching rss files, or the default
  local m
  m=$(cat "$PV"/rss/$1.txt 2>/dev/null | sort -n | tail -1)
  if [ -z "${m:-}" ]; then echo $RSS_DEFAULT_GB; else echo $(( (m + 1048575) / 1048576 )); fi
}

ram_min_gb() {   # <name> -> MemAvailable floor for this job = measured peak RSS of that job class + 10 GB (operator decision 2026-09-15 12:58Z:
  # the 28 GB pretraining rule assumed the corpus is held in host RAM; measured pretraining VmHWM is 3.6 GB (the corpus is a GPU tensor) and the
  # DMS extractor's 13.4 GB RSS left MemAvailable at 15-17 GB, so the coordinator's need-based formula is applied to both job classes; RAM_PRE_GB
  # is kept as an upper clamp)
  local e
  case "$1" in *_pretrain_*) e=$(( $(est_rss_gb "*_pretrain_*") + RAM_FT_MARGIN_GB )); [ "$e" -gt "$RAM_PRE_GB" ] && e=$RAM_PRE_GB; echo $e ;;
                *) echo $(( $(est_rss_gb "*_v2") + RAM_FT_MARGIN_GB )) ;; esac
}

gate_ok() {   # gate_ok <need_gb> <name>: 0 iff the launch gate is satisfied now; prints the state
  local need=$1 name=$2 free_mib avail_kb ntr ram ok=1
  free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | tr -d ' ')
  avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo)
  ntr=$(pgrep -fc "python.*(train_cwm3|train_kp|pretrain)")
  ram=$(ram_min_gb "$name")
  [ "${free_mib:-0}" -lt $(( (need + 4) * 1024 )) ] && ok=0
  [ "${avail_kb:-0}" -lt $(( ram * 1024 * 1024 )) ] && ok=0
  [ "${ntr:-9}" -ge "$MAXTR" ] && ok=0
  GATE_STATE="free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= ${ram}), trainers ${ntr} (< $MAXTR)"
  [ "$ok" = 1 ]
}

gate() {   # gate <need_gb> <name>: block (WITHOUT the launch lock) until the gate is satisfied
  local n=0
  until gate_ok "$1" "$2"; do
    [ $((n % 10)) -eq 0 ] && log "gate WAIT [$2]: $GATE_STATE"
    n=$((n + 1)); sleep 30
  done
  log "gate OK [$2]: $GATE_STATE"
}

track_rss() {   # track_rss <pid> <name>: record the job's peak RSS (VmHWM, kB) and peak VRAM (MiB, nvidia-smi) every 15 s while it lives
  local pid=$1 name=$2 hwm vram best=0
  while kill -0 "$pid" 2>/dev/null; do
    hwm=$(awk '/VmHWM/{print $2}' /proc/"$pid"/status 2>/dev/null)
    [ -n "${hwm:-}" ] && echo "$hwm" > "$PV/rss/$name.txt"
    vram=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits 2>/dev/null | awk -F', ' -v p="$pid" '$1==p{print $2}')
    if [ -n "${vram:-}" ] && [ "$vram" -gt "$best" ]; then best=$vram; echo "$best" > "$PV/rss/$name.vram"; fi
    sleep 15
  done
}

run() {    # run <name> <need_gb> <cmd...>: skip if done, attach if running, gate (no lock), lock + re-check + launch + settle, record pid, wait
  local name=$1 need=$2; shift 2
  local pid live rc
  if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists; existing runs are never re-run)"; return 0; fi
  live=$(pgrep -f -- "python.*--name ${name}( |$)" | head -1)
  if [ -n "${live:-}" ]; then
    log "attach to the running $name (pid $live) instead of re-launching"
    echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$live ATTACH" >> "$PIDS"
    track_rss "$live" "$name" &
    while kill -0 "$live" 2>/dev/null; do sleep 30; done
    wait 2>/dev/null
    log "attached $name finished (rc unknown; run json check follows)"
    echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$live END(attached) rss_kb=$(cat "$PV/rss/$name.txt" 2>/dev/null)" >> "$PIDS"
    ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; return $?
  fi
  while :; do
    gate "$need" "$name"
    exec 9>>"$PV/launch.lock"
    flock 9
    if gate_ok "$need" "$name"; then break; fi
    flock -u 9; exec 9>&-
    log "gate re-check under the lock failed [$name]: $GATE_STATE — back to waiting"
    sleep 30
  done
  local extra=""
  case "$name" in *_pretrain_*) extra="--min-avail-ram-gb $(ram_min_gb "$name")" ;; esac    # the script's own RAM floor follows the same need-based value
  "$@" $extra > "$RUNS/${name}_train.log" 2>&1 &
  pid=$!
  echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid START" >> "$PIDS"
  log "launched $name pid $pid: $*"
  track_rss "$pid" "$name" &
  sleep 45
  flock -u 9; exec 9>&-
  wait "$pid"; rc=$?
  wait 2>/dev/null
  echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid END rc=$rc rss_kb=$(cat "$PV/rss/$name.txt" 2>/dev/null)" >> "$PIDS"
  log "finished $name rc=$rc (peak RSS $(( $(cat "$PV/rss/$name.txt" 2>/dev/null || echo 0) / 1024 )) MB)"
  return $rc
}

pretrain_ok() { [ -f "$RUNS/$1/ckpt_best.pt" ] && [ -f "$RUNS/$1/TRAIN_DONE" ]; }

k_variant() {   # <tag>: CockpitWM pretraining variant + 5 trunk-only fine-tunes
  local tag=$1 pre="cwm3d_kp_batonpre_pretrain_${tag}_s123" s
  run "$pre" $NEED_PRE_K "$PY" aide/train/pretrain_cwm3d_baton.py --model cwm3d --name "$pre" $PRE_COMMON $(variant_flags "$tag") --max-minutes 90 --min-free-vram-gb $((NEED_PRE_K + 4)) \
      || log "pretraining $pre failed (rc != 0)"
  pretrain_ok "$pre" || { log "SKIP fine-tunes of $tag: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; }
  for s in $SEEDS5; do
    run "cwm3d_kp_batonpre_${tag}_s${s}_v2" $NEED_FT "$PY" aide/train/train_kp.py --model cwm3d --seed "$s" --name "cwm3d_kp_batonpre_${tag}_s${s}_v2" $RECIPE \
        --init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \
        --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2" --no-lock --max-minutes 25 \
        --notes "pretrain_variants;P1:${tag};init=${pre}:trunk;stage_A_recipe_verbatim" || log "fine-tune ${tag} s${s} failed"
  done
}

arch_variant() {   # <model> <tag> <need_gb> : matched pretraining of a keypoint baseline architecture + 5 trunk-only fine-tunes (--loss euc recipe)
  local model=$1 tag=$2 need=$3 pre="${1}_batonpre_pretrain_${2}_s123" s refs
  run "$pre" "$need" "$PY" aide/train/pretrain_cwm3d_baton.py --model "$model" --name "$pre" $PRE_COMMON $(variant_flags "$tag") --max-minutes 400 --min-free-vram-gb $((need + 4)) \
      || log "pretraining $pre failed (rc != 0)"
  pretrain_ok "$pre" || { log "SKIP fine-tunes of $model/$tag: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; }
  for s in $SEEDS5; do
    refs="zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2$(ref_if_exists seedinit "${model}_s${s}_v2")"
    run "${model}_batonpre_${tag}_s${s}_v2" $NEED_FT "$PY" aide/train/train_kp.py --model "$model" --seed "$s" --name "${model}_batonpre_${tag}_s${s}_v2" $RECIPE \
        --init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk --refs "$refs" --no-lock --max-minutes 45 \
        --notes "pretrain_variants;P3:matched_${tag};init=${pre}:trunk;hardening_D_recipe_verbatim" || log "fine-tune ${model}/${tag} s${s} failed"
  done
}

v2_variant() {   # <tag>: lean-read-out CockpitWM (cwm3d_v2 --no-v2-spatial) pretraining variant + 5 trunk-only fine-tunes (K-v2 S2 recipe verbatim)
  local tag=$1 pre="cwm3d_v2_nospatial_batonpre_pretrain_${tag}_s123" s
  run "$pre" $NEED_PRE_V2 "$PY" aide/train/pretrain_cwm3d_baton.py --model cwm3d_v2 --no-v2-spatial --name "$pre" $PRE_COMMON $(variant_flags "$tag") --max-minutes 150 --min-free-vram-gb $((NEED_PRE_V2 + 4)) \
      || log "pretraining $pre failed (rc != 0)"
  pretrain_ok "$pre" || { log "SKIP fine-tunes of v2/$tag: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; }
  for s in $SEEDS5; do
    run "cwm3d_v2_nospatial_batonpre_${tag}_s${s}_v2" $NEED_FT "$PY" aide/train/train_kp.py --model cwm3d_v2 --no-v2-spatial --seed "$s" --name "cwm3d_v2_nospatial_batonpre_${tag}_s${s}_v2" $RECIPE \
        --init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \
        --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,mb=motionbert_kp_s${s}_v2,seedinit=cwm3d_v2_nospatial_s${s}_v2,kppre=cwm3d_v2_nospatial_batonpre_trunkp123_s${s}_v2" \
        --no-lock --max-minutes 45 --notes "pretrain_variants;P1f:${tag};cfg=nospatial;init=${pre}:trunk;kv2_S2_recipe_verbatim" || log "fine-tune v2/${tag} s${s} failed"
  done
}

case "$LANE" in
  K)
    for tag in euc conv conveuc; do k_variant "$tag"; done
    wait_file "$PV/GO_scale"
    for tag in scale convscale; do k_variant "$tag"; done ;;
  MB) arch_variant motionbert_kp conv $NEED_PRE_MB ;;
  ST) wait_file "$PV/GO_st"; arch_variant stgcn_kp conv $NEED_PRE_ST ;;
  MBX) wait_file "$PV/GO_mbx"; arch_variant motionbert_kp convscale $NEED_PRE_MB ;;
  STX) wait_file "$PV/GO_stx"; arch_variant stgcn_kp convscale $NEED_PRE_ST ;;
  P2)
    wait_file "$PV/GO_p2"
    pre="driverwm_kp_batonpre_pretrain_s123"
    run "$pre" $NEED_PRE_DWM "$PY" aide/train/pretrain_cwm3d_baton.py --model driverwm_kp --name "$pre" --seed 123 --bs "$P2_BS" --lr 3e-4 --ema 0.999 --zeroshot 0 --epochs 6 --max-minutes 150 --min-free-vram-gb $((NEED_PRE_DWM + 4)) \
        || log "pretraining $pre failed (rc != 0)"
    if pretrain_ok "$pre"; then
      for s in 42 123 456; do
        run "driverwm_kp_batonpre_trunkp123_s${s}_v2" $NEED_FT_DWM "$PY" aide/train/train_kp.py --model driverwm_kp --seed "$s" --name "driverwm_kp_batonpre_trunkp123_s${s}_v2" $RECIPE \
            --init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \
            --refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${s}_v2,kp=cwm3d_kp_s${s}_v2,mb=motionbert_kp_s${s}_v2,dwm=driverwm_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2" \
            --no-lock --max-minutes 60 --notes "pretrain_variants;P2:driverwm_kp_batonpre;init=${pre}:trunk;shared_recipe_verbatim;exterior_stream_zero_input_in_pretraining" \
            || log "fine-tune driverwm_kp s${s} failed"
      done
    else
      log "SKIP P2 fine-tunes: $pre has no ckpt_best.pt / TRAIN_DONE"
    fi ;;
  F)
    wait_file "$PV/SELECTED"
    for tag in $(cat "$PV/SELECTED"); do v2_variant "$tag"; done ;;
  *) echo "unknown lane $LANE"; exit 2 ;;
esac
log "lane $LANE done"
echo "$(date -u +%FT%TZ) lane=$LANE LANE_DONE" >> "$PIDS"

Xet Storage Details

Size:: 16.4 kB
Xet hash:: 066b7d2ee50b63903282da8a87a67ef6959da3bf7c5852406412df68ac8c2d4c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	#!/usr/bin/env bash
	# Pre-registered pretraining experiments P1 / P1-ext / P2 / P3 (docs/decisions.md 2026-09-15 rows "PRE-REGISTRATION — CockpitWM pretraining
	# variants …" and "… Driver-WM architecture on CockpitWM tokens + BATON pretraining") — detached lane runner.
	#
	# setsid nohup bash aide/train/run_pretrain_variants.sh <K\|MB\|ST\|P2\|F\|MBX\|STX> > aide/results/pretrain_variants/lane_<X>.log 2>&1 &
	#
	# Lane K CockpitWM (cwm3d_kp) pretraining variants on the SAME corpus (sha256 465dac1cf0a8265a…), seed 123, bs 256, lr 3e-4, EMA 0.999,
	# held-out-route selection, --zeroshot 0: euc (6 ep, Euclidean only) -> conv (<= 36 ep, patience 4, all four loss terms)
	# -> conveuc (conv + Euclidean only) -> scale (6 ep, all terms, --motion-scale-max 2.5) -> convscale (conv + --motion-scale-max 2.5);
	# after each pretraining the 5 trunk-only AIDE fine-tunes cwm3d_kp_batonpre_<tag>_s<k>_v2 (Stage-A recipe verbatim as
	# cwm3d_kp_batonpre_trunkp123_s<k>_v2, refs incl. kppre = that adopted arm). The scale variants wait for GO_scale (code + tests landed).
	# Lane MB MotionBERT architecture, converged matched recipe: motionbert_kp_batonpre_pretrain_conv_s123 (<= 36 ep, patience 4, --max-minutes 400)
	# -> 5 trunk-only fine-tunes motionbert_kp_batonpre_conv_s<k>_v2 (hardening-D recipe verbatim, --loss euc). For tensor-output
	# architectures conv == conveuc (the nll / bone / jerk weights are inert), so this run is the matched run for either selection.
	# Lane ST the same for the ST-GCN architecture (stgcn_kp_batonpre_pretrain_conv_s123 -> stgcn_kp_batonpre_conv_s<k>_v2); waits for GO_st.
	# Lane P2 Driver-WM architecture on CockpitWM tokens: driverwm_kp_batonpre_pretrain_s123 (6 ep, bs $P2_BS, --max-minutes 150) -> 3 trunk-only
	# fine-tunes driverwm_kp_batonpre_trunkp123_s{42,123,456}_v2 (shared recipe verbatim, --loss euc); waits for GO_p2 (code + tests + row).
	# Lane F lean-read-out CockpitWM (cwm3d_v2 --no-v2-spatial): the best two K variants by the pre-registered val rule (file SELECTED, one tag per
	# line, written by report_pretrain_variants.py --select) -> cwm3d_v2_nospatial_batonpre_pretrain_<tag>_s123 -> 5 trunk-only fine-tunes.
	# Lane MBX / STX only if the selected recipe is convscale: motionbert_kp / stgcn_kp pretraining with --motion-scale-max 2.5 (waits for GO_mbx / GO_stx).
	#
	# The pretraining script's own resource gate receives the same floor (--min-free-vram-gb need + 4; its default 12 GB aborted the conveuc / scale launches at 06:08Z).
	# Launch gate (coordinator 2026-09-15 05:45Z revision): free VRAM >= need + 4 GB (need = the job's measured peak footprint, rounded up) and
	# trainers alive (`pgrep -fc "python.*(train_cwm3\|train_kp\|pretrain)"`) < MAXTR (default 3; project cap < 4) for every job; host RAM:
	# pretraining processes (name contains _pretrain_, they load the corpus) keep the project rule MemAvailable >= 28 GB; the small AIDE fine-tunes are
	# need-based: MemAvailable >= est. RSS + 10 GB, est. RSS = the largest VmHWM recorded so far for a fine-tune of this lane runner
	# (aide/results/pretrain_variants/rss/<run>.txt, sampled every 15 s while the job runs) or 4 GB before any measurement. The gate is polled WITHOUT
	# the launch lock; the lock (aide/results/pretrain_variants/launch.lock) is taken only for re-check + launch + 45 s settle, so a lane whose gate cannot
	# be met never blocks the others (the 04:31Z–05:46Z stall). A lane restarted while its job is still running ATTACHES to that process instead of
	# re-launching. Existing runs are never touched: a run whose run json exists is skipped; new run dirs only. PIDs -> pids.txt.
	# Selection is val-only inside train_kp / held-out-route only inside the pretraining script; the official test split is evaluated once per run.
	set -u
	LANE=${1:?lane K\|MB\|ST\|P2\|F\|MBX\|STX}
	ROOT=/home/USER/cwm3
	PY=python
	RUNS=$ROOT/aide/results/runs
	PV=$ROOT/aide/results/pretrain_variants
	PIDS=$PV/pids.txt
	MAXTR=${MAXTR:-3}
	P2_BS=${P2_BS:-256}
	RAM_PRE_GB=28 # project rule for corpus-loading pretraining processes
	RAM_FT_MARGIN_GB=10 # fine-tunes: MemAvailable >= est. RSS + this
	RSS_DEFAULT_GB=4
	export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4
	mkdir -p "$PV/rss"
	cd "$ROOT" \|\| exit 1
	# Stage-A / hardening-D fine-tune recipe (aide/train/run_stage_bc.RECIPE + --loss euc) — verbatim as aide/train/run_arch_attribution.sh
	RECIPE="--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 --translate 20 --scale 0.05 --flip-p 0 --loss euc"
	# pretraining constants shared by every variant (== the adopted run cwm3d_kp_batonpre_pretrain_s123 except the varied flags)
	PRE_COMMON="--seed 123 --bs 256 --lr 3e-4 --ema 0.999 --zeroshot 0"
	SEEDS5="42 123 456 789 1011"
	# measured peak footprints (GiB reserved, rounded up): cwm3d pretraining 3.98 -> 4; cwm3d fine-tune ~1 -> 3 (conservative); driverwm_kp fine-tune
	# 2.46 -> 3; motionbert_kp pretraining 12.7 -> 13 (bs 256 + bs-1024 held-out eval cache); stgcn_kp pretraining 14.0 -> 14; cwm3d_v2 pretraining unmeasured -> 8
	NEED_PRE_K=4; NEED_FT=3; NEED_FT_DWM=3; NEED_PRE_MB=13; NEED_PRE_ST=14; NEED_PRE_V2=5; NEED_PRE_DWM=7 # V2: estimate (cwm3d footprint + 1 GB; 2.7 M params), measured VRAM is written to rss/<run>.vram

	log() { echo "$(date -u +%FT%TZ) [lane $LANE] $*"; }

	variant_flags() { # <tag> -> the flags that define the variant (everything else = the adopted run's defaults)
	case "$1" in
	euc) echo "--epochs 6 --w-nll 0 --w-bone 0 --w-jerk 0" ;;
	scale) echo "--epochs 6 --motion-scale-max 2.5" ;;
	conv) echo "--epochs 36 --patience 4" ;;
	conveuc) echo "--epochs 36 --patience 4 --w-nll 0 --w-bone 0 --w-jerk 0" ;;
	convscale) echo "--epochs 36 --patience 4 --motion-scale-max 2.5" ;;
	*) echo "unknown variant $1" >&2; return 1 ;;
	esac
	}

	wait_file() { # block until the file exists (code / pre-registration landed, selection written)
	while [ ! -f "$1" ]; do log "waiting for $1"; sleep 60; done
	log "found $1: $(tr '\n' ' ' < "$1")"
	}

	ref_if_exists() { # <key> <run> -> ",key=run" if the run has a test prediction, else ""
	[ -f "$RUNS/$2/pred_test.npz" ] && echo ",$1=$2" \|\| echo ""
	}

	est_rss_gb() { # <glob> -> largest recorded VmHWM (GB, rounded up) among the matching rss files, or the default
	local m
	m=$(cat "$PV"/rss/$1.txt 2>/dev/null \| sort -n \| tail -1)
	if [ -z "${m:-}" ]; then echo $RSS_DEFAULT_GB; else echo $(( (m + 1048575) / 1048576 )); fi
	}

	ram_min_gb() { # <name> -> MemAvailable floor for this job = measured peak RSS of that job class + 10 GB (operator decision 2026-09-15 12:58Z:
	# the 28 GB pretraining rule assumed the corpus is held in host RAM; measured pretraining VmHWM is 3.6 GB (the corpus is a GPU tensor) and the
	# DMS extractor's 13.4 GB RSS left MemAvailable at 15-17 GB, so the coordinator's need-based formula is applied to both job classes; RAM_PRE_GB
	# is kept as an upper clamp)
	local e
	case "$1" in _pretrain_) e=$(( $(est_rss_gb "_pretrain_") + RAM_FT_MARGIN_GB )); [ "$e" -gt "$RAM_PRE_GB" ] && e=$RAM_PRE_GB; echo $e ;;
	) echo $(( $(est_rss_gb "_v2") + RAM_FT_MARGIN_GB )) ;; esac
	}

	gate_ok() { # gate_ok <need_gb> <name>: 0 iff the launch gate is satisfied now; prints the state
	local need=$1 name=$2 free_mib avail_kb ntr ram ok=1
	free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits \| head -1 \| tr -d ' ')
	avail_kb=$(awk '/MemAvailable/{print $2}' /proc/meminfo)
	ntr=$(pgrep -fc "python.*(train_cwm3\|train_kp\|pretrain)")
	ram=$(ram_min_gb "$name")
	[ "${free_mib:-0}" -lt $(( (need + 4) * 1024 )) ] && ok=0
	[ "${avail_kb:-0}" -lt $(( ram * 1024 * 1024 )) ] && ok=0
	[ "${ntr:-9}" -ge "$MAXTR" ] && ok=0
	GATE_STATE="free VRAM ${free_mib} MiB (need $((need + 4)) GB), MemAvailable $((avail_kb / 1048576)) GB (>= ${ram}), trainers ${ntr} (< $MAXTR)"
	[ "$ok" = 1 ]
	}

	gate() { # gate <need_gb> <name>: block (WITHOUT the launch lock) until the gate is satisfied
	local n=0
	until gate_ok "$1" "$2"; do
	[ $((n % 10)) -eq 0 ] && log "gate WAIT [$2]: $GATE_STATE"
	n=$((n + 1)); sleep 30
	done
	log "gate OK [$2]: $GATE_STATE"
	}

	track_rss() { # track_rss <pid> <name>: record the job's peak RSS (VmHWM, kB) and peak VRAM (MiB, nvidia-smi) every 15 s while it lives
	local pid=$1 name=$2 hwm vram best=0
	while kill -0 "$pid" 2>/dev/null; do
	hwm=$(awk '/VmHWM/{print $2}' /proc/"$pid"/status 2>/dev/null)
	[ -n "${hwm:-}" ] && echo "$hwm" > "$PV/rss/$name.txt"
	vram=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits 2>/dev/null \| awk -F', ' -v p="$pid" '$1==p{print $2}')
	if [ -n "${vram:-}" ] && [ "$vram" -gt "$best" ]; then best=$vram; echo "$best" > "$PV/rss/$name.vram"; fi
	sleep 15
	done
	}

	run() { # run <name> <need_gb> <cmd...>: skip if done, attach if running, gate (no lock), lock + re-check + launch + settle, record pid, wait
	local name=$1 need=$2; shift 2
	local pid live rc
	if ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; then log "skip $name (run json exists; existing runs are never re-run)"; return 0; fi
	live=$(pgrep -f -- "python.*--name ${name}( \|$)" \| head -1)
	if [ -n "${live:-}" ]; then
	log "attach to the running $name (pid $live) instead of re-launching"
	echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$live ATTACH" >> "$PIDS"
	track_rss "$live" "$name" &
	while kill -0 "$live" 2>/dev/null; do sleep 30; done
	wait 2>/dev/null
	log "attached $name finished (rc unknown; run json check follows)"
	echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$live END(attached) rss_kb=$(cat "$PV/rss/$name.txt" 2>/dev/null)" >> "$PIDS"
	ls "$RUNS/$name"/run_"$name"_2*.json >/dev/null 2>&1; return $?
	fi
	while :; do
	gate "$need" "$name"
	exec 9>>"$PV/launch.lock"
	flock 9
	if gate_ok "$need" "$name"; then break; fi
	flock -u 9; exec 9>&-
	log "gate re-check under the lock failed [$name]: $GATE_STATE — back to waiting"
	sleep 30
	done
	local extra=""
	case "$name" in _pretrain_) extra="--min-avail-ram-gb $(ram_min_gb "$name")" ;; esac # the script's own RAM floor follows the same need-based value
	"$@" $extra > "$RUNS/${name}_train.log" 2>&1 &
	pid=$!
	echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid START" >> "$PIDS"
	log "launched $name pid $pid: $*"
	track_rss "$pid" "$name" &
	sleep 45
	flock -u 9; exec 9>&-
	wait "$pid"; rc=$?
	wait 2>/dev/null
	echo "$(date -u +%FT%TZ) lane=$LANE name=$name pid=$pid END rc=$rc rss_kb=$(cat "$PV/rss/$name.txt" 2>/dev/null)" >> "$PIDS"
	log "finished $name rc=$rc (peak RSS $(( $(cat "$PV/rss/$name.txt" 2>/dev/null \|\| echo 0) / 1024 )) MB)"
	return $rc
	}

	pretrain_ok() { [ -f "$RUNS/$1/ckpt_best.pt" ] && [ -f "$RUNS/$1/TRAIN_DONE" ]; }

	k_variant() { # <tag>: CockpitWM pretraining variant + 5 trunk-only fine-tunes
	local tag=$1 pre="cwm3d_kp_batonpre_pretrain_${tag}_s123" s
	run "$pre" $NEED_PRE_K "$PY" aide/train/pretrain_cwm3d_baton.py --model cwm3d --name "$pre" $PRE_COMMON $(variant_flags "$tag") --max-minutes 90 --min-free-vram-gb $((NEED_PRE_K + 4)) \
	\|\| log "pretraining $pre failed (rc != 0)"
	pretrain_ok "$pre" \|\| { log "SKIP fine-tunes of $tag: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; }
	for s in $SEEDS5; do
	run "cwm3d_kp_batonpre_${tag}_s${s}_v2" $NEED_FT "$PY" aide/train/train_kp.py --model cwm3d --seed "$s" --name "cwm3d_kp_batonpre_${tag}_s${s}_v2" $RECIPE \
	--init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \
	--refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2" --no-lock --max-minutes 25 \
	--notes "pretrain_variants;P1:${tag};init=${pre}:trunk;stage_A_recipe_verbatim" \|\| log "fine-tune ${tag} s${s} failed"
	done
	}

	arch_variant() { # <model> <tag> <need_gb> : matched pretraining of a keypoint baseline architecture + 5 trunk-only fine-tunes (--loss euc recipe)
	local model=$1 tag=$2 need=$3 pre="${1}_batonpre_pretrain_${2}_s123" s refs
	run "$pre" "$need" "$PY" aide/train/pretrain_cwm3d_baton.py --model "$model" --name "$pre" $PRE_COMMON $(variant_flags "$tag") --max-minutes 400 --min-free-vram-gb $((need + 4)) \
	\|\| log "pretraining $pre failed (rc != 0)"
	pretrain_ok "$pre" \|\| { log "SKIP fine-tunes of $model/$tag: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; }
	for s in $SEEDS5; do
	refs="zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2$(ref_if_exists seedinit "${model}_s${s}_v2")"
	run "${model}_batonpre_${tag}_s${s}_v2" $NEED_FT "$PY" aide/train/train_kp.py --model "$model" --seed "$s" --name "${model}_batonpre_${tag}_s${s}_v2" $RECIPE \
	--init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk --refs "$refs" --no-lock --max-minutes 45 \
	--notes "pretrain_variants;P3:matched_${tag};init=${pre}:trunk;hardening_D_recipe_verbatim" \|\| log "fine-tune ${model}/${tag} s${s} failed"
	done
	}

	v2_variant() { # <tag>: lean-read-out CockpitWM (cwm3d_v2 --no-v2-spatial) pretraining variant + 5 trunk-only fine-tunes (K-v2 S2 recipe verbatim)
	local tag=$1 pre="cwm3d_v2_nospatial_batonpre_pretrain_${tag}_s123" s
	run "$pre" $NEED_PRE_V2 "$PY" aide/train/pretrain_cwm3d_baton.py --model cwm3d_v2 --no-v2-spatial --name "$pre" $PRE_COMMON $(variant_flags "$tag") --max-minutes 150 --min-free-vram-gb $((NEED_PRE_V2 + 4)) \
	\|\| log "pretraining $pre failed (rc != 0)"
	pretrain_ok "$pre" \|\| { log "SKIP fine-tunes of v2/$tag: $pre has no ckpt_best.pt / TRAIN_DONE"; return 1; }
	for s in $SEEDS5; do
	run "cwm3d_v2_nospatial_batonpre_${tag}_s${s}_v2" $NEED_FT "$PY" aide/train/train_kp.py --model cwm3d_v2 --no-v2-spatial --seed "$s" --name "cwm3d_v2_nospatial_batonpre_${tag}_s${s}_v2" $RECIPE \
	--init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \
	--refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s${s}_v2,mb=motionbert_kp_s${s}_v2,seedinit=cwm3d_v2_nospatial_s${s}_v2,kppre=cwm3d_v2_nospatial_batonpre_trunkp123_s${s}_v2" \
	--no-lock --max-minutes 45 --notes "pretrain_variants;P1f:${tag};cfg=nospatial;init=${pre}:trunk;kv2_S2_recipe_verbatim" \|\| log "fine-tune v2/${tag} s${s} failed"
	done
	}

	case "$LANE" in
	K)
	for tag in euc conv conveuc; do k_variant "$tag"; done
	wait_file "$PV/GO_scale"
	for tag in scale convscale; do k_variant "$tag"; done ;;
	MB) arch_variant motionbert_kp conv $NEED_PRE_MB ;;
	ST) wait_file "$PV/GO_st"; arch_variant stgcn_kp conv $NEED_PRE_ST ;;
	MBX) wait_file "$PV/GO_mbx"; arch_variant motionbert_kp convscale $NEED_PRE_MB ;;
	STX) wait_file "$PV/GO_stx"; arch_variant stgcn_kp convscale $NEED_PRE_ST ;;
	P2)
	wait_file "$PV/GO_p2"
	pre="driverwm_kp_batonpre_pretrain_s123"
	run "$pre" $NEED_PRE_DWM "$PY" aide/train/pretrain_cwm3d_baton.py --model driverwm_kp --name "$pre" --seed 123 --bs "$P2_BS" --lr 3e-4 --ema 0.999 --zeroshot 0 --epochs 6 --max-minutes 150 --min-free-vram-gb $((NEED_PRE_DWM + 4)) \
	\|\| log "pretraining $pre failed (rc != 0)"
	if pretrain_ok "$pre"; then
	for s in 42 123 456; do
	run "driverwm_kp_batonpre_trunkp123_s${s}_v2" $NEED_FT_DWM "$PY" aide/train/train_kp.py --model driverwm_kp --seed "$s" --name "driverwm_kp_batonpre_trunkp123_s${s}_v2" $RECIPE \
	--init-ckpt "aide/results/runs/$pre/ckpt_best.pt" --init-parts trunk \
	--refs "zvshrink=zv_shrink_val,simlpe=simlpe_kp_s${s}_v2,kp=cwm3d_kp_s${s}_v2,mb=motionbert_kp_s${s}_v2,dwm=driverwm_kp_s${s}_v2,kppre=cwm3d_kp_batonpre_trunkp123_s${s}_v2" \
	--no-lock --max-minutes 60 --notes "pretrain_variants;P2:driverwm_kp_batonpre;init=${pre}:trunk;shared_recipe_verbatim;exterior_stream_zero_input_in_pretraining" \
	\|\| log "fine-tune driverwm_kp s${s} failed"
	done
	else
	log "SKIP P2 fine-tunes: $pre has no ckpt_best.pt / TRAIN_DONE"
	fi ;;
	F)
	wait_file "$PV/SELECTED"
	for tag in $(cat "$PV/SELECTED"); do v2_variant "$tag"; done ;;
	*) echo "unknown lane $LANE"; exit 2 ;;
	esac
	log "lane $LANE done"
	echo "$(date -u +%FT%TZ) lane=$LANE LANE_DONE" >> "$PIDS"