Buckets:
| # Automation companion of aide/train/run_pretrain_variants.sh (pre-registered rows docs/decisions.md 2026-09-15): releases the lane GO files | |
| # in the pre-registered order and performs the recipe selection exactly once, from run outputs only. | |
| # | |
| # setsid nohup bash aide/train/run_pretrain_variants_auto.sh > aide/results/pretrain_variants/auto.log 2>&1 & | |
| # | |
| # A. when all 5 CockpitWM variants have their 5 fine-tune run jsons -> report_pretrain_variants.py --select (writes SELECTED / P3_TAG / BEST_CONV | |
| # and the first block of aide/results/tables/pretrain_variants.md); lane F proceeds on SELECTED; if P3_TAG == convscale the MBX / STX lanes are | |
| # launched (they wait for GO_mbx / GO_stx, released immediately). | |
| # B. when lane F has launched its second lean-read-out pretraining (the last pretraining that needs >= 12 GB free VRAM), release GO_mb after a | |
| # 2-min settle: the MotionBERT matched pretraining (12.7 GB VRAM) then runs while only fine-tunes remain in the other lanes. | |
| set -u | |
| ROOT=/home/USER/cwm3 | |
| PY=python | |
| RUNS=$ROOT/aide/results/runs | |
| PV=$ROOT/aide/results/pretrain_variants | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4 | |
| cd "$ROOT" || exit 1 | |
| log() { echo "$(date -u +%FT%TZ) [auto] $*"; } | |
| n_ft_done() { # number of finished CockpitWM variant fine-tunes (run json present) out of 25 | |
| local n=0 tag s | |
| for tag in euc scale conv conveuc convscale; do for s in 42 123 456 789 1011; do | |
| ls "$RUNS/cwm3d_kp_batonpre_${tag}_s${s}_v2"/run_cwm3d_kp_batonpre_${tag}_s${s}_v2_2*.json >/dev/null 2>&1 && n=$((n + 1)) | |
| done; done | |
| echo $n | |
| } | |
| # ---- B runs in the background so that GO_mb does not wait for the selection | |
| ( # ---- B. GO_mb when the ST-GCN matched pretraining ends (coordinator 05:45Z: do not leave the MotionBERT lane idle; its 12.7 GB re-uses the | |
| # 14 GB slot ST-GCN vacates; the remaining CockpitWM / lean-read-out / Driver-WM jobs (<= 5 GB each) run alongside it in the rest of the GPU) | |
| while :; do | |
| if ls "$RUNS/stgcn_kp_batonpre_pretrain_conv_s123"/run_stgcn_kp_batonpre_pretrain_conv_s123_2*.json >/dev/null 2>&1 || grep -q "lane=ST LANE_DONE" "$PV/pids.txt" 2>/dev/null; then break; fi | |
| sleep 60 | |
| done | |
| sleep 60 | |
| [ -f "$PV/GO_mb" ] || { date -u +%FT%TZ > "$PV/GO_mb"; log "GO_mb released (ST-GCN pretraining finished)"; } | |
| if [ "$(cat "$PV/P3_TAG" 2>/dev/null)" = "convscale" ]; then | |
| while ! grep -q "lane=MB LANE_DONE" "$PV/pids.txt" 2>/dev/null; do sleep 300; done | |
| date -u +%FT%TZ > "$PV/GO_mbx"; log "GO_mbx released" | |
| fi | |
| ) & | |
| # ---- A. selection | |
| while :; do | |
| n=$(n_ft_done) | |
| if [ "$n" -ge 25 ]; then break; fi | |
| if ! pgrep -f "run_pretrain_variants.sh K$" >/dev/null; then log "no lane K process alive with only $n/25 fine-tunes — selecting on the finished ones is NOT allowed by the rule; stopping automation A"; exit 1; fi | |
| sleep 120 | |
| done | |
| log "all 25 CockpitWM variant fine-tunes finished -> selection (val rule) + first table block" | |
| "$PY" aide/eval/report_pretrain_variants.py --select > "$PV/report_select.log" 2>&1 || log "report --select rc=$?" | |
| log "SELECTED=$(tr '\n' ' ' < "$PV/SELECTED" 2>/dev/null) P3_TAG=$(cat "$PV/P3_TAG" 2>/dev/null) BEST_CONV=$(cat "$PV/BEST_CONV" 2>/dev/null)" | |
| if [ "$(cat "$PV/P3_TAG" 2>/dev/null)" = "convscale" ]; then | |
| log "convscale selected -> launching MBX / STX lanes (matched pretraining with --motion-scale-max 2.5)" | |
| setsid nohup bash aide/train/run_pretrain_variants.sh MBX > "$PV/lane_MBX.log" 2>&1 < /dev/null & | |
| setsid nohup bash aide/train/run_pretrain_variants.sh STX > "$PV/lane_STX.log" 2>&1 < /dev/null & | |
| sleep 2; date -u +%FT%TZ > "$PV/GO_stx" | |
| # MBX (12.7 GB) is released together with GO_mb below | |
| fi | |
| wait | |
| log "automation done" | |
Xet Storage Details
- Size:
- 3.73 kB
- Xet hash:
- 6b26920162a7c2c34f2cd09ef5f41c79d50b87816d6a82477e56e5573b3f521c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.