Buckets:
| # cwm3/train/run_queue_udwm.sh — gated sequential queue for the experiment-C trainers (2026-09-17), = train/run_queue_cmds.sh (same job-line format | |
| # "<out_dir>||<trainer script + args>", same gates, same log / retry conventions) PLUS one interleaving rule: the job HOLDS /tmp/s13_gpu.lock for its whole | |
| # duration. The WP5 holdout V-JEPA2 extractor (results/holdout/queue_holdout_stage2.sh) takes that lock around every ~25-min chunk, so (i) this queue cannot | |
| # launch while a chunk runs (the blocking flock waits for it) and (ii) the extractor's next chunk waits until the trainer is finished instead of starting into | |
| # a contended GPU (its routes fail with CUDA OOM under contention — results/holdout/logs/s5_chunk_20260916T230015Z.log). Nothing is ever killed. | |
| # bash train/run_queue_udwm.sh <queue_name> <queue_file> [min_avail_gb=28] [min_vram_gb=19] [passes=3] | |
| # Gates (checked under flock /tmp/cwm3_joint_launch.lock, shared with the other launchers so two cannot race, AFTER the s13 lock is held): | |
| # (a) trainer PARENT processes matching "python … (train_cwm3|train_kp|pretrain)" < MAX_TRAINERS (4) across ALL agents, (b) MemAvailable >= min_avail_gb, | |
| # (c) nvidia-smi free VRAM >= min_vram_gb (= the job's need + 4 GB: U0 udwm ≈ 15 GB -> 19; joint udwm ≈ 12 GB -> 16), (d) df free >= MIN_DISK_GB (60). | |
| # Threads: OMP/MKL 4. Jobs whose out_dir has PRED_DONE are skipped; a failed job is retried on the next pass (the trainers resume from their checkpoints). | |
| # Log: results/runs/<queue_name>_<utc>.log (append-only); per job <out_dir>.stdout.log. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| NAME="$1"; QUEUE="$2"; MIN_AVAIL_GB="${3:-28}"; MIN_VRAM_GB="${4:-19}"; PASSES="${5:-3}" | |
| MIN_DISK_GB="${MIN_DISK_GB:-60}" | |
| LOG="results/runs/${NAME}_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; } | |
| MAX_TRAINERS="${MAX_TRAINERS:-4}"; TRAINER_PAT="${TRAINER_PAT:-^[^ ]*python[^ ]* [^ ]*(train_cwm3|train_kp|pretrain)}" | |
| n_trainers() { | |
| local pids n=0 pp; pids=$(pgrep -f "$TRAINER_PAT" || true) | |
| for p in $pids; do pp=$(awk '/^PPid/{print $2}' /proc/$p/status 2>/dev/null); grep -qx -- "$pp" <<< "$pids" || n=$((n+1)); done; echo $n; } | |
| avail_gb() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_vram_gb() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| disk_gb() { df -BG --output=avail / | tail -1 | tr -dc '0-9'; } | |
| LAUNCH_LOCK=/tmp/cwm3_joint_launch.lock | |
| GPU_LOCK=/tmp/s13_gpu.lock | |
| log "queue $NAME from $QUEUE: gates trainers<${MAX_TRAINERS} avail>=${MIN_AVAIL_GB}G vram_free>=${MIN_VRAM_GB}G disk>=${MIN_DISK_GB}G; holds $GPU_LOCK per job (interleave with the holdout extractor)" | |
| for pass in $(seq 1 "$PASSES"); do | |
| pending=0 | |
| while IFS= read -r line; do | |
| [[ -z "$line" || "$line" =~ ^# ]] && continue | |
| out="${line%%||*}"; args="${line#*||}" | |
| [[ -f "$out/PRED_DONE" ]] && continue | |
| cmd="$PY $args --out $out" | |
| # ---- hold the GPU lock for the whole job (blocks while an extractor chunk runs; the extractor blocks while we run) | |
| exec 8>"$GPU_LOCK" | |
| log "pass $pass: waiting for $GPU_LOCK before $(basename $out)" | |
| flock 8 | |
| log "pass $pass: holding $GPU_LOCK for $(basename $out)" | |
| while true; do | |
| exec 9>"$LAUNCH_LOCK"; flock 9 | |
| nt=$(n_trainers); av=$(avail_gb); vr=$(free_vram_gb); dk=$(disk_gb) | |
| if [[ "$nt" -lt "$MAX_TRAINERS" && "$av" -ge "$MIN_AVAIL_GB" && "$vr" -ge "$MIN_VRAM_GB" && "$dk" -ge "$MIN_DISK_GB" ]]; then | |
| log "pass $pass: start $(basename $out): $cmd (trainers=$nt avail=${av}G vram_free=${vr}G disk=${dk}G)" | |
| t0=$(date +%s) | |
| mkdir -p "$out" | |
| $cmd >> "$out.stdout.log" 2>&1 & | |
| pid=$!; sleep 20; flock -u 9; exec 9>&- | |
| wait $pid; code=$? | |
| break | |
| fi | |
| flock -u 9; exec 9>&- | |
| log "pass $pass: wait $(basename $out) (trainers=$nt avail=${av}G vram_free=${vr}G disk=${dk}G)"; sleep 60 | |
| done | |
| flock -u 8; exec 8>&- | |
| log "pass $pass: end $(basename $out): exit $code in $(( ($(date +%s)-t0) )) s; released $GPU_LOCK" | |
| [[ $code -ne 0 ]] && pending=1 | |
| done < "$QUEUE" | |
| [[ $pending -eq 0 ]] && { log "QUEUE DONE"; exit 0; } | |
| log "pass $pass done; failed jobs remain -> sleeping 5 min"; sleep 300 | |
| done | |
| log "QUEUE EXHAUSTED ($PASSES passes) with pending jobs"; exit 1 | |
Xet Storage Details
- Size:
- 4.41 kB
- Xet hash:
- 8366235fb3af14794ef365524d0f542eebc678a7d0b40de27c0afb476f4efd6d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.