Buckets:
| # cwm3/train/run_queue_gpu_baselines.sh — gated sequential queue of base-trainer runs for the GPU-baselines experiments (B: branch-architecture ablation), | |
| # WITHOUT the GPU lock (project rule: concurrent trainers, gated launches). Each job line: "<out_dir>||<train_cwm3.py args>". | |
| # bash train/run_queue_gpu_baselines.sh <queue_name> <queue_file> [min_avail_gb=24] [min_vram_gb=8] [passes=3] | |
| # Gates (under flock /tmp/cwm3_joint_launch.lock, shared with the other launchers so two cannot race): | |
| # (a) trainer python processes matching (train_cwm3|train_kp|pretrain) across ALL agents <= MAX_OTHERS (2) — i.e. < 4 alive after the launch, | |
| # (b) MemAvailable >= min_avail_gb (= job RSS + 14 GB headroom: U0 ~10 GB -> 24, V0 ~7 GB -> 21, E0 ~14 GB -> 28), | |
| # (c) nvidia-smi free VRAM >= min_vram_gb (job need + 4 GB), (d) df free >= MIN_DISK_GB (30 GB, owner rule 2026-09-17). | |
| # Jobs whose out_dir has PRED_DONE are skipped; a failed job is retried on the next pass (the trainers resume from their checkpoints). | |
| # Trainers run inline (--no-lock, OMP/MKL threads 4). Log: results/runs/<queue_name>_<utc>.log (append-only); per job <out_dir>.stdout.log. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| NAME="$1"; QUEUE="$2"; MIN_AVAIL_GB="${3:-24}"; MIN_VRAM_GB="${4:-8}"; PASSES="${5:-3}" | |
| MAX_OTHERS="${MAX_OTHERS:-2}"; MIN_DISK_GB="${MIN_DISK_GB:-30}" | |
| LOG="results/runs/${NAME}_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; } | |
| n_trainers() { | |
| local pids n=0 pp; pids=$(pgrep -f "^[^ ]*python[^ ]* [^ ]*(train_cwm3|train_kp|pretrain)" || true) | |
| for p in $pids; do pp=$(awk '/^PPid/{print $2}' /proc/$p/status 2>/dev/null); grep -qx -- "$pp" <<< "$pids" || n=$((n+1)); done; echo $n; } | |
| avail_gb() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_vram_gb() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| disk_gb() { df -BG --output=avail / | tail -1 | tr -dc '0-9'; } | |
| LAUNCH_LOCK=/tmp/cwm3_joint_launch.lock | |
| log "queue $NAME from $QUEUE: gates others<=${MAX_OTHERS} avail>=${MIN_AVAIL_GB}G vram_free>=${MIN_VRAM_GB}G disk>=${MIN_DISK_GB}G" | |
| for pass in $(seq 1 "$PASSES"); do | |
| pending=0 | |
| while IFS= read -r line; do | |
| [[ -z "$line" || "$line" =~ ^# ]] && continue | |
| out="${line%%||*}"; args="${line#*||}" | |
| [[ -f "$out/PRED_DONE" ]] && continue | |
| if [[ "$args" == train/* ]]; then cmd="$PY $args --out $out --no-lock"; else cmd="$PY train/train_cwm3.py $args --out $out --no-lock"; fi # a line may name its trainer script (joint runs) | |
| while true; do | |
| exec 9>"$LAUNCH_LOCK"; flock 9 | |
| nt=$(n_trainers); av=$(avail_gb); vr=$(free_vram_gb); dk=$(disk_gb) | |
| if [[ "$nt" -le "$MAX_OTHERS" && "$av" -ge "$MIN_AVAIL_GB" && "$vr" -ge "$MIN_VRAM_GB" && "$dk" -ge "$MIN_DISK_GB" ]]; then | |
| log "pass $pass: start $(basename $out): $cmd (others=$nt avail=${av}G vram_free=${vr}G disk=${dk}G)" | |
| t0=$(date +%s) | |
| mkdir -p "$out" | |
| $cmd >> "$out.stdout.log" 2>&1 & | |
| pid=$!; sleep 25; flock -u 9; exec 9>&- | |
| wait $pid; code=$? | |
| break | |
| fi | |
| flock -u 9; exec 9>&- | |
| log "pass $pass: wait $(basename $out) (others=$nt avail=${av}G vram_free=${vr}G disk=${dk}G)"; sleep 90 | |
| done | |
| log "pass $pass: end $(basename $out): exit $code in $(( ($(date +%s)-t0) )) s" | |
| [[ $code -ne 0 ]] && pending=1 | |
| done < "$QUEUE" | |
| [[ $pending -eq 0 ]] && { log "QUEUE DONE"; exit 0; } | |
| log "pass $pass done; failed jobs remain -> sleeping 5 min"; sleep 300 | |
| done | |
| log "QUEUE EXHAUSTED ($PASSES passes) with pending jobs"; exit 1 | |
Xet Storage Details
- Size:
- 3.68 kB
- Xet hash:
- 449e014b20add9d6b30dd251b4d194663879fc92f4a18126db120a9bf0a3bf35
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.