Buckets:
| # cwm3/train/run_v2_followups.sh — gated sequential queue of base-trainer runs for the v2 follow-ups (K4 on the v2 causal packs, E0 regionpca256), | |
| # WITHOUT the GPU lock (project rule: concurrent trainers, gated launches). Each job line of the queue file: "<out_dir>||<train_cwm3.py args>". | |
| # bash train/run_v2_followups.sh <queue_name> <queue_file> [min_avail_gb=26] [min_vram_gb=14] [passes=4] | |
| # Gates (checked under flock /tmp/cwm3_joint_launch.lock, shared with run_queue_joint.sh so two launchers cannot race): | |
| # (a) trainer PARENT processes matching "python … train_cwm3" < 2 across ALL agents, (b) MemAvailable >= min_avail_gb (= job RSS + 14 GB headroom), | |
| # (c) nvidia-smi free VRAM >= min_vram_gb (= job need + 4 GB). Jobs whose out_dir has PRED_DONE are skipped; a failed job is retried on the next pass. | |
| # Trainers run inline (--no-lock, OMP/MKL threads 4). Log: results/runs/<queue_name>_<utc>.log (append-only); per job <out_dir>.stdout.log. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| NAME="$1"; QUEUE="$2"; MIN_AVAIL_GB="${3:-26}"; MIN_VRAM_GB="${4:-14}"; PASSES="${5:-4}" | |
| LOG="results/runs/${NAME}_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; } | |
| n_trainers() { | |
| local pids n=0 pp; pids=$(pgrep -f "^[^ ]*python[^ ]* [^ ]*train_cwm3" || true) | |
| for p in $pids; do pp=$(awk '/^PPid/{print $2}' /proc/$p/status 2>/dev/null); grep -qx -- "$pp" <<< "$pids" || n=$((n+1)); done; echo $n; } | |
| avail_gb() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_gb() { awk '/MemFree/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| MIN_FREE_GB="${MIN_FREE_GB:-0}" # optional MemFree gate (the host's low-memory guard reacts to FREE memory; loaded packs are evicted from the page cache) | |
| free_vram_gb() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| LAUNCH_LOCK=/tmp/cwm3_joint_launch.lock | |
| log "queue $NAME from $QUEUE: gates trainers<2 avail>=${MIN_AVAIL_GB}G vram_free>=${MIN_VRAM_GB}G" | |
| for pass in $(seq 1 "$PASSES"); do | |
| pending=0 | |
| while IFS= read -r line; do | |
| [[ -z "$line" || "$line" =~ ^# ]] && continue | |
| out="${line%%||*}"; args="${line#*||}" | |
| [[ -f "$out/PRED_DONE" ]] && continue | |
| cmd="$PY train/train_cwm3.py $args --out $out --no-lock" | |
| while true; do | |
| exec 9>"$LAUNCH_LOCK"; flock 9 | |
| nt=$(n_trainers); av=$(avail_gb); vr=$(free_vram_gb); fr=$(free_gb) | |
| if [[ "$nt" -lt 2 && "$av" -ge "$MIN_AVAIL_GB" && "$vr" -ge "$MIN_VRAM_GB" && "$fr" -ge "$MIN_FREE_GB" ]]; then | |
| log "pass $pass: start $(basename $out): $cmd (trainers=$nt avail=${av}G vram_free=${vr}G)" | |
| t0=$(date +%s) | |
| mkdir -p "$out" | |
| $cmd >> "$out.stdout.log" 2>&1 & | |
| pid=$!; sleep 20; flock -u 9; exec 9>&- | |
| wait $pid; code=$? | |
| break | |
| fi | |
| flock -u 9; exec 9>&- | |
| log "pass $pass: wait $(basename $out) (trainers=$nt avail=${av}G free=${fr}G vram_free=${vr}G)"; sleep 60 | |
| done | |
| log "pass $pass: end $(basename $out): exit $code in $(( ($(date +%s)-t0) )) s" | |
| [[ $code -ne 0 ]] && pending=1 | |
| done < "$QUEUE" | |
| [[ $pending -eq 0 ]] && { log "QUEUE DONE"; exit 0; } | |
| log "pass $pass done; failed jobs remain -> sleeping 5 min"; sleep 300 | |
| done | |
| log "QUEUE EXHAUSTED ($PASSES passes) with pending jobs"; exit 1 | |
Xet Storage Details
- Size:
- 3.38 kB
- Xet hash:
- cc631f49f62d5fa86513814ad2c9a84f2a1b6152751aaee82502870fa3cf9278
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.