Buckets:
| # cwm3/train/run_queue_cmds.sh — gated sequential queue of ARBITRARY trainer commands (2026-09-11 final assembly; the joint runs of E1 / E3 / E5 use it), | |
| # WITHOUT the GPU lock (project rule: concurrent trainers, gated launches). Each job line of the queue file: "<out_dir>||<trainer script + args>", | |
| # e.g. "results/runs/I1_indep_s0_final||train/train_cwm3_joint.py --stage I1 --variant indep --seed 0 --no-lock --minutes 600 ...". | |
| # bash train/run_queue_cmds.sh <queue_name> <queue_file> [min_avail_gb=28] [min_vram_gb=14] [passes=3] | |
| # Gates (checked under flock /tmp/cwm3_joint_launch.lock, shared with run_queue_joint.sh / run_v2_followups.sh so two launchers cannot race): | |
| # (a) trainer PARENT processes matching "python … (train_cwm3|train_kp|pretrain)" < MAX_TRAINERS (4; coordinator 2026-09-11 23:3x) across ALL agents, (b) MemAvailable >= min_avail_gb (job RSS ≈ 12.5 GB for the joint | |
| # trainer + 14 GB headroom), (c) nvidia-smi free VRAM >= min_vram_gb (joint ≈ 10 GB + 4 GB), (d) optional MemFree >= $MIN_FREE_GB, (e) df free >= $MIN_DISK_GB | |
| # (default 60 GB; a joint run writes 3–8 GB). Jobs whose out_dir has PRED_DONE are skipped; a failed job is retried on the next pass (the trainers resume | |
| # from their checkpoints). Log: results/runs/<queue_name>_<utc>.log (append-only); per job <out_dir>.stdout.log. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| NAME="$1"; QUEUE="$2"; MIN_AVAIL_GB="${3:-28}"; MIN_VRAM_GB="${4:-14}"; PASSES="${5:-3}" | |
| MIN_FREE_GB="${MIN_FREE_GB:-0}"; MIN_DISK_GB="${MIN_DISK_GB:-30}" | |
| LOG="results/runs/${NAME}_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; } | |
| # 2026-09-11 23:3x EDT (coordinator, GPU sharing with the AIDE-hardening / GPU-baselines agents): the global cap is trainers matching | |
| # "train_cwm3|train_kp|pretrain" < MAX_TRAINERS (4) with host RAM >= 14 GB available after the job's RSS (MIN_AVAIL_GB keeps that); parent-process dedupe kept. | |
| MAX_TRAINERS="${MAX_TRAINERS:-4}"; TRAINER_PAT="${TRAINER_PAT:-^[^ ]*python[^ ]* [^ ]*(train_cwm3|train_kp|pretrain)}" | |
| n_trainers() { | |
| local pids n=0 pp; pids=$(pgrep -f "$TRAINER_PAT" || true) | |
| for p in $pids; do pp=$(awk '/^PPid/{print $2}' /proc/$p/status 2>/dev/null); grep -qx -- "$pp" <<< "$pids" || n=$((n+1)); done; echo $n; } | |
| avail_gb() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_gb() { awk '/MemFree/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_vram_gb() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| disk_gb() { df -BG --output=avail / | tail -1 | tr -dc '0-9'; } | |
| LAUNCH_LOCK=/tmp/cwm3_joint_launch.lock | |
| log "queue $NAME from $QUEUE: gates trainers<${MAX_TRAINERS} (pattern ${TRAINER_PAT}) avail>=${MIN_AVAIL_GB}G vram_free>=${MIN_VRAM_GB}G free>=${MIN_FREE_GB}G disk>=${MIN_DISK_GB}G" | |
| for pass in $(seq 1 "$PASSES"); do | |
| pending=0 | |
| while IFS= read -r line; do | |
| [[ -z "$line" || "$line" =~ ^# ]] && continue | |
| out="${line%%||*}"; args="${line#*||}" | |
| [[ -f "$out/PRED_DONE" ]] && continue | |
| cmd="$PY $args --out $out" | |
| while true; do | |
| exec 9>"$LAUNCH_LOCK"; flock 9 | |
| nt=$(n_trainers); av=$(avail_gb); vr=$(free_vram_gb); fr=$(free_gb); dk=$(disk_gb) | |
| if [[ "$nt" -lt "$MAX_TRAINERS" && "$av" -ge "$MIN_AVAIL_GB" && "$vr" -ge "$MIN_VRAM_GB" && "$fr" -ge "$MIN_FREE_GB" && "$dk" -ge "$MIN_DISK_GB" ]]; then | |
| log "pass $pass: start $(basename $out): $cmd (trainers=$nt avail=${av}G vram_free=${vr}G disk=${dk}G)" | |
| t0=$(date +%s) | |
| mkdir -p "$out" | |
| $cmd >> "$out.stdout.log" 2>&1 & | |
| pid=$!; sleep 20; flock -u 9; exec 9>&- | |
| wait $pid; code=$? | |
| break | |
| fi | |
| flock -u 9; exec 9>&- | |
| log "pass $pass: wait $(basename $out) (trainers=$nt avail=${av}G free=${fr}G vram_free=${vr}G disk=${dk}G)"; sleep 60 | |
| done | |
| log "pass $pass: end $(basename $out): exit $code in $(( ($(date +%s)-t0) )) s" | |
| [[ $code -ne 0 ]] && pending=1 | |
| done < "$QUEUE" | |
| [[ $pending -eq 0 ]] && { log "QUEUE DONE"; exit 0; } | |
| log "pass $pass done; failed jobs remain -> sleeping 5 min"; sleep 300 | |
| done | |
| log "QUEUE EXHAUSTED ($PASSES passes) with pending jobs"; exit 1 | |
Xet Storage Details
- Size:
- 4.25 kB
- Xet hash:
- 438e3e4ea34eef51bd9eecdcbcf97b905707c89456d160437cc58ef44c5a3ca1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.