Buckets:
| # cwm3/train/run_queue_joint.sh — sequential queue of joint-model runs (I1 / F1) WITHOUT the GPU lock (owner directive 2026-09-10 13:58), each | |
| # launch gated (under flock /tmp/cwm3_joint_launch.lock so two queue instances cannot race) on: (a) `pgrep -c -f train_cwm3` < 2 (at most two | |
| # BATON-size trainers alive across all agents; the week-1 base queue counts), (b) MemAvailable >= 26 GB (>= 14 GB headroom after the job's | |
| # ≈ 12 GB RSS: inline mode loads the prediction packs only after training), (c) nvidia-smi free VRAM >= 14 GB (job ≈ 10 GB + 4 GB), (d) the base checkpoints of the run's seed (BEST + PRED_DONE of | |
| # V0/U0/E0) — a run whose base is missing is skipped and retried on the next pass (the queue loops until every job is done or --passes is | |
| # exhausted). Jobs whose run dir has PRED_DONE are skipped. Trainers run inline (--no-lock, torch/OMP threads 4). | |
| # Base-stage jobs (V0/U0/E0:variant:seed) run the UNCHANGED train/train_cwm3.py — used for base checkpoints the week-1 queue does not produce | |
| # (U0:kin_can_noK for seeds 1-2). | |
| # bash train/run_queue_joint.sh "I1:indep:0 I1:all:0 I1:none:0 I1:naive:0 F1:chunked:0 F1:direct_fresh:0 I1:all:1 ..." [passes=6] [extra trainer args] | |
| # Log: results/runs/queue_joint_<utc>.log (append-only). The trainer itself takes the GPU lock in <= 18-min chunks. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| JOBS="$1"; PASSES="${2:-6}"; EXTRA="${3:-}" | |
| LOG="results/runs/queue_joint_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; } | |
| n_trainers() { # trainer PARENT processes only: the week-1 trainer forks a GPU child with the same cmdline (would count twice); shell wrappers excluded | |
| local pids n=0 pp; pids=$(pgrep -f "^[^ ]*python[^ ]* [^ ]*train_cwm3" || true) # cmdline starts with the python binary (not shell wrappers quoting it) | |
| for p in $pids; do pp=$(awk '/^PPid/{print $2}' /proc/$p/status 2>/dev/null); grep -qx -- "$pp" <<< "$pids" || n=$((n+1)); done; echo $n; } | |
| avail_gb() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_vram_gb() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| MIN_AVAIL_GB=26; MIN_VRAM_GB=14; LAUNCH_LOCK=/tmp/cwm3_joint_launch.lock | |
| base_ok() { # seed, base-v | |
| local s="$1" v="$2"; for d in "V0_${v}_s${s}" "U0_kin_can_noK_s${s}" "E0_tok8s_s${s}"; do | |
| [[ -f "results/runs/$d/BEST" && -f "results/runs/$d/PRED_DONE" ]] || return 1; done; return 0; } | |
| BASEV=can4s; for a in $EXTRA; do case "$a" in --base-v=*) BASEV="${a#--base-v=}";; esac; done | |
| for pass in $(seq 1 "$PASSES"); do | |
| pending=0 | |
| for job in $JOBS; do | |
| IFS=: read -r stage variant seed pv <<< "$job"; pv="${pv:-v1}" | |
| out="results/runs/${stage}_${variant}_s${seed}"; [[ "$BASEV" == "can8s" ]] && out="${out}_v8"; [[ "$pv" == "v2" ]] && out="${out}_v2" | |
| if [[ -f "$out/PRED_DONE" ]]; then continue; fi | |
| if [[ ! "$stage" =~ ^(V0|U0|E0)$ ]] && ! base_ok "$seed" "$BASEV"; then log "pass $pass: skip $job (base checkpoints for seed $seed / $BASEV not ready)"; pending=1; continue; fi | |
| if [[ "$stage" =~ ^(V0|U0|E0)$ ]]; then # a missing BASE checkpoint (e.g. U0:kin_can_noK:1, absent from the week-1 queue): the unchanged base trainer | |
| cmd="$PY train/train_cwm3.py --stage $stage --variant $variant --seed $seed --no-lock --pack-version $pv" | |
| else | |
| cmd="$PY train/train_cwm3_joint.py --stage $stage --variant $variant --seed $seed --base-v $BASEV --no-lock --minutes 600 --pack-version $pv $EXTRA" | |
| cmd="${cmd//--base-v=$BASEV/}" | |
| fi | |
| # gated launch under a flock: check resources, spawn, wait 20 s so pgrep sees the new trainer, release | |
| while true; do | |
| exec 9>"$LAUNCH_LOCK"; flock 9 | |
| if [[ "$(n_trainers)" -lt 2 && "$(avail_gb)" -ge $MIN_AVAIL_GB && "$(free_vram_gb)" -ge $MIN_VRAM_GB ]]; then | |
| log "pass $pass: start $job: $cmd (trainers=$(n_trainers) avail=$(avail_gb)G vram_free=$(free_vram_gb)G)" | |
| t0=$(date +%s) | |
| $cmd >> "results/runs/${stage}_${variant}_s${seed}$([[ "$pv" == "v2" ]] && echo _v2).joint.stdout.log" 2>&1 & | |
| pid=$!; sleep 20; flock -u 9; exec 9>&- | |
| wait $pid; code=$? | |
| break | |
| fi | |
| flock -u 9; exec 9>&-; sleep 60 | |
| done | |
| log "pass $pass: end $job: exit $code in $(( ($(date +%s)-t0)/60 )) min" | |
| [[ $code -ne 0 ]] && pending=1 | |
| done | |
| [[ $pending -eq 0 ]] && { log "QUEUE DONE"; exit 0; } | |
| log "pass $pass done; pending jobs remain -> sleeping 15 min before the next pass"; sleep 900 | |
| done | |
| log "QUEUE EXHAUSTED ($PASSES passes) with pending jobs" | |
Xet Storage Details
- Size:
- 4.65 kB
- Xet hash:
- fc5b4e1c88ea8ea5ab2f80c5b0981e206c8b5aae0c2324804aedc40350678ee7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.