Buckets:
| # cwm3/train/run_innov_queue.sh — sequential launcher of the INNOVATION trainer runs (innovation agent, 2026-09-11) with the project-wide launch guards: | |
| # before EACH launch wait until free VRAM >= NEED_GB + 4, host MemAvailable >= 14 GB, live trainers (python.*train_cwm3) < 2 (final-assembly agent has priority). | |
| # Queue file lines: <out_dir>||<trainer args> (skipped when <out_dir>/PRED_DONE exists). bash train/run_innov_queue.sh train/queue_identity.txt [NEED_GB=8] | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| Q="$1"; NEED_GB="${2:-8}" | |
| LOG="results/runs/innov_queue_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; } | |
| guards_ok() { | |
| local free_mib avail_kb ntr | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1) | |
| avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*train_cwm3") | |
| [[ $free_mib -ge $(( (NEED_GB + 4) * 1024 )) && $avail_kb -ge $((14 * 1024 * 1024)) && $ntr -lt 2 ]] | |
| } | |
| while IFS= read -r line; do | |
| [[ -z "$line" || "$line" == \#* ]] && continue | |
| out="${line%%||*}"; args="${line##*||}" | |
| if [[ -f "$out/PRED_DONE" ]]; then log "skip $out (PRED_DONE)"; continue; fi | |
| n=0 | |
| until guards_ok; do | |
| (( n % 12 == 0 )) && log "waiting for launch guards ($out): VRAM free $(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1) MiB, MemAvail $(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo) GB, trainers $(pgrep -fc 'python.*train_cwm3')" | |
| sleep 30; n=$((n + 1)) | |
| done | |
| log "start $out :: $args" | |
| t0=$(date +%s) | |
| $PY train/train_cwm3.py $args --out "$out" --no-lock >> "$out.stdout.log" 2>&1 | |
| code=$? | |
| log "end $out exit $code in $(( ($(date +%s) - t0) / 60 )) min" | |
| sleep 5 | |
| done < "$Q" | |
| log "INNOV_QUEUE_DONE $Q" | |
Xet Storage Details
- Size:
- 1.86 kB
- Xet hash:
- 33ba4a21cec69e3c1249d41bcd95bccce1c19559b8ec76980a795cab3b322002
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.