Buckets:
| # cwm3/train/run_oof.sh — fold-by-fold launcher for train/train_cwm3_oof.py: the WAITING happens here (a bash process, not counted as a trainer by | |
| # the queue gates), each fold runs as its own python process with --no-gate once the gates pass (pgrep trainers < 2, MemAvailable >= 26 GB, | |
| # free VRAM >= 12 GB), then the merge runs (no GPU). bash train/run_oof.sh <seed> [folds=5] [variant=all] | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| SEED="$1"; K="${2:-5}"; VARIANT="${3:-all}" | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| LOG="results/runs/oof_s${SEED}.stdout.log" | |
| n_trainers() { local pids n=0 pp; pids=$(pgrep -f "^[^ ]*python[^ ]* [^ ]*train_cwm3" || true) | |
| for p in $pids; do pp=$(awk '/^PPid/{print $2}' /proc/$p/status 2>/dev/null); grep -qx -- "$pp" <<< "$pids" || n=$((n+1)); done; echo $n; } | |
| avail_gb() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| free_vram_gb() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| for k in $(seq 0 $((K-1))); do | |
| [[ -f "results/runs/oof/I1_${VARIANT}_s${SEED}_f${k}/PRED_DONE" ]] && continue | |
| while true; do | |
| exec 9>/tmp/cwm3_joint_launch.lock; flock 9 | |
| if [[ "$(n_trainers)" -lt 2 && "$(avail_gb)" -ge 26 && "$(free_vram_gb)" -ge 12 ]]; then | |
| echo "[$(date +%H:%M:%S)] [oof-launcher] fold $k start (trainers=$(n_trainers) avail=$(avail_gb)G vram=$(free_vram_gb)G)" | tee -a "$LOG" | |
| $PY train/train_cwm3_oof.py --seed "$SEED" --variant "$VARIANT" --folds "$K" --fold "$k" --no-gate >> "$LOG" 2>&1 & | |
| pid=$!; sleep 20; flock -u 9; exec 9>&-; wait $pid; code=$? | |
| echo "[$(date +%H:%M:%S)] [oof-launcher] fold $k end exit $code" | tee -a "$LOG"; break | |
| fi | |
| flock -u 9; exec 9>&-; sleep 60 | |
| done | |
| done | |
| $PY train/train_cwm3_oof.py --seed "$SEED" --variant "$VARIANT" --folds "$K" --no-gate >> "$LOG" 2>&1 # every fold present -> merge only | |
| echo "[$(date +%H:%M:%S)] [oof-launcher] DONE" | tee -a "$LOG" | |
Xet Storage Details
- Size:
- 1.98 kB
- Xet hash:
- 795132c97afc14ed11a1f13869fa08ed3d8301683b3c95ba4181d6853d0f11be
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.