Buckets:
| # F8 lane launcher (2026-09-17, pre-reg 2026-09-17T19:32:07Z): gated launch of ONE train_overlap_head.py command. | |
| # bash train/run_overlap_f8.sh <tag> <need_vram_gb> -- <args of train_overlap_head.py> | |
| # Gate (polled every 30 s, lock-free): free VRAM >= need + 4 GB, MemAvailable >= 28 GB, < 4 python train processes, disk >= 30 GB. | |
| # Logs: results/runs/f8_<tag>.log, pid file results/runs/f8_<tag>.pid. OMP / torch threads 4 (set by the trainer). | |
| set -u | |
| CWM3=/home/USER/cwm3 | |
| PY=python | |
| TAG=$1; NEED=$2; shift 3 | |
| LOG=$CWM3/results/runs/f8_$TAG.log; PID=$CWM3/results/runs/f8_$TAG.pid | |
| echo $$ > "$PID" | |
| free_vram() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1 | awk '{printf "%d", $1/1024}'; } | |
| avail() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } | |
| ntrain() { pgrep -fc "^[^ ]*python[^ ]* [^ ]*train" || true; } | |
| disk() { df -BG /home/USER | tail -1 | awk '{gsub("G","",$4); print $4}'; } | |
| while true; do | |
| v=$(free_vram); m=$(avail); n=$(ntrain); d=$(disk) | |
| if [[ $v -ge $((NEED + 4)) && $m -ge 28 && $n -lt 4 && $d -ge 30 ]]; then | |
| echo "[$(date -u +%H:%M:%SZ)] gate pass: vram_free=${v}G avail=${m}G trainers=$n disk=${d}G -> $PY $CWM3/train/train_overlap_head.py $*" >> "$LOG" | |
| break | |
| fi | |
| echo "[$(date -u +%H:%M:%SZ)] gate wait: vram_free=${v}G (need $((NEED + 4))) avail=${m}G trainers=$n disk=${d}G" >> "$LOG" | |
| sleep 30 | |
| done | |
| cd "$CWM3" && OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 "$PY" train/train_overlap_head.py "$@" >> "$LOG" 2>&1 | |
| rc=$? # captured BEFORE the date substitution (the first version echoed $(date)'s status: an OOM crash was logged as exit 0, 2026-09-17 20:01Z) | |
| echo "[$(date -u +%H:%M:%SZ)] exit $rc" >> "$LOG" | |
| exit $rc | |
Xet Storage Details
- Size:
- 1.75 kB
- Xet hash:
- 3599e950b50cd7edd5729161b808044faa87c886c0c28edb709b53d92b0309e4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.