Buckets:
| # cwm3/baselines_dms/run_queue_dms.sh — sequential launcher of the DMS-baseline layer 1-3 queue (copy of train/run_innov_queue.sh with the project-wide launch | |
| # guards of 2026-09-14): before EACH launch wait until free VRAM >= NEED_GB + 4 GB, host MemAvailable >= 28 GB, live trainers (python.*train_cwm3|chain_style) < 4. | |
| # Queue file lines: <done_marker>||<python script + args>[||<need_gb>] (# comments and blank lines skipped; a line is skipped when <done_marker> exists; | |
| # need_gb 0 = CPU-only step: the VRAM guard is not applied, the memory / trainer guards are). Every step is launched as `setsid nohup $PY <args>` with its own | |
| # log under results/baselines_dms/logs/ and its pid appended to results/baselines_dms/logs/pids.txt; the launcher waits for it and records the exit code. | |
| # bash baselines_dms/run_queue_dms.sh baselines_dms/queue_dms.txt [NEED_GB=8] (never started automatically; launch with setsid nohup yourself) | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| Q="$1"; NEED_GB_DEFAULT="${2:-8}" | |
| LOGDIR=results/baselines_dms/logs | |
| mkdir -p "$LOGDIR" | |
| LOG="$LOGDIR/queue_dms_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| PIDS="$LOGDIR/pids.txt" | |
| log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; } | |
| guards_ok() { # $1 = need_gb (0 -> no VRAM guard) | |
| local need=$1 free_mib avail_kb ntr | |
| avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) | |
| ntr=$(pgrep -fc "python.*(train_cwm3|chain_style|fit_u_base.py --baseline .* --train)") | |
| [[ $avail_kb -ge $((28 * 1024 * 1024)) && $ntr -lt 4 ]] || return 1 | |
| if [[ $need -gt 0 ]]; then | |
| free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1) | |
| [[ $free_mib -ge $(( (need + 4) * 1024 )) ]] || return 1 | |
| fi | |
| return 0 | |
| } | |
| while IFS= read -r line; do | |
| [[ -z "$line" || "$line" == \#* ]] && continue | |
| marker="${line%%||*}"; rest="${line#*||}"; args="${rest%%||*}"; need="$NEED_GB_DEFAULT" | |
| [[ "$rest" == *"||"* ]] && need="${rest##*||}" | |
| if [[ -e "$marker" ]]; then log "skip $marker (exists)"; continue; fi | |
| n=0 | |
| until guards_ok "$need"; do | |
| (( n % 12 == 0 )) && log "waiting for launch guards ($marker, need ${need} GB): VRAM free $(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1) MiB, MemAvail $(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo) GB, trainers $(pgrep -fc 'python.*(train_cwm3|chain_style|fit_u_base.py --baseline .* --train)')" | |
| sleep 30; n=$((n + 1)) | |
| done | |
| name=$(echo "$args" | tr ' /' '__' | cut -c1-120) | |
| step_log="$LOGDIR/step_${name}_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| log "start $marker :: $PY $args (need ${need} GB) -> $step_log" | |
| t0=$(date +%s) | |
| setsid nohup $PY $args >> "$step_log" 2>&1 & | |
| pid=$! | |
| echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) pid=$pid $args" >> "$PIDS" | |
| wait $pid | |
| code=$? | |
| log "end $marker exit $code in $(( ($(date +%s) - t0) / 60 )) min" | |
| [[ -e "$marker" ]] || log "WARNING: $marker not produced (exit $code); later lines that need it will fail on their own asserts" | |
| sleep 5 | |
| done < "$Q" | |
| log "DMS_QUEUE_DONE $Q" | |
Xet Storage Details
- Size:
- 3.09 kB
- Xet hash:
- e2569472cf2b575cb641e24687a8f6a8af747f01efc1ef1f71fbfb32310d645e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.