Buckets:
| # cwm3/train/run_gaze_queue.sh — gated sequential launcher of the gaze-chain U0 runs (gaze/robustness agent, 2026-09-12; docs/gpu_gaze_report.md). | |
| # bash train/run_gaze_queue.sh <variant: kin_can_noK_clip8_ln_gz|kin_can_noK_clip8_ln_gzfc> <suffix: gz|gzfc> "<seeds>" [need_vram_mb=4096] | |
| # Gate before every run (coordinator rules, no GPU lock): free VRAM >= need + 4 GB, MemAvailable >= 14 GB, python trainers matching | |
| # "train_cwm3|train_kp|pretrain" < 4 (python processes only, not shell wrappers), OMP/MKL threads 4. Idempotent: a run dir with PRED_DONE is skipped. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| VAR="$1"; SUF="$2"; SEEDS="${3:-0 1 2}"; NEED="${4:-4096}" | |
| gate() { | |
| while :; do | |
| vram=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1) | |
| avail=$(awk '/MemAvailable/{print int($2/1024/1024)}' /proc/meminfo) | |
| ntr=$(pgrep -f "train_cwm3|train_kp|pretrain" | xargs -r ps -o comm= -p 2>/dev/null | grep -c python) | |
| if (( vram >= NEED + 4096 && avail >= 14 && ntr < 4 )); then echo "[$(date +%H:%M:%S)] gate open: vram_free ${vram} MB, avail ${avail} GB, trainers ${ntr}"; return; fi | |
| echo "[$(date +%H:%M:%S)] gate closed (vram_free ${vram} MB, avail ${avail} GB, trainers ${ntr}); waiting 30 s"; sleep 30 | |
| done | |
| } | |
| for s in $SEEDS; do | |
| OUT=results/runs/U0_kin_can_noK_s${s}_kcv2_clip8_ln_${SUF} | |
| if [[ -f $OUT/PRED_DONE ]]; then echo "skip $OUT (PRED_DONE)"; continue; fi | |
| gate | |
| echo "[$(date +%H:%M:%S)] == $OUT" | |
| $PY train/train_cwm3.py --stage U0 --variant "$VAR" --seed "$s" --kin-source causal_routes --kin-version v2 --no-lock --out "$OUT" > "$OUT.stdout.log" 2>&1 | |
| echo "[$(date +%H:%M:%S)] exit $? ($OUT)" | |
| done | |
| echo "GAZE_QUEUE_DONE $SUF" | |
Xet Storage Details
- Size:
- 1.77 kB
- Xet hash:
- 0ae7496cb05d5daa7a2e693788938d7d4df476eda2c288b348e808a278635810
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.