Buckets:
| # cwm3/eval/run_robustness.sh — test-time robustness of the frozen main model (gaze/robustness agent, 2026-09-12; results/tables/robustness_final.md). | |
| # bash eval/run_robustness.sh predict "<seeds>" "<perts>" # GPU inference, gated (free VRAM >= 4 + 4 GB, MemAvailable >= 14 GB, python trainers < 4), sequential | |
| # bash eval/run_robustness.sh pairs "<seeds>" "<perts>" # CPU: paired_compare_joint --only-onset (A = perturbed, B = frozen; v2 causal driver targets) + pressure_eval (P cells) | |
| # Tags: robust_s<k>_<pert>_vs_clean (onset cells incl. veh_any_on), robustP_s<k>_<pert>_vs_clean (p_onset|<chain>|3s). Idempotent. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| MODE="$1"; SEEDS="${2:-0 1 2}"; PERTS="${3:-kin_drop20 kin_drop50 can_noise road_off kh_trunc_old2s kh_mask_recent2s}" | |
| NEED=${NEED_VRAM_MB:-7700} # measured 14:15 EDT: one joint prediction = 7.6 GB VRAM, 12.8 GB RSS -> gate free VRAM >= 7.7 + 4 GB | |
| MIN_AVAIL=${MIN_AVAIL_GB:-14} # coordinator rule >= 14 GB; the parallel per-seed chains (2026-09-12 14:15) use 20 so a 12.8 GB RSS launch never breaks the rule | |
| # The gate is SERIALISED across concurrent chains (flock on /tmp/cwm3_robust_gate.lock, held from the check through the first ALLOC_S seconds of the launch): | |
| # 14:13 EDT three chains passed the check in the same second (each saw the 16.7 GB free before any had allocated) and collided in CUDA OOM. | |
| ALLOC_S=${ALLOC_S:-75} | |
| gate() { | |
| exec 9>/tmp/cwm3_robust_gate.lock; flock 9 | |
| while :; do | |
| vram=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1) | |
| avail=$(awk '/MemAvailable/{print int($2/1024/1024)}' /proc/meminfo) | |
| ntr=$(pgrep -f "train_cwm3|train_kp|pretrain" | xargs -r ps -o comm= -p 2>/dev/null | grep -c python) | |
| if (( vram >= NEED + 4096 && avail >= MIN_AVAIL && ntr < 4 )); then echo "[$(date +%H:%M:%S)] gate open: vram_free ${vram} MB, avail ${avail} GB, trainers ${ntr}"; return; fi | |
| echo "[$(date +%H:%M:%S)] gate closed (vram_free ${vram} MB, avail ${avail} GB, trainers ${ntr}); waiting 30 s"; sleep 30 | |
| done | |
| } | |
| ungate() { flock -u 9; exec 9>&-; } | |
| have_tag() { grep -q ",$1,$2,test," results/paired_compare.csv 2>/dev/null; } | |
| for s in $SEEDS; do | |
| RUN=I1_all_minus_road2drv_s${s}_final | |
| for p in $PERTS; do | |
| D=results/runs/${RUN}_pert_${p} | |
| if [[ $MODE == predict ]]; then | |
| [[ -f $D/PRED_DONE ]] && { echo "skip $D (PRED_DONE)"; continue; } | |
| for attempt in 1 2 3; do | |
| gate | |
| echo "[$(date +%H:%M:%S)] == predict $D (attempt $attempt)" | |
| $PY eval/robustness_perturb.py --run $RUN --pert $p >> results/runs/${RUN}_pert_${p}.stdout.log 2>&1 & | |
| pid=$!; sleep $ALLOC_S; ungate | |
| wait $pid; rc=$? | |
| stamp=$(date +%H:%M:%S); echo "[$stamp] exit $rc ($D, attempt $attempt)" | |
| [[ $rc == 0 ]] && break | |
| sleep 60 | |
| done | |
| else | |
| [[ -f $D/PRED_DONE ]] || { echo "skip pairs $D (not predicted)"; continue; } | |
| for reg in manual engaged; do | |
| tag=robust_s${s}_${p}_vs_clean | |
| if ! have_tag "$tag" "$reg"; then | |
| echo "== $tag $reg" | |
| $PY eval/paired_compare_joint.py --regime $reg --a $D --b results/runs/$RUN --tag $tag --kin-targets causal_routes --kin-version v2 --only-onset 2>&1 | grep -E "appended|Error|Traceback|File \"|LAYOUT GUARD" | |
| fi | |
| tag=robustP_s${s}_${p}_vs_clean | |
| if ! have_tag "$tag" "$reg"; then | |
| echo "== $tag $reg" | |
| $PY eval/pressure_eval.py --regime $reg --a $D --b results/runs/$RUN --tag $tag 2>&1 | grep -E "appended|Error|Traceback|File \"|LAYOUT GUARD" | |
| fi | |
| done | |
| fi | |
| done | |
| done | |
| echo "ROBUSTNESS_${MODE}_DONE" | |
Xet Storage Details
- Size:
- 3.71 kB
- Xet hash:
- 91830ec8fd8fb71f4a240c96ec2c70d3920b4040972a820c029923294e92b4a0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.