Buckets:
| # cwm3/train/run_can_arms.sh — gated launcher of the OPTIONAL BATON-side CAN arms on the driver chains (plan Part A §A.5; pre-registration docs/decisions.md | |
| # 2026-09-15T01:48:39Z). Three phases, idempotent: | |
| # 1. 6 trainings (arms nocan / cangate x seeds 0 1 2) with the survivor U0 argv (== the recorded command of results/runs/U0_kin_can_noK_s<k>_kcv2_clip8_ln + | |
| # the explicit default --stats-version 2, as the DMS layer-1 driver does), <= MAX_OWN of them concurrently; before EACH launch the project gate: | |
| # free VRAM >= NEED_GB + 4 GB, host MemAvailable >= 28 GB, live trainers (python.*train_cwm3|train_kp|pretrain) < 4, OMP/MKL 4. Every trainer is launched as | |
| # `setsid nohup $PY ...`, its pid (verified with pgrep on its --out) appended to results/can_arms/pids.txt; a run dir with PRED_DONE is never re-run. | |
| # 2. 12 CPU pairings vs the frozen base with the UNCHANGED eval/paired_compare_joint.py (the B1 argv of eval/run_joint_evals_final_k0.sh), | |
| # tags canU_s<k>_<arm>_vs_u0base, both regimes; a tag/regime already in results/paired_compare.csv is skipped. | |
| # 3. eval/report_can_arms.py -> results/tables/can_arms.md. | |
| # setsid nohup bash train/run_can_arms.sh > results/can_arms/launcher_stdout.log 2>&1 & | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| NEED_GB="${NEED_GB:-6}"; MAX_OWN="${MAX_OWN:-2}" | |
| OUT=results/can_arms; mkdir -p "$OUT" | |
| PIDS="$OUT/pids.txt" | |
| LOG="$OUT/launcher_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG"; } | |
| declare -A VARIANT=([nocan]=kin_can_noK_clip8_ln_nocan [cangate]=kin_can_noK_clip8_ln_cangate) | |
| run_dir() { echo "results/runs/U0_${1}_s${2}_kcv2_clip8_ln"; } | |
| trainers() { pgrep -fc "python.*(train_cwm3|train_kp|pretrain)"; } | |
| vram_free() { nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | head -1; } | |
| mem_avail_gb() { awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo; } | |
| guards_ok() { | |
| local avail_kb ntr free_mib | |
| avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) | |
| ntr=$(trainers) | |
| free_mib=$(vram_free) | |
| [[ $avail_kb -ge $((28 * 1024 * 1024)) && $ntr -lt 4 && $free_mib -ge $(( (NEED_GB + 4) * 1024 )) ]] | |
| } | |
| OWN=() | |
| own_running() { local n=0 p; for p in "${OWN[@]:-}"; do [[ -n "$p" ]] && kill -0 "$p" 2>/dev/null && n=$((n + 1)); done; echo $n; } | |
| log "run_can_arms start (pid $$) NEED_GB=$NEED_GB MAX_OWN=$MAX_OWN" | |
| # ---------------------------------------------------------------- phase 1: trainings | |
| for arm in nocan cangate; do | |
| for s in 0 1 2; do | |
| d=$(run_dir "$arm" "$s") | |
| if [[ -f "$d/PRED_DONE" ]]; then log "skip $d (PRED_DONE)"; continue; fi | |
| n=0 | |
| until [[ $(own_running) -lt $MAX_OWN ]] && guards_ok; do | |
| (( n % 10 == 0 )) && log "waiting ($d): own $(own_running)/$MAX_OWN, VRAM free $(vram_free) MiB (need $(( (NEED_GB + 4) * 1024 ))), MemAvail $(mem_avail_gb) GB (need 28), trainers $(trainers) (< 4)" | |
| sleep 30; n=$((n + 1)) | |
| done | |
| mkdir -p "$d" | |
| step_log="$OUT/train_${arm}_s${s}_$(date -u +%Y%m%dT%H%M%SZ).log" | |
| args="train/train_cwm3.py --stage U0 --variant ${VARIANT[$arm]} --seed $s --kin-source causal_routes --kin-version v2 --stats-version 2 --out $d --no-lock" | |
| setsid nohup $PY $args > "$step_log" 2>&1 & | |
| pid=$! | |
| sleep 5 | |
| real=$(pgrep -f -- "--out $d --no-lock" | head -1) | |
| [[ -n "$real" ]] && pid="$real" | |
| OWN+=("$pid") | |
| echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) pid=$pid arm=$arm seed=$s $PY $args -> $step_log" >> "$PIDS" | |
| log "start pid $pid :: $PY $args -> $step_log" | |
| sleep 25 # let the parent load its arrays before the next gate check | |
| done | |
| done | |
| # wait for every own trainer (poll; the processes are in their own sessions) | |
| while [[ $(own_running) -gt 0 ]]; do sleep 20; done | |
| for arm in nocan cangate; do for s in 0 1 2; do d=$(run_dir "$arm" "$s"); [[ -f "$d/PRED_DONE" ]] && log "done $d" || log "WARNING: $d has no PRED_DONE (see its train log)"; done; done | |
| # ---------------------------------------------------------------- phase 2: pairings (CPU, sequential, unchanged evaluator) | |
| have_tag() { grep -q ",$1,$2,test," results/paired_compare.csv 2>/dev/null; } | |
| for arm in nocan cangate; do | |
| for s in 0 1 2; do | |
| d=$(run_dir "$arm" "$s") | |
| [[ -f "$d/PRED_DONE" && -f "$d/TRAIN_DONE" ]] || { log "skip pairing $d (not finished)"; continue; } | |
| for reg in manual engaged; do | |
| tag="canU_s${s}_${arm}_vs_u0base" | |
| if have_tag "$tag" "$reg"; then log "skip $tag $reg (in csv)"; continue; fi | |
| log "pair $tag $reg" | |
| $PY eval/paired_compare_joint.py --regime "$reg" --a "$d" --b "results/runs/U0_kin_can_noK_s${s}_kcv2_clip8_ln" --tag "$tag" --kin-targets causal_routes --kin-version v2 \ | |
| >> "$OUT/pair_${tag}_${reg}.log" 2>&1 || log "WARNING: pairing $tag $reg exited non-zero" | |
| grep -E "appended|LAYOUT GUARD|Traceback" "$OUT/pair_${tag}_${reg}.log" | tail -3 | sed 's/^/ /' | tee -a "$LOG" | |
| done | |
| done | |
| done | |
| # ---------------------------------------------------------------- phase 3: table | |
| $PY eval/report_can_arms.py --out results/tables/can_arms.md >> "$LOG" 2>&1 && log "table written results/tables/can_arms.md" || log "WARNING: report_can_arms.py failed" | |
| log CAN_ARMS_DONE | |
Xet Storage Details
- Size:
- 5.24 kB
- Xet hash:
- 7a7a5d4a5fca409482dea74c5e245b0ed275a6acbf342c9277075aa8bda8f420
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.