Buckets:
| # cwm3/train/run_udwm_chain.sh — owner experiment C (2026-09-17; pre-registration docs/decisions.md 2026-09-17T07:00:22Z part (C)) step chain AFTER the U0 queue | |
| # (train/queue_udwm_u0.txt via train/run_queue_udwm.sh, launched separately): | |
| # 1. wait until the three U0_kin_can_noK_s<k>_kcv2_clip8_ln_udwm runs carry TRAIN_DONE + PRED_DONE (poll 120 s); | |
| # 2. identity gate: train/queue_udwm_gate.txt (I1_indep_s0_final_udwm, prediction only; gated launch, holds /tmp/s13_gpu.lock per job) then | |
| # eval/verify_identity_final.py -> results/tables/identity_final_udwm.md; exit 0 = PASS -> marker results/runs/udwm_gate.PASS, else udwm_gate.FAIL and STOP; | |
| # 3. joint queue train/queue_udwm.txt (all_minus_road2drv s0-2 + indep s1-2; FINAL argv, only the U base swapped; gated launch); | |
| # 4. pairings + report: eval/run_joint_evals_final_udwm.sh "0 1 2" 1 3 (CPU, <= 3 parallel, MemAvailable >= 8 GB) -> results/tables/stage_final_udwm.md. | |
| # Nothing here kills anything; every step is idempotent (finished runs / existing tags are skipped). Log: results/runs/udwm_chain.log (append-only). | |
| # setsid nohup bash train/run_udwm_chain.sh > results/runs/udwm_chain.launcher.log 2>&1 < /dev/null & | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| LOG=results/runs/udwm_chain.log | |
| log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; } | |
| done_run() { [[ -f results/runs/$1/PRED_DONE && -f results/runs/$1/TRAIN_DONE ]]; } | |
| log "chain start: waiting for the three U0 udwm runs" | |
| until done_run U0_kin_can_noK_s0_kcv2_clip8_ln_udwm && done_run U0_kin_can_noK_s1_kcv2_clip8_ln_udwm && done_run U0_kin_can_noK_s2_kcv2_clip8_ln_udwm; do sleep 120; done | |
| log "U0 udwm s0/s1/s2 finished" | |
| # ---- 2. identity gate | |
| if ! done_run I1_indep_s0_final_udwm; then | |
| log "gate queue start"; bash train/run_queue_udwm.sh udwm_gate train/queue_udwm_gate.txt 28 17 3; log "gate queue exit $?" | |
| fi | |
| rm -f results/runs/udwm_gate.PASS results/runs/udwm_gate.FAIL | |
| $PY eval/verify_identity_final.py --run I1_indep_s0_final_udwm --bases V0_can4s_s0_stats2 U0_kin_can_noK_s0_kcv2_clip8_ln_udwm E0_pooled_P_s0_v2 \ | |
| --out results/tables/identity_final_udwm.md > results/runs/udwm_identity.log 2>&1 | |
| code=$? | |
| if [[ $code -ne 0 ]]; then | |
| date -u +%FT%TZ > results/runs/udwm_gate.FAIL; log "IDENTITY GATE FAIL (exit $code): see results/runs/udwm_identity.log — joint queue NOT launched"; exit 2 | |
| fi | |
| date -u +%FT%TZ > results/runs/udwm_gate.PASS; log "identity gate PASS -> results/tables/identity_final_udwm.md" | |
| # ---- 3. joint queue | |
| log "joint queue start"; bash train/run_queue_udwm.sh udwm_joint train/queue_udwm.txt 28 17 3; log "joint queue exit $?" | |
| # ---- 4. pairings + report | |
| log "pairings start"; bash eval/run_joint_evals_final_udwm.sh "0 1 2" 1 3 > results/runs/udwm_pairings.log 2>&1; log "pairings exit $? -> results/runs/udwm_pairings.log" | |
| log "CHAIN DONE" | |
Xet Storage Details
- Size:
- 2.89 kB
- Xet hash:
- acef0b0a1a347f8d81c53034da910113d9a8512beb55710df64c1496289e820a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.