Buckets:
| # cwm3/eval/run_warning_head_oof.sh — the learned warning head on the OUT-OF-FOLD label-row packs of the FINAL main model (coordinator request 2026-09-12; gaze/robustness agent), CPU. | |
| # bash eval/run_warning_head_oof.sh ["0 1 2"] [n_boot_train=1000] [n_boot_eval=2000] | |
| # Seeds whose `results/runs/I1_all_minus_road2drv_s<k>_final/pred_{manual,engaged}_train_labelrows_oof.npz` exist (train/train_cwm3_oof.py, 5 device folds; docs/gpu_baselines_report.md §D). | |
| # Per seed: (1) train/train_warning_head.py --oof (feature sets current / future / both; W dir `_oof`, columns *_n979oof); (2) --oof --feature-sets current --extra-features dwell,face | |
| # --w-suffix _oofdwfc (columns *_n979oofdwfc); (3) eval/merge_w_layouts.py `_oof` + `_oofdwfc` + the IN-SAMPLE `` and `_dwfc` families (final-assembly / warning-head agents) -> `_oofmerged`, | |
| # so that ONE warning_gold item `cwm3_final_oof` scores OOF and in-sample columns on identical rows and the regime report pairs them (OOF − in-sample per feature set; future / both − current under OOF). | |
| # Memory gate MemAvailable >= 18 GB before each step; idempotent (re-run when seeds 1–2 land). Every step uses the unchanged evaluators. | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=python | |
| export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 | |
| SEEDS="${1:-0 1 2}"; NBT="${2:-1000}"; NBE="${3:-2000}" | |
| gate() { until [ "$(awk '/MemAvailable/{print int($2/1024/1024)}' /proc/meminfo)" -ge 18 ]; do echo "[$(date +%H:%M:%S)] memory gate closed"; sleep 30; done; } | |
| have_w() { for d in $(ls -d results/warning_head/$1_* 2>/dev/null); do $PY -c "import json,sys; d=json.load(open('$d/selected.json')); sys.exit(0 if d.get('w_suffix')=='$2' else 1)" 2>/dev/null && return 0; done; return 1; } | |
| RUNS=() | |
| for s in $SEEDS; do | |
| R=I1_all_minus_road2drv_s${s}_final | |
| [[ -f results/runs/$R/pred_manual_train_labelrows_oof.npz && -f results/runs/$R/pred_engaged_train_labelrows_oof.npz ]] || { echo "skip seed $s (no OOF packs yet)"; continue; } | |
| [[ -f results/runs/W_$R/pred_layout_manual_test.npz && -f results/runs/W_${R}_dwfc/pred_layout_manual_test.npz ]] || { echo "skip seed $s (in-sample W_$R / W_${R}_dwfc layouts missing)"; continue; } | |
| RUNS+=("$R") | |
| if have_w "$R" "_oof"; then echo "have OOF head for $R"; else | |
| gate; echo "[$(date +%H:%M:%S)] == train_warning_head --oof $R (current / future / both)" | |
| $PY train/train_warning_head.py --run "$R" --demand P --n-boot "$NBT" --oof 2>&1 | grep -E "DONE|Error|Traceback|File \"|selected|gold clean_test AUROC|train labels|paired" | cut -c1-220 | |
| fi | |
| if have_w "$R" "_oofdwfc"; then echo "have OOF dwfc head for $R"; else | |
| gate; echo "[$(date +%H:%M:%S)] == train_warning_head --oof $R (+dwell,face)" | |
| $PY train/train_warning_head.py --run "$R" --demand P --n-boot "$NBT" --oof --feature-sets current --extra-features dwell,face --w-suffix _oofdwfc 2>&1 | grep -E "DONE|Error|Traceback|File \"|selected|gold clean_test AUROC|train labels" | cut -c1-220 | |
| fi | |
| $PY eval/merge_w_layouts.py --run "$R" --suffixes _oof _oofdwfc "" _dwfc --out-suffix _oofmerged --force 2>&1 | grep -E "wrote|Error|Traceback" | cut -c1-160 | |
| done | |
| [[ ${#RUNS[@]} -gt 0 ]] || { echo "no seed with OOF packs"; echo WH_OOF_DONE; exit 0; } | |
| HL=W_learned_bothhgb_t15_w10_n979oof | |
| gate; echo "[$(date +%H:%M:%S)] == warning_head_eval --item cwm3_final_oof (${RUNS[*]})" | |
| $PY eval/warning_head_eval.py --runs "${RUNS[@]}" --item cwm3_final_oof --n-boot "$NBE" --w-suffix _oofmerged --dir-suffixes _oof _oofdwfc "" _dwfc --headline $HL --out-name warning_head_final_oof.md \ | |
| --feature-label "OUT-OF-FOLD train features of the final model (5 device folds; feature sets current / future / both) — the in-sample rows of warning_head_final*.md are the reference" 2>&1 | grep -E "B4|B5|written|Error|Traceback|File \"" | head -12 | |
| C=W_learned_curhgb_t15_w10_n979; F=W_learned_futhgb_t15_w10_n979; B=W_learned_bothhgb_t15_w10_n979; D=W_learned_curhgb_t15_w10_n979dwfc | |
| gate; echo "[$(date +%H:%M:%S)] == regime report" | |
| $PY eval/warning_head_regime_report.py --item cwm3_final_oof --cols ${C}oof ${F}oof ${B}oof ${D%dwfc}oofdwfc $C $F $B $D W_learned_bothsel_t15_w10_n979oof W_learned_bothsel_t15_w10_n979 W_learned_futsel_t15_w10_n979oof W_learned_futsel_t15_w10_n979 \ | |
| --pairs "oof_minus_insample_cur:${C}oof-${C}" "oof_minus_insample_fut:${F}oof-${F}" "oof_minus_insample_both:${B}oof-${B}" "oof_minus_insample_dwfc:${D%dwfc}oofdwfc-${D}" \ | |
| "oof_minus_insample_bothsel:W_learned_bothsel_t15_w10_n979oof-W_learned_bothsel_t15_w10_n979" "oof_minus_insample_futsel:W_learned_futsel_t15_w10_n979oof-W_learned_futsel_t15_w10_n979" \ | |
| "both_minus_cur_OOF:${B}oof-${C}oof" "fut_minus_cur_OOF:${F}oof-${C}oof" "both_minus_cur_insample:${B}-${C}" "fut_minus_cur_insample:${F}-${C}" \ | |
| --ensemble-cols ${B}oof ${F}oof ${C}oof --n-boot "$NBE" --out results/tables/warning_head_final_oof.md \ | |
| --title "OUT-OF-FOLD warning heads of the final model vs the in-sample heads — per-regime strata, paired OOF − in-sample per feature set, predicted-future effect under OOF" 2>&1 | grep -E "appended|written|Error|Traceback|File \"" | head | |
| $PY eval/finish_warning_head_oof.py --seeds "${RUNS[*]}" 2>&1 | tail -2 | |
| echo WH_OOF_DONE | |
Xet Storage Details
- Size:
- 5.2 kB
- Xet hash:
- dd75eab1d3c205bfc767c1b6b799ab47b6eac7b948099ec2f639170e0a5aca71
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.