| # Halt Stage-B after the step-108 cadence save (micro_00009180). | |
| # Dated entry 2026-09-07: horizon cut 500 -> 108. Do not kill before the save. | |
| set -euo pipefail | |
| ROOT=/apps/development/rlhf/nano-mistral-o1 | |
| CKPT="$ROOT/runs/grpo_stageb/checkpoints/micro_00009180/resume.pt" | |
| H="$ROOT/runs/grpo_stageb/history.jsonl" | |
| W="$ROOT/runs/grpo_stageb_halt108.log" | |
| cd "$ROOT" | |
| echo "$(date '+%H:%M %d-%b') halt-at-108 armed (wait for $CKPT)" >> "$W" | |
| while :; do | |
| sleep 120 | |
| TS=$(date '+%H:%M %d-%b') | |
| if [ -f "$CKPT" ]; then | |
| echo "$TS checkpoint 108 present — SIGTERM torchrun" >> "$W" | |
| pkill -TERM -f 'torchrun --nproc_per_node=3 scripts/grpo.py' || true | |
| sleep 15 | |
| pkill -TERM -f 'scripts/grpo.py configs/grpo_stageb.toml' || true | |
| pkill -TERM -f 'rollout_worker.py --spool' || true | |
| sleep 10 | |
| pkill -KILL -f 'VLLM::EngineCore' || true | |
| echo "$TS halt complete. GPUs:" >> "$W" | |
| nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader >> "$W" || true | |
| break | |
| fi | |
| if ! pgrep -f 'torchrun --nproc_per_node=3 scripts/grpo.py' >/dev/null; then | |
| echo "$TS torchrun already gone before step 108 — not killing" >> "$W" | |
| break | |
| fi | |
| done | |