#!/usr/bin/env bash
# Halt Stage-B after the step-108 cadence save (micro_00009180).
# Dated entry 2026-09-07: horizon cut 500 -> 108. Do not kill before the save.
set -euo pipefail
ROOT=/apps/development/rlhf/nano-mistral-o1
CKPT="$ROOT/runs/grpo_stageb/checkpoints/micro_00009180/resume.pt"
H="$ROOT/runs/grpo_stageb/history.jsonl"
W="$ROOT/runs/grpo_stageb_halt108.log"
cd "$ROOT"
echo "$(date '+%H:%M %d-%b') halt-at-108 armed (wait for $CKPT)" >> "$W"
while :; do
  sleep 120
  TS=$(date '+%H:%M %d-%b')
  if [ -f "$CKPT" ]; then
    echo "$TS checkpoint 108 present — SIGTERM torchrun" >> "$W"
    pkill -TERM -f 'torchrun --nproc_per_node=3 scripts/grpo.py' || true
    sleep 15
    pkill -TERM -f 'scripts/grpo.py configs/grpo_stageb.toml' || true
    pkill -TERM -f 'rollout_worker.py --spool' || true
    sleep 10
    pkill -KILL -f 'VLLM::EngineCore' || true
    echo "$TS halt complete. GPUs:" >> "$W"
    nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader >> "$W" || true
    break
  fi
  if ! pgrep -f 'torchrun --nproc_per_node=3 scripts/grpo.py' >/dev/null; then
    echo "$TS torchrun already gone before step 108 — not killing" >> "$W"
    break
  fi
done