Blame for scripts/b4_continue.sh

#!/usr/bin/env bash
# =============================================================================
# b4_continue.sh -- CONTINUE ext32k through the rest of b4 at a 32k window.
#
#   ext32k already trained 750 steps = 0.393 B tokens = 34% of b4's train split.
#   This runs steps 750 -> 2227, the remaining 1,477 steps = 0.775 B tokens,
#   finishing exactly one pass over b4 (1.168 B of 1.171 B available, no wrap).
#
# WHY CP_CKPT_RESUME AND NOT CP_INIT_FROM:
#   CP_INIT_FROM loads WEIGHTS ONLY and starts at step 0, so the DataLoader would
#   restart at token 0 and replay the 0.393 B ext32k already saw. CP_CKPT_RESUME
#   restores start_step=750, and bluscriptCP.cpp:2084 then calls
#   skip_batches(750 * 16 = 12,000) to fast-forward the stream to 0.393 B. That
#   is what makes this a true continuation of the epoch rather than a replay.
#
# KNOWN DEGRADATION -- the ZeRO optimizer sidecar is missing.
#   Yesterday's teardown uploaded the two .ckpt files but NOT the .zero_g1
#   sidecar, and that box is gone. Per bluscriptCP.cpp:2032, "without the sidecar
#   a bf16 run resumes with group 1's Adam m/v at zero -- it trains on, so the
#   loss curve only looks slightly wrong." Weights and group-0 optimizer state
#   restore normally; group 1's momentum rebuilds over tens of steps. Expect a
#   small loss transient right after step 750 that is an artifact, not a signal.
#
# LEARNING RATE -- CP_MAX_LR / CP_MIN_LR DO NOT EXIST.
#   bluscriptCP.cpp hardcodes max_lr=6e-4 (line 131) and min_lr=6e-5 (line 132),
#   and the env-parsing block never reads CP_MAX_LR or CP_MIN_LR -- there is not
#   even a float env helper. Every script in this project passed them as a no-op.
#   So the real schedule is 6e-4 * CP_REWARMUP_PEAK(0.3) = 1.8e-4 peak, 6e-5 floor.
#   ext32k's first 750 steps ran at exactly that (verified from its log: 1.8000e-04
#   at steps 99..649, 6.0030e-05 at 749), so this continuation MUST keep the same
#   values or it would not be a continuation. Do not "fix" the LR here.
#
# SCHEDULE: re-warm 100 steps from 750 back to 1.8e-4, flat to 2027, then a
#   200-step decay tail. decay_start = 750+100+1177 = 2027. The guard at
#   bluscriptCP.cpp:1006 needs CP_WARMUP+CP_STABLE < CP_MAX_STEPS-1:
#   100+1177 = 1277 < 2226. This is a SECOND schedule on top of ext32k's
#   completed one, so "ext32k@750 vs this" is not a steps-only contrast.
# =============================================================================
set -uo pipefail
cd "$(dirname "$0")"
ROOT="$(cd ../.. && pwd)"
export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1

PY=/root/Adhi/BluTrain/envv/bin/python
EXE=./build/bluscriptCP_exec
CK=checkpoints_bluscriptcp/rank_0
LOG=b4cont_logs; mkdir -p "$LOG"
M="$LOG/master.log"
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
die(){ say "FATAL: $*"; exit 1; }

T=32768; NP=2; GPUS=0,1; ORIG=16384; SCALE=2
RUN=2; START=750; STEPS=2227; WARM=100; STABLE=1177; FREQ=50; KEEP=2
EVALS="1100 1650 2227"          # 750 is already evaluated (the ext32k result)
DATA=lendata/b4_32768_65536
HAY="$PWD/lendata/haystack/b2val_haystack.bin"
ARCH="CP_N_EMBD=1536 CP_N_LAYER=36 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=0"
BASE="CP_SIZE=$NP CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG \
CP_ZERO=1 CP_ZERO_V2=1 CP_ZERO_SYMMETRIC=1 CP_BF16=1 CP_OPTIMIZER=muon \
CP_WARMUP=$WARM CP_REWARMUP=$WARM \
CP_REWARMUP_PEAK=0.3 CP_REWARMUP_ANCHOR=$START CP_STABLE=$STABLE \
CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"

$PY -c "import tiktoken" 2>/dev/null || die "tiktoken missing"
[ -x "$EXE" ] || die "not built"
for r in 0 1; do
  [ -f "checkpoints_bluscriptcp/rank_$r/blumodelcp_run${RUN}_step_${START}.ckpt" ] || \
    die "rank_$r ext32k step-$START checkpoint missing"
done
[ $((SCALE*ORIG)) -eq $T ] || die "YaRN invariant"
say "=== CONTINUE ext32k through the rest of b4 ==="
say "resume run $RUN at step $START -> $STEPS  ($((STEPS-START)) new steps, $(echo "scale=3; ($STEPS-$START)*524288/1000000000" | bc) B tokens)"
say "loader fast-forwards $((START*16)) batches = $(echo "scale=3; $START*524288/1000000000" | bc) B; reads to $(echo "scale=3; $STEPS*524288/1000000000" | bc) B of 1.171 B -- no wrap"
say "WARNING: .zero_g1 sidecar absent -- group 1 Adam moments start at zero (small transient)"
say "eval points: $EVALS"

run_seg(){
  local TGT="$1"
  [ -f "$LOG/seg_${TGT}.done" ] && { say "segment to $TGT done"; return 0; }
  say "training to step $TGT"
  ( for i in $(seq 1 6000); do
      n=$(ls checkpoints_bluscriptcp/rank_*/blumodelcp_run${RUN}_step_${TGT}.ckpt 2>/dev/null | wc -l)
      if [ "$n" -ge "$NP" ]; then
        sleep 5
        for p in $(pgrep -f 'mpirun.*bluscriptCP_exec') $(pgrep -f 'build/bluscriptCP_exec'); do kill "$p" 2>/dev/null; done
        exit 0
      fi
      sleep 20
    done ) &
  local WPID=$!
  env $ARCH $BASE CP_CKPT_RESUME=$RUN CP_DATA_ROOT="$PWD/$DATA" \
      CP_B=1 CP_GLOBAL_BATCH=524288 CP_MAX_STEPS=$STEPS \
      CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" \
      >> "$LOG/train.log" 2>&1
  kill $WPID 2>/dev/null; wait $WPID 2>/dev/null
  grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM"
  local SRC="blumodelcp_run${RUN}_step_${TGT}.ckpt"
  local n; n=$(ls checkpoints_bluscriptcp/rank_*/"$SRC" 2>/dev/null | wc -l)
  [ "$n" -ge "$NP" ] || die "step $TGT checkpoint missing ($n/$NP)"
  local SAFE="b4c${TGT}_${SRC}"
  for r in 0 1; do
    [ -f "checkpoints_bluscriptcp/rank_$r/$SAFE" ] || \
      cp "checkpoints_bluscriptcp/rank_$r/$SRC" "checkpoints_bluscriptcp/rank_$r/$SAFE"
  done
  echo "$SAFE" > "$LOG/seg_${TGT}.ckpt"; touch "$LOG/seg_${TGT}.done"
  say "step $TGT reached, preserved $SAFE, loss $(grep -aE '^step ' "$LOG/train.log" | tail -1 | grep -oE 'loss: [0-9.]+')"
}

eval_seg(){
  local TGT="$1"
  [ -f "$LOG/eval_${TGT}.done" ] && return 0
  local SAFE; SAFE=$(cat "$LOG/seg_${TGT}.ckpt")
  say "EVAL @ step $TGT"
  for LEN in 8192 16360 32744; do
    GEN_DATA_ROOT=$DATA HAY="$HAY" PY=$PY OUT=b4c_${TGT}_len_$LEN TASKS="niah_single" \
    CKPTS="b4c$TGT" N=100 PROMPT_LEN=$LEN MAX_NEW=24 T=$T \
    DEPTHS="0.0,0.25,0.5,0.75,1.0" YARN_ORIG=$ORIG YARN=$SCALE \
    GPUS=0,1 JOBS_PER_GPU=1 BF16=1 GREEDY=1 \
    CKPT_EXTRA="b4c$TGT=$SAFE" ./run_longeval.sh >> "$LOG/eval_${TGT}_${LEN}.log" 2>&1 \
      || say "  WARN eval @$TGT len $LEN"
    [ -s "b4c_${TGT}_len_$LEN/summary.csv" ] && \
      say "  step $TGT len $LEN -> $(sed -n 2p b4c_${TGT}_len_$LEN/summary.csv)"
  done
  touch "$LOG/eval_${TGT}.done"
}

for TGT in $EVALS; do run_seg "$TGT" || die "segment $TGT failed"; eval_seg "$TGT"; done
say "=== B4 CONTINUATION DONE ==="
say "progression vs ext32k@750 (niah_single):"
for LEN in 8192 16360 32744; do
  line="  len $LEN:  750=(see ext32k)"
  for TGT in $EVALS; do
    v=$(sed -n 2p "b4c_${TGT}_len_$LEN/summary.csv" 2>/dev/null | cut -d, -f2)
    line="$line  ${TGT}=${v:-?}"
  done
  say "$line"
done