Upload scripts/mixcont_h100.sh with huggingface_hub

scripts/mixcont_h100.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Continue the mixed-corpus 64k run to a FULL b5 EPOCH. 8x H100, cp=8 dp=1.
+#
+# THE QUESTION
+#   mix64k reached step 1200 with retrieval INTACT (83/70/50/31 vs the parent's
+#   67/65/52/29) where 100% b5 had collapsed to 45/17/2/1. But it had only seen
+#   39% of b5. This run carries it to 100% of b5 inside the mix, so the model
+#   sees the SAME 0.866 B of long-document data that destroyed cont64k -- just
+#   interleaved with b3/b1. If it holds, MIXTURE is the mechanism. If it
+#   collapses, EXPOSURE was.
+#
+# THE DATA IS A VERIFIED CONTINUATION, NOT A RESTART
+#   CP_INIT_FROM bypasses skip_batches (bluscriptCP.cpp:2047), so the loader
+#   would otherwise replay from token 0. mixcont was built with --skip-tokens
+#   629,342,677 -- the EXACT stream position mix50's loader held at step 1200,
+#   including the 197,077 tokens of shard tails the loader discards. The
+#   junction was checked byte-for-byte against a no-skip reference build.
+#
+# WHY cp=8 AND NOT cp=4 dp=2
+#   These are H100 80GB, not the H200s of the last run. 16,384 tok/rank
+#   MEASURED 105.6 GB and will not fit. cp=8 halves it to 8,192 tok/rank
+#   (~70 GB) at a measured 7.81 s/step.
+#
+# CHECKPOINT POLICY (ckpt_shipper.sh runs alongside)
+#   FULL resumable (8 ranks, .ckpt + .zero_g1) at the eval steps and the final;
+#   rank_0 only elsewhere; every upload byte-verified against HF before the
+#   local copy is freed. Keeping everything would need 647 GB on a 193 GB disk.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+
+PY=/root/Adhi/BluTrain/envv/bin/python
+HF=/root/Adhi/BluTrain/envv/bin/hf
+EXE=./build/bluscriptCP_exec
+CK=checkpoints_bluscriptcp/rank_0
+LOG=mixcont_logs; mkdir -p "$LOG"
+M="$LOG/master.log"
+REPO=unparallelled/BluScriptCP
+
+T=65536; NP=8; CPSZ=8; GPUS=0,1,2,3,4,5,6,7      # dp = 1
+ORIG=32768; SCALE=2
+BASE_STEP=1200                                    # where the parent checkpoint sits
+NEW=${NEW:-2100}                                  # 2102 usable; 2100 keeps FREQ alignment
+STEPS=$(( BASE_STEP + NEW ))                      # 3300 global, for reporting only
+WARM=100; TAIL=2; FREQ=200; KEEP=2
+# CP_INIT_FROM restarts the step counter at 0, so the trainer sees max_steps=NEW
+# and its checkpoints are numbered LOCALLY (200..2100). STABLE must therefore be
+# computed against NEW, not the global 3300 -- against 3300 the guard at
+# bluscriptCP.cpp:1006 (warmup+stable >= max_steps-1) would kill the run.
+STABLE=$(( NEW - WARM - TAIL ))
+GB=524288
+DATA=Data_Loader/Data/mixcont
+PARENT="$CK/mix64k_step1200.ckpt"; PARENT_SZ=3750897182
+HAY="$PWD/Data_Loader/Data/b2_8192_16384/val_shard_00000.bin"
+EVALS_GLOBAL=${EVALS_GLOBAL:-"2000 2600 3300"}
+# local = global - BASE_STEP, because checkpoints are named locally
+EVALS=""; for g in $EVALS_GLOBAL; do EVALS="$EVALS $(( g - 1200 ))"; done; EVALS=${EVALS# }
+
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+# ---------------------------------------------------------------- preflight --
+$PY -c "import tiktoken" 2>/dev/null || die "tiktoken missing (eval would score nothing, silently)"
+[ -x "$EXE" ] || die "$EXE not built"
+S=build/objects/.cp_rope_flags
+grep -q 'CP_FUSED_ROPE=1' "$S" 2>/dev/null && grep -q 'CP_ATTN_SPLIT=1' "$S" || \
+  die "binary lacks the split kernels ($(cat "$S" 2>/dev/null)); CP_ATTN_FUSION=split throws at the first forward"
+[ -f "$PARENT" ] || die "parent missing: $PARENT"
+[ "$(stat -c%s "$PARENT")" = "$PARENT_SZ" ] || die "parent wrong size"
+[ -f "$HAY" ] || die "haystack missing"
+[ -d "$DATA" ] || die "$DATA missing"
+[ $((SCALE*ORIG)) -eq $T ] || die "YaRN invariant"
+U=$(awk '/^usable_steps/{print $2}' "$DATA/MANIFEST.txt")
+[ "$NEW" -le "$U" ] || die "NEW=$NEW exceeds usable_steps=$U -- the loader would WRAP silently"
+[ $(( WARM + STABLE )) -lt $(( NEW - 1 )) ] || die "illegal WSD: $WARM+$STABLE >= $NEW-1"
+FREEG=$(df -BG --output=avail / | tail -1 | tr -dc 0-9)
+[ "$FREEG" -ge 110 ] || die "only ${FREEG} GB free; need ~110"
+
+say "=== mix -> FULL b5 EPOCH (8x H100, cp=$CPSZ dp=$((NP/CPSZ))) ==="
+say "parent: mix64k step $BASE_STEP (83/70/50/31), weights-only via CP_INIT_FROM"
+say "data:   $DATA -- continuation from stream offset 629,342,677 (junction verified)"
+say "        b5 reaches 100% of its split in this run"
+say "steps:  $BASE_STEP -> $STEPS ($NEW new, $U usable)"
+say "WSD:    warmup $WARM, stable $STABLE, tail $TAIL"
+say "evals:  global $EVALS_GLOBAL = local $EVALS  (full resumable at these)"
+say "disk:   ${FREEG} GB free"
+
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=36 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=0"
+BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG \
+CP_ZERO=1 CP_ZERO_V2=1 CP_ZERO_SYMMETRIC=1 CP_BF16=1 CP_OPTIMIZER=muon \
+CP_WARMUP=$WARM CP_REWARMUP=$WARM CP_REWARMUP_PEAK=0.3 \
+CP_STABLE=$STABLE CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"
+
+# ------------------------------------------------------------------- probe ---
+if [ ! -f "$LOG/probe.ok" ]; then
+  say "PHASE 1: 3-step probe (8,192 tok/rank; expect ~70 GB of 81.5)"
+  env $ARCH $BASE CP_STABLE=0 CP_CKPT=0 \
+      CP_INIT_FROM="$PWD/$PARENT" CP_DATA_ROOT="$PWD/$DATA" \
+      CP_B=1 CP_GLOBAL_BATCH=$(( T * NP / CPSZ )) CP_MAX_STEPS=3 \
+      CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" \
+      > "$LOG/probe.log" 2>&1
+  RC=$?
+  grep -qaiE 'out of memory|bad_alloc' "$LOG/probe.log" && { say "PROBE OOM at cp=$CPSZ"; exit 1; }
+  grep -qaE 'terminate called|std::runtime_error|Signal: Aborted' "$LOG/probe.log" && {
+    say "PROBE CRASHED:"; grep -aA1 'terminate called' "$LOG/probe.log" | grep -a 'what()' | head -1 >&2; exit 1; }
+  N=$(grep -acE '^step +[0-9]+ \|' "$LOG/probe.log"); P=$(grep -aoE 'mem: [0-9]+MB' "$LOG/probe.log" | tail -1)
+  [ "$RC" -eq 0 ] && [ "$N" -ge 1 ] && [ -n "$P" ] || { say "PROBE FAILED (rc=$RC steps=$N peak='$P')"; tail -15 "$LOG/probe.log" >&2; exit 1; }
+  say "PROBE OK: $N steps, peak $P"; touch "$LOG/probe.ok"
+fi
+
+# ---- one continuous run; the shipper handles checkpoints, evals come after ---
+say "PHASE 2: training $BASE_STEP -> $STEPS"
+env $ARCH $BASE CP_INIT_FROM="$PWD/$PARENT" CP_CKPT_NEW_RUN=1 CP_DATA_ROOT="$PWD/$DATA" \
+    CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=$NEW \
+    CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" \
+    >> "$LOG/train.log" 2>&1
+grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM"
+RN=$(grep -aoE 'CP_Training_log[0-9]+' "$LOG/train.log" | head -1 | grep -oE '[0-9]+')
+[ -n "$RN" ] || die "no run number"; echo "$RN" > "$LOG/runnum"
+say "training done, run $RN, $(grep -aE '^step ' "$LOG/train.log" | tail -1 | grep -oE 'loss: [0-9.]+')"
+say "  peak mem $(grep -aoE 'mem: [0-9]+MB' "$LOG/train.log" | tail -1)"
+say "=== PHASE 2 COMPLETE -- finisher takes over for evals and upload ==="