Upload scripts/t128c_train.sh with huggingface_hub

scripts/t128c_train.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# =============================================================================
+# 128k run #3 (t128c) -- 8x H100 80GB, from t64, SAME b6-35 mix as run #2.
+#
+# ONE variable changes from run #2: GB 524288 -> 1048576, i.e. 8 sequences per
+# optimizer step instead of 4. That matches t64 exactly, and t64 at this same
+# LR IMPROVED (90->95->99 @32,744) where run #2 degraded (89->77).
+# Evidence: grad clip (1.0, bluscriptCP.cpp:148) fired on 0.2% of t64 steps and
+# 12.4% of run #2 steps, pre-clip norms to 7.38.
+#
+# Memory: grad_accum does NOT change peak activation memory (B=1 x T per
+# micro-step). H100 80GB needs CP_SAVE_KV_BF16=1 CP_RING_BF16=1 -- run #2 used
+# 87,025 MB on H200 and will not fit here without them.
+#
+# LR: UNCHANGED at 1.8e-4 (6e-4 hardcoded x REWARMUP_PEAK 0.3). Deliberate --
+# changing batch AND lr would confound the test.
+#
+# Stops pre-anneal (decay_start=597 of 600) so the stable phase can be extended
+# when the new data lands.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+PY=/root/Adhi/BluTrain/envv/bin/python
+EXE=./build/bluscriptCP_exec
+LOG=t128c_logs; mkdir -p "$LOG"; M="$LOG/master.log"
+
+T=131072; NP=8; CPSZ=8; SCALE=32; ORIG=4096
+GB=1048576                                   # 8 seqs/step (was 524288 = 4)
+STEPS=600; WARM=60; STABLE=537; FREQ=50; KEEP=4   # decay_start=597 -> no anneal
+DATA=data/mix128_b6_35_640M
+CK=checkpoints_bluscriptcp/rank_0
+PARENT="${PARENT_OVERRIDE:-$CK/t64.ckpt}"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+[ -x "$EXE" ] || die "binary missing"
+grep -q 'CP_FUSED_ROPE=1' build/objects/.cp_rope_flags && grep -q 'CP_ATTN_SPLIT=1' build/objects/.cp_rope_flags || die "binary lacks split kernels"
+[ -s "$PARENT" ] || die "parent missing: $PARENT"
+[ -d "$DATA" ] || die "$DATA missing"
+[ $((SCALE*ORIG)) -eq $T ] || die "YaRN INVARIANT BROKEN: $SCALE*$ORIG != $T"
+[ $(( WARM + STABLE )) -lt $(( STEPS - 1 )) ] || die "illegal WSD"
+# KEEP x 17.6 GB must fit the disk
+AV=$(df -BG --output=avail . | tail -1 | tr -dc 0-9)
+[ "$AV" -gt $(( KEEP * 18 + 20 )) ] || die "disk: ${AV}G free, KEEP=$KEEP needs $(( KEEP*18+20 ))G"
+
+U=$($PY - "$DATA" "$T" "$GB" <<'PY'
+import glob,os,sys
+d,BT,G=sys.argv[1],int(sys.argv[2]),int(sys.argv[3]); GA=G//BT; t=0
+for p in sorted(glob.glob(f"{d}/train_shard_*.bin")):
+    n=os.path.getsize(p)//2; pos=0; b=0
+    while pos+BT+1<=n:
+        b+=1; pos+=BT
+        if pos+(BT+1)>n: break
+    t+=b
+print(t//GA)
+PY
+)
+[ "$STEPS" -le "$U" ] || die "STEPS=$STEPS > usable=$U at GB=$GB"
+
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+CP_SAVE_KV_BF16=1 CP_RING_BF16=1 \
+YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG CP_ROPE_THETA=500000 \
+CP_ZERO=1 CP_ZERO_V2=1 CP_BF16=1 CP_OPTIMIZER=muon \
+CP_WARMUP=$WARM CP_REWARMUP=$WARM CP_REWARMUP_PEAK=0.3 \
+CP_STABLE=$STABLE CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"
+
+say "=== 128k run #3 (t128c) -- 8x H100, batch fix only ==="
+say "parent : $PARENT"
+say "data   : $DATA (UNCHANGED from run #2)  usable $U steps at GB=$GB"
+say "batch  : $GB = $(( GB / T )) seqs/step   <-- the only change (run #2 was 4)"
+say "LR     : 1.8e-4 flat, decay_start=$(( WARM + STABLE )) of $STEPS -> pre-anneal stop"
+say "ckpts  : every $FREQ steps, KEEP=$KEEP (disk ${AV}G free)"
+say "GATE   : eval at every ckpt. If niah_single @4096 < 98, STOP and diagnose."
+env $ARCH $BASE CP_INIT_FROM="$PWD/$PARENT" CP_CKPT_NEW_RUN=1 CP_DATA_ROOT="$PWD/$DATA" \
+    CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=$STEPS \
+    CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 mpirun --allow-run-as-root -np $NP "$EXE" >> "$LOG/train.log" 2>&1
+grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM -- check CP_SAVE_KV_BF16/CP_RING_BF16 took effect"
+RN=$(grep -aoE 'CP_Training_log[0-9]+' "$LOG/train.log" | head -1 | grep -oE '[0-9]+')
+echo "${RN:-?}" > "$LOG/runnum"
+say "done, run ${RN:-?}, $(grep -aE '^step ' "$LOG/train.log"|tail -1|grep -oE 'loss: [0-9.]+')"