Upload scripts/p1_680m.sh with huggingface_hub

scripts/p1_680m.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Phase 1 of the 680M ladder: 4k -> 16k via YaRN, on a 50/30/20 mix.
+#
+# WHY THIS MODEL
+#   varybatch step 3400: 681,328,128 params at 10.5 tok/param, vs the 1.06B's
+#   0.99. On identical data it scores 3.05 where the 1.06B scores 3.17. The
+#   whole long-context programme has pointed at the pretrain deficit as the root
+#   cause of the retrieval collapse; this is the first base good enough to test
+#   that.
+#
+#   It required fixing a real bug first: GPT::forward's tied-embedding path cast
+#   a TRANSPOSED (strided) tensor to bf16, and the cast ignores strides, so the
+#   tied head multiplied a garbled weight (NLL 12.13 vs 3.05). Invisible until
+#   now because every model trained here uses weight_tying=0. Fixed by
+#   cast -> transpose -> contiguous(); the 1.06B is unchanged (3.1717 -> 3.1719).
+#
+# DATA
+#   b3 50% (docs 16k-32k: >= the NEW window, contiguous)
+#   b2 30% / b1 20% (sub-window: supply the document boundaries that 100%
+#   long-data lacked -- the composition that collapsed twice before).
+#   Phases 2 and 3 use fresh token ranges via --skip-tokens; b3 is only 20%
+#   consumed here so it remains available as filler later.
+#
+# YaRN: 4 * 4096 == 16384 exactly. ORIG_MAXPOS must be the length the base was
+# actually trained at (4096), not the 1024 default.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+PY=/root/Adhi/BluTrain/envv/bin/python
+EXE=./build/bluscriptCP_exec
+CK=checkpoints_680m/rank_0
+LOG=p1_logs; mkdir -p "$LOG"; M="$LOG/master.log"
+T=16384; NP=2; CPSZ=2; GPUS=0,1
+ORIG=4096; SCALE=4
+STEPS=${STEPS:-1200}; WARM=100; TAIL=2; FREQ=200; KEEP=3
+STABLE=$(( STEPS - WARM - TAIL ))
+GB=524288
+DATA=Data_Loader/Data/p1mix
+PARENT="$CK/varybatch.ckpt"; PARENT_SZ=1362674564
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+$PY -c "import tiktoken" 2>/dev/null || die "tiktoken missing"
+[ -x "$EXE" ] || die "$EXE not built"
+S=build/objects/.cp_rope_flags
+grep -q 'CP_FUSED_ROPE=1' "$S" && grep -q 'CP_ATTN_SPLIT=1' "$S" || die "binary lacks split kernels ($(cat $S))"
+[ -f "$PARENT" ] || die "parent missing"
+[ "$(stat -c%s "$PARENT")" = "$PARENT_SZ" ] || die "parent wrong size"
+[ -d "$DATA" ] || die "$DATA missing"
+[ $((SCALE*ORIG)) -eq $T ] || die "YaRN invariant: $SCALE*$ORIG != $T"
+U=$($PY - <<'PY'
+import glob,os
+BT=16384;GA=32;t=0
+for p in sorted(glob.glob("Data_Loader/Data/p1mix/train_shard_*.bin")):
+    n=os.path.getsize(p)//2;pos=0;b=0
+    while pos+BT+1<=n:
+        b+=1;pos+=BT
+        if pos+(BT+1)>n:break
+    t+=b
+print(t//GA)
+PY
+)
+[ "$STEPS" -le "$U" ] || die "STEPS=$STEPS exceeds usable=$U (loader would WRAP silently)"
+[ $(( WARM + STABLE )) -lt $(( STEPS - 1 )) ] || die "illegal WSD"
+
+say "=== 680M phase 1: 4k -> 16k (2x H100, cp=$CPSZ) ==="
+say "parent: varybatch step 3400, 10.5 tok/param, scores 3.05 (1.06B scores 3.17)"
+say "data:   $DATA  b3=50% b2=30% b1=20%   usable $U steps, running $STEPS"
+say "YaRN:   SCALE=$SCALE ORIG=$ORIG -> $((SCALE*ORIG)) == CP_T=$T  OK"
+say "WSD:    warmup $WARM stable $STABLE tail $TAIL"
+
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG CP_ROPE_THETA=500000 \
+CP_ZERO=1 CP_ZERO_V2=1 CP_BF16=1 CP_OPTIMIZER=muon \
+CP_WARMUP=$WARM CP_REWARMUP=$WARM CP_REWARMUP_PEAK=0.3 \
+CP_STABLE=$STABLE CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"
+
+if [ ! -f "$LOG/probe.ok" ]; then
+  say "PHASE 1a: 3-step probe ($((T/CPSZ)) tok/rank)"
+  env $ARCH $BASE CP_STABLE=0 CP_CKPT=0 CP_INIT_FROM="$PWD/$PARENT" \
+      CP_DATA_ROOT="$PWD/$DATA" CP_B=1 CP_GLOBAL_BATCH=$(( T * NP / CPSZ )) CP_MAX_STEPS=3 \
+      CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" > "$LOG/probe.log" 2>&1
+  RC=$?
+  grep -qaiE 'out of memory|bad_alloc' "$LOG/probe.log" && die "PROBE OOM"
+  grep -qaE 'terminate called|Signal: Aborted' "$LOG/probe.log" && {
+    say "PROBE CRASHED:"; grep -aA1 'terminate called' "$LOG/probe.log"|grep -a 'what()'|head -1 >&2; exit 1; }
+  N=$(grep -acE '^step +[0-9]+ \|' "$LOG/probe.log"); P=$(grep -aoE 'mem: [0-9]+MB' "$LOG/probe.log"|tail -1)
+  [ "$RC" -eq 0 ] && [ "$N" -ge 1 ] && [ -n "$P" ] || { say "PROBE FAILED rc=$RC steps=$N peak='$P'"; tail -12 "$LOG/probe.log" >&2; exit 1; }
+  say "PROBE OK: $N steps, peak $P of 81559 MiB/card"; touch "$LOG/probe.ok"
+fi
+
+say "PHASE 1b: training $STEPS steps"
+env $ARCH $BASE CP_INIT_FROM="$PWD/$PARENT" CP_CKPT_NEW_RUN=1 CP_DATA_ROOT="$PWD/$DATA" \
+    CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=$STEPS \
+    CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" >> "$LOG/train.log" 2>&1
+grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM"
+RN=$(grep -aoE 'CP_Training_log[0-9]+' "$LOG/train.log" | head -1 | grep -oE '[0-9]+')
+[ -n "$RN" ] || die "no run number"; echo "$RN" > "$LOG/runnum"
+say "training done, run $RN, $(grep -aE '^step ' "$LOG/train.log" | tail -1 | grep -oE 'loss: [0-9.]+')"
+say "  peak mem $(grep -aoE 'mem: [0-9]+MB' "$LOG/train.log" | tail -1)"
+say "=== PHASE 1 TRAINING COMPLETE ==="