Upload scripts/t64_train.sh with huggingface_hub

scripts/t64_train.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# =============================================================================
+# 64k extension, 8x H100, with the CORRECT configuration.
+#
+# THE MISTAKE THIS AVOIDS: the 32k run trained at CP_T=32,768 with SCALE=32,
+# which squeezed the whole context into virtual positions 0..1,023 -- 32 tokens
+# per virtual unit. The model re-conditioned to that compressed regime and lost
+# the positional sensitivity that made compression useful: 90 -> 49 by step 800.
+#
+# HERE THE INVARIANT HOLDS:  SCALE x ORIG == CP_T  ->  16 x 4,096 == 65,536
+#   -> positions span the FULL virtual range 0..4,095
+#   -> 16 tokens per virtual unit, the minimum compression 64k allows
+#   -> this is what p1final was trained under (4 x 4,096 == 16,384) and p1final
+#      is the best model this project has produced.
+# Evaluate afterwards at x64; compression helps at INFERENCE, not at TRAINING.
+#
+# LR: CP_MAX_LR / CP_MIN_LR are NO-OPS (0 occurrences in bluscriptCP.cpp).
+#   max_lr 6e-4 hardcoded :151, x REWARMUP_PEAK 0.3 = 1.8e-4 peak, min 6e-5.
+#   TAIL=200 gives a REAL cosine decay. Every prior run used TAIL=2 and never
+#   annealed -- the 2026-09-09 log flagged this ("all of it UNDECAYED").
+#
+# DATA: mix, never pure b5. b5 50 / b4 25 / b3 15 / b1 10. b5 documents are
+#   65,536-131,072 tokens, at or above the window, so every window is contiguous
+#   single-document text. b5 ends at 36% of its epoch; both prior collapses were
+#   at ~80%.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+PY=/root/Adhi/BluTrain/envv/bin/python
+EXE=./build/bluscriptCP_exec
+LOG=t64_logs; mkdir -p "$LOG"; M="$LOG/master.log"
+T=65536; NP=8; SCALE=16; ORIG=4096
+CPSZ=${CPSZ:-8}; DP=$(( NP / CPSZ ))
+STEPS=${STEPS:-1200}; WARM=100; TAIL=200; FREQ=200; KEEP=2
+STABLE=$(( STEPS - WARM - TAIL ))
+GB=524288
+DATA=Data_Loader/Data/mix64
+CK=checkpoints_bluscriptcp/rank_0
+PARENT="$CK/p1final.ckpt"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+[ -x "$EXE" ] || die "$EXE not built"
+S=build/objects/.cp_rope_flags
+grep -q 'CP_FUSED_ROPE=1' "$S" && grep -q 'CP_ATTN_SPLIT=1' "$S" || die "binary lacks split kernels ($(cat $S))"
+[ -f "$PARENT" ] || die "parent missing"
+[ "$(stat -c%s "$PARENT")" = 5141244514 ] || die "parent wrong size"
+[ -d "$DATA" ] || die "$DATA missing"
+[ $((SCALE*ORIG)) -eq $T ] || die "YaRN INVARIANT BROKEN: $SCALE*$ORIG != $T -- this is the 32k mistake"
+[ $(( WARM + STABLE )) -lt $(( STEPS - 1 )) ] || die "illegal WSD"
+$PY -c "import tiktoken" 2>/dev/null || die "tiktoken missing"
+U=$($PY - "$DATA" "$T" "$GB" <<'PY'
+import glob,os,sys
+d,BT,G=sys.argv[1],int(sys.argv[2]),int(sys.argv[3]); GA=G//BT; t=0
+for p in sorted(glob.glob(f"{d}/train_shard_*.bin")):
+    n=os.path.getsize(p)//2; pos=0; b=0
+    while pos+BT+1<=n:
+        b+=1; pos+=BT
+        if pos+(BT+1)>n: break
+    t+=b
+print(t//GA)
+PY
+)
+[ "$STEPS" -le "$U" ] || die "STEPS=$STEPS > usable=$U (loader would WRAP silently)"
+
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG CP_ROPE_THETA=500000 \
+CP_ZERO=1 CP_ZERO_V2=1 CP_BF16=1 CP_OPTIMIZER=muon \
+CP_WARMUP=$WARM CP_REWARMUP=$WARM CP_REWARMUP_PEAK=0.3 \
+CP_STABLE=$STABLE CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"
+
+say "=== 680M 64k extension, 8x H100, cp=$CPSZ dp=$DP ==="
+say "parent : p1final (95/90/82 at x64 -- best model we have)"
+say "YaRN   : SCALE=$SCALE x ORIG=$ORIG = $((SCALE*ORIG)) == CP_T  INVARIANT HOLDS"
+say "         virtual range 0..$(( (T-1)/SCALE )) (full), $SCALE tokens/virtual unit"
+say "LR     : peak 1.8e-4 (6e-4 x 0.3), warmup $WARM / stable $STABLE / DECAY $TAIL -> 6e-5"
+say "data   : $DATA  b5=50 b4=25 b3=15 b1=10, usable $U steps, running $STEPS"
+
+if [ ! -f "$LOG/probe_cp${CPSZ}.ok" ]; then
+  say "PROBE cp=$CPSZ ($((T/CPSZ)) tok/rank, grad_accum $((GB/(T*DP))))"
+  env $ARCH $BASE CP_STABLE=0 CP_CKPT=0 CP_INIT_FROM="$PWD/$PARENT" \
+      CP_DATA_ROOT="$PWD/$DATA" CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=3 \
+      CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 mpirun --allow-run-as-root -np $NP "$EXE" > "$LOG/probe_cp${CPSZ}.log" 2>&1
+  RC=$?
+  grep -qaiE 'out of memory|bad_alloc' "$LOG/probe_cp${CPSZ}.log" && die "PROBE OOM at cp=$CPSZ"
+  N=$(grep -acE '^step +[0-9]+ \|' "$LOG/probe_cp${CPSZ}.log"); P=$(grep -aoE 'mem: [0-9]+MB' "$LOG/probe_cp${CPSZ}.log"|tail -1)
+  [ "$RC" -eq 0 ] && [ "$N" -ge 1 ] && [ -n "$P" ] || { say "PROBE FAILED rc=$RC steps=$N"; tail -14 "$LOG/probe_cp${CPSZ}.log" >&2; exit 1; }
+  say "PROBE OK cp=$CPSZ: peak $P of 81559 MiB, dt $(grep -aoE 'dt: [0-9.]+ms' "$LOG/probe_cp${CPSZ}.log"|tail -1)"
+  touch "$LOG/probe_cp${CPSZ}.ok"
+fi
+
+say "TRAINING $STEPS steps"
+env $ARCH $BASE CP_INIT_FROM="$PWD/$PARENT" CP_CKPT_NEW_RUN=1 CP_DATA_ROOT="$PWD/$DATA" \
+    CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=$STEPS \
+    CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 mpirun --allow-run-as-root -np $NP "$EXE" >> "$LOG/train.log" 2>&1
+grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM mid-run"
+RN=$(grep -aoE 'CP_Training_log[0-9]+' "$LOG/train.log" | head -1 | grep -oE '[0-9]+')
+[ -n "$RN" ] || die "no run number"; echo "$RN" > "$LOG/runnum"
+say "done, run $RN, $(grep -aE '^step ' "$LOG/train.log"|tail -1|grep -oE 'loss: [0-9.]+')"
+say "=== 64K TRAINING COMPLETE ==="