Upload scripts/t128d_train.sh with huggingface_hub

scripts/t128d_train.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# =============================================================================
+# 128k run #4 (t128d) -- the long-range-density fix.  4x H200.
+#
+# DIAGNOSIS (measured 2026-09-23: BM25 cosine between 16,384-token chunks of a
+# real 131,072-token training window; "far" = ~98k apart, vs a cross-window floor)
+#
+#   b6 >=131k institutional   3.01x   <-- the ONLY bucket with 128k-range structure
+#   b5 64-131k CommonCrawl    1.26x
+#   b4 32-64k                 0.97x   \
+#   b3 16-32k                 1.01x    >  AT THE RANDOM FLOOR
+#   b1 4-8k                   0.99x   /
+#
+# t128c trained on 35% b6, so 65% of its windows held NO long-range dependency.
+# The model learns from the majority that distant context is irrelevant and NIAH
+# erodes with training: 92 @300 -> 75 @400 -> 69 @450.
+#
+# t64 never collapsed because at its 65,536 window, b5 documents (65,536-131,072)
+# SPANNED the window. Same data, different window, different long-range density.
+#
+# FIX: b6 60 / b5 20 / b2 10 / b1 10.  Measured mix coherence:
+#   mix128_splice30  1.91x   (SPLiCe -- worse than what we had, not used)
+#   mix128_b6_35     2.20x   (what t128c trained on)
+#   mix128_b6_60     2.58x   <-- this
+#
+# ONE variable changes vs t128c: the mix. Batch stays 1,048,576 (8 seq/step);
+# LR stays 1.8e-4. Anneal over the last 60 steps instead of stopping dead.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+PY=${PY:-/root/Adhi/BluTrain/envv/bin/python}
+EXE=./build/bluscriptCP_exec
+LOG=t128d_logs; mkdir -p "$LOG"; M="$LOG/master.log"
+
+NP=${NP:-4}; CPSZ=$NP
+T=131072; SCALE=32; ORIG=4096
+GB=1048576                                        # 8 seq/step
+STEPS=360; WARM=60; STABLE=240; FREQ=50; KEEP=4   # decay_start=300 -> 60-step anneal
+DATA=data/mix128_b6_60
+CK=checkpoints_bluscriptcp/rank_0
+PARENT="${PARENT_OVERRIDE:-$CK/t64.ckpt}"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+[ -x "$EXE" ] || die "binary missing"
+grep -q 'CP_ATTN_SPLIT=1' build/objects/.cp_rope_flags || die "binary lacks split kernels"
+[ "$(stat -c%s "$PARENT")" = 2423335522 ] || die "parent t64 wrong size"
+[ -d "$DATA" ] || die "$DATA missing"
+[ $((SCALE*ORIG)) -eq $T ] || die "YaRN INVARIANT BROKEN: $SCALE*$ORIG != $T"
+[ $(( WARM + STABLE )) -lt $(( STEPS - 1 )) ] || die "illegal WSD"
+AV=$(df -BG --output=avail . | tail -1 | tr -dc 0-9)
+[ "$AV" -gt $(( KEEP * 18 + 20 )) ] || die "disk ${AV}G < $(( KEEP*18+20 ))G"
+U=$($PY - "$DATA" "$T" "$GB" <<'PY'
+import glob,os,sys
+d,BT,G=sys.argv[1],int(sys.argv[2]),int(sys.argv[3]); GA=G//BT; t=0
+for p in sorted(glob.glob(f"{d}/train_shard_*.bin")):
+    n=os.path.getsize(p)//2; pos=0; b=0
+    while pos+BT+1<=n:
+        b+=1; pos+=BT
+        if pos+(BT+1)>n: break
+    t+=b
+print(t//GA)
+PY
+)
+[ "$STEPS" -le "$U" ] || die "STEPS=$STEPS > usable=$U"
+
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+CP_SAVE_KV_BF16=1 CP_RING_BF16=1 \
+YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG CP_ROPE_THETA=500000 \
+CP_ZERO=1 CP_ZERO_V2=1 CP_BF16=1 CP_OPTIMIZER=muon \
+CP_WARMUP=$WARM CP_REWARMUP=$WARM CP_REWARMUP_PEAK=${PEAK:-0.3} \
+CP_STABLE=$STABLE CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"
+
+say "=== t128d: long-range-density fix, NP=$NP (4x H200) ==="
+say "parent : $PARENT (t64 step 1200)"
+say "data   : $DATA  b6 60 / b5 20 / b2 10 / b1 10  (2.58x far/cross vs 2.20x)  usable $U"
+say "batch  : $GB = $((GB/T)) seq/step   peak = ${PEAK:-0.3} x 6e-4"
+say "WSD    : warm $WARM, stable $STABLE, decay_start $((WARM+STABLE)) of $STEPS -> 60-step anneal"
+say "GATE   : ckpt every $FREQ. If niah_single @4096 < 98, STOP and diagnose."
+env $ARCH $BASE CP_INIT_FROM="$PWD/$PARENT" CP_CKPT_NEW_RUN=1 CP_DATA_ROOT="$PWD/$DATA" \
+    CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=$STEPS \
+    mpirun --allow-run-as-root -np $NP "$EXE" >> "$LOG/train.log" 2>&1
+grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM -- lower NP or check bf16 flags"
+RN=$(grep -aoE 'CP_Training_log[0-9]+' "$LOG/train.log" | head -1 | grep -oE '[0-9]+')
+echo "${RN:-?}" > "$LOG/runnum"
+say "done, run ${RN:-?}: $(grep -aE '^step ' "$LOG/train.log"|tail -1|grep -oE 'loss: [0-9.]+')"