Upload scripts/mix64k_h200.sh with huggingface_hub

scripts/mix64k_h200.sh

@@ -0,0 +1,343 @@
+#!/usr/bin/env bash
+# =============================================================================
+# 32k -> 64k extension on the MIXED corpus. 8x H200, cp=4 dp=2.
+#
+# THE EXPERIMENT
+#   The 2026-09-12 run extended to 64k on 100% b5 (every doc >= 65,536 tokens)
+#   and destroyed retrieval: 67/65/52/29 -> 45/17/2/1 at 8k/16k/32k/64k, while
+#   training loss FELL 2.89 -> 1.14. Data composition is the leading suspect
+#   (100% long-context data is documented to cause severe degradation).
+#   This run changes ONE variable: the data is now 50% b5 / 30% b3 / 20% b1.
+#   LR, schedule shape, parent and window are all held identical so a null
+#   result is attributable.
+#
+#   PARENT is ext64k step 695 -- the SAME starting point the failed cont64k run
+#   used, so the only changed variable is the data. Its own measured scores
+#   (67/65/52/29, logs/ext64k_len_*/summary.csv on HF) are the step-0 baseline.
+#   Starting instead from the 32k model would top out at 1,714 cumulative 64k
+#   steps, short of the 1,894 where the cliff actually appeared -- the run could
+#   miss the failure regime entirely and prove nothing.
+#
+# WHY cp=4 dp=2 AND NOT cp=8
+#   16,384 tok/rank MEASURED 108,493 MB, which fits H200's 143 GB with room.
+#   dp=2 halves the micro-steps per optimizer step (8 -> 4) instead of adding
+#   ring hops: ~6.75 s/step projected vs 7.81 measured for cp=8 on H100.
+#
+# SEGMENTED, BECAUSE EVAL NEEDS THE GPUS
+#   CP_MAX_STEPS is ALWAYS the full 1714 -- never the segment target. That
+#   keeps the WSD schedule identical across segments and keeps the guard at
+#   bluscriptCP.cpp:1006 (warmup+stable >= max_steps-1) satisfied. A watcher
+#   stops the trainer when the segment's checkpoint lands. Resumes pin
+#   CP_REWARMUP_ANCHOR=0; its default (-1) re-anchors warmup to start_step and
+#   would re-ramp the LR at every resume.
+#
+# WHAT KILLED THE LAST RUN'S POST-MORTEM
+#   CP_CKPT_KEEP=2 pruned every checkpoint before the collapse, so the last
+#   good model was unrecoverable and the failure could not be bisected. Here a
+#   janitor preserves rank_0 of EVERY save (3.75 GB; eval only ever loads
+#   rank_0 at NP=1) under a prefix the pruner cannot match, and uploads on a
+#   coarser cadence. Worst case you lose 100 steps.
+#
+# COLLAPSE DETECTION IS FREE AND CONTINUOUS
+#   The cliff is visible in training loss long before any eval: the failed run
+#   went 2.886 -> 2.642 -> 2.185 across steps 1100-1249 on a 50-step mean.
+#   detect_cliff() compares the current 50-step mean to the 200-700 baseline.
+#   Loss CANNOT prove retrieval is intact -- that is the whole lesson -- so the
+#   evals still run; the detector only decides WHICH checkpoints to evaluate.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+
+PY=${PY:-/root/Adhi/BluTrain/envv/bin/python3}
+HF=${HF:-/root/Adhi/BluTrain/envv/bin/hf}
+EXE=./build/bluscriptCP_exec
+CK=checkpoints_bluscriptcp/rank_0
+LOG=mix64k_logs; mkdir -p "$LOG"
+M="$LOG/master.log"
+REPO=unparallelled/BluScriptCP
+
+T=65536; NP=8; CPSZ=4; GPUS=0,1,2,3,4,5,6,7    # dp = NP/CPSZ = 2
+ORIG=32768; SCALE=2                             # 2 * 32768 == 65536
+STEPS=${STEPS:-1300}                            # past the cliff with margin; mix50 allows 1716
+WARM=100; TAIL=2; FREQ=100; KEEP=2
+STABLE=$(( STEPS - WARM - TAIL ))               # plateau throughout, like the old ext64k
+GB=524288
+DATA=Data_Loader/Data/mix50
+PARENT="$CK/ext64k_step695.ckpt"
+PARENT_SZ=5225752094
+HAY="$PWD/Data_Loader/Data/b2_8192_16384/val_shard_00000.bin"
+EVAL_A=${EVAL_A:-695}                           # A/B point vs the failed ext64k
+EVAL_B=${EVAL_B:-1200}                          # collapse check
+UPLOAD_EVERY=${UPLOAD_EVERY:-300}
+
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+# ---------------------------------------------------------------- preflight --
+$PY -c "import tiktoken" 2>/dev/null || die "tiktoken missing (eval would score nothing, silently)"
+[ -x "$EXE" ] || die "$EXE not built"
+STAMP=build/objects/.cp_rope_flags
+[ -f "$STAMP" ] || die "no build-flag stamp; rebuild: make CP_FUSED_ROPE=1 CP_ATTN_SPLIT=1 SM_ARCH=90 bluscript-cp -j\$(nproc)"
+grep -q 'CP_FUSED_ROPE=1' "$STAMP" && grep -q 'CP_ATTN_SPLIT=1' "$STAMP" || \
+  die "binary lacks the split kernels ($(cat "$STAMP")). CP_ATTN_FUSION=split will throw at the first forward. Rebuild: make CP_FUSED_ROPE=1 CP_ATTN_SPLIT=1 SM_ARCH=90 bluscript-cp -j\$(nproc)"
+[ -f "$PARENT" ] || die "parent missing: $PARENT"
+[ "$(stat -c%s "$PARENT")" = "$PARENT_SZ" ] || die "parent wrong size: $(stat -c%s "$PARENT") (want $PARENT_SZ)"
+[ -f "$HAY" ] || die "haystack missing: $HAY"
+[ -d "$DATA" ] || die "$DATA missing -- run mix_buckets.py first"
+[ $((SCALE*ORIG)) -eq $T ] || die "YaRN invariant: $SCALE * $ORIG != $T"
+[ $((NP % CPSZ)) -eq 0 ] || die "NP=$NP not divisible by CP_SIZE=$CPSZ"
+[ $(( WARM + STABLE )) -lt $(( STEPS - 1 )) ] || die "illegal WSD: $WARM+$STABLE >= $STEPS-1"
+# the manifest counts batches the way DataLoaderLite actually consumes them;
+# total_tokens//global_batch silently overshoots and the loader wraps in silence
+if [ -f "$DATA/MANIFEST.txt" ]; then
+  USABLE=$(awk '/^usable_steps/{print $2}' "$DATA/MANIFEST.txt")
+  [ "$STEPS" -le "$USABLE" ] || die "STEPS=$STEPS exceeds usable_steps=$USABLE -- the loader would WRAP silently"
+  say "manifest: usable_steps=$USABLE, running $STEPS"
+else
+  say "WARN: no MANIFEST.txt in $DATA -- cannot rule out a silent loader wrap"
+fi
+FREEG=$(df -BG --output=avail / | tail -1 | tr -dc 0-9)
+[ "$FREEG" -ge 120 ] || die "only ${FREEG} GB free; need ~120 (30 GB live ckpts + ~64 GB preserved rank_0)"
+
+say "=== 32k -> 64k on the MIXED corpus (8x H200, cp=$CPSZ dp=$((NP/CPSZ))) ==="
+say "parent: ext64k step 695 (64k model, 67/65/52/29) -- weights-only: its 4-rank ZeRO sidecars cannot load at world_size 8"
+say "data:   $DATA  $(awk '/^mix /{printf "%s=%s ", $2, $4}' "$DATA/MANIFEST.txt" 2>/dev/null)"
+say "YaRN:   SCALE=$SCALE ORIG=$ORIG -> $((SCALE*ORIG)) == CP_T=$T  OK"
+say "WSD:    warmup $WARM, stable $STABLE, tail $TAIL -> plateau at 1.8e-4 throughout"
+say "evals:  $EVAL_A (mid-run health), $EVAL_B (the failed run cliffed at its step 1199)"
+say "disk:   ${FREEG} GB free"
+
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=36 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=0"
+BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+YARN_SCALE=$SCALE YARN_ORIG_MAXPOS=$ORIG \
+CP_ZERO=1 CP_ZERO_V2=1 CP_ZERO_SYMMETRIC=1 CP_BF16=1 CP_OPTIMIZER=muon \
+CP_WARMUP=$WARM CP_REWARMUP=$WARM CP_REWARMUP_PEAK=0.3 \
+CP_STABLE=$STABLE CP_CKPT=1 CP_CKPT_FREQ=$FREQ CP_CKPT_KEEP=$KEEP"
+
+# ------------------------------------------------------------- upload helper --
+# A missing source is a LOUD failure and every upload is confirmed by the byte
+# size HF reports back. The old helper was `[ -e "$1" ] || return 0`, which
+# returned SUCCESS when the file was absent -- that is how 17 GB of checkpoints
+# were lost with nothing in the log.
+up_verify(){ # src dst
+  local src=$1 dst=$2
+  [ -f "$src" ] || { say "  MISSING SOURCE (NOT UPLOADED): $src"; return 1; }
+  $HF upload "$REPO" "$src" "$dst" >>"$LOG/upload.log" 2>&1 || { say "  UPLOAD FAILED $dst"; return 1; }
+  $PY - "$REPO" "$src" "$dst" <<'PYEOF'
+import sys, os
+from huggingface_hub import HfApi
+repo, loc, rem = sys.argv[1], sys.argv[2], sys.argv[3]
+want = os.path.getsize(loc)
+got = {f.rfilename: (f.size or 0) for f in HfApi().repo_info(repo, files_metadata=True).siblings}.get(rem)
+print(("  VERIFIED " if got == want else "  *** MISMATCH/ABSENT ") + f"{rem} local={want:,} remote={got}")
+sys.exit(0 if got == want else 1)
+PYEOF
+}
+
+# ---------------------------------------------------------------- janitor ----
+# Preserve rank_0 of EVERY save before CP_CKPT_KEEP can prune it, and upload on
+# a coarser cadence (HF deletes do not reclaim LFS storage in this repo).
+start_janitor(){
+  ( mkdir -p "$LOG/keep"
+    while sleep 45; do
+      RN=$(cat "$LOG/runnum" 2>/dev/null) || continue
+      for f in $CK/blumodelcp_run${RN}_step_*.ckpt; do
+        [ -e "$f" ] || continue
+        case "$f" in *keep_*) continue;; esac
+        S=$(basename "$f" | grep -oE 'step_[0-9]+' | cut -d_ -f2)
+        [ -n "$S" ] || continue
+        K="$CK/keep_step_${S}.ckpt"
+        if [ ! -f "$K" ]; then
+          cp "$f" "$K" 2>/dev/null && echo "$S" >> "$LOG/keep/preserved.txt"
+          if [ $(( S % UPLOAD_EVERY )) -eq 0 ]; then
+            up_verify "$K" "checkpoints/mix64k_step_${S}.ckpt" >>"$M" 2>&1 &
+          fi
+        fi
+      done
+      # drop orphaned sidecars: CP_CKPT_KEEP prunes .ckpt but never .zero_g*,
+      # the leak that filled a 193 GB disk on 2026-09-09
+      for s in checkpoints_bluscriptcp/rank_*/*.zero_g*; do
+        [ -e "$s" ] || continue
+        [ -f "${s%.zero_g*}.ckpt" ] || rm -f "$s"
+      done
+    done ) & JANITOR=$!
+}
+stop_janitor(){ kill "${JANITOR:-}" 2>/dev/null; wait "${JANITOR:-}" 2>/dev/null; }
+trap 'stop_janitor' EXIT
+
+# ------------------------------------------------------------ cliff detector --
+# Verdict from the 50-step rolling mean vs the 200-700 baseline. The failed run:
+# baseline ~2.87, then 2.642 (steps 1150-1199) and 2.185 (1200-1249). Per-step
+# SD is ~0.16, so a 50-step mean has SE ~0.023 and a 0.20 drop is ~9 sigma --
+# this cannot fire on noise.
+detect_cliff(){   # prints: CLEAN|CLIFF <onset_step> <baseline> <current>
+  $PY - "$LOG/train.log" <<'PYEOF'
+import re, sys, statistics as st
+rows = [(int(a), float(b)) for a, b in
+        re.findall(r"^step +(\d+) \| loss: ([0-9.]+)", open(sys.argv[1], errors="ignore").read(), re.M)]
+# NODATA, never CLEAN, when the baseline cannot be computed. Reporting "clean"
+# because the evidence is missing is the failure direction that matters here --
+# b4cont/train.log does exactly this, since it resumes at step 750 and has no
+# steps in the 200-700 baseline window.
+if len(rows) < 250:
+    print("NODATA 0 0 0"); raise SystemExit
+d = dict(rows)
+base = [d[s] for s in range(200, 701) if s in d]
+if len(base) < 100:
+    print("NODATA 0 0 0"); raise SystemExit
+B = st.mean(base)
+THRESH = B - 0.20
+onset = 0
+for s in range(750, max(d) + 1):
+    w = [d[k] for k in range(s - 49, s + 1) if k in d]
+    if len(w) == 50 and st.mean(w) < THRESH:
+        onset = s; break
+cur = st.mean([d[k] for k in range(max(d) - 49, max(d) + 1) if k in d] or [B])
+print(f"{'CLIFF' if onset else 'CLEAN'} {onset} {B:.3f} {cur:.3f}")
+PYEOF
+}
+
+# ------------------------------------------------------------------ probe ----
+if [ ! -f "$LOG/probe.ok" ]; then
+  # grad_accum=1 means global_batch == B*T*dp_size, NOT T. At dp=2 the minimum
+  # micro-batch is 131,072 tokens; passing T alone dies in the divisibility
+  # guard (bluscriptCP.cpp:1462) before the first step, and MPI_ABORT eats the
+  # message so the log shows only "MPI_ABORT ... errorcode 1".
+  say "PHASE 1: 3-step probe ($(( T / CPSZ )) tok/rank, global_batch $(( T * NP / CPSZ )))"
+  env $ARCH $BASE CP_STABLE=0 CP_CKPT=0 \
+      CP_INIT_FROM="$PWD/$PARENT" CP_DATA_ROOT="$PWD/$DATA" \
+      CP_B=1 CP_GLOBAL_BATCH=$(( T * NP / CPSZ )) CP_MAX_STEPS=3 \
+      CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" \
+      > "$LOG/probe.log" 2>&1
+  RC=$?
+  # The probe must PROVE it trained. An earlier version only grepped for OOM,
+  # so a probe that aborted on every rank was called "PROBE OK" and the real
+  # run died 73 s later.
+  grep -qaiE 'out of memory|bad_alloc' "$LOG/probe.log" && { say "PROBE OOM at cp=$CPSZ"; exit 1; }
+  grep -qaE 'terminate called|std::runtime_error|Signal: Aborted' "$LOG/probe.log" && {
+    say "PROBE CRASHED:"; grep -aA1 'terminate called' "$LOG/probe.log" | grep -a 'what()' | head -1 >&2; exit 1; }
+  NSTEP=$(grep -acE '^step +[0-9]+ \|' "$LOG/probe.log")
+  PEAK=$(grep -aoE 'mem: [0-9]+MB' "$LOG/probe.log" | tail -1)
+  [ "$RC" -eq 0 ] && [ "$NSTEP" -ge 1 ] && [ -n "$PEAK" ] || {
+    say "PROBE FAILED (rc=$RC steps=$NSTEP peak='$PEAK')"; tail -15 "$LOG/probe.log" >&2; exit 1; }
+  say "PROBE OK: $NSTEP steps, peak $PEAK"
+  touch "$LOG/probe.ok"
+fi
+
+# ------------------------------------------------------------- run a segment --
+run_seg(){          # $1 = target step
+  local TGT="$1"
+  [ -f "$LOG/seg_${TGT}.done" ] && { say "segment to $TGT already done"; return 0; }
+  local MODE
+  if [ -f "$LOG/runnum" ]; then
+    MODE="CP_CKPT_RESUME=$(cat "$LOG/runnum") CP_REWARMUP_ANCHOR=0"
+    say "resuming to step $TGT"
+  else
+    MODE="CP_INIT_FROM=$PWD/$PARENT CP_CKPT_NEW_RUN=1"
+    say "starting fresh from $(basename "$PARENT"), target step $TGT"
+  fi
+  ( for i in $(seq 1 6000); do
+      RN=$(cat "$LOG/runnum" 2>/dev/null || \
+           grep -aoE 'CP_Training_log[0-9]+' "$LOG/train.log" 2>/dev/null | tail -1 | grep -oE '[0-9]+')
+      [ -n "$RN" ] || { sleep 20; continue; }
+      echo "$RN" > "$LOG/runnum"
+      n=$(ls checkpoints_bluscriptcp/rank_*/blumodelcp_run${RN}_step_${TGT}.ckpt 2>/dev/null | wc -l)
+      if [ "$n" -ge "$NP" ]; then
+        sleep 5
+        for p in $(pgrep -f 'mpirun.*bluscriptCP_exec') $(pgrep -f 'build/bluscriptCP_exec'); do kill "$p" 2>/dev/null; done
+        exit 0
+      fi
+      sleep 20
+    done ) & WPID=$!
+  env $ARCH $BASE $MODE CP_DATA_ROOT="$PWD/$DATA" \
+      CP_B=1 CP_GLOBAL_BATCH=$GB CP_MAX_STEPS=$STEPS \
+      CUDA_VISIBLE_DEVICES=$GPUS mpirun --allow-run-as-root -np $NP "$EXE" \
+      >> "$LOG/train.log" 2>&1
+  kill $WPID 2>/dev/null; wait $WPID 2>/dev/null
+  grep -qaiE 'out of memory|bad_alloc' "$LOG/train.log" && die "OOM"
+  local RN; RN=$(cat "$LOG/runnum" 2>/dev/null) || die "no run number"
+  local SRC="blumodelcp_run${RN}_step_${TGT}.ckpt"
+  local n; n=$(ls checkpoints_bluscriptcp/rank_*/"$SRC" 2>/dev/null | wc -l)
+  [ "$n" -ge "$NP" ] || die "step $TGT checkpoint missing ($n/$NP ranks)"
+  local SAFE="mix64k${TGT}_${SRC}"
+  [ -f "$CK/$SAFE" ] || cp "$CK/$SRC" "$CK/$SAFE"
+  echo "$SAFE" > "$LOG/seg_${TGT}.ckpt"; touch "$LOG/seg_${TGT}.done"
+  say "step $TGT reached (run $RN), preserved as $SAFE, $(grep -aE '^step ' "$LOG/train.log" | tail -1 | grep -oE 'loss: [0-9.]+')"
+  up_verify "$CK/$SAFE" "checkpoints/mix64k_step_${TGT}.ckpt" >>"$M" 2>&1 &
+}
+
+# --------------------------------------------------------------- run an eval --
+eval_at(){          # $1 = label   $2 = checkpoint basename in rank_0
+  local TAG="$1" CKF="$2"
+  [ -f "$CK/$CKF" ] || { say "eval $TAG: $CKF not present, SKIP"; return 1; }
+  [ -f "$LOG/eval_${TAG}.done" ] && { say "eval $TAG already done"; return 0; }
+  say "EVAL $TAG -- 4 lengths, one GPU each ($CKF)"
+  for LEN in 8192 16360 32744 65512; do
+    ( GEN_DATA_ROOT=$DATA HAY="$HAY" PY=$PY OUT=mix64k_${TAG}_len_$LEN TASKS="niah_single" \
+      CKPTS="m$TAG" N=100 PROMPT_LEN=$LEN MAX_NEW=24 T=$T \
+      DEPTHS="0.0,0.25,0.5,0.75,1.0" YARN_ORIG=$ORIG YARN=$SCALE \
+      GPUS=$(( (LEN==8192)?0:(LEN==16360)?1:(LEN==32744)?2:3 )) JOBS_PER_GPU=1 \
+      BF16=1 GREEDY=1 CKPT_EXTRA="m$TAG=$CKF" ./run_longeval.sh \
+      >> "$LOG/eval_${TAG}_$LEN.log" 2>&1 ) &
+  done
+  wait
+  for LEN in 8192 16360 32744 65512; do
+    [ -s "mix64k_${TAG}_len_$LEN/summary.csv" ] \
+      && say "  $TAG len $LEN -> $(sed -n 2p mix64k_${TAG}_len_$LEN/summary.csv)" \
+      || say "  $TAG len $LEN -> NO RESULT"
+  done
+  touch "$LOG/eval_${TAG}.done"
+}
+
+# =============================== the experiment ==============================
+start_janitor
+
+run_seg "$EVAL_A" || die "segment to $EVAL_A failed"
+eval_at "$EVAL_A" "$(cat "$LOG/seg_${EVAL_A}.ckpt")"
+say "baseline for all of these = this run's own parent, measured: 67.0/65.0/52.0/29.0"
+
+run_seg "$EVAL_B" || die "segment to $EVAL_B failed"
+eval_at "$EVAL_B" "$(cat "$LOG/seg_${EVAL_B}.ckpt")"
+
+read -r VERDICT ONSET BASE_L CUR_L <<<"$(detect_cliff)"
+say "cliff detector: $VERDICT (onset=$ONSET baseline=$BASE_L current=$CUR_L)"
+
+[ "$VERDICT" = "NODATA" ] && die "cliff detector has no baseline (needs steps 200-700 in $LOG/train.log). Refusing to call this run clean on missing evidence -- inspect the loss curve by hand."
+
+if [ "$VERDICT" = "CLIFF" ]; then
+  # BRANCH A -- collapsed. Evaluate the last preserved checkpoint BEFORE onset,
+  # which is exactly what CP_CKPT_KEEP=2 destroyed last time.
+  # ONSET is the END of the first 50-step window under threshold, so the window
+  # opened at ONSET-49 and the last save known-clean is below that.
+  PRE=$(( ((ONSET - 49) / FREQ) * FREQ ))
+  say "BRANCH A: collapse detected at step $ONSET -- evaluating the pre-cliff checkpoint at $PRE"
+  if [ -f "$CK/keep_step_${PRE}.ckpt" ]; then
+    eval_at "pre${PRE}" "keep_step_${PRE}.ckpt"
+    up_verify "$CK/keep_step_${PRE}.ckpt" "checkpoints/mix64k_pre_step_${PRE}.ckpt" >>"$M" 2>&1
+  else
+    say "  keep_step_${PRE}.ckpt missing; preserved steps: $(sort -n "$LOG/keep/preserved.txt" 2>/dev/null | tr '\n' ' ')"
+  fi
+  say "RESULT: the mix did NOT prevent the collapse. Data composition is not the (only) cause."
+else
+  # BRANCH B -- clean at EVAL_B. Run out the mix and evaluate the endpoint.
+  say "BRANCH B: no cliff by $EVAL_B (the pure-b5 run was already at val 2.00 here) -- continuing to $STEPS"
+  run_seg "$STEPS" || die "segment to $STEPS failed"
+  eval_at "$STEPS" "$(cat "$LOG/seg_${STEPS}.ckpt")"
+  read -r V2 O2 B2 C2 <<<"$(detect_cliff)"
+  say "cliff detector after full pass: $V2 (onset=$O2 baseline=$B2 current=$C2)"
+  if [ "$V2" = "CLIFF" ]; then
+    PRE=$(( ((O2 - 49) / FREQ) * FREQ ))
+    say "collapse appeared later, at $O2 -- evaluating the pre-cliff checkpoint at $PRE"
+    [ -f "$CK/keep_step_${PRE}.ckpt" ] && eval_at "pre${PRE}" "keep_step_${PRE}.ckpt"
+  else
+    say "RESULT: a FULL pass on the mix with no cliff. The pure-b5 run collapsed at ~1150."
+  fi
+fi
+
+stop_janitor
+say "=== preserved checkpoints: $(ls $CK/keep_step_*.ckpt 2>/dev/null | wc -l) ==="
+say "=== DONE -- see $M; every upload above was byte-verified against HF ==="