Upload scripts/run_longeval.sh with huggingface_hub

scripts/run_longeval.sh

@@ -111,7 +111,9 @@ fi
 say "longeval: T=$T prompt_len=$PROMPT_LEN max_new=$MAX_NEW n=$N greedy=$GREEDY gpus=$GPUS"
 
 # ---- 1. prompts (shared across checkpoints; identical bytes => fair compare) --
-HAY=$(ls Data_Loader/Data/flux_val_*.bin 2>/dev/null | head -1)
+# HAY is overridable: the length-bucket experiment needs a haystack held out from
+# every training arm, and flux is not present on every box.
+HAY=${HAY:-$(ls Data_Loader/Data/flux_val_*.bin 2>/dev/null | head -1)}
 [ -z "$HAY" ] && HAY=$(ls Data_Loader/Data/flux_train_*.bin 2>/dev/null | head -1)
 say "haystack: ${HAY:-<synthetic fallback>}"
 [ -z "$HAY" ] && say "  WARN: synthetic filler is a weaker distractor than real text -- it INFLATES accuracy. Record this."
@@ -122,7 +124,7 @@ say "haystack: ${HAY:-<synthetic fallback>}"
 # scored and marked .done, permanently). The signature covers everything that
 # changes prompt content -- N, length, depths, seed, AND the haystack, so a
 # synthetic-filler run is not reused once the flux shard arrives.
-SIG="N=$N|LEN=$PROMPT_LEN|DEPTHS=$DEPTHS|SEED=0|HAY=$(basename "${HAY:-none}")|HAYSZ=$(stat -c %s "${HAY:-/nonexistent}" 2>/dev/null || echo 0)"
+SIG="N=$N|LEN=$PROMPT_LEN|DEPTHS=$DEPTHS|SEED=0|HAY=$(basename "${HAY:-none}")|HAYSZ=$(stat -Lc %s "${HAY:-/nonexistent}" 2>/dev/null || echo 0)"
 if [ "$(cat "$OUT/prompts/.signature" 2>/dev/null)" != "$SIG" ]; then
   say "prompt signature changed -- regenerating prompts and clearing state"
   rm -f "$OUT"/prompts/*_prompts.bin "$OUT"/prompts/*_meta.json
@@ -200,6 +202,7 @@ for J in "${jobs[@]:-}"; do
         ${BF16:+CP_BF16=$BF16} \
           ${YARN_ORIG:+YARN_ORIG_MAXPOS=$YARN_ORIG} \
         CP_INIT_FROM="$PWD/$CK/$F" \
+        CP_DATA_ROOT="$PWD/${GEN_DATA_ROOT:-Data_Loader/Data}" \
         CP_GENERATE=1 CP_GEN_TOKENS_BIN="$PWD/$PB" \
         CP_GEN_PROMPT_LEN=$PROMPT_LEN CP_GEN_N=$N CP_GEN_MAX_NEW=$MAX_NEW \
         CP_GEN_GREEDY=$GREEDY CP_GEN_OUT="$PWD/$G" \