model-3/scripts/progress.sh

#!/usr/bin/env bash
# Appends a one-line pipeline status every INTERVAL seconds to logs/progress.log,
# so anyone can see history without needing to reconstruct it from raw logs.
#
#   tail -f logs/progress.log        # follow
#   tail -20 logs/progress.log       # last few hours
set -uo pipefail
cd "$(dirname "$0")/.."
ROOT=$(pwd)
LOGS=${M3_LOGS:-$ROOT/logs}
INTERVAL=${INTERVAL:-900}            # 15 min
OUT="$LOGS/progress.log"

line() { echo "$*" >> "$OUT"; }

line "=== progress logger start $(date '+%Y-%m-%d %H:%M:%S') (every ${INTERVAL}s) ==="
while true; do
  ts=$(date '+%m-%d %H:%M')
  phase=$(cat "$LOGS/run_all.status" 2>/dev/null || echo "?")

  # most recent meaningful line from whichever stage is live
  if [ -f "$LOGS/run_test.log" ] && [ "$phase" = "PHASE_test" ]; then
    last=$(grep -aE "^\[" "$LOGS/run_test.log" 2>/dev/null | tail -1 | cut -c1-110)
  else
    last=$(grep -aE "^\[" "$LOGS/train_eval.log" 2>/dev/null | tail -1 | cut -c1-110)
  fi

  ck=$(python3 - <<'PY' 2>/dev/null || echo "-"
import json, glob
f = sorted(glob.glob("work/ce_model_*/ckpt/state.json"))
if f:
    d = json.load(open(f[-1]))
    print(f"{d['step']}/{d['total_steps']}")
else:
    print("-")
PY
)
  gpu=$(nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader 2>/dev/null | tr '\n' ' ')
  retries=$(grep -ch "^.*--- attempt" "$LOGS"/supervisor_*.log 2>/dev/null | awk '{s+=$1} END{print s+0}')
  hangs=$(grep -c "HANG" "$LOGS"/watchdog_*.log 2>/dev/null | awk -F: '{s+=$2} END{print s+0}')
  alive="runall=$(pgrep -cf '[r]un_all.sh') sup=$(pgrep -cf '[s]upervise.sh') wd=$(pgrep -cf '[w]atchdog.sh')"

  line "[$ts] $phase | ckpt $ck | attempts=$retries hangs=$hangs | $alive | gpu: $gpu"
  line "          $last"

  if [ "$phase" = "DONE" ] || [ "${phase#FAILED}" != "$phase" ]; then
    line "[$ts] terminal state '$phase' - logger exiting"
    exit 0
  fi
  sleep "$INTERVAL"
done