File size: 1,916 Bytes
e79d917 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #!/usr/bin/env bash
# Appends a one-line pipeline status every INTERVAL seconds to logs/progress.log,
# so anyone can see history without needing to reconstruct it from raw logs.
#
# tail -f logs/progress.log # follow
# tail -20 logs/progress.log # last few hours
set -uo pipefail
cd "$(dirname "$0")/.."
ROOT=$(pwd)
LOGS=${M3_LOGS:-$ROOT/logs}
INTERVAL=${INTERVAL:-900} # 15 min
OUT="$LOGS/progress.log"
line() { echo "$*" >> "$OUT"; }
line "=== progress logger start $(date '+%Y-%m-%d %H:%M:%S') (every ${INTERVAL}s) ==="
while true; do
ts=$(date '+%m-%d %H:%M')
phase=$(cat "$LOGS/run_all.status" 2>/dev/null || echo "?")
# most recent meaningful line from whichever stage is live
if [ -f "$LOGS/run_test.log" ] && [ "$phase" = "PHASE_test" ]; then
last=$(grep -aE "^\[" "$LOGS/run_test.log" 2>/dev/null | tail -1 | cut -c1-110)
else
last=$(grep -aE "^\[" "$LOGS/train_eval.log" 2>/dev/null | tail -1 | cut -c1-110)
fi
ck=$(python3 - <<'PY' 2>/dev/null || echo "-"
import json, glob
f = sorted(glob.glob("work/ce_model_*/ckpt/state.json"))
if f:
d = json.load(open(f[-1]))
print(f"{d['step']}/{d['total_steps']}")
else:
print("-")
PY
)
gpu=$(nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader 2>/dev/null | tr '\n' ' ')
retries=$(grep -ch "^.*--- attempt" "$LOGS"/supervisor_*.log 2>/dev/null | awk '{s+=$1} END{print s+0}')
hangs=$(grep -c "HANG" "$LOGS"/watchdog_*.log 2>/dev/null | awk -F: '{s+=$2} END{print s+0}')
alive="runall=$(pgrep -cf '[r]un_all.sh') sup=$(pgrep -cf '[s]upervise.sh') wd=$(pgrep -cf '[w]atchdog.sh')"
line "[$ts] $phase | ckpt $ck | attempts=$retries hangs=$hangs | $alive | gpu: $gpu"
line " $last"
if [ "$phase" = "DONE" ] || [ "${phase#FAILED}" != "$phase" ]; then
line "[$ts] terminal state '$phase' - logger exiting"
exit 0
fi
sleep "$INTERVAL"
done
|