| # Every minute for 1 hour: RssAnon (GB) of our biggest training program on this node, with the step. | |
| # Run on a training node: nohup ./memtrace.sh <jobid> > logs/mem_trace.log 2>&1 & | |
| cd /fsx/ajay-stu/AR | |
| J=$1 | |
| # pid= prints no header; tr removes the spaces ps puts in front of the number | |
| P=$(ps -u ajay-stu -o pid= --sort=-rss | head -1 | tr -d ' ') | |
| for i in $(seq 60); do | |
| a=$(awk '/RssAnon/ {printf "%.1f", $2 / 1048576}' /proc/$P/status 2>/dev/null) | |
| s=$(grep "^step" logs/real_$J.log | tail -1 | awk '{print $2}') | |
| echo "$(date -u +%H:%M) step $s RssAnon ${a:-gone} GB" | |
| sleep 60 | |
| done | |