File size: 605 Bytes
2ae6691 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | #!/bin/bash
# Every minute for 1 hour: RssAnon (GB) of our biggest training program on this node, with the step.
# Run on a training node: nohup ./memtrace.sh <jobid> > logs/mem_trace.log 2>&1 &
cd /fsx/ajay-stu/AR
J=$1
# pid= prints no header; tr removes the spaces ps puts in front of the number
P=$(ps -u ajay-stu -o pid= --sort=-rss | head -1 | tr -d ' ')
for i in $(seq 60); do
a=$(awk '/RssAnon/ {printf "%.1f", $2 / 1048576}' /proc/$P/status 2>/dev/null)
s=$(grep "^step" logs/real_$J.log | tail -1 | awk '{print $2}')
echo "$(date -u +%H:%M) step $s RssAnon ${a:-gone} GB"
sleep 60
done
|