#!/bin/bash
# Every minute for 1 hour: RssAnon (GB) of our biggest training program on this node, with the step.
# Run on a training node: nohup ./memtrace.sh <jobid> > logs/mem_trace.log 2>&1 &
cd /fsx/ajay-stu/AR
J=$1
# pid= prints no header; tr removes the spaces ps puts in front of the number
P=$(ps -u ajay-stu -o pid= --sort=-rss | head -1 | tr -d ' ')
for i in $(seq 60); do
  a=$(awk '/RssAnon/ {printf "%.1f", $2 / 1048576}' /proc/$P/status 2>/dev/null)
  s=$(grep "^step" logs/real_$J.log | tail -1 | awk '{print $2}')
  echo "$(date -u +%H:%M)  step $s  RssAnon ${a:-gone} GB"
  sleep 60
done