#!/bin/bash
# While a memory leak is suspected: ask training to restart (after a checkpoint, same nodes)
# every 6 hours, which frees the memory the training programs have built up.
# Start: nohup ./autoreload.sh >> logs/autoreload.log 2>&1 &     Stop: pkill -f autoreload.sh
cd /fsx/ajay-stu/AR
while true; do
  sleep $((6 * 3600))
  touch logs/RELOAD && echo "$(date -u) RELOAD requested (memory reset)"
done