AR / autoreload.sh

Upload AR-4B pretraining codebase, configs, and cluster scripts

2ae6691 verified 18 days ago

411 Bytes

#!/bin/bash
# While a memory leak is suspected: ask training to restart (after a checkpoint, same nodes)
# every 6 hours, which frees the memory the training programs have built up.
# Start: nohup ./autoreload.sh >> logs/autoreload.log 2>&1 &     Stop: pkill -f autoreload.sh
cd /fsx/ajay-stu/AR
while true; do
  sleep $((6 * 3600))
  touch logs/RELOAD && echo "$(date -u) RELOAD requested (memory reset)"
done

	#!/bin/bash
	# While a memory leak is suspected: ask training to restart (after a checkpoint, same nodes)
	# every 6 hours, which frees the memory the training programs have built up.
	# Start: nohup ./autoreload.sh >> logs/autoreload.log 2>&1 & Stop: pkill -f autoreload.sh
	cd /fsx/ajay-stu/AR
	while true; do
	sleep $((6 * 3600))
	touch logs/RELOAD && echo "$(date -u) RELOAD requested (memory reset)"
	done