Full state: code, corpora, 8 adapters, gate results, dataset refs, session log

e7d8c8b verified 2 months ago

431 Bytes

#!/bin/bash
# Kills any downloader if free space drops below 25G. Training is never touched.
while true; do
  FREE=$(df --output=avail -BG / | tail -1 | tr -dc '0-9')
  if [ "${FREE:-999}" -lt 25 ]; then
    echo "$(date -Is) DISK GUARD: ${FREE}G free - killing downloaders" >> /workspace/aditi/logs/disk_guard.log
    pkill -f "[s]weep2"; pkill -f "[r]etry_gated"; pkill -f "[d]l.py"; pkill -f "[a]cquire.py"
  fi
  sleep 30
done

	#!/bin/bash
	# Kills any downloader if free space drops below 25G. Training is never touched.
	while true; do
	FREE=$(df --output=avail -BG / \| tail -1 \| tr -dc '0-9')
	if [ "${FREE:-999}" -lt 25 ]; then
	echo "$(date -Is) DISK GUARD: ${FREE}G free - killing downloaders" >> /workspace/aditi/logs/disk_guard.log
	pkill -f "[s]weep2"; pkill -f "[r]etry_gated"; pkill -f "[d]l.py"; pkill -f "[a]cquire.py"
	fi
	sleep 30
	done