#!/bin/bash
# Kills any downloader if free space drops below 25G. Training is never touched.
while true; do
  FREE=$(df --output=avail -BG / | tail -1 | tr -dc '0-9')
  if [ "${FREE:-999}" -lt 25 ]; then
    echo "$(date -Is) DISK GUARD: ${FREE}G free - killing downloaders" >> /workspace/aditi/logs/disk_guard.log
    pkill -f "[s]weep2"; pkill -f "[r]etry_gated"; pkill -f "[d]l.py"; pkill -f "[a]cquire.py"
  fi
  sleep 30
done