scripts/utils/gpu_guard.sh

#!/usr/bin/env bash
set -euo pipefail

GPU_MEM_USED_LIMIT_MB="${GPU_MEM_USED_LIMIT_MB:-2048}"
GPU_UTIL_LIMIT="${GPU_UTIL_LIMIT:-10}"

require_nvidia_smi() {
  if ! command -v nvidia-smi >/dev/null 2>&1; then
    echo "nvidia-smi is required for GPU guard checks." >&2
    return 1
  fi
}

_gpu_query_once() {
  nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader,nounits
}

list_idle_gpus() {
  require_nvidia_smi || return 1
  local sample idx mem util
  sample="$(_gpu_query_once)"
  while IFS=',' read -r idx mem util; do
    idx="${idx// /}"
    mem="${mem// /}"
    util="${util// /}"
    if [[ -n "$idx" && "$mem" -le "$GPU_MEM_USED_LIMIT_MB" && "$util" -le "$GPU_UTIL_LIMIT" ]]; then
      printf '%s\n' "$idx"
    fi
  done <<< "$sample"
}

count_idle_gpus() {
  list_idle_gpus | sed '/^$/d' | wc -l | tr -d ' '
}

join_first_n_idle_gpus() {
  local need="$1"
  list_idle_gpus | sed '/^$/d' | head -n "$need" | paste -sd, -
}

ensure_idle_gpu_count() {
  local need="$1"
  local have
  have="$(count_idle_gpus)"
  if [[ "$have" -lt "$need" ]]; then
    echo "Need $need idle GPU(s), but only found $have under guard limits: mem<=${GPU_MEM_USED_LIMIT_MB}MB util<=${GPU_UTIL_LIMIT}%." >&2
    return 1
  fi
}

print_gpu_guard_summary() {
  require_nvidia_smi || return 1
  echo "GPU guard limits: mem<=${GPU_MEM_USED_LIMIT_MB}MB util<=${GPU_UTIL_LIMIT}%"
  nvidia-smi --query-gpu=index,name,memory.used,memory.free,utilization.gpu --format=csv,noheader
}