AR / preflight.sh

Upload AR-4B pretraining codebase, configs, and cluster scripts

2ae6691 verified 18 days ago

301 Bytes

#!/bin/bash
# Runs inside a job: fail if any node's GPUs are already in use by someone else.
srun --ntasks-per-node=1 bash -c '
  u=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk "{s+=\$1} END {print s}")
  echo "preflight $(hostname) gpu_used=${u}MiB"
  [ "$u" -lt 2000 ]
'

	#!/bin/bash
	# Runs inside a job: fail if any node's GPUs are already in use by someone else.
	srun --ntasks-per-node=1 bash -c '
	u=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \| awk "{s+=\$1} END {print s}")
	echo "preflight $(hostname) gpu_used=${u}MiB"
	[ "$u" -lt 2000 ]
	'