AR / find_nodes.sh

Upload AR-4B pretraining codebase, configs, and cluster scripts

2ae6691 verified 21 days ago

497 Bytes

#!/bin/bash
# Prints nodes that are Slurm-idle AND have all 8 GPUs free, one hostname per line.
cd /fsx/ajay-stu/AR
/usr/bin/python3 /fsx/ajay-stu/gpu_check.py 2>/dev/null | grep -F "8/8" | grep -F "Users: []" \
  | awk '{print $1}' | sed 's/\./-/g; s/^/ip-/' | sort -u > logs/gpufree.txt
# -N = one line per node (without it sinfo groups names like ip-10-0-192-[64,124])
sinfo -p dev -h -N -o "%N %t" | awk '$2=="idle"{print $1}' | sort -u > logs/idle.txt
comm -12 logs/gpufree.txt logs/idle.txt

	#!/bin/bash
	# Prints nodes that are Slurm-idle AND have all 8 GPUs free, one hostname per line.
	cd /fsx/ajay-stu/AR
	/usr/bin/python3 /fsx/ajay-stu/gpu_check.py 2>/dev/null \| grep -F "8/8" \| grep -F "Users: []" \
	\| awk '{print $1}' \| sed 's/\./-/g; s/^/ip-/' \| sort -u > logs/gpufree.txt
	# -N = one line per node (without it sinfo groups names like ip-10-0-192-[64,124])
	sinfo -p dev -h -N -o "%N %t" \| awk '$2=="idle"{print $1}' \| sort -u > logs/idle.txt
	comm -12 logs/gpufree.txt logs/idle.txt