File size: 651 Bytes
47ea602 | 1 2 3 4 5 6 7 8 9 10 11 12 | #!/usr/bin/env bash
# On the pod: let the second teacher run go until DEADLINE (UTC, e.g. 21:50), stop it, then train DMAD on all GPUs.
DEADLINE=${1:?usage: pod_dmad.sh HH:MM}
until [[ "$(date -u +%H:%M)" > "$DEADLINE" || "$(date -u +%H:%M)" == "$DEADLINE" ]]; do sleep 20; done
pkill -f "rapid_moss.teachergen"
while pgrep -f "rapid_moss.teachergen" > /dev/null; do sleep 5; done
echo "== teacher samples: $(ls /workspace/teacher /workspace/teacher2 | grep -c safetensors) shards $(date -u +%T)"
cd /workspace/code
N=$(nvidia-smi -L | wc -l)
torchrun --nproc-per-node "$N" -m rapid_moss.train --config configs/dmad.json
echo "== done $(date -u +%T)"
|