File size: 1,110 Bytes
29f25be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | #!/usr/bin/env bash
# Real-data performance experiment; copied model updates are discarded.
set -euo pipefail
cd "$(dirname "$0")/../.."
export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 PYTHONUNBUFFERED=1
export TORCHINDUCTOR_CACHE_DIR=/root/autodl-tmp/torchinductor-cache
export TRITON_CACHE_DIR=/root/autodl-tmp/triton-cache
python_bin="${VIMEML_PYTHON:-/root/miniconda3/bin/python}"
output_dir="${1:-$PWD/outputs/model-checks/v21-real-data-performance-$(date +%Y%m%d-%H%M%S)}"
if [[ -e "$output_dir" ]]; then
echo "Use a fresh output directory: $output_dir" >&2
exit 2
fi
mkdir -p "$output_dir" "$TORCHINDUCTOR_CACHE_DIR" "$TRITON_CACHE_DIR"
nvidia-smi --query-gpu=uuid,utilization.gpu,utilization.memory,memory.used,power.draw,clocks.sm,temperature.gpu --format=csv -l 1 > "$output_dir/gpu.csv" &
gpu_monitor_pid=$!
trap 'kill "$gpu_monitor_pid" 2>/dev/null || true' EXIT
"$python_bin" -X utf8 -u scripts/training/benchmark_v21.py \
--config configs/train-v21.toml --output "$output_dir/measurements" \
--batch-sizes 256 512 --warmup 32 --steps 128 --workers 4 \
2>&1 | tee "$output_dir/benchmark.log"
|