File size: 1,885 Bytes
8101696 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | #!/usr/bin/env bash
# Launch the 60-epoch pack_phone uniform run.
#
# Global batch 128 (per-device 4 x 8 GPUs x accumulation 4), 874 optimizer
# steps per data epoch, 52380 steps total, checkpoints every 10 epochs at
# [8730, 17460, 26190, 34920, 43650, 52380]. Roughly 80 hours at the measured
# ~5.4 s per optimizer step.
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.."
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
export OPTR_PROJECT_ROOT=$PWD
export OPTR_PYTHON="${OPTR_PYTHON:-python}"
export OPTR_RUN_ROOT=/path/to/runs
export FLORENCE_MODEL_PATH=/path/to/models/Florence-2-large
export XTRAINER2_DATA_ROOT=/path/to/data/lerobot
export XTRAINER2_PACK_PHONE_PROVENANCE_PATH=$XTRAINER2_DATA_ROOT/xtrainer2_optr_support/pack_phone/data_provenance.json
export XTRAINER2_PACK_PHONE_FINGERPRINT_PATH=$XTRAINER2_DATA_ROOT/xtrainer2_optr_support/pack_phone/content_fingerprints.json
export OPTR_ACCELERATE_CONFIG=$OPTR_PROJECT_ROOT/configs/deepspeed_florence_8gpu.yaml
export OPTR_NUM_PROCESSES=8
# Stream metrics to wandb.ai. Requires the proxy for the whole run, not just
# the initial model read, and a key in ~/.netrc (wandb >= 0.30 -- 0.21.x
# rejects the newer wandb_v1_ key format as "must be 40 characters").
# Set WANDB_MODE=offline to fall back to local files plus a later `wandb sync`.
export WANDB_MODE="${WANDB_MODE:-offline}"
# Resume the same wandb run if the job is relaunched, so a restart continues
# one set of curves instead of starting a second run.
export WANDB_RUN_ID="${WANDB_RUN_ID:-pack-phone-uniform-60ep}"
export WANDB_RESUME=allow
run_id=florence-xtrainer2-pack-phone-uniform
log=$OPTR_RUN_ROOT/${run_id}.log
mkdir -p "$OPTR_RUN_ROOT"
echo "launching ${run_id}"
echo " log: ${log}"
exec bash examples/Xtrainer2/train_files/run_xtrainer2_train.sh \
configs/florence_xtrainer2_pack_phone_uniform.yaml \
"$run_id" >> "$log" 2>&1
|