#!/usr/bin/env bash
# Launch the 60-epoch pack_phone uniform run.
#
# Global batch 128 (per-device 4 x 8 GPUs x accumulation 4), 874 optimizer
# steps per data epoch, 52380 steps total, checkpoints every 10 epochs at
# [8730, 17460, 26190, 34920, 43650, 52380]. Roughly 80 hours at the measured
# ~5.4 s per optimizer step.
set -euo pipefail

cd "$(dirname "${BASH_SOURCE[0]}")/.."

export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True

export OPTR_PROJECT_ROOT=$PWD
export OPTR_PYTHON="${OPTR_PYTHON:-python}"
export OPTR_RUN_ROOT=/path/to/runs
export FLORENCE_MODEL_PATH=/path/to/models/Florence-2-large
export XTRAINER2_DATA_ROOT=/path/to/data/lerobot
export XTRAINER2_PACK_PHONE_PROVENANCE_PATH=$XTRAINER2_DATA_ROOT/xtrainer2_optr_support/pack_phone/data_provenance.json
export XTRAINER2_PACK_PHONE_FINGERPRINT_PATH=$XTRAINER2_DATA_ROOT/xtrainer2_optr_support/pack_phone/content_fingerprints.json
export OPTR_ACCELERATE_CONFIG=$OPTR_PROJECT_ROOT/configs/deepspeed_florence_8gpu.yaml
export OPTR_NUM_PROCESSES=8

# Stream metrics to wandb.ai. Requires the proxy for the whole run, not just
# the initial model read, and a key in ~/.netrc (wandb >= 0.30 -- 0.21.x
# rejects the newer wandb_v1_ key format as "must be 40 characters").
# Set WANDB_MODE=offline to fall back to local files plus a later `wandb sync`.
export WANDB_MODE="${WANDB_MODE:-offline}"

# Resume the same wandb run if the job is relaunched, so a restart continues
# one set of curves instead of starting a second run.
export WANDB_RUN_ID="${WANDB_RUN_ID:-pack-phone-uniform-60ep}"
export WANDB_RESUME=allow

run_id=florence-xtrainer2-pack-phone-uniform
log=$OPTR_RUN_ROOT/${run_id}.log
mkdir -p "$OPTR_RUN_ROOT"

echo "launching ${run_id}"
echo "  log: ${log}"
exec bash examples/Xtrainer2/train_files/run_xtrainer2_train.sh \
  configs/florence_xtrainer2_pack_phone_uniform.yaml \
  "$run_id" >> "$log" 2>&1