File size: 6,276 Bytes
8101696 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | #!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 2 || $# -gt 4 ]]; then
echo "usage: $0 <uniform|shuffle|residual> <run-id> [proposal.json] [config.yaml]" >&2
exit 64
fi
arm=$1
run_id=$2
proposal=${3:-${OPTR_TEMPORAL_PROPOSAL_PATH:-}}
project="${OPTR_PROJECT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
python="${OPTR_PYTHON:-python}"
config=${4:-"${project}/configs/florence_xtrainer2_pack_phone_residual_20ep_to_60ep.yaml"}
accelerate_config="${OPTR_ACCELERATE_CONFIG:-${project}/configs/deepspeed_florence_8gpu.yaml}"
num_processes="${OPTR_NUM_PROCESSES:-8}"
missing=()
for var in OPTR_RUN_ROOT FLORENCE_MODEL_PATH XTRAINER2_DATA_ROOT \
XTRAINER2_PACK_PHONE_PROVENANCE_PATH XTRAINER2_PACK_PHONE_FINGERPRINT_PATH \
XTRAINER2_PACK_PHONE_MANIFEST_PATH XTRAINER2_PACK_PHONE_RESUME_DIR \
XTRAINER2_PACK_PHONE_RESUME_INTEGRITY_PATH; do
[[ -n "${!var:-}" ]] || missing+=("$var")
done
if (( ${#missing[@]} )); then
echo "error: required environment variables are unset: ${missing[*]}" >&2
exit 78
fi
[[ "$num_processes" == 8 ]] || {
echo "error: OPTR continuation requires exactly 8 processes (got $num_processes)" >&2
exit 78
}
[[ "${FLORENCE_MODEL_PATH,,}" == *florence* && -d "$FLORENCE_MODEL_PATH" ]] || {
echo "error: FLORENCE_MODEL_PATH must be an existing Florence directory" >&2
exit 66
}
[[ -d "$XTRAINER2_DATA_ROOT" ]] || {
echo "error: XTRAINER2_DATA_ROOT is not a directory: $XTRAINER2_DATA_ROOT" >&2
exit 66
}
for path in "$config" "$accelerate_config" \
"$XTRAINER2_PACK_PHONE_PROVENANCE_PATH" \
"$XTRAINER2_PACK_PHONE_FINGERPRINT_PATH" \
"$XTRAINER2_PACK_PHONE_MANIFEST_PATH" \
"$XTRAINER2_PACK_PHONE_RESUME_INTEGRITY_PATH"; do
[[ -f "$path" ]] || { echo "error: required file missing: $path" >&2; exit 66; }
done
[[ -d "$XTRAINER2_PACK_PHONE_RESUME_DIR" ]] || {
echo "error: resume checkpoint directory missing: $XTRAINER2_PACK_PHONE_RESUME_DIR" >&2
exit 66
}
[[ "$(basename "$XTRAINER2_PACK_PHONE_RESUME_DIR")" == steps_17460_accelerate ]] || {
echo "error: exact common-start must be steps_17460_accelerate" >&2
exit 78
}
if ! command -v "$python" >/dev/null 2>&1 && [[ ! -x "$python" ]]; then
echo "error: python interpreter not found: $python" >&2
exit 69
fi
case "$arm" in
uniform)
[[ -z "$proposal" ]] || {
echo "error: uniform arm does not accept a proposal" >&2
exit 64
}
proposal_args=(--datasets.vla_data.temporal_proposal_path=null)
;;
shuffle|residual)
[[ -n "$proposal" && -f "$proposal" ]] || {
echo "error: $arm requires a proposal file via CLI or OPTR_TEMPORAL_PROPOSAL_PATH" >&2
exit 66
}
proposal_args=(--datasets.vla_data.temporal_proposal_path="$proposal")
;;
*)
echo "error: unknown arm: $arm" >&2
exit 64
;;
esac
output_dir="${OPTR_RUN_ROOT}/${run_id}"
[[ ! -e "$output_dir" ]] || {
echo "error: refusing to reuse run directory: $output_dir" >&2
exit 73
}
export PYTHONPATH="${project}:${PYTHONPATH:-}"
"$python" - "$XTRAINER2_PACK_PHONE_RESUME_DIR" \
"$XTRAINER2_PACK_PHONE_RESUME_INTEGRITY_PATH" \
"$XTRAINER2_PACK_PHONE_MANIFEST_PATH" "${proposal:-}" "$arm" <<'PY'
import json
import sys
from pathlib import Path
checkpoint, inventory_path, manifest_path, proposal_path = map(Path, sys.argv[1:5])
arm = sys.argv[5]
commit_path = checkpoint / "checkpoint.json"
try:
commit = json.loads(commit_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SystemExit(f"error: invalid checkpoint commit {commit_path}: {exc}")
if commit.get("schema_version") != 1:
raise SystemExit("error: unsupported checkpoint schema")
if int(commit.get("completed_steps", -1)) != 17460:
raise SystemExit("error: checkpoint is not global step 17460")
if int(commit.get("world_size", -1)) != 8:
raise SystemExit("error: checkpoint world size is not 8")
files = commit.get("files")
if not isinstance(files, list) or not files:
raise SystemExit("error: checkpoint commit has no file inventory")
for entry in files:
path = checkpoint / entry["name"]
if not path.is_file() or path.stat().st_size != int(entry["bytes"]):
raise SystemExit(f"error: incomplete checkpoint file: {path}")
try:
inventory = json.loads(inventory_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SystemExit(f"error: invalid checkpoint integrity manifest: {exc}")
expected = [(x["name"], int(x["bytes"])) for x in files]
if inventory.get("schema_version") != 1:
raise SystemExit("error: checkpoint integrity manifest must use schema 1")
if inventory.get("status") == "locked":
observed = [(x["name"], int(x["bytes"])) for x in inventory.get("files", [])]
if observed != expected:
raise SystemExit("error: checkpoint integrity manifest differs from checkpoint.json")
elif inventory.get("status") == "pass":
if (
inventory.get("errors") != []
or int(inventory.get("completed_steps", -1)) != 17460
or int(inventory.get("world_size", -1)) != 8
or int(inventory.get("inventory_files", -1)) != len(files)
):
raise SystemExit("error: common-start preflight report is inconsistent")
else:
raise SystemExit("error: checkpoint integrity manifest did not pass")
try:
from starVLA.dataloader.infoflow_sampler import ValidAnchorManifest
from starVLA.dataloader.temporal_proposal import TemporalProposal
manifest = ValidAnchorManifest.load(manifest_path)
if arm != "uniform":
proposal = TemporalProposal.load(proposal_path, manifest)
if proposal.score_kind != arm:
raise ValueError(
f"{arm} arm received {proposal.score_kind!r} proposal"
)
except Exception as exc:
raise SystemExit(f"error: manifest/proposal validation failed: {exc}")
PY
if [[ "${OPTR_PREFLIGHT_ONLY:-0}" == 1 ]]; then
echo "preflight passed: $arm -> $output_dir"
exit 0
fi
cd "$project"
export WANDB_MODE="${WANDB_MODE:-offline}"
export PYTHONUNBUFFERED=1
exec "$python" -m accelerate.commands.launch \
--config_file "$accelerate_config" \
--num_processes 8 \
starVLA/training/train_starvla.py \
--config_yaml "$config" \
--run_id "$run_id" \
--run_root_dir "$OPTR_RUN_ROOT" \
"${proposal_args[@]}"
|