Add history-w3 HTML world model train
Browse files
training/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Full SFT for the history-aware world model (direct_full_html_history, window=3).
|
| 3 |
+
# 2-node x 8-GPU (16 GPU total) PAI DLC multi-node launch, mirroring
|
| 4 |
+
# examples/train_full/run_gui_agent_mobileworld_action_only_v2_qwen_official_pai.sh.
|
| 5 |
+
#
|
| 6 |
+
# Usage (on each of the 2 PAI DLC task replicas):
|
| 7 |
+
# bash examples/train_full/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
|
| 8 |
+
#
|
| 9 |
+
# Optional:
|
| 10 |
+
# CONDA_ENV=gui NNODES=2 NPROC_PER_NODE=8 bash examples/train_full/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
|
| 11 |
+
# DRY_RUN=1 bash examples/train_full/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
|
| 12 |
+
|
| 13 |
+
set -euo pipefail
|
| 14 |
+
|
| 15 |
+
ROOT_DIR="${ROOT_DIR:-/data/shared/public/weikai.xu/NIPS2026-guiagent/train_utils/LlamaFactory}"
|
| 16 |
+
CONFIG="${CONFIG:-examples/train_full/qwen3vl_full_sft_world_model_direct_full_html_history_w3.yaml}"
|
| 17 |
+
DATA_DIR="${DATA_DIR:-${ROOT_DIR}/data/appdelta_world_model_train_direct_full_html_history_w3}"
|
| 18 |
+
LOG_DIR="${LOG_DIR:-/data/shared/public/weikai.xu/AAAI2026-AppDeltaWorld/training/logs}"
|
| 19 |
+
LOG_FILE="${LOG_FILE:-${LOG_DIR}/world_model_direct_full_html_history_w3_16gpu_$(date +%Y%m%d_%H%M%S).log}"
|
| 20 |
+
|
| 21 |
+
cd "${ROOT_DIR}"
|
| 22 |
+
|
| 23 |
+
export NNODES="${NNODES:-2}"
|
| 24 |
+
export NPROC_PER_NODE="${NPROC_PER_NODE:-8}"
|
| 25 |
+
export NODE_RANK="${NODE_RANK:-${PAI_CURRENT_TASK_ROLE_CURRENT_REPLICA_INDEX:-${PAI_NODE_RANK:-${RANK:-0}}}}"
|
| 26 |
+
export MASTER_ADDR="${MASTER_ADDR:-${PAI_MASTER_ADDR:-127.0.0.1}}"
|
| 27 |
+
export MASTER_PORT="${MASTER_PORT:-29501}"
|
| 28 |
+
export NCCL_DEBUG="${NCCL_DEBUG:-INFO}"
|
| 29 |
+
export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
|
| 30 |
+
export PYTHONUNBUFFERED=1
|
| 31 |
+
export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
|
| 32 |
+
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
|
| 33 |
+
# gui 环境里 transformers/datasets/accelerate/peft 版本均高于 LlamaFactory
|
| 34 |
+
# pyproject 里写的历史上限(这是为了支持 Qwen3-VL 才装的新版本),实测可用,
|
| 35 |
+
# 跳过 LlamaFactory 自带的非强制版本闸门,避免逐个版本报错反复重跑分布式任务。
|
| 36 |
+
export DISABLE_VERSION_CHECK="${DISABLE_VERSION_CHECK:-1}"
|
| 37 |
+
|
| 38 |
+
if [[ -n "${CONDA_ENV:-}" ]]; then
|
| 39 |
+
# 计算节点上常常没有 `conda` 可执行文件/未 init shell,不能依赖 `conda activate`。
|
| 40 |
+
# 直接把目标 env 的 bin/ 前置到 PATH,等价于激活该环境。
|
| 41 |
+
CONDA_BASE="${CONDA_BASE:-/data/shared/public/weikai.xu/AAAI2026-AppDeltaWorld/miniconda3}"
|
| 42 |
+
CONDA_ENV_BIN="${CONDA_ENV_BIN:-${CONDA_BASE}/envs/${CONDA_ENV}/bin}"
|
| 43 |
+
if [[ -x "${CONDA_ENV_BIN}/python" ]]; then
|
| 44 |
+
export PATH="${CONDA_ENV_BIN}:${PATH}"
|
| 45 |
+
unset PYTHONHOME
|
| 46 |
+
else
|
| 47 |
+
printf '[train][error] conda env not found or has no python: %s\n' "${CONDA_ENV_BIN}" >&2
|
| 48 |
+
printf '[train][hint] 用 CONDA_BASE=<miniconda根目录> 或 CONDA_ENV_BIN=<env/bin目录> 指定正确路径。\n' >&2
|
| 49 |
+
exit 1
|
| 50 |
+
fi
|
| 51 |
+
fi
|
| 52 |
+
|
| 53 |
+
# LlamaFactory 若未作为包安装(当前镜像/环境常见情况),直接从仓库 src/ 导入。
|
| 54 |
+
if ! python -c "import importlib.util as u, sys; sys.exit(0 if u.find_spec('llamafactory') else 1)" 2>/dev/null; then
|
| 55 |
+
export PYTHONPATH="${ROOT_DIR}/src${PYTHONPATH:+:${PYTHONPATH}}"
|
| 56 |
+
fi
|
| 57 |
+
|
| 58 |
+
# 完整依赖预检(对齐 pyproject.toml 中的硬依赖),避免训练启动到一半才因缺包失败。
|
| 59 |
+
missing="$(python - <<'PY'
|
| 60 |
+
import importlib.util
|
| 61 |
+
required = [
|
| 62 |
+
"llamafactory", "torch", "transformers", "accelerate", "datasets",
|
| 63 |
+
"peft", "deepspeed", "omegaconf", "trl",
|
| 64 |
+
]
|
| 65 |
+
print(" ".join(n for n in required if importlib.util.find_spec(n) is None))
|
| 66 |
+
PY
|
| 67 |
+
)"
|
| 68 |
+
if [[ -n "${missing}" ]]; then
|
| 69 |
+
printf '[train][error] 当前解释器缺少依赖: %s\n' "${missing}" >&2
|
| 70 |
+
printf '[train][error] python=%s\n' "$(command -v python)" >&2
|
| 71 |
+
printf '[train][hint] 用 CONDA_ENV=<env> 指定已装齐依赖的环境,或先 pip install 上述包。\n' >&2
|
| 72 |
+
exit 1
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [[ ! -f "${CONFIG}" ]]; then
|
| 76 |
+
printf '[train][error] config not found: %s\n' "${CONFIG}" >&2
|
| 77 |
+
exit 1
|
| 78 |
+
fi
|
| 79 |
+
if [[ ! -d "${DATA_DIR}" ]]; then
|
| 80 |
+
printf '[train][error] dataset not found: %s\n' "${DATA_DIR}" >&2
|
| 81 |
+
exit 1
|
| 82 |
+
fi
|
| 83 |
+
mkdir -p "${LOG_DIR}"
|
| 84 |
+
|
| 85 |
+
printf '[train] ROOT_DIR=%s\n' "${ROOT_DIR}"
|
| 86 |
+
printf '[train] CONFIG=%s\n' "${CONFIG}"
|
| 87 |
+
printf '[train] DATA_DIR=%s\n' "${DATA_DIR}"
|
| 88 |
+
printf '[train] LOG_FILE=%s\n' "${LOG_FILE}"
|
| 89 |
+
printf '[train] NNODES=%s NPROC_PER_NODE=%s NODE_RANK=%s (total GPUs=%s)\n' \
|
| 90 |
+
"${NNODES}" "${NPROC_PER_NODE}" "${NODE_RANK}" "$((NNODES * NPROC_PER_NODE))"
|
| 91 |
+
printf '[train] MASTER_ADDR=%s MASTER_PORT=%s\n' "${MASTER_ADDR}" "${MASTER_PORT}"
|
| 92 |
+
|
| 93 |
+
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
| 94 |
+
printf '[train] DRY_RUN=1, skip launch.\n'
|
| 95 |
+
exit 0
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
python -m llamafactory.cli train "${CONFIG}" 2>&1 | tee "${LOG_FILE}"
|