xwk123 commited on
Commit
b8491a3
·
verified ·
1 Parent(s): b81f55f

Add history-w3 HTML world model train

Browse files
training/run_world_model_direct_full_html_history_w3_16gpu_pai.sh ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Full SFT for the history-aware world model (direct_full_html_history, window=3).
3
+ # 2-node x 8-GPU (16 GPU total) PAI DLC multi-node launch, mirroring
4
+ # examples/train_full/run_gui_agent_mobileworld_action_only_v2_qwen_official_pai.sh.
5
+ #
6
+ # Usage (on each of the 2 PAI DLC task replicas):
7
+ # bash examples/train_full/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
8
+ #
9
+ # Optional:
10
+ # CONDA_ENV=gui NNODES=2 NPROC_PER_NODE=8 bash examples/train_full/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
11
+ # DRY_RUN=1 bash examples/train_full/run_world_model_direct_full_html_history_w3_16gpu_pai.sh
12
+
13
+ set -euo pipefail
14
+
15
+ ROOT_DIR="${ROOT_DIR:-/data/shared/public/weikai.xu/NIPS2026-guiagent/train_utils/LlamaFactory}"
16
+ CONFIG="${CONFIG:-examples/train_full/qwen3vl_full_sft_world_model_direct_full_html_history_w3.yaml}"
17
+ DATA_DIR="${DATA_DIR:-${ROOT_DIR}/data/appdelta_world_model_train_direct_full_html_history_w3}"
18
+ LOG_DIR="${LOG_DIR:-/data/shared/public/weikai.xu/AAAI2026-AppDeltaWorld/training/logs}"
19
+ LOG_FILE="${LOG_FILE:-${LOG_DIR}/world_model_direct_full_html_history_w3_16gpu_$(date +%Y%m%d_%H%M%S).log}"
20
+
21
+ cd "${ROOT_DIR}"
22
+
23
+ export NNODES="${NNODES:-2}"
24
+ export NPROC_PER_NODE="${NPROC_PER_NODE:-8}"
25
+ export NODE_RANK="${NODE_RANK:-${PAI_CURRENT_TASK_ROLE_CURRENT_REPLICA_INDEX:-${PAI_NODE_RANK:-${RANK:-0}}}}"
26
+ export MASTER_ADDR="${MASTER_ADDR:-${PAI_MASTER_ADDR:-127.0.0.1}}"
27
+ export MASTER_PORT="${MASTER_PORT:-29501}"
28
+ export NCCL_DEBUG="${NCCL_DEBUG:-INFO}"
29
+ export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
30
+ export PYTHONUNBUFFERED=1
31
+ export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
32
+ export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
33
+ # gui 环境里 transformers/datasets/accelerate/peft 版本均高于 LlamaFactory
34
+ # pyproject 里写的历史上限(这是为了支持 Qwen3-VL 才装的新版本),实测可用,
35
+ # 跳过 LlamaFactory 自带的非强制版本闸门,避免逐个版本报错反复重跑分布式任务。
36
+ export DISABLE_VERSION_CHECK="${DISABLE_VERSION_CHECK:-1}"
37
+
38
+ if [[ -n "${CONDA_ENV:-}" ]]; then
39
+ # 计算节点上常常没有 `conda` 可执行文件/未 init shell,不能依赖 `conda activate`。
40
+ # 直接把目标 env 的 bin/ 前置到 PATH,等价于激活该环境。
41
+ CONDA_BASE="${CONDA_BASE:-/data/shared/public/weikai.xu/AAAI2026-AppDeltaWorld/miniconda3}"
42
+ CONDA_ENV_BIN="${CONDA_ENV_BIN:-${CONDA_BASE}/envs/${CONDA_ENV}/bin}"
43
+ if [[ -x "${CONDA_ENV_BIN}/python" ]]; then
44
+ export PATH="${CONDA_ENV_BIN}:${PATH}"
45
+ unset PYTHONHOME
46
+ else
47
+ printf '[train][error] conda env not found or has no python: %s\n' "${CONDA_ENV_BIN}" >&2
48
+ printf '[train][hint] 用 CONDA_BASE=<miniconda根目录> 或 CONDA_ENV_BIN=<env/bin目录> 指定正确路径。\n' >&2
49
+ exit 1
50
+ fi
51
+ fi
52
+
53
+ # LlamaFactory 若未作为包安装(当前镜像/环境常见情况),直接从仓库 src/ 导入。
54
+ if ! python -c "import importlib.util as u, sys; sys.exit(0 if u.find_spec('llamafactory') else 1)" 2>/dev/null; then
55
+ export PYTHONPATH="${ROOT_DIR}/src${PYTHONPATH:+:${PYTHONPATH}}"
56
+ fi
57
+
58
+ # 完整依赖预检(对齐 pyproject.toml 中的硬依赖),避免训练启动到一半才因缺包失败。
59
+ missing="$(python - <<'PY'
60
+ import importlib.util
61
+ required = [
62
+ "llamafactory", "torch", "transformers", "accelerate", "datasets",
63
+ "peft", "deepspeed", "omegaconf", "trl",
64
+ ]
65
+ print(" ".join(n for n in required if importlib.util.find_spec(n) is None))
66
+ PY
67
+ )"
68
+ if [[ -n "${missing}" ]]; then
69
+ printf '[train][error] 当前解释器缺少依赖: %s\n' "${missing}" >&2
70
+ printf '[train][error] python=%s\n' "$(command -v python)" >&2
71
+ printf '[train][hint] 用 CONDA_ENV=<env> 指定已装齐依赖的环境,或先 pip install 上述包。\n' >&2
72
+ exit 1
73
+ fi
74
+
75
+ if [[ ! -f "${CONFIG}" ]]; then
76
+ printf '[train][error] config not found: %s\n' "${CONFIG}" >&2
77
+ exit 1
78
+ fi
79
+ if [[ ! -d "${DATA_DIR}" ]]; then
80
+ printf '[train][error] dataset not found: %s\n' "${DATA_DIR}" >&2
81
+ exit 1
82
+ fi
83
+ mkdir -p "${LOG_DIR}"
84
+
85
+ printf '[train] ROOT_DIR=%s\n' "${ROOT_DIR}"
86
+ printf '[train] CONFIG=%s\n' "${CONFIG}"
87
+ printf '[train] DATA_DIR=%s\n' "${DATA_DIR}"
88
+ printf '[train] LOG_FILE=%s\n' "${LOG_FILE}"
89
+ printf '[train] NNODES=%s NPROC_PER_NODE=%s NODE_RANK=%s (total GPUs=%s)\n' \
90
+ "${NNODES}" "${NPROC_PER_NODE}" "${NODE_RANK}" "$((NNODES * NPROC_PER_NODE))"
91
+ printf '[train] MASTER_ADDR=%s MASTER_PORT=%s\n' "${MASTER_ADDR}" "${MASTER_PORT}"
92
+
93
+ if [[ "${DRY_RUN:-0}" == "1" ]]; then
94
+ printf '[train] DRY_RUN=1, skip launch.\n'
95
+ exit 0
96
+ fi
97
+
98
+ python -m llamafactory.cli train "${CONFIG}" 2>&1 | tee "${LOG_FILE}"