maliang / scripts /train_sft.sh
teawhite's picture
Add files using upload-large-folder tool
51ac7cc verified
Raw History Blame Contribute Delete
7.26 kB
#!/usr/bin/env bash
# gc25d 双向 SFT 启动器(SPEC §7):accelerate + DeepSpeed ZeRO-2,按 NGPU / GA 选(没有就生成)configs/accelerate_zero2*.yaml,
# 输出 tee -a 到 $OUT/train.log。同一个 OUT 已有训练在跑 ⇒ 拒绝启动(flock 锁 + 进程检查)。
#
# 用法(参数全走环境变量;其余 train.py 参数放 EXTRA,或直接跟在命令后面):
# OUT=outputs/sft_v10 bash scripts/train_sft.sh # 8 卡 × micro 1 × GA 2 = 16,20000 步
# OUT=outputs/sft_v10 RESUME=outputs/sft_v10/state bash scripts/train_sft.sh # 崩溃续训(accelerator.load_state)
# GC_RES=1280x704 DATA=data/gc25d_v10_704 OUT=outputs/sft_v10_704 bash scripts/train_sft.sh # 704p,从头训
# OUT=outputs/_smoke/sft NGPU=4 GPUS=2,3,4,5 GA=2 MAX_STEPS=10 DATA=data/gc25d_v10_mini \
# EXTRA="--save_every 5 --val_every 5 --val_n 4 --warmup_steps 2" bash scripts/train_sft.sh
#
# OUT 必填;相对路径相对 maliang 根目录;必须落在 $M/outputs/ 下(根盘只有 100G)
# NGPU 进程数(默认 8;给了 GPUS 时必须与 GPUS 的个数一致)
# GPUS CUDA_VISIBLE_DEVICES(默认 0..NGPU-1)
# GA 梯度累积(默认 2;全局 batch = NGPU × GA)
# MAX_STEPS 默认 20000
# RESUME state 目录(续训)或 *.safetensors(热启动:gc25d 导出 / 普通 Wan 导出 ⇒ gc_sem_cond 零初始化),可空
# VOCAB parts(默认)/ ids
# DATA 数据根(默认 $M/data/gc25d_v10);它的 index/prep_args.json "res"(缺失 = 832x480)必须 = GC_RES
# GC_RES 训练分辨率 <W>x<H>(默认 832x480;1280x704 = v10 原生分辨率),透传给 train.py(code/gc25d/geometry.py)
# PORT main_process_port(默认自动挑一个空闲端口,免得与同机别的 accelerate 任务撞)
# EXTRA 其余 train.py 参数
# ALLOW_EXISTING=1 OUT 里已有 step-*.safetensors / state 却没给 RESUME 时也照样从 step 0 开(默认拒绝,防止覆盖)
# 不要在训练运行中编辑本文件(bash 边读边执行)。
set -euo pipefail
M=/opt/dlami/nvme/zhiyangdeng/maliang
PY="$M/.venv/bin/python"
abspath() { case "$1" in /*) echo "$1" ;; *) echo "$M/$1" ;; esac; }
: "${OUT:?OUT 必填,例如 OUT=outputs/sft_v10}"
OUT=$(abspath "$OUT")
OUT=$(realpath -m "$OUT")
case "$OUT/" in "$M/outputs/"?*) ;; *) echo "[train_sft] OUT=$OUT 不在 $M/outputs/ 下,拒绝" >&2; exit 2 ;; esac
GA="${GA:-2}"
MAX_STEPS="${MAX_STEPS:-20000}"
VOCAB="${VOCAB:-parts}"
export GC_RES="${GC_RES:-832x480}"
DATA=$(abspath "${DATA:-$M/data/gc25d_v10}")
RESUME="${RESUME:-}"
[ -n "$RESUME" ] && RESUME=$(abspath "$RESUME")
EXTRA="${EXTRA:-}"
if [ -n "${GPUS:-}" ]; then
N_LIST=$(echo "$GPUS" | tr ',' '\n' | grep -c .)
NGPU="${NGPU:-$N_LIST}"
[ "$NGPU" = "$N_LIST" ] || { echo "[train_sft] NGPU=$NGPU 与 GPUS=$GPUS 的个数 $N_LIST 不一致" >&2; exit 2; }
else
NGPU="${NGPU:-8}"
GPUS=$(seq -s, 0 $((NGPU - 1)))
fi
PORT="${PORT:-$("$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')}"
[ -f "$DATA/index/clips.jsonl" ] || { echo "[train_sft] $DATA/index/clips.jsonl 不存在(prep 没跑完?)" >&2; exit 2; }
[ -f "$DATA/READY" ] || echo "[train_sft] 注意:$DATA 没有 READY 标记(prep_v10 / verify_prep 可能没跑完)" >&2
# 分辨率:GC_RES 合法(geometry.parse_res)后规范成 RES_TAG(" 1280X704" ⇒ 1280x704),且 = 数据根的 res
# (index/prep_args.json,缺失 = 832x480),否则不启动
RES_PAIR=$(PYTHONPATH="$M/code/gc25d" "$PY" -c "import sys, geometry; print(geometry.RES_TAG, geometry.root_res(sys.argv[1]))" "$DATA") \
|| { echo "[train_sft] GC_RES=$GC_RES 不合法(见 code/gc25d/geometry.py::parse_res)" >&2; exit 2; }
read -r GC_RES DATA_RES <<< "$RES_PAIR"
export GC_RES
[ "$DATA_RES" = "$GC_RES" ] || { echo "[train_sft] DATA=$DATA 是 $DATA_RES 的数据,GC_RES=$GC_RES:设 GC_RES=$DATA_RES 或换 DATA" >&2; exit 2; }
# ---- accelerate yaml:8 卡用现成的两份,其它组合从 ga2 模板生成(只改 num_processes / gradient_accumulation_steps)----
if [ "$NGPU" = 8 ] && [ "$GA" = 2 ]; then
CFG="$M/configs/accelerate_zero2_ga2.yaml"
elif [ "$NGPU" = 8 ] && [ "$GA" = 1 ]; then
CFG="$M/configs/accelerate_zero2.yaml"
else
CFG="$M/configs/accelerate_zero2_${NGPU}gpu_ga${GA}.yaml"
if [ ! -f "$CFG" ]; then
{
echo "# ${NGPU} 卡 × GA ${GA}(全局 batch $((NGPU * GA))):由 accelerate_zero2_ga2.yaml 只改 num_processes / gradient_accumulation_steps"
echo "# 生成(scripts/train_sft.sh)。train.py 的 --grad_accum 必须等于这里的 ${GA},否则 SystemExit。"
sed -e "s/^ gradient_accumulation_steps: 2$/ gradient_accumulation_steps: ${GA}/" \
-e "s/^num_processes: 8$/num_processes: ${NGPU}/" "$M/configs/accelerate_zero2_ga2.yaml"
} > "$CFG.tmp.$$"
grep -q "^ gradient_accumulation_steps: ${GA}$" "$CFG.tmp.$$" && grep -q "^num_processes: ${NGPU}$" "$CFG.tmp.$$" \
|| { echo "[train_sft] 生成 $CFG 失败(模板格式变了?)" >&2; rm -f "$CFG.tmp.$$"; exit 2; }
mv "$CFG.tmp.$$" "$CFG"
echo "[train_sft] 生成 $CFG"
fi
fi
# ---- 同一个 OUT 只能有一个训练 ----
mkdir -p "$OUT"
exec 9>"$OUT/.train_sft.lock"
flock -n 9 || { echo "[train_sft] $OUT 已有训练在跑(锁 $OUT/.train_sft.lock 被占用:$(cat "$OUT/.train_sft.pid" 2>/dev/null)),拒绝启动" >&2; exit 3; }
if pgrep -af -- "train.py .*--output $OUT( |$)" >/dev/null; then
echo "[train_sft] 发现写 $OUT 的 train.py 进程,拒绝启动:" >&2
pgrep -af -- "train.py .*--output $OUT( |$)" >&2
exit 3
fi
if [ -z "$RESUME" ] && [ "${ALLOW_EXISTING:-0}" != 1 ] && { compgen -G "$OUT/step-*.safetensors" >/dev/null || [ -e "$OUT/state" ]; }; then
echo "[train_sft] $OUT 里已有 checkpoint / state,但没给 RESUME:续训请 RESUME=$OUT/state,确要从 0 开请 ALLOW_EXISTING=1" >&2
exit 3
fi
echo "$$ $(date '+%F %T') GPUS=$GPUS" > "$OUT/.train_sft.pid"
# ---- 环境:缓存 / 临时文件一律落 nvme ----
export DIFFSYNTH_SKIP_DOWNLOAD=True
export PYTHONPATH="$M/code/gc25d:$M/code/arope"
export HF_HOME=/opt/dlami/nvme/zhiyangdeng/.cache/huggingface
export TMPDIR="$M/.tmp" TORCHINDUCTOR_CACHE_DIR="$M/.tmp/inductor" TRITON_CACHE_DIR="$M/.tmp/triton"
export TORCH_EXTENSIONS_DIR="$M/.tmp/torch_extensions" TORCH_HOME="$M/.tmp/torch_home" XDG_CACHE_HOME="$M/.tmp/xdg_cache"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}" PYTHONUNBUFFERED=1
export CUDA_VISIBLE_DEVICES="$GPUS"
mkdir -p "$TMPDIR"
CMD=("$M/.venv/bin/accelerate" launch --config_file "$CFG" --main_process_port "$PORT"
"$M/code/gc25d/train.py"
--data_root "$DATA" --vocab "$VOCAB" --grad_accum "$GA" --max_steps "$MAX_STEPS" --output "$OUT")
[ -n "$RESUME" ] && CMD+=(--resume "$RESUME")
# shellcheck disable=SC2206
[ -n "$EXTRA" ] && CMD+=($EXTRA)
CMD+=("$@")
{
echo "[train_sft] $(date '+%F %T') host=$(hostname) GPUS=$GPUS NGPU=$NGPU GA=$GA(全局 batch $((NGPU * GA)))PORT=$PORT GC_RES=$GC_RES"
echo "[train_sft] cfg=$CFG"
echo "[train_sft] ${CMD[*]}"
} | tee -a "$OUT/train.log"
"${CMD[@]}" 2>&1 | tee -a "$OUT/train.log"