CharlesZhang-USTC's picture
Add files using upload-large-folder tool
ea1b5d0 verified
Raw History Blame Contribute Delete
16.2 kB
#!/usr/bin/env bash
# =============================================================================
# SNF 五模型实验环境 一键部署脚本(原生运行,不依赖 Docker)
#
# 目标机器:Linux x86_64 + NVIDIA GPU(驱动 >= 535,CUDA 12.8/12.9 运行时)+
# 宿主 Python 3.12 且已装 torch 2.7.1(+cu128/cu129)
#
# 本脚本只做“部署”,不跑任何实验:
# 1. 下载并解压官方构建源码包,校验 397 个上游文件
# 2. 建立 /weights /data /outputs 存储布局
# 3. 建立五个模型各自隔离的 /opt/envs/<model> 虚拟环境(继承宿主 torch)
# 4. 应用两处必要的 harness 兼容补丁(不改动上游源码)
# 5. 下载并锁定全部模型权重(HF + MoMask 官方 Google Drive)
# 6. 验收:源码校验 + 五模型导入 + doctor(GPU 实测)
#
# 用法:
# export HF_TOKEN='hf_xxx' # 必需(SD3.5 为 gated 仓库)
# bash deploy.sh # 全量部署
# SNF_SKIP_WEIGHTS=1 bash deploy.sh # 只装环境不下载权重
# SNF_WITH_TASKS=1 bash deploy.sh # 额外生成默认任务清单
# SNF_STORAGE=/mnt/snf-runtime bash deploy.sh # 换存储根目录
#
# 前置条件(脚本会检查并在不满足时退出):
# - HF_TOKEN 对应的账号已在 https://huggingface.co/stabilityai/stable-diffusion-3.5-medium
# 页面点击 "Agree and access repository",否则 SD3.5 权重会 403
# - 磁盘可用空间 >= 150 GB(实测权重 + 缓存约 100 GB)
# - 能访问 huggingface.co / github.com / drive.google.com(脚本继承宿主 http_proxy 配置)
#
# 幂等:每一步都有完成标记,中断后重跑会跳过已完成部分。
# =============================================================================
set -euo pipefail
SNF_DEPLOY_ROOT="${SNF_DEPLOY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
SNF_CODE_ROOT="${SNF_CODE_ROOT:-$SNF_DEPLOY_ROOT/snf-five-models}"
SNF_STORAGE="${SNF_STORAGE:-$SNF_DEPLOY_ROOT/runtime}"
SNF_ZIP_URL="${SNF_ZIP_URL:-https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/snf-five-models-cu128-build.zip}"
SNF_ZIP="$SNF_DEPLOY_ROOT/snf-five-models-cu128-build.zip"
HF_ENDPOINT="${HF_ENDPOINT:-https://huggingface.co}"
PYTHON="${PYTHON:-python3}"
MODELS=(meissonic self_forcing momask sd35 showo)
SNF_SKIP_WEIGHTS="${SNF_SKIP_WEIGHTS:-0}"
SNF_WITH_TASKS="${SNF_WITH_TASKS:-0}"
SNF_DOCTOR="${SNF_DOCTOR:-1}"
LOG_DIR="$SNF_DEPLOY_ROOT/logs"; mkdir -p "$LOG_DIR"
LOG="$LOG_DIR/deploy.$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1
step(){ printf '\n\033[1;36m[%s] === %s ===\033[0m\n' "$(date +%H:%M:%S)" "$*"; }
ok(){ printf ' \033[32m✓\033[0m %s\n' "$*"; }
warn(){ printf ' \033[33m!\033[0m %s\n' "$*"; }
die(){ printf ' \033[31m✗ %s\033[0m\n' "$*"; exit 1; }
step "0/7 前置检查"
[ "$(id -u)" = 0 ] || warn "非 root:/opt/envs 与 /weights 可能无法写入"
command -v "$PYTHON" >/dev/null || die "找不到 python3(可用 PYTHON= 指定)"
command -v nvidia-smi >/dev/null || die "找不到 nvidia-smi:本机无 GPU 驱动"
GPU_COUNT="$(nvidia-smi -L 2>/dev/null | wc -l)"
[ "$GPU_COUNT" -gt 0 ] || die "nvidia-smi 未识别到 GPU"
ok "GPU: ${GPU_COUNT} 张 -> $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | tr -d '\r')"
PYV="$($PYTHON -c 'import sys;print("%d.%d"%sys.version_info[:2])')"
ok "Python $PYV ($(command -v $PYTHON))"
TORCHV="$($PYTHON -c 'import torch;print(torch.__version__,torch.version.cuda)' 2>/dev/null || echo none)"
[ "$TORCHV" = none ] && die "宿主 Python 没有 torch;镜像要求 torch==2.7.1(+cu128/cu129),五个虚拟环境都继承它"
ok "宿主 torch $TORCHV"
for tool in curl unzip git; do command -v $tool >/dev/null || die "缺少系统工具 $tool"; done
ok "curl / unzip / git 就绪"
FREE_GB="$(df -Pk "$SNF_DEPLOY_ROOT" | awk 'NR==2{print int($4/1048576)}')"
if [ "$FREE_GB" -lt 150 ]; then warn "可用空间仅 ${FREE_GB}GB,建议 >=150GB(权重+缓存约100GB)"; else ok "可用空间 ${FREE_GB}GB"; fi
if [ -z "${HF_TOKEN:-}" ]; then die "未设置 HF_TOKEN:SD3.5 是 gated 仓库,且匿名访问会被 429 限流"; fi
WHOAMI="$(curl -s --max-time 30 -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/api/whoami-v2" | head -c 200)"
case "$WHOAMI" in *'"name"'*) ok "HF_TOKEN 有效 -> $(printf '%s' "$WHOAMI" | sed -n 's/.*"name":"\([^"]*\)".*/\1/p')";; *) die "HF_TOKEN 无效或网络不通: $WHOAMI";; esac
SD35_CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -L -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/stabilityai/stable-diffusion-3.5-medium/resolve/main/model_index.json")"
if [ "$SD35_CODE" != "200" ]; then
die "SD3.5 权重不可访问(HTTP $SD35_CODE)。请先在浏览器打开
https://huggingface.co/stabilityai/stable-diffusion-3.5-medium 点 Agree and access repository"
fi
ok "SD3.5 权重可访问(授权已生效)"
step "1/7 下载并解压构建源码包"
mkdir -p "$SNF_DEPLOY_ROOT"
if [ ! -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ]; then
[ -f "$SNF_ZIP" ] || curl -fL --retry 3 -o "$SNF_ZIP" "$SNF_ZIP_URL"
unzip -q -o "$SNF_ZIP" -d "$SNF_DEPLOY_ROOT"
else
ok "源码已存在,跳过下载"
fi
[ -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ] || die "解压失败:缺少 sources.manifest.json"
ok "源码包就绪 $SNF_CODE_ROOT"
step "2/7 建立存储布局"
mkdir -p "$SNF_STORAGE"/{weights,data,outputs}
for pair in "weights:/weights" "data:/data" "outputs:/outputs"; do
name="${pair%%:*}"; link="${pair##*:}"; target="$SNF_STORAGE/$name"
if [ -L "$link" ] && [ "$(readlink -f "$link")" = "$(readlink -f "$target")" ]; then continue; fi
if [ -d "$link" ] && [ ! -L "$link" ] && [ -n "$(ls -A "$link" 2>/dev/null)" ]; then
warn "$link 已存在且非空(宿主目录),保留原目录不建软链"
continue
fi
rm -rf "$link"; ln -sfn "$target" "$link"
done
ok "存储布局就绪 -> $SNF_STORAGE/{weights,data,outputs}"
export PYTHONPATH="$SNF_CODE_ROOT" HF_HOME=/weights/huggingface \
TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache \
TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton \
MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 \
WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT
mkdir -p /weights/huggingface
cd "$SNF_CODE_ROOT"
snf(){ ( cd "$SNF_CODE_ROOT" && $PYTHON -m suite.cli "$@" ); }
VERIFY="$(snf source-check 2>/dev/null | tr -d '\n' || true)"
case "$VERIFY" in *'"verified_files": 397'*) ok "源码完整性校验通过(397 个上游文件)";; *) die "源码校验失败: ${VERIFY:0:200}";; esac
step "3/7 创建五个隔离模型环境 /opt/envs/<model>"
REQ="$SNF_CODE_ROOT/packaging/requirements-common.txt"
REQ_SUM="$(sha256sum "$REQ" | cut -c1-16)"
mkdir -p /opt/envs
for model in "${MODELS[@]}"; do
ENV="/opt/envs/$model"; STAMP="$ENV/.req.$REQ_SUM"
if [ -x "$ENV/bin/python" ] && [ -f "$ENV/installed.freeze.txt" ] && [ -f "$STAMP" ]; then
ok "$model 环境已就绪,跳过"; continue
fi
echo " · 创建 $model 环境(继承宿主 torch,只隔离 Python API 层)"
rm -rf "$ENV"
"$PYTHON" -m venv --system-site-packages "$ENV"
"$ENV/bin/pip" install --no-cache-dir -r "$REQ" >"$LOG_DIR/env_$model.log" 2>&1 \
|| { echo " pip 失败,详见 $LOG_DIR/env_$model.log"; continue; }
touch "$STAMP"
ok "$model 依赖安装完成"
done
echo " · Show-o 专属旧版栈(transformers 4.41.2 / diffusers 0.30.1)"
/opt/envs/showo/bin/pip install --no-cache-dir \
transformers==4.41.2 tokenizers==0.19.1 peft==0.11.1 diffusers==0.30.1 \
>"$LOG_DIR/env_showo_override.log" 2>&1 || echo " Show-o 覆盖安装失败,详见日志"
echo " · MoMask 官方 CLIP(固定 commit)"
/opt/envs/momask/bin/pip install --no-cache-dir \
'git+https://github.com/openai/CLIP.git@dcba3cb2e2827b402d2701e7e1c7d9fed8a20ef1' \
>"$LOG_DIR/env_momask_clip.log" 2>&1 || echo " CLIP 安装失败,详见日志"
MISSING_ENV=0
for model in "${MODELS[@]}"; do
[ -x "/opt/envs/$model/bin/python" ] || { warn "/opt/envs/$model 缺失"; MISSING_ENV=1; continue; }
"/opt/envs/$model/bin/pip" freeze >"/opt/envs/$model/installed.freeze.txt"
done
[ "$MISSING_ENV" = 0 ] || die "有模型环境未建成,请查看 $LOG_DIR/env_*.log"
step "4/7 应用 harness 兼容补丁(不改动任何上游源码)"
$PYTHON - "$SNF_CODE_ROOT" <<'PATCH'
import sys
from pathlib import Path
root = Path(sys.argv[1])
# 补丁一:MoMask 上游代码使用 NumPy 1.24 已移除的 np.float / np.int 等旧别名。
# 只在 harness 进程内恢复别名,vendor/ 下 397 个上游文件的校验值保持不变。
momask = root / 'suite/momask_loading.py'
anchor = "def loading_functions():"
shim = '''def numpy_legacy_aliases():
"""Restore aliases NumPy removed in 1.24; upstream MoMask still uses np.float/np.int.
Applied in the harness process only. Vendored upstream files are untouched, so
packaging/sources.manifest.json keeps verifying all 397 upstream files.
"""
import numpy as np
for name, replacement in (('float', float), ('int', int), ('bool', bool),
('object', object), ('str', str), ('complex', complex)):
if not hasattr(np, name):
setattr(np, name, replacement)
return np
numpy_legacy_aliases()
'''
if 'numpy_legacy_aliases' in momask.read_text():
print(' ✓ momask numpy 兼容补丁已存在')
else:
text = momask.read_text()
assert anchor in text, 'momask_loading.py 结构变化,请人工确认'
momask.write_text(text.replace(anchor, shim + anchor, 1))
print(' ✓ 已写入 momask numpy 兼容补丁')
# 补丁二:SD3.5 上游只提供 fp16 的 CLIP 文本编码器,diffusers 只把 text_projection
# 转成 bf16,前向时 dtype 冲突(Half != BFloat16)。统一到管线声明的 bf16。
adapters = root / 'suite/adapters.py'
old = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
self.pipe.set_progress_bar_config(disable=True)"""
new = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
# Upstream ships the two CLIP encoders as fp16 only; diffusers casts their
# text_projection to bf16 but leaves the encoder body fp16, which raises a
# dtype mismatch in the forward. Align them with the declared pipeline dtype.
for name in ('text_encoder','text_encoder_2','text_encoder_3'):
module=getattr(self.pipe,name,None)
if module is not None:module.to(torch.bfloat16)
self.pipe.set_progress_bar_config(disable=True)"""
text = adapters.read_text()
if 'Upstream ships the two CLIP encoders' in text:
print(' ✓ sd35 dtype 补丁已存在')
else:
assert old in text, 'adapters.py 结构变化,请人工确认'
adapters.write_text(text.replace(old, new, 1))
print(' ✓ 已写入 sd35 dtype 补丁')
PATCH
cat > "$SNF_DEPLOY_ROOT/snf.sh" <<EOS
#!/usr/bin/env bash
# 实验入口(原生替代 packaging/start.sh):bash snf.sh <command> ...
set -euo pipefail
export PYTHONPATH="$SNF_CODE_ROOT"
export HF_HOME=/weights/huggingface TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache
export TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton
export MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1
export WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT="\${HF_ENDPOINT:-https://huggingface.co}"
# HF_TOKEN 由调用方内联传入,不落盘
cd "$SNF_CODE_ROOT"
exec $PYTHON -m suite.cli "\$@"
EOS
chmod +x "$SNF_DEPLOY_ROOT/snf.sh"
ok "实验入口已生成 $SNF_DEPLOY_ROOT/snf.sh"
step "5/7 下载并锁定模型权重"
if [ "$SNF_SKIP_WEIGHTS" = 1 ]; then
warn "SNF_SKIP_WEIGHTS=1,跳过权重下载"
else
LOCK=/weights/weights.lock.json
[ -f "$LOCK" ] || echo '{}' > "$LOCK"
# 串行执行:fetch 会对 weights.lock.json 做读-改-写,并发会丢条目
for model in "${MODELS[@]}"; do
echo " · fetch $model"
if ! snf fetch --models "$model" --weights /weights >"$LOG_DIR/fetch_$model.log" 2>&1; then
echo " fetch $model 失败,详见 $LOG_DIR/fetch_$model.log"; tail -3 "$LOG_DIR/fetch_$model.log" || true
fi
done
python3 - "$LOCK" <<'CHECK'
import json, sys, os
lock = json.load(open(sys.argv[1]))
expect = {
'MeissonFlow/Meissonic': None, 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K': None,
'Wan-AI/Wan2.1-T2V-1.3B': None, 'gdhe17/Self-Forcing': None,
'showlab/show-o': None, 'showlab/magvitv2': None, 'microsoft/phi-1_5': None,
'stabilityai/stable-diffusion-3.5-medium': None, 'momask_archive_sha256': None,
}
missing = [k for k in expect if k not in lock]
print(' 已锁定 %d/%d 个权重源' % (len(lock), len(expect)))
if missing:
print(' 缺失:', missing)
sys.exit(1)
checks = {
'MoMask 主模型': '/weights/momask/checkpoints/t2m/t2m_nlayer8_nhead6_ld384_ff1024_cdp0.1_rvq6ns/opt.txt',
'MoMask 残差模型': '/weights/momask/checkpoints/t2m/tres_nlayer8_ld384_ff1024_rvq6ns_cdp0.2_sw/opt.txt',
'Self-Forcing 学生权重': '/weights/self_forcing/checkpoints/self_forcing_dmd.pt',
'Wan2.1 基础模型': '/weights/self_forcing/wan_models/Wan2.1-T2V-1.3B/config.json',
}
for name, path in checks.items():
print(' %s %s' % ('✓' if os.path.exists(path) else '✗', name))
if not os.path.exists(path):
sys.exit(1)
CHECK
[ $? -eq 0 ] || die "权重不完整,请查看 $LOG_DIR/fetch_*.log"
ok "全部权重就绪(HF revision 已锁定在 /weights/weights.lock.json)"
rm -rf /weights/huggingface/xet 2>/dev/null && ok "已清理 xet 分块缓存(约 9GB,不影响已下载权重)"
fi
step "6/7 验收:源码 / 五模型导入 / GPU"
snf source-check >/dev/null && ok "源码校验 397 文件通过"
for model in "${MODELS[@]}"; do
OUT="$(/opt/envs/$model/bin/python -m suite.cli import-check --model "$model" 2>&1 | tail -1)"
case "$OUT" in *'"imports": "passed"'*) ok "$model 导入通过";; *) warn "$model 导入失败: ${OUT:0:200}";; esac
done
if [ "$SNF_DOCTOR" = 1 ]; then
if snf doctor >"$LOG_DIR/doctor.json" 2>&1; then
$PYTHON - "$LOG_DIR/doctor.json" <<'DOC'
import json, sys
raw = open(sys.argv[1]).read()
d = json.loads(raw[raw.index('{'):])
print(' ✓ CUDA available=%s kernel_finite=%s' % (d['cuda_available'], d['kernel_finite']))
print(' ✓ GPU: %s' % ', '.join('%s %.0fGiB' % (g['name'], g['memory_gib']) for g in d['gpus']))
bad = [k for k, v in d['models'].items() if v['returncode']]
print(' ✓ 五模型环境导入: %s' % ('全部通过' if not bad else '失败 ' + str(bad)))
DOC
else
warn "doctor 未通过,详见 $LOG_DIR/doctor.json"
fi
fi
if [ "$SNF_WITH_TASKS" = 1 ]; then
step "7/7 生成默认任务清单(可选,属于实验输入)"
snf create-tasks --output /data/tasks.curated.jsonl && ok "已生成 /data/tasks.curated.jsonl(120 test + 8 dev)"
else
step "7/7 完成(未生成任务清单,SNF_WITH_TASKS=1 可开启)"
fi
cat <<SUMMARY
==================== 部署完成 ====================
源码 : $SNF_CODE_ROOT
存储 : $SNF_STORAGE/{weights,data,outputs} (/weights /data /outputs 为软链)
模型环境 : /opt/envs/{meissonic,self_forcing,momask,sd35,showo}(各含 installed.freeze.txt)
权重锁 : /weights/weights.lock.json
实验入口 : $SNF_DEPLOY_ROOT/snf.sh
部署日志 : $LOG
跑实验(需内联传入 HF_TOKEN):
cd $SNF_DEPLOY_ROOT
HF_TOKEN=xxx bash snf.sh plan --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full-plan
HF_TOKEN=xxx bash snf.sh run --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full \\
--render-motion --resume --gpus 0,1,2
参考耗时(本机 3×H20 实测):Meissonic 4.2s/条、Show-o 2.2s/条、MoMask 0.15s/条、
SD3.5 11.6s/条、Self-Forcing 17.1s/条;full 主实验 27000 条 ≈ 12h,含 extras 49660 条 ≈ 24h。
==================================================
SUMMARY