File size: 16,211 Bytes
ea1b5d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 | #!/usr/bin/env bash
# =============================================================================
# SNF 五模型实验环境 一键部署脚本(原生运行,不依赖 Docker)
#
# 目标机器:Linux x86_64 + NVIDIA GPU(驱动 >= 535,CUDA 12.8/12.9 运行时)+
# 宿主 Python 3.12 且已装 torch 2.7.1(+cu128/cu129)
#
# 本脚本只做“部署”,不跑任何实验:
# 1. 下载并解压官方构建源码包,校验 397 个上游文件
# 2. 建立 /weights /data /outputs 存储布局
# 3. 建立五个模型各自隔离的 /opt/envs/<model> 虚拟环境(继承宿主 torch)
# 4. 应用两处必要的 harness 兼容补丁(不改动上游源码)
# 5. 下载并锁定全部模型权重(HF + MoMask 官方 Google Drive)
# 6. 验收:源码校验 + 五模型导入 + doctor(GPU 实测)
#
# 用法:
# export HF_TOKEN='hf_xxx' # 必需(SD3.5 为 gated 仓库)
# bash deploy.sh # 全量部署
# SNF_SKIP_WEIGHTS=1 bash deploy.sh # 只装环境不下载权重
# SNF_WITH_TASKS=1 bash deploy.sh # 额外生成默认任务清单
# SNF_STORAGE=/mnt/snf-runtime bash deploy.sh # 换存储根目录
#
# 前置条件(脚本会检查并在不满足时退出):
# - HF_TOKEN 对应的账号已在 https://huggingface.co/stabilityai/stable-diffusion-3.5-medium
# 页面点击 "Agree and access repository",否则 SD3.5 权重会 403
# - 磁盘可用空间 >= 150 GB(实测权重 + 缓存约 100 GB)
# - 能访问 huggingface.co / github.com / drive.google.com(脚本继承宿主 http_proxy 配置)
#
# 幂等:每一步都有完成标记,中断后重跑会跳过已完成部分。
# =============================================================================
set -euo pipefail
SNF_DEPLOY_ROOT="${SNF_DEPLOY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
SNF_CODE_ROOT="${SNF_CODE_ROOT:-$SNF_DEPLOY_ROOT/snf-five-models}"
SNF_STORAGE="${SNF_STORAGE:-$SNF_DEPLOY_ROOT/runtime}"
SNF_ZIP_URL="${SNF_ZIP_URL:-https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/snf-five-models-cu128-build.zip}"
SNF_ZIP="$SNF_DEPLOY_ROOT/snf-five-models-cu128-build.zip"
HF_ENDPOINT="${HF_ENDPOINT:-https://huggingface.co}"
PYTHON="${PYTHON:-python3}"
MODELS=(meissonic self_forcing momask sd35 showo)
SNF_SKIP_WEIGHTS="${SNF_SKIP_WEIGHTS:-0}"
SNF_WITH_TASKS="${SNF_WITH_TASKS:-0}"
SNF_DOCTOR="${SNF_DOCTOR:-1}"
LOG_DIR="$SNF_DEPLOY_ROOT/logs"; mkdir -p "$LOG_DIR"
LOG="$LOG_DIR/deploy.$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1
step(){ printf '\n\033[1;36m[%s] === %s ===\033[0m\n' "$(date +%H:%M:%S)" "$*"; }
ok(){ printf ' \033[32m✓\033[0m %s\n' "$*"; }
warn(){ printf ' \033[33m!\033[0m %s\n' "$*"; }
die(){ printf ' \033[31m✗ %s\033[0m\n' "$*"; exit 1; }
step "0/7 前置检查"
[ "$(id -u)" = 0 ] || warn "非 root:/opt/envs 与 /weights 可能无法写入"
command -v "$PYTHON" >/dev/null || die "找不到 python3(可用 PYTHON= 指定)"
command -v nvidia-smi >/dev/null || die "找不到 nvidia-smi:本机无 GPU 驱动"
GPU_COUNT="$(nvidia-smi -L 2>/dev/null | wc -l)"
[ "$GPU_COUNT" -gt 0 ] || die "nvidia-smi 未识别到 GPU"
ok "GPU: ${GPU_COUNT} 张 -> $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | tr -d '\r')"
PYV="$($PYTHON -c 'import sys;print("%d.%d"%sys.version_info[:2])')"
ok "Python $PYV ($(command -v $PYTHON))"
TORCHV="$($PYTHON -c 'import torch;print(torch.__version__,torch.version.cuda)' 2>/dev/null || echo none)"
[ "$TORCHV" = none ] && die "宿主 Python 没有 torch;镜像要求 torch==2.7.1(+cu128/cu129),五个虚拟环境都继承它"
ok "宿主 torch $TORCHV"
for tool in curl unzip git; do command -v $tool >/dev/null || die "缺少系统工具 $tool"; done
ok "curl / unzip / git 就绪"
FREE_GB="$(df -Pk "$SNF_DEPLOY_ROOT" | awk 'NR==2{print int($4/1048576)}')"
if [ "$FREE_GB" -lt 150 ]; then warn "可用空间仅 ${FREE_GB}GB,建议 >=150GB(权重+缓存约100GB)"; else ok "可用空间 ${FREE_GB}GB"; fi
if [ -z "${HF_TOKEN:-}" ]; then die "未设置 HF_TOKEN:SD3.5 是 gated 仓库,且匿名访问会被 429 限流"; fi
WHOAMI="$(curl -s --max-time 30 -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/api/whoami-v2" | head -c 200)"
case "$WHOAMI" in *'"name"'*) ok "HF_TOKEN 有效 -> $(printf '%s' "$WHOAMI" | sed -n 's/.*"name":"\([^"]*\)".*/\1/p')";; *) die "HF_TOKEN 无效或网络不通: $WHOAMI";; esac
SD35_CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -L -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/stabilityai/stable-diffusion-3.5-medium/resolve/main/model_index.json")"
if [ "$SD35_CODE" != "200" ]; then
die "SD3.5 权重不可访问(HTTP $SD35_CODE)。请先在浏览器打开
https://huggingface.co/stabilityai/stable-diffusion-3.5-medium 点 Agree and access repository"
fi
ok "SD3.5 权重可访问(授权已生效)"
step "1/7 下载并解压构建源码包"
mkdir -p "$SNF_DEPLOY_ROOT"
if [ ! -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ]; then
[ -f "$SNF_ZIP" ] || curl -fL --retry 3 -o "$SNF_ZIP" "$SNF_ZIP_URL"
unzip -q -o "$SNF_ZIP" -d "$SNF_DEPLOY_ROOT"
else
ok "源码已存在,跳过下载"
fi
[ -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ] || die "解压失败:缺少 sources.manifest.json"
ok "源码包就绪 $SNF_CODE_ROOT"
step "2/7 建立存储布局"
mkdir -p "$SNF_STORAGE"/{weights,data,outputs}
for pair in "weights:/weights" "data:/data" "outputs:/outputs"; do
name="${pair%%:*}"; link="${pair##*:}"; target="$SNF_STORAGE/$name"
if [ -L "$link" ] && [ "$(readlink -f "$link")" = "$(readlink -f "$target")" ]; then continue; fi
if [ -d "$link" ] && [ ! -L "$link" ] && [ -n "$(ls -A "$link" 2>/dev/null)" ]; then
warn "$link 已存在且非空(宿主目录),保留原目录不建软链"
continue
fi
rm -rf "$link"; ln -sfn "$target" "$link"
done
ok "存储布局就绪 -> $SNF_STORAGE/{weights,data,outputs}"
export PYTHONPATH="$SNF_CODE_ROOT" HF_HOME=/weights/huggingface \
TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache \
TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton \
MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 \
WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT
mkdir -p /weights/huggingface
cd "$SNF_CODE_ROOT"
snf(){ ( cd "$SNF_CODE_ROOT" && $PYTHON -m suite.cli "$@" ); }
VERIFY="$(snf source-check 2>/dev/null | tr -d '\n' || true)"
case "$VERIFY" in *'"verified_files": 397'*) ok "源码完整性校验通过(397 个上游文件)";; *) die "源码校验失败: ${VERIFY:0:200}";; esac
step "3/7 创建五个隔离模型环境 /opt/envs/<model>"
REQ="$SNF_CODE_ROOT/packaging/requirements-common.txt"
REQ_SUM="$(sha256sum "$REQ" | cut -c1-16)"
mkdir -p /opt/envs
for model in "${MODELS[@]}"; do
ENV="/opt/envs/$model"; STAMP="$ENV/.req.$REQ_SUM"
if [ -x "$ENV/bin/python" ] && [ -f "$ENV/installed.freeze.txt" ] && [ -f "$STAMP" ]; then
ok "$model 环境已就绪,跳过"; continue
fi
echo " · 创建 $model 环境(继承宿主 torch,只隔离 Python API 层)"
rm -rf "$ENV"
"$PYTHON" -m venv --system-site-packages "$ENV"
"$ENV/bin/pip" install --no-cache-dir -r "$REQ" >"$LOG_DIR/env_$model.log" 2>&1 \
|| { echo " pip 失败,详见 $LOG_DIR/env_$model.log"; continue; }
touch "$STAMP"
ok "$model 依赖安装完成"
done
echo " · Show-o 专属旧版栈(transformers 4.41.2 / diffusers 0.30.1)"
/opt/envs/showo/bin/pip install --no-cache-dir \
transformers==4.41.2 tokenizers==0.19.1 peft==0.11.1 diffusers==0.30.1 \
>"$LOG_DIR/env_showo_override.log" 2>&1 || echo " Show-o 覆盖安装失败,详见日志"
echo " · MoMask 官方 CLIP(固定 commit)"
/opt/envs/momask/bin/pip install --no-cache-dir \
'git+https://github.com/openai/CLIP.git@dcba3cb2e2827b402d2701e7e1c7d9fed8a20ef1' \
>"$LOG_DIR/env_momask_clip.log" 2>&1 || echo " CLIP 安装失败,详见日志"
MISSING_ENV=0
for model in "${MODELS[@]}"; do
[ -x "/opt/envs/$model/bin/python" ] || { warn "/opt/envs/$model 缺失"; MISSING_ENV=1; continue; }
"/opt/envs/$model/bin/pip" freeze >"/opt/envs/$model/installed.freeze.txt"
done
[ "$MISSING_ENV" = 0 ] || die "有模型环境未建成,请查看 $LOG_DIR/env_*.log"
step "4/7 应用 harness 兼容补丁(不改动任何上游源码)"
$PYTHON - "$SNF_CODE_ROOT" <<'PATCH'
import sys
from pathlib import Path
root = Path(sys.argv[1])
# 补丁一:MoMask 上游代码使用 NumPy 1.24 已移除的 np.float / np.int 等旧别名。
# 只在 harness 进程内恢复别名,vendor/ 下 397 个上游文件的校验值保持不变。
momask = root / 'suite/momask_loading.py'
anchor = "def loading_functions():"
shim = '''def numpy_legacy_aliases():
"""Restore aliases NumPy removed in 1.24; upstream MoMask still uses np.float/np.int.
Applied in the harness process only. Vendored upstream files are untouched, so
packaging/sources.manifest.json keeps verifying all 397 upstream files.
"""
import numpy as np
for name, replacement in (('float', float), ('int', int), ('bool', bool),
('object', object), ('str', str), ('complex', complex)):
if not hasattr(np, name):
setattr(np, name, replacement)
return np
numpy_legacy_aliases()
'''
if 'numpy_legacy_aliases' in momask.read_text():
print(' ✓ momask numpy 兼容补丁已存在')
else:
text = momask.read_text()
assert anchor in text, 'momask_loading.py 结构变化,请人工确认'
momask.write_text(text.replace(anchor, shim + anchor, 1))
print(' ✓ 已写入 momask numpy 兼容补丁')
# 补丁二:SD3.5 上游只提供 fp16 的 CLIP 文本编码器,diffusers 只把 text_projection
# 转成 bf16,前向时 dtype 冲突(Half != BFloat16)。统一到管线声明的 bf16。
adapters = root / 'suite/adapters.py'
old = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
self.pipe.set_progress_bar_config(disable=True)"""
new = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
# Upstream ships the two CLIP encoders as fp16 only; diffusers casts their
# text_projection to bf16 but leaves the encoder body fp16, which raises a
# dtype mismatch in the forward. Align them with the declared pipeline dtype.
for name in ('text_encoder','text_encoder_2','text_encoder_3'):
module=getattr(self.pipe,name,None)
if module is not None:module.to(torch.bfloat16)
self.pipe.set_progress_bar_config(disable=True)"""
text = adapters.read_text()
if 'Upstream ships the two CLIP encoders' in text:
print(' ✓ sd35 dtype 补丁已存在')
else:
assert old in text, 'adapters.py 结构变化,请人工确认'
adapters.write_text(text.replace(old, new, 1))
print(' ✓ 已写入 sd35 dtype 补丁')
PATCH
cat > "$SNF_DEPLOY_ROOT/snf.sh" <<EOS
#!/usr/bin/env bash
# 实验入口(原生替代 packaging/start.sh):bash snf.sh <command> ...
set -euo pipefail
export PYTHONPATH="$SNF_CODE_ROOT"
export HF_HOME=/weights/huggingface TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache
export TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton
export MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1
export WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT="\${HF_ENDPOINT:-https://huggingface.co}"
# HF_TOKEN 由调用方内联传入,不落盘
cd "$SNF_CODE_ROOT"
exec $PYTHON -m suite.cli "\$@"
EOS
chmod +x "$SNF_DEPLOY_ROOT/snf.sh"
ok "实验入口已生成 $SNF_DEPLOY_ROOT/snf.sh"
step "5/7 下载并锁定模型权重"
if [ "$SNF_SKIP_WEIGHTS" = 1 ]; then
warn "SNF_SKIP_WEIGHTS=1,跳过权重下载"
else
LOCK=/weights/weights.lock.json
[ -f "$LOCK" ] || echo '{}' > "$LOCK"
# 串行执行:fetch 会对 weights.lock.json 做读-改-写,并发会丢条目
for model in "${MODELS[@]}"; do
echo " · fetch $model"
if ! snf fetch --models "$model" --weights /weights >"$LOG_DIR/fetch_$model.log" 2>&1; then
echo " fetch $model 失败,详见 $LOG_DIR/fetch_$model.log"; tail -3 "$LOG_DIR/fetch_$model.log" || true
fi
done
python3 - "$LOCK" <<'CHECK'
import json, sys, os
lock = json.load(open(sys.argv[1]))
expect = {
'MeissonFlow/Meissonic': None, 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K': None,
'Wan-AI/Wan2.1-T2V-1.3B': None, 'gdhe17/Self-Forcing': None,
'showlab/show-o': None, 'showlab/magvitv2': None, 'microsoft/phi-1_5': None,
'stabilityai/stable-diffusion-3.5-medium': None, 'momask_archive_sha256': None,
}
missing = [k for k in expect if k not in lock]
print(' 已锁定 %d/%d 个权重源' % (len(lock), len(expect)))
if missing:
print(' 缺失:', missing)
sys.exit(1)
checks = {
'MoMask 主模型': '/weights/momask/checkpoints/t2m/t2m_nlayer8_nhead6_ld384_ff1024_cdp0.1_rvq6ns/opt.txt',
'MoMask 残差模型': '/weights/momask/checkpoints/t2m/tres_nlayer8_ld384_ff1024_rvq6ns_cdp0.2_sw/opt.txt',
'Self-Forcing 学生权重': '/weights/self_forcing/checkpoints/self_forcing_dmd.pt',
'Wan2.1 基础模型': '/weights/self_forcing/wan_models/Wan2.1-T2V-1.3B/config.json',
}
for name, path in checks.items():
print(' %s %s' % ('✓' if os.path.exists(path) else '✗', name))
if not os.path.exists(path):
sys.exit(1)
CHECK
[ $? -eq 0 ] || die "权重不完整,请查看 $LOG_DIR/fetch_*.log"
ok "全部权重就绪(HF revision 已锁定在 /weights/weights.lock.json)"
rm -rf /weights/huggingface/xet 2>/dev/null && ok "已清理 xet 分块缓存(约 9GB,不影响已下载权重)"
fi
step "6/7 验收:源码 / 五模型导入 / GPU"
snf source-check >/dev/null && ok "源码校验 397 文件通过"
for model in "${MODELS[@]}"; do
OUT="$(/opt/envs/$model/bin/python -m suite.cli import-check --model "$model" 2>&1 | tail -1)"
case "$OUT" in *'"imports": "passed"'*) ok "$model 导入通过";; *) warn "$model 导入失败: ${OUT:0:200}";; esac
done
if [ "$SNF_DOCTOR" = 1 ]; then
if snf doctor >"$LOG_DIR/doctor.json" 2>&1; then
$PYTHON - "$LOG_DIR/doctor.json" <<'DOC'
import json, sys
raw = open(sys.argv[1]).read()
d = json.loads(raw[raw.index('{'):])
print(' ✓ CUDA available=%s kernel_finite=%s' % (d['cuda_available'], d['kernel_finite']))
print(' ✓ GPU: %s' % ', '.join('%s %.0fGiB' % (g['name'], g['memory_gib']) for g in d['gpus']))
bad = [k for k, v in d['models'].items() if v['returncode']]
print(' ✓ 五模型环境导入: %s' % ('全部通过' if not bad else '失败 ' + str(bad)))
DOC
else
warn "doctor 未通过,详见 $LOG_DIR/doctor.json"
fi
fi
if [ "$SNF_WITH_TASKS" = 1 ]; then
step "7/7 生成默认任务清单(可选,属于实验输入)"
snf create-tasks --output /data/tasks.curated.jsonl && ok "已生成 /data/tasks.curated.jsonl(120 test + 8 dev)"
else
step "7/7 完成(未生成任务清单,SNF_WITH_TASKS=1 可开启)"
fi
cat <<SUMMARY
==================== 部署完成 ====================
源码 : $SNF_CODE_ROOT
存储 : $SNF_STORAGE/{weights,data,outputs} (/weights /data /outputs 为软链)
模型环境 : /opt/envs/{meissonic,self_forcing,momask,sd35,showo}(各含 installed.freeze.txt)
权重锁 : /weights/weights.lock.json
实验入口 : $SNF_DEPLOY_ROOT/snf.sh
部署日志 : $LOG
跑实验(需内联传入 HF_TOKEN):
cd $SNF_DEPLOY_ROOT
HF_TOKEN=xxx bash snf.sh plan --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full-plan
HF_TOKEN=xxx bash snf.sh run --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full \\
--render-motion --resume --gpus 0,1,2
参考耗时(本机 3×H20 实测):Meissonic 4.2s/条、Show-o 2.2s/条、MoMask 0.15s/条、
SD3.5 11.6s/条、Self-Forcing 17.1s/条;full 主实验 27000 条 ≈ 12h,含 extras 49660 条 ≈ 24h。
==================================================
SUMMARY
|