#!/usr/bin/env bash # ============================================================================= # SNF 五模型实验环境 一键部署脚本(原生运行,不依赖 Docker) # # 目标机器:Linux x86_64 + NVIDIA GPU(驱动 >= 535,CUDA 12.8/12.9 运行时)+ # 宿主 Python 3.12 且已装 torch 2.7.1(+cu128/cu129) # # 本脚本只做“部署”,不跑任何实验: # 1. 下载并解压官方构建源码包,校验 397 个上游文件 # 2. 建立 /weights /data /outputs 存储布局 # 3. 建立五个模型各自隔离的 /opt/envs/ 虚拟环境(继承宿主 torch) # 4. 应用两处必要的 harness 兼容补丁(不改动上游源码) # 5. 下载并锁定全部模型权重(HF + MoMask 官方 Google Drive) # 6. 验收:源码校验 + 五模型导入 + doctor(GPU 实测) # # 用法: # export HF_TOKEN='hf_xxx' # 必需(SD3.5 为 gated 仓库) # bash deploy.sh # 全量部署 # SNF_SKIP_WEIGHTS=1 bash deploy.sh # 只装环境不下载权重 # SNF_WITH_TASKS=1 bash deploy.sh # 额外生成默认任务清单 # SNF_STORAGE=/mnt/snf-runtime bash deploy.sh # 换存储根目录 # # 前置条件(脚本会检查并在不满足时退出): # - HF_TOKEN 对应的账号已在 https://huggingface.co/stabilityai/stable-diffusion-3.5-medium # 页面点击 "Agree and access repository",否则 SD3.5 权重会 403 # - 磁盘可用空间 >= 150 GB(实测权重 + 缓存约 100 GB) # - 能访问 huggingface.co / github.com / drive.google.com(脚本继承宿主 http_proxy 配置) # # 幂等:每一步都有完成标记,中断后重跑会跳过已完成部分。 # ============================================================================= set -euo pipefail SNF_DEPLOY_ROOT="${SNF_DEPLOY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" SNF_CODE_ROOT="${SNF_CODE_ROOT:-$SNF_DEPLOY_ROOT/snf-five-models}" SNF_STORAGE="${SNF_STORAGE:-$SNF_DEPLOY_ROOT/runtime}" SNF_ZIP_URL="${SNF_ZIP_URL:-https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/snf-five-models-cu128-build.zip}" SNF_ZIP="$SNF_DEPLOY_ROOT/snf-five-models-cu128-build.zip" HF_ENDPOINT="${HF_ENDPOINT:-https://huggingface.co}" PYTHON="${PYTHON:-python3}" MODELS=(meissonic self_forcing momask sd35 showo) SNF_SKIP_WEIGHTS="${SNF_SKIP_WEIGHTS:-0}" SNF_WITH_TASKS="${SNF_WITH_TASKS:-0}" SNF_DOCTOR="${SNF_DOCTOR:-1}" LOG_DIR="$SNF_DEPLOY_ROOT/logs"; mkdir -p "$LOG_DIR" LOG="$LOG_DIR/deploy.$(date +%Y%m%d_%H%M%S).log" exec > >(tee -a "$LOG") 2>&1 step(){ printf '\n\033[1;36m[%s] === %s ===\033[0m\n' "$(date +%H:%M:%S)" "$*"; } ok(){ printf ' \033[32m✓\033[0m %s\n' "$*"; } warn(){ printf ' \033[33m!\033[0m %s\n' "$*"; } die(){ printf ' \033[31m✗ %s\033[0m\n' "$*"; exit 1; } step "0/7 前置检查" [ "$(id -u)" = 0 ] || warn "非 root:/opt/envs 与 /weights 可能无法写入" command -v "$PYTHON" >/dev/null || die "找不到 python3(可用 PYTHON= 指定)" command -v nvidia-smi >/dev/null || die "找不到 nvidia-smi:本机无 GPU 驱动" GPU_COUNT="$(nvidia-smi -L 2>/dev/null | wc -l)" [ "$GPU_COUNT" -gt 0 ] || die "nvidia-smi 未识别到 GPU" ok "GPU: ${GPU_COUNT} 张 -> $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | tr -d '\r')" PYV="$($PYTHON -c 'import sys;print("%d.%d"%sys.version_info[:2])')" ok "Python $PYV ($(command -v $PYTHON))" TORCHV="$($PYTHON -c 'import torch;print(torch.__version__,torch.version.cuda)' 2>/dev/null || echo none)" [ "$TORCHV" = none ] && die "宿主 Python 没有 torch;镜像要求 torch==2.7.1(+cu128/cu129),五个虚拟环境都继承它" ok "宿主 torch $TORCHV" for tool in curl unzip git; do command -v $tool >/dev/null || die "缺少系统工具 $tool"; done ok "curl / unzip / git 就绪" FREE_GB="$(df -Pk "$SNF_DEPLOY_ROOT" | awk 'NR==2{print int($4/1048576)}')" if [ "$FREE_GB" -lt 150 ]; then warn "可用空间仅 ${FREE_GB}GB,建议 >=150GB(权重+缓存约100GB)"; else ok "可用空间 ${FREE_GB}GB"; fi if [ -z "${HF_TOKEN:-}" ]; then die "未设置 HF_TOKEN:SD3.5 是 gated 仓库,且匿名访问会被 429 限流"; fi WHOAMI="$(curl -s --max-time 30 -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/api/whoami-v2" | head -c 200)" case "$WHOAMI" in *'"name"'*) ok "HF_TOKEN 有效 -> $(printf '%s' "$WHOAMI" | sed -n 's/.*"name":"\([^"]*\)".*/\1/p')";; *) die "HF_TOKEN 无效或网络不通: $WHOAMI";; esac SD35_CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -L -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/stabilityai/stable-diffusion-3.5-medium/resolve/main/model_index.json")" if [ "$SD35_CODE" != "200" ]; then die "SD3.5 权重不可访问(HTTP $SD35_CODE)。请先在浏览器打开 https://huggingface.co/stabilityai/stable-diffusion-3.5-medium 点 Agree and access repository" fi ok "SD3.5 权重可访问(授权已生效)" step "1/7 下载并解压构建源码包" mkdir -p "$SNF_DEPLOY_ROOT" if [ ! -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ]; then [ -f "$SNF_ZIP" ] || curl -fL --retry 3 -o "$SNF_ZIP" "$SNF_ZIP_URL" unzip -q -o "$SNF_ZIP" -d "$SNF_DEPLOY_ROOT" else ok "源码已存在,跳过下载" fi [ -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ] || die "解压失败:缺少 sources.manifest.json" ok "源码包就绪 $SNF_CODE_ROOT" step "2/7 建立存储布局" mkdir -p "$SNF_STORAGE"/{weights,data,outputs} for pair in "weights:/weights" "data:/data" "outputs:/outputs"; do name="${pair%%:*}"; link="${pair##*:}"; target="$SNF_STORAGE/$name" if [ -L "$link" ] && [ "$(readlink -f "$link")" = "$(readlink -f "$target")" ]; then continue; fi if [ -d "$link" ] && [ ! -L "$link" ] && [ -n "$(ls -A "$link" 2>/dev/null)" ]; then warn "$link 已存在且非空(宿主目录),保留原目录不建软链" continue fi rm -rf "$link"; ln -sfn "$target" "$link" done ok "存储布局就绪 -> $SNF_STORAGE/{weights,data,outputs}" export PYTHONPATH="$SNF_CODE_ROOT" HF_HOME=/weights/huggingface \ TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache \ TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton \ MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 \ WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT mkdir -p /weights/huggingface cd "$SNF_CODE_ROOT" snf(){ ( cd "$SNF_CODE_ROOT" && $PYTHON -m suite.cli "$@" ); } VERIFY="$(snf source-check 2>/dev/null | tr -d '\n' || true)" case "$VERIFY" in *'"verified_files": 397'*) ok "源码完整性校验通过(397 个上游文件)";; *) die "源码校验失败: ${VERIFY:0:200}";; esac step "3/7 创建五个隔离模型环境 /opt/envs/" REQ="$SNF_CODE_ROOT/packaging/requirements-common.txt" REQ_SUM="$(sha256sum "$REQ" | cut -c1-16)" mkdir -p /opt/envs for model in "${MODELS[@]}"; do ENV="/opt/envs/$model"; STAMP="$ENV/.req.$REQ_SUM" if [ -x "$ENV/bin/python" ] && [ -f "$ENV/installed.freeze.txt" ] && [ -f "$STAMP" ]; then ok "$model 环境已就绪,跳过"; continue fi echo " · 创建 $model 环境(继承宿主 torch,只隔离 Python API 层)" rm -rf "$ENV" "$PYTHON" -m venv --system-site-packages "$ENV" "$ENV/bin/pip" install --no-cache-dir -r "$REQ" >"$LOG_DIR/env_$model.log" 2>&1 \ || { echo " pip 失败,详见 $LOG_DIR/env_$model.log"; continue; } touch "$STAMP" ok "$model 依赖安装完成" done echo " · Show-o 专属旧版栈(transformers 4.41.2 / diffusers 0.30.1)" /opt/envs/showo/bin/pip install --no-cache-dir \ transformers==4.41.2 tokenizers==0.19.1 peft==0.11.1 diffusers==0.30.1 \ >"$LOG_DIR/env_showo_override.log" 2>&1 || echo " Show-o 覆盖安装失败,详见日志" echo " · MoMask 官方 CLIP(固定 commit)" /opt/envs/momask/bin/pip install --no-cache-dir \ 'git+https://github.com/openai/CLIP.git@dcba3cb2e2827b402d2701e7e1c7d9fed8a20ef1' \ >"$LOG_DIR/env_momask_clip.log" 2>&1 || echo " CLIP 安装失败,详见日志" MISSING_ENV=0 for model in "${MODELS[@]}"; do [ -x "/opt/envs/$model/bin/python" ] || { warn "/opt/envs/$model 缺失"; MISSING_ENV=1; continue; } "/opt/envs/$model/bin/pip" freeze >"/opt/envs/$model/installed.freeze.txt" done [ "$MISSING_ENV" = 0 ] || die "有模型环境未建成,请查看 $LOG_DIR/env_*.log" step "4/7 应用 harness 兼容补丁(不改动任何上游源码)" $PYTHON - "$SNF_CODE_ROOT" <<'PATCH' import sys from pathlib import Path root = Path(sys.argv[1]) # 补丁一:MoMask 上游代码使用 NumPy 1.24 已移除的 np.float / np.int 等旧别名。 # 只在 harness 进程内恢复别名,vendor/ 下 397 个上游文件的校验值保持不变。 momask = root / 'suite/momask_loading.py' anchor = "def loading_functions():" shim = '''def numpy_legacy_aliases(): """Restore aliases NumPy removed in 1.24; upstream MoMask still uses np.float/np.int. Applied in the harness process only. Vendored upstream files are untouched, so packaging/sources.manifest.json keeps verifying all 397 upstream files. """ import numpy as np for name, replacement in (('float', float), ('int', int), ('bool', bool), ('object', object), ('str', str), ('complex', complex)): if not hasattr(np, name): setattr(np, name, replacement) return np numpy_legacy_aliases() ''' if 'numpy_legacy_aliases' in momask.read_text(): print(' ✓ momask numpy 兼容补丁已存在') else: text = momask.read_text() assert anchor in text, 'momask_loading.py 结构变化,请人工确认' momask.write_text(text.replace(anchor, shim + anchor, 1)) print(' ✓ 已写入 momask numpy 兼容补丁') # 补丁二:SD3.5 上游只提供 fp16 的 CLIP 文本编码器,diffusers 只把 text_projection # 转成 bf16,前向时 dtype 冲突(Half != BFloat16)。统一到管线声明的 bf16。 adapters = root / 'suite/adapters.py' old = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda') self.pipe.set_progress_bar_config(disable=True)""" new = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda') # Upstream ships the two CLIP encoders as fp16 only; diffusers casts their # text_projection to bf16 but leaves the encoder body fp16, which raises a # dtype mismatch in the forward. Align them with the declared pipeline dtype. for name in ('text_encoder','text_encoder_2','text_encoder_3'): module=getattr(self.pipe,name,None) if module is not None:module.to(torch.bfloat16) self.pipe.set_progress_bar_config(disable=True)""" text = adapters.read_text() if 'Upstream ships the two CLIP encoders' in text: print(' ✓ sd35 dtype 补丁已存在') else: assert old in text, 'adapters.py 结构变化,请人工确认' adapters.write_text(text.replace(old, new, 1)) print(' ✓ 已写入 sd35 dtype 补丁') PATCH cat > "$SNF_DEPLOY_ROOT/snf.sh" < ... set -euo pipefail export PYTHONPATH="$SNF_CODE_ROOT" export HF_HOME=/weights/huggingface TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache export TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton export MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 export WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT="\${HF_ENDPOINT:-https://huggingface.co}" # HF_TOKEN 由调用方内联传入,不落盘 cd "$SNF_CODE_ROOT" exec $PYTHON -m suite.cli "\$@" EOS chmod +x "$SNF_DEPLOY_ROOT/snf.sh" ok "实验入口已生成 $SNF_DEPLOY_ROOT/snf.sh" step "5/7 下载并锁定模型权重" if [ "$SNF_SKIP_WEIGHTS" = 1 ]; then warn "SNF_SKIP_WEIGHTS=1,跳过权重下载" else LOCK=/weights/weights.lock.json [ -f "$LOCK" ] || echo '{}' > "$LOCK" # 串行执行:fetch 会对 weights.lock.json 做读-改-写,并发会丢条目 for model in "${MODELS[@]}"; do echo " · fetch $model" if ! snf fetch --models "$model" --weights /weights >"$LOG_DIR/fetch_$model.log" 2>&1; then echo " fetch $model 失败,详见 $LOG_DIR/fetch_$model.log"; tail -3 "$LOG_DIR/fetch_$model.log" || true fi done python3 - "$LOCK" <<'CHECK' import json, sys, os lock = json.load(open(sys.argv[1])) expect = { 'MeissonFlow/Meissonic': None, 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K': None, 'Wan-AI/Wan2.1-T2V-1.3B': None, 'gdhe17/Self-Forcing': None, 'showlab/show-o': None, 'showlab/magvitv2': None, 'microsoft/phi-1_5': None, 'stabilityai/stable-diffusion-3.5-medium': None, 'momask_archive_sha256': None, } missing = [k for k in expect if k not in lock] print(' 已锁定 %d/%d 个权重源' % (len(lock), len(expect))) if missing: print(' 缺失:', missing) sys.exit(1) checks = { 'MoMask 主模型': '/weights/momask/checkpoints/t2m/t2m_nlayer8_nhead6_ld384_ff1024_cdp0.1_rvq6ns/opt.txt', 'MoMask 残差模型': '/weights/momask/checkpoints/t2m/tres_nlayer8_ld384_ff1024_rvq6ns_cdp0.2_sw/opt.txt', 'Self-Forcing 学生权重': '/weights/self_forcing/checkpoints/self_forcing_dmd.pt', 'Wan2.1 基础模型': '/weights/self_forcing/wan_models/Wan2.1-T2V-1.3B/config.json', } for name, path in checks.items(): print(' %s %s' % ('✓' if os.path.exists(path) else '✗', name)) if not os.path.exists(path): sys.exit(1) CHECK [ $? -eq 0 ] || die "权重不完整,请查看 $LOG_DIR/fetch_*.log" ok "全部权重就绪(HF revision 已锁定在 /weights/weights.lock.json)" rm -rf /weights/huggingface/xet 2>/dev/null && ok "已清理 xet 分块缓存(约 9GB,不影响已下载权重)" fi step "6/7 验收:源码 / 五模型导入 / GPU" snf source-check >/dev/null && ok "源码校验 397 文件通过" for model in "${MODELS[@]}"; do OUT="$(/opt/envs/$model/bin/python -m suite.cli import-check --model "$model" 2>&1 | tail -1)" case "$OUT" in *'"imports": "passed"'*) ok "$model 导入通过";; *) warn "$model 导入失败: ${OUT:0:200}";; esac done if [ "$SNF_DOCTOR" = 1 ]; then if snf doctor >"$LOG_DIR/doctor.json" 2>&1; then $PYTHON - "$LOG_DIR/doctor.json" <<'DOC' import json, sys raw = open(sys.argv[1]).read() d = json.loads(raw[raw.index('{'):]) print(' ✓ CUDA available=%s kernel_finite=%s' % (d['cuda_available'], d['kernel_finite'])) print(' ✓ GPU: %s' % ', '.join('%s %.0fGiB' % (g['name'], g['memory_gib']) for g in d['gpus'])) bad = [k for k, v in d['models'].items() if v['returncode']] print(' ✓ 五模型环境导入: %s' % ('全部通过' if not bad else '失败 ' + str(bad))) DOC else warn "doctor 未通过,详见 $LOG_DIR/doctor.json" fi fi if [ "$SNF_WITH_TASKS" = 1 ]; then step "7/7 生成默认任务清单(可选,属于实验输入)" snf create-tasks --output /data/tasks.curated.jsonl && ok "已生成 /data/tasks.curated.jsonl(120 test + 8 dev)" else step "7/7 完成(未生成任务清单,SNF_WITH_TASKS=1 可开启)" fi cat <