Download paper_export/evidence/code/deploy.sh from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 16.2 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/paper_export/evidence/code/deploy.sh
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/paper_export/evidence/code/deploy.sh
-
curl -L -o deploy.sh https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/paper_export/evidence/code/deploy.sh
16.2 kB
| # ============================================================================= | |
| # SNF 五模型实验环境 一键部署脚本(原生运行,不依赖 Docker) | |
| # | |
| # 目标机器:Linux x86_64 + NVIDIA GPU(驱动 >= 535,CUDA 12.8/12.9 运行时)+ | |
| # 宿主 Python 3.12 且已装 torch 2.7.1(+cu128/cu129) | |
| # | |
| # 本脚本只做“部署”,不跑任何实验: | |
| # 1. 下载并解压官方构建源码包,校验 397 个上游文件 | |
| # 2. 建立 /weights /data /outputs 存储布局 | |
| # 3. 建立五个模型各自隔离的 /opt/envs/<model> 虚拟环境(继承宿主 torch) | |
| # 4. 应用两处必要的 harness 兼容补丁(不改动上游源码) | |
| # 5. 下载并锁定全部模型权重(HF + MoMask 官方 Google Drive) | |
| # 6. 验收:源码校验 + 五模型导入 + doctor(GPU 实测) | |
| # | |
| # 用法: | |
| # export HF_TOKEN='hf_xxx' # 必需(SD3.5 为 gated 仓库) | |
| # bash deploy.sh # 全量部署 | |
| # SNF_SKIP_WEIGHTS=1 bash deploy.sh # 只装环境不下载权重 | |
| # SNF_WITH_TASKS=1 bash deploy.sh # 额外生成默认任务清单 | |
| # SNF_STORAGE=/mnt/snf-runtime bash deploy.sh # 换存储根目录 | |
| # | |
| # 前置条件(脚本会检查并在不满足时退出): | |
| # - HF_TOKEN 对应的账号已在 https://huggingface.co/stabilityai/stable-diffusion-3.5-medium | |
| # 页面点击 "Agree and access repository",否则 SD3.5 权重会 403 | |
| # - 磁盘可用空间 >= 150 GB(实测权重 + 缓存约 100 GB) | |
| # - 能访问 huggingface.co / github.com / drive.google.com(脚本继承宿主 http_proxy 配置) | |
| # | |
| # 幂等:每一步都有完成标记,中断后重跑会跳过已完成部分。 | |
| # ============================================================================= | |
| set -euo pipefail | |
| SNF_DEPLOY_ROOT="${SNF_DEPLOY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" | |
| SNF_CODE_ROOT="${SNF_CODE_ROOT:-$SNF_DEPLOY_ROOT/snf-five-models}" | |
| SNF_STORAGE="${SNF_STORAGE:-$SNF_DEPLOY_ROOT/runtime}" | |
| SNF_ZIP_URL="${SNF_ZIP_URL:-https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/snf-five-models-cu128-build.zip}" | |
| SNF_ZIP="$SNF_DEPLOY_ROOT/snf-five-models-cu128-build.zip" | |
| HF_ENDPOINT="${HF_ENDPOINT:-https://huggingface.co}" | |
| PYTHON="${PYTHON:-python3}" | |
| MODELS=(meissonic self_forcing momask sd35 showo) | |
| SNF_SKIP_WEIGHTS="${SNF_SKIP_WEIGHTS:-0}" | |
| SNF_WITH_TASKS="${SNF_WITH_TASKS:-0}" | |
| SNF_DOCTOR="${SNF_DOCTOR:-1}" | |
| LOG_DIR="$SNF_DEPLOY_ROOT/logs"; mkdir -p "$LOG_DIR" | |
| LOG="$LOG_DIR/deploy.$(date +%Y%m%d_%H%M%S).log" | |
| exec > >(tee -a "$LOG") 2>&1 | |
| step(){ printf '\n\033[1;36m[%s] === %s ===\033[0m\n' "$(date +%H:%M:%S)" "$*"; } | |
| ok(){ printf ' \033[32m✓\033[0m %s\n' "$*"; } | |
| warn(){ printf ' \033[33m!\033[0m %s\n' "$*"; } | |
| die(){ printf ' \033[31m✗ %s\033[0m\n' "$*"; exit 1; } | |
| step "0/7 前置检查" | |
| [ "$(id -u)" = 0 ] || warn "非 root:/opt/envs 与 /weights 可能无法写入" | |
| command -v "$PYTHON" >/dev/null || die "找不到 python3(可用 PYTHON= 指定)" | |
| command -v nvidia-smi >/dev/null || die "找不到 nvidia-smi:本机无 GPU 驱动" | |
| GPU_COUNT="$(nvidia-smi -L 2>/dev/null | wc -l)" | |
| [ "$GPU_COUNT" -gt 0 ] || die "nvidia-smi 未识别到 GPU" | |
| ok "GPU: ${GPU_COUNT} 张 -> $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | tr -d '\r')" | |
| PYV="$($PYTHON -c 'import sys;print("%d.%d"%sys.version_info[:2])')" | |
| ok "Python $PYV ($(command -v $PYTHON))" | |
| TORCHV="$($PYTHON -c 'import torch;print(torch.__version__,torch.version.cuda)' 2>/dev/null || echo none)" | |
| [ "$TORCHV" = none ] && die "宿主 Python 没有 torch;镜像要求 torch==2.7.1(+cu128/cu129),五个虚拟环境都继承它" | |
| ok "宿主 torch $TORCHV" | |
| for tool in curl unzip git; do command -v $tool >/dev/null || die "缺少系统工具 $tool"; done | |
| ok "curl / unzip / git 就绪" | |
| FREE_GB="$(df -Pk "$SNF_DEPLOY_ROOT" | awk 'NR==2{print int($4/1048576)}')" | |
| if [ "$FREE_GB" -lt 150 ]; then warn "可用空间仅 ${FREE_GB}GB,建议 >=150GB(权重+缓存约100GB)"; else ok "可用空间 ${FREE_GB}GB"; fi | |
| if [ -z "${HF_TOKEN:-}" ]; then die "未设置 HF_TOKEN:SD3.5 是 gated 仓库,且匿名访问会被 429 限流"; fi | |
| WHOAMI="$(curl -s --max-time 30 -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/api/whoami-v2" | head -c 200)" | |
| case "$WHOAMI" in *'"name"'*) ok "HF_TOKEN 有效 -> $(printf '%s' "$WHOAMI" | sed -n 's/.*"name":"\([^"]*\)".*/\1/p')";; *) die "HF_TOKEN 无效或网络不通: $WHOAMI";; esac | |
| SD35_CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -L -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/stabilityai/stable-diffusion-3.5-medium/resolve/main/model_index.json")" | |
| if [ "$SD35_CODE" != "200" ]; then | |
| die "SD3.5 权重不可访问(HTTP $SD35_CODE)。请先在浏览器打开 | |
| https://huggingface.co/stabilityai/stable-diffusion-3.5-medium 点 Agree and access repository" | |
| fi | |
| ok "SD3.5 权重可访问(授权已生效)" | |
| step "1/7 下载并解压构建源码包" | |
| mkdir -p "$SNF_DEPLOY_ROOT" | |
| if [ ! -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ]; then | |
| [ -f "$SNF_ZIP" ] || curl -fL --retry 3 -o "$SNF_ZIP" "$SNF_ZIP_URL" | |
| unzip -q -o "$SNF_ZIP" -d "$SNF_DEPLOY_ROOT" | |
| else | |
| ok "源码已存在,跳过下载" | |
| fi | |
| [ -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ] || die "解压失败:缺少 sources.manifest.json" | |
| ok "源码包就绪 $SNF_CODE_ROOT" | |
| step "2/7 建立存储布局" | |
| mkdir -p "$SNF_STORAGE"/{weights,data,outputs} | |
| for pair in "weights:/weights" "data:/data" "outputs:/outputs"; do | |
| name="${pair%%:*}"; link="${pair##*:}"; target="$SNF_STORAGE/$name" | |
| if [ -L "$link" ] && [ "$(readlink -f "$link")" = "$(readlink -f "$target")" ]; then continue; fi | |
| if [ -d "$link" ] && [ ! -L "$link" ] && [ -n "$(ls -A "$link" 2>/dev/null)" ]; then | |
| warn "$link 已存在且非空(宿主目录),保留原目录不建软链" | |
| continue | |
| fi | |
| rm -rf "$link"; ln -sfn "$target" "$link" | |
| done | |
| ok "存储布局就绪 -> $SNF_STORAGE/{weights,data,outputs}" | |
| export PYTHONPATH="$SNF_CODE_ROOT" HF_HOME=/weights/huggingface \ | |
| TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache \ | |
| TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton \ | |
| MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 \ | |
| WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT | |
| mkdir -p /weights/huggingface | |
| cd "$SNF_CODE_ROOT" | |
| snf(){ ( cd "$SNF_CODE_ROOT" && $PYTHON -m suite.cli "$@" ); } | |
| VERIFY="$(snf source-check 2>/dev/null | tr -d '\n' || true)" | |
| case "$VERIFY" in *'"verified_files": 397'*) ok "源码完整性校验通过(397 个上游文件)";; *) die "源码校验失败: ${VERIFY:0:200}";; esac | |
| step "3/7 创建五个隔离模型环境 /opt/envs/<model>" | |
| REQ="$SNF_CODE_ROOT/packaging/requirements-common.txt" | |
| REQ_SUM="$(sha256sum "$REQ" | cut -c1-16)" | |
| mkdir -p /opt/envs | |
| for model in "${MODELS[@]}"; do | |
| ENV="/opt/envs/$model"; STAMP="$ENV/.req.$REQ_SUM" | |
| if [ -x "$ENV/bin/python" ] && [ -f "$ENV/installed.freeze.txt" ] && [ -f "$STAMP" ]; then | |
| ok "$model 环境已就绪,跳过"; continue | |
| fi | |
| echo " · 创建 $model 环境(继承宿主 torch,只隔离 Python API 层)" | |
| rm -rf "$ENV" | |
| "$PYTHON" -m venv --system-site-packages "$ENV" | |
| "$ENV/bin/pip" install --no-cache-dir -r "$REQ" >"$LOG_DIR/env_$model.log" 2>&1 \ | |
| || { echo " pip 失败,详见 $LOG_DIR/env_$model.log"; continue; } | |
| touch "$STAMP" | |
| ok "$model 依赖安装完成" | |
| done | |
| echo " · Show-o 专属旧版栈(transformers 4.41.2 / diffusers 0.30.1)" | |
| /opt/envs/showo/bin/pip install --no-cache-dir \ | |
| transformers==4.41.2 tokenizers==0.19.1 peft==0.11.1 diffusers==0.30.1 \ | |
| >"$LOG_DIR/env_showo_override.log" 2>&1 || echo " Show-o 覆盖安装失败,详见日志" | |
| echo " · MoMask 官方 CLIP(固定 commit)" | |
| /opt/envs/momask/bin/pip install --no-cache-dir \ | |
| 'git+https://github.com/openai/CLIP.git@dcba3cb2e2827b402d2701e7e1c7d9fed8a20ef1' \ | |
| >"$LOG_DIR/env_momask_clip.log" 2>&1 || echo " CLIP 安装失败,详见日志" | |
| MISSING_ENV=0 | |
| for model in "${MODELS[@]}"; do | |
| [ -x "/opt/envs/$model/bin/python" ] || { warn "/opt/envs/$model 缺失"; MISSING_ENV=1; continue; } | |
| "/opt/envs/$model/bin/pip" freeze >"/opt/envs/$model/installed.freeze.txt" | |
| done | |
| [ "$MISSING_ENV" = 0 ] || die "有模型环境未建成,请查看 $LOG_DIR/env_*.log" | |
| step "4/7 应用 harness 兼容补丁(不改动任何上游源码)" | |
| $PYTHON - "$SNF_CODE_ROOT" <<'PATCH' | |
| import sys | |
| from pathlib import Path | |
| root = Path(sys.argv[1]) | |
| # 补丁一:MoMask 上游代码使用 NumPy 1.24 已移除的 np.float / np.int 等旧别名。 | |
| # 只在 harness 进程内恢复别名,vendor/ 下 397 个上游文件的校验值保持不变。 | |
| momask = root / 'suite/momask_loading.py' | |
| anchor = "def loading_functions():" | |
| shim = '''def numpy_legacy_aliases(): | |
| """Restore aliases NumPy removed in 1.24; upstream MoMask still uses np.float/np.int. | |
| Applied in the harness process only. Vendored upstream files are untouched, so | |
| packaging/sources.manifest.json keeps verifying all 397 upstream files. | |
| """ | |
| import numpy as np | |
| for name, replacement in (('float', float), ('int', int), ('bool', bool), | |
| ('object', object), ('str', str), ('complex', complex)): | |
| if not hasattr(np, name): | |
| setattr(np, name, replacement) | |
| return np | |
| numpy_legacy_aliases() | |
| ''' | |
| if 'numpy_legacy_aliases' in momask.read_text(): | |
| print(' ✓ momask numpy 兼容补丁已存在') | |
| else: | |
| text = momask.read_text() | |
| assert anchor in text, 'momask_loading.py 结构变化,请人工确认' | |
| momask.write_text(text.replace(anchor, shim + anchor, 1)) | |
| print(' ✓ 已写入 momask numpy 兼容补丁') | |
| # 补丁二:SD3.5 上游只提供 fp16 的 CLIP 文本编码器,diffusers 只把 text_projection | |
| # 转成 bf16,前向时 dtype 冲突(Half != BFloat16)。统一到管线声明的 bf16。 | |
| adapters = root / 'suite/adapters.py' | |
| old = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda') | |
| self.pipe.set_progress_bar_config(disable=True)""" | |
| new = """ self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda') | |
| # Upstream ships the two CLIP encoders as fp16 only; diffusers casts their | |
| # text_projection to bf16 but leaves the encoder body fp16, which raises a | |
| # dtype mismatch in the forward. Align them with the declared pipeline dtype. | |
| for name in ('text_encoder','text_encoder_2','text_encoder_3'): | |
| module=getattr(self.pipe,name,None) | |
| if module is not None:module.to(torch.bfloat16) | |
| self.pipe.set_progress_bar_config(disable=True)""" | |
| text = adapters.read_text() | |
| if 'Upstream ships the two CLIP encoders' in text: | |
| print(' ✓ sd35 dtype 补丁已存在') | |
| else: | |
| assert old in text, 'adapters.py 结构变化,请人工确认' | |
| adapters.write_text(text.replace(old, new, 1)) | |
| print(' ✓ 已写入 sd35 dtype 补丁') | |
| PATCH | |
| cat > "$SNF_DEPLOY_ROOT/snf.sh" <<EOS | |
| #!/usr/bin/env bash | |
| # 实验入口(原生替代 packaging/start.sh):bash snf.sh <command> ... | |
| set -euo pipefail | |
| export PYTHONPATH="$SNF_CODE_ROOT" | |
| export HF_HOME=/weights/huggingface TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache | |
| export TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton | |
| export MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 | |
| export WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT="\${HF_ENDPOINT:-https://huggingface.co}" | |
| # HF_TOKEN 由调用方内联传入,不落盘 | |
| cd "$SNF_CODE_ROOT" | |
| exec $PYTHON -m suite.cli "\$@" | |
| EOS | |
| chmod +x "$SNF_DEPLOY_ROOT/snf.sh" | |
| ok "实验入口已生成 $SNF_DEPLOY_ROOT/snf.sh" | |
| step "5/7 下载并锁定模型权重" | |
| if [ "$SNF_SKIP_WEIGHTS" = 1 ]; then | |
| warn "SNF_SKIP_WEIGHTS=1,跳过权重下载" | |
| else | |
| LOCK=/weights/weights.lock.json | |
| [ -f "$LOCK" ] || echo '{}' > "$LOCK" | |
| # 串行执行:fetch 会对 weights.lock.json 做读-改-写,并发会丢条目 | |
| for model in "${MODELS[@]}"; do | |
| echo " · fetch $model" | |
| if ! snf fetch --models "$model" --weights /weights >"$LOG_DIR/fetch_$model.log" 2>&1; then | |
| echo " fetch $model 失败,详见 $LOG_DIR/fetch_$model.log"; tail -3 "$LOG_DIR/fetch_$model.log" || true | |
| fi | |
| done | |
| python3 - "$LOCK" <<'CHECK' | |
| import json, sys, os | |
| lock = json.load(open(sys.argv[1])) | |
| expect = { | |
| 'MeissonFlow/Meissonic': None, 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K': None, | |
| 'Wan-AI/Wan2.1-T2V-1.3B': None, 'gdhe17/Self-Forcing': None, | |
| 'showlab/show-o': None, 'showlab/magvitv2': None, 'microsoft/phi-1_5': None, | |
| 'stabilityai/stable-diffusion-3.5-medium': None, 'momask_archive_sha256': None, | |
| } | |
| missing = [k for k in expect if k not in lock] | |
| print(' 已锁定 %d/%d 个权重源' % (len(lock), len(expect))) | |
| if missing: | |
| print(' 缺失:', missing) | |
| sys.exit(1) | |
| checks = { | |
| 'MoMask 主模型': '/weights/momask/checkpoints/t2m/t2m_nlayer8_nhead6_ld384_ff1024_cdp0.1_rvq6ns/opt.txt', | |
| 'MoMask 残差模型': '/weights/momask/checkpoints/t2m/tres_nlayer8_ld384_ff1024_rvq6ns_cdp0.2_sw/opt.txt', | |
| 'Self-Forcing 学生权重': '/weights/self_forcing/checkpoints/self_forcing_dmd.pt', | |
| 'Wan2.1 基础模型': '/weights/self_forcing/wan_models/Wan2.1-T2V-1.3B/config.json', | |
| } | |
| for name, path in checks.items(): | |
| print(' %s %s' % ('✓' if os.path.exists(path) else '✗', name)) | |
| if not os.path.exists(path): | |
| sys.exit(1) | |
| CHECK | |
| [ $? -eq 0 ] || die "权重不完整,请查看 $LOG_DIR/fetch_*.log" | |
| ok "全部权重就绪(HF revision 已锁定在 /weights/weights.lock.json)" | |
| rm -rf /weights/huggingface/xet 2>/dev/null && ok "已清理 xet 分块缓存(约 9GB,不影响已下载权重)" | |
| fi | |
| step "6/7 验收:源码 / 五模型导入 / GPU" | |
| snf source-check >/dev/null && ok "源码校验 397 文件通过" | |
| for model in "${MODELS[@]}"; do | |
| OUT="$(/opt/envs/$model/bin/python -m suite.cli import-check --model "$model" 2>&1 | tail -1)" | |
| case "$OUT" in *'"imports": "passed"'*) ok "$model 导入通过";; *) warn "$model 导入失败: ${OUT:0:200}";; esac | |
| done | |
| if [ "$SNF_DOCTOR" = 1 ]; then | |
| if snf doctor >"$LOG_DIR/doctor.json" 2>&1; then | |
| $PYTHON - "$LOG_DIR/doctor.json" <<'DOC' | |
| import json, sys | |
| raw = open(sys.argv[1]).read() | |
| d = json.loads(raw[raw.index('{'):]) | |
| print(' ✓ CUDA available=%s kernel_finite=%s' % (d['cuda_available'], d['kernel_finite'])) | |
| print(' ✓ GPU: %s' % ', '.join('%s %.0fGiB' % (g['name'], g['memory_gib']) for g in d['gpus'])) | |
| bad = [k for k, v in d['models'].items() if v['returncode']] | |
| print(' ✓ 五模型环境导入: %s' % ('全部通过' if not bad else '失败 ' + str(bad))) | |
| DOC | |
| else | |
| warn "doctor 未通过,详见 $LOG_DIR/doctor.json" | |
| fi | |
| fi | |
| if [ "$SNF_WITH_TASKS" = 1 ]; then | |
| step "7/7 生成默认任务清单(可选,属于实验输入)" | |
| snf create-tasks --output /data/tasks.curated.jsonl && ok "已生成 /data/tasks.curated.jsonl(120 test + 8 dev)" | |
| else | |
| step "7/7 完成(未生成任务清单,SNF_WITH_TASKS=1 可开启)" | |
| fi | |
| cat <<SUMMARY | |
| ==================== 部署完成 ==================== | |
| 源码 : $SNF_CODE_ROOT | |
| 存储 : $SNF_STORAGE/{weights,data,outputs} (/weights /data /outputs 为软链) | |
| 模型环境 : /opt/envs/{meissonic,self_forcing,momask,sd35,showo}(各含 installed.freeze.txt) | |
| 权重锁 : /weights/weights.lock.json | |
| 实验入口 : $SNF_DEPLOY_ROOT/snf.sh | |
| 部署日志 : $LOG | |
| 跑实验(需内联传入 HF_TOKEN): | |
| cd $SNF_DEPLOY_ROOT | |
| HF_TOKEN=xxx bash snf.sh plan --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full-plan | |
| HF_TOKEN=xxx bash snf.sh run --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full \\ | |
| --render-motion --resume --gpus 0,1,2 | |
| 参考耗时(本机 3×H20 实测):Meissonic 4.2s/条、Show-o 2.2s/条、MoMask 0.15s/条、 | |
| SD3.5 11.6s/条、Self-Forcing 17.1s/条;full 主实验 27000 条 ≈ 12h,含 extras 49660 条 ≈ 24h。 | |
| ================================================== | |
| SUMMARY | |