File size: 16,211 Bytes
ea1b5d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
#!/usr/bin/env bash
# =============================================================================
# SNF 五模型实验环境 一键部署脚本(原生运行,不依赖 Docker)
#
# 目标机器:Linux x86_64 + NVIDIA GPU(驱动 >= 535,CUDA 12.8/12.9 运行时)+
#           宿主 Python 3.12 且已装 torch 2.7.1(+cu128/cu129)
#
# 本脚本只做“部署”,不跑任何实验:
#   1. 下载并解压官方构建源码包,校验 397 个上游文件
#   2. 建立 /weights /data /outputs 存储布局
#   3. 建立五个模型各自隔离的 /opt/envs/<model> 虚拟环境(继承宿主 torch)
#   4. 应用两处必要的 harness 兼容补丁(不改动上游源码)
#   5. 下载并锁定全部模型权重(HF + MoMask 官方 Google Drive)
#   6. 验收:源码校验 + 五模型导入 + doctor(GPU 实测)
#
# 用法:
#   export HF_TOKEN='hf_xxx'                       # 必需(SD3.5 为 gated 仓库)
#   bash deploy.sh                                  # 全量部署
#   SNF_SKIP_WEIGHTS=1 bash deploy.sh               # 只装环境不下载权重
#   SNF_WITH_TASKS=1 bash deploy.sh                 # 额外生成默认任务清单
#   SNF_STORAGE=/mnt/snf-runtime bash deploy.sh     # 换存储根目录
#
# 前置条件(脚本会检查并在不满足时退出):
#   - HF_TOKEN 对应的账号已在 https://huggingface.co/stabilityai/stable-diffusion-3.5-medium
#     页面点击 "Agree and access repository",否则 SD3.5 权重会 403
#   - 磁盘可用空间 >= 150 GB(实测权重 + 缓存约 100 GB)
#   - 能访问 huggingface.co / github.com / drive.google.com(脚本继承宿主 http_proxy 配置)
#
# 幂等:每一步都有完成标记,中断后重跑会跳过已完成部分。
# =============================================================================
set -euo pipefail

SNF_DEPLOY_ROOT="${SNF_DEPLOY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
SNF_CODE_ROOT="${SNF_CODE_ROOT:-$SNF_DEPLOY_ROOT/snf-five-models}"
SNF_STORAGE="${SNF_STORAGE:-$SNF_DEPLOY_ROOT/runtime}"
SNF_ZIP_URL="${SNF_ZIP_URL:-https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/snf-five-models-cu128-build.zip}"
SNF_ZIP="$SNF_DEPLOY_ROOT/snf-five-models-cu128-build.zip"
HF_ENDPOINT="${HF_ENDPOINT:-https://huggingface.co}"
PYTHON="${PYTHON:-python3}"
MODELS=(meissonic self_forcing momask sd35 showo)
SNF_SKIP_WEIGHTS="${SNF_SKIP_WEIGHTS:-0}"
SNF_WITH_TASKS="${SNF_WITH_TASKS:-0}"
SNF_DOCTOR="${SNF_DOCTOR:-1}"

LOG_DIR="$SNF_DEPLOY_ROOT/logs"; mkdir -p "$LOG_DIR"
LOG="$LOG_DIR/deploy.$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1

step(){ printf '\n\033[1;36m[%s] === %s ===\033[0m\n' "$(date +%H:%M:%S)" "$*"; }
ok(){ printf '  \033[32m✓\033[0m %s\n' "$*"; }
warn(){ printf '  \033[33m!\033[0m %s\n' "$*"; }
die(){ printf '  \033[31m✗ %s\033[0m\n' "$*"; exit 1; }

step "0/7 前置检查"
[ "$(id -u)" = 0 ] || warn "非 root:/opt/envs 与 /weights 可能无法写入"
command -v "$PYTHON" >/dev/null || die "找不到 python3(可用 PYTHON= 指定)"
command -v nvidia-smi >/dev/null || die "找不到 nvidia-smi:本机无 GPU 驱动"
GPU_COUNT="$(nvidia-smi -L 2>/dev/null | wc -l)"
[ "$GPU_COUNT" -gt 0 ] || die "nvidia-smi 未识别到 GPU"
ok "GPU: ${GPU_COUNT} 张 -> $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | tr -d '\r')"

PYV="$($PYTHON -c 'import sys;print("%d.%d"%sys.version_info[:2])')"
ok "Python $PYV ($(command -v $PYTHON))"
TORCHV="$($PYTHON -c 'import torch;print(torch.__version__,torch.version.cuda)' 2>/dev/null || echo none)"
[ "$TORCHV" = none ] && die "宿主 Python 没有 torch;镜像要求 torch==2.7.1(+cu128/cu129),五个虚拟环境都继承它"
ok "宿主 torch $TORCHV"

for tool in curl unzip git; do command -v $tool >/dev/null || die "缺少系统工具 $tool"; done
ok "curl / unzip / git 就绪"

FREE_GB="$(df -Pk "$SNF_DEPLOY_ROOT" | awk 'NR==2{print int($4/1048576)}')"
if [ "$FREE_GB" -lt 150 ]; then warn "可用空间仅 ${FREE_GB}GB,建议 >=150GB(权重+缓存约100GB)"; else ok "可用空间 ${FREE_GB}GB"; fi

if [ -z "${HF_TOKEN:-}" ]; then die "未设置 HF_TOKEN:SD3.5 是 gated 仓库,且匿名访问会被 429 限流"; fi
WHOAMI="$(curl -s --max-time 30 -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/api/whoami-v2" | head -c 200)"
case "$WHOAMI" in *'"name"'*) ok "HF_TOKEN 有效 -> $(printf '%s' "$WHOAMI" | sed -n 's/.*"name":"\([^"]*\)".*/\1/p')";; *) die "HF_TOKEN 无效或网络不通: $WHOAMI";; esac
SD35_CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -L -H "Authorization: Bearer $HF_TOKEN" "$HF_ENDPOINT/stabilityai/stable-diffusion-3.5-medium/resolve/main/model_index.json")"
if [ "$SD35_CODE" != "200" ]; then
  die "SD3.5 权重不可访问(HTTP $SD35_CODE)。请先在浏览器打开
       https://huggingface.co/stabilityai/stable-diffusion-3.5-medium 点 Agree and access repository"
fi
ok "SD3.5 权重可访问(授权已生效)"

step "1/7 下载并解压构建源码包"
mkdir -p "$SNF_DEPLOY_ROOT"
if [ ! -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ]; then
  [ -f "$SNF_ZIP" ] || curl -fL --retry 3 -o "$SNF_ZIP" "$SNF_ZIP_URL"
  unzip -q -o "$SNF_ZIP" -d "$SNF_DEPLOY_ROOT"
else
  ok "源码已存在,跳过下载"
fi
[ -f "$SNF_CODE_ROOT/packaging/sources.manifest.json" ] || die "解压失败:缺少 sources.manifest.json"
ok "源码包就绪 $SNF_CODE_ROOT"

step "2/7 建立存储布局"
mkdir -p "$SNF_STORAGE"/{weights,data,outputs}
for pair in "weights:/weights" "data:/data" "outputs:/outputs"; do
  name="${pair%%:*}"; link="${pair##*:}"; target="$SNF_STORAGE/$name"
  if [ -L "$link" ] && [ "$(readlink -f "$link")" = "$(readlink -f "$target")" ]; then continue; fi
  if [ -d "$link" ] && [ ! -L "$link" ] && [ -n "$(ls -A "$link" 2>/dev/null)" ]; then
    warn "$link 已存在且非空(宿主目录),保留原目录不建软链"
    continue
  fi
  rm -rf "$link"; ln -sfn "$target" "$link"
done
ok "存储布局就绪 -> $SNF_STORAGE/{weights,data,outputs}"

export PYTHONPATH="$SNF_CODE_ROOT" HF_HOME=/weights/huggingface \
       TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache \
       TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton \
       MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1 \
       WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT
mkdir -p /weights/huggingface
cd "$SNF_CODE_ROOT"
snf(){ ( cd "$SNF_CODE_ROOT" && $PYTHON -m suite.cli "$@" ); }

VERIFY="$(snf source-check 2>/dev/null | tr -d '\n' || true)"
case "$VERIFY" in *'"verified_files": 397'*) ok "源码完整性校验通过(397 个上游文件)";; *) die "源码校验失败: ${VERIFY:0:200}";; esac

step "3/7 创建五个隔离模型环境 /opt/envs/<model>"
REQ="$SNF_CODE_ROOT/packaging/requirements-common.txt"
REQ_SUM="$(sha256sum "$REQ" | cut -c1-16)"
mkdir -p /opt/envs
for model in "${MODELS[@]}"; do
  ENV="/opt/envs/$model"; STAMP="$ENV/.req.$REQ_SUM"
  if [ -x "$ENV/bin/python" ] && [ -f "$ENV/installed.freeze.txt" ] && [ -f "$STAMP" ]; then
    ok "$model 环境已就绪,跳过"; continue
  fi
  echo "  · 创建 $model 环境(继承宿主 torch,只隔离 Python API 层)"
  rm -rf "$ENV"
  "$PYTHON" -m venv --system-site-packages "$ENV"
  "$ENV/bin/pip" install --no-cache-dir -r "$REQ" >"$LOG_DIR/env_$model.log" 2>&1 \
    || { echo "    pip 失败,详见 $LOG_DIR/env_$model.log"; continue; }
  touch "$STAMP"
  ok "$model 依赖安装完成"
done

echo "  · Show-o 专属旧版栈(transformers 4.41.2 / diffusers 0.30.1)"
/opt/envs/showo/bin/pip install --no-cache-dir \
  transformers==4.41.2 tokenizers==0.19.1 peft==0.11.1 diffusers==0.30.1 \
  >"$LOG_DIR/env_showo_override.log" 2>&1 || echo "    Show-o 覆盖安装失败,详见日志"
echo "  · MoMask 官方 CLIP(固定 commit)"
/opt/envs/momask/bin/pip install --no-cache-dir \
  'git+https://github.com/openai/CLIP.git@dcba3cb2e2827b402d2701e7e1c7d9fed8a20ef1' \
  >"$LOG_DIR/env_momask_clip.log" 2>&1 || echo "    CLIP 安装失败,详见日志"

MISSING_ENV=0
for model in "${MODELS[@]}"; do
  [ -x "/opt/envs/$model/bin/python" ] || { warn "/opt/envs/$model 缺失"; MISSING_ENV=1; continue; }
  "/opt/envs/$model/bin/pip" freeze >"/opt/envs/$model/installed.freeze.txt"
done
[ "$MISSING_ENV" = 0 ] || die "有模型环境未建成,请查看 $LOG_DIR/env_*.log"

step "4/7 应用 harness 兼容补丁(不改动任何上游源码)"
$PYTHON - "$SNF_CODE_ROOT" <<'PATCH'
import sys
from pathlib import Path
root = Path(sys.argv[1])

# 补丁一:MoMask 上游代码使用 NumPy 1.24 已移除的 np.float / np.int 等旧别名。
# 只在 harness 进程内恢复别名,vendor/ 下 397 个上游文件的校验值保持不变。
momask = root / 'suite/momask_loading.py'
anchor = "def loading_functions():"
shim = '''def numpy_legacy_aliases():
    """Restore aliases NumPy removed in 1.24; upstream MoMask still uses np.float/np.int.

    Applied in the harness process only. Vendored upstream files are untouched, so
    packaging/sources.manifest.json keeps verifying all 397 upstream files.
    """
    import numpy as np
    for name, replacement in (('float', float), ('int', int), ('bool', bool),
                              ('object', object), ('str', str), ('complex', complex)):
        if not hasattr(np, name):
            setattr(np, name, replacement)
    return np

numpy_legacy_aliases()

'''
if 'numpy_legacy_aliases' in momask.read_text():
    print('  ✓ momask numpy 兼容补丁已存在')
else:
    text = momask.read_text()
    assert anchor in text, 'momask_loading.py 结构变化,请人工确认'
    momask.write_text(text.replace(anchor, shim + anchor, 1))
    print('  ✓ 已写入 momask numpy 兼容补丁')

# 补丁二:SD3.5 上游只提供 fp16 的 CLIP 文本编码器,diffusers 只把 text_projection
# 转成 bf16,前向时 dtype 冲突(Half != BFloat16)。统一到管线声明的 bf16。
adapters = root / 'suite/adapters.py'
old = """        self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
        self.pipe.set_progress_bar_config(disable=True)"""
new = """        self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
        # Upstream ships the two CLIP encoders as fp16 only; diffusers casts their
        # text_projection to bf16 but leaves the encoder body fp16, which raises a
        # dtype mismatch in the forward. Align them with the declared pipeline dtype.
        for name in ('text_encoder','text_encoder_2','text_encoder_3'):
            module=getattr(self.pipe,name,None)
            if module is not None:module.to(torch.bfloat16)
        self.pipe.set_progress_bar_config(disable=True)"""
text = adapters.read_text()
if 'Upstream ships the two CLIP encoders' in text:
    print('  ✓ sd35 dtype 补丁已存在')
else:
    assert old in text, 'adapters.py 结构变化,请人工确认'
    adapters.write_text(text.replace(old, new, 1))
    print('  ✓ 已写入 sd35 dtype 补丁')
PATCH

cat > "$SNF_DEPLOY_ROOT/snf.sh" <<EOS
#!/usr/bin/env bash
# 实验入口(原生替代 packaging/start.sh):bash snf.sh <command> ...
set -euo pipefail
export PYTHONPATH="$SNF_CODE_ROOT"
export HF_HOME=/weights/huggingface TORCH_HOME=/weights/torch XDG_CACHE_HOME=/weights/cache
export TORCHINDUCTOR_CACHE_DIR=/weights/torchinductor TRITON_CACHE_DIR=/weights/triton
export MPLBACKEND=Agg TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_TELEMETRY=1
export WANDB_MODE=disabled PYTHONUNBUFFERED=1 HF_ENDPOINT="\${HF_ENDPOINT:-https://huggingface.co}"
# HF_TOKEN 由调用方内联传入,不落盘
cd "$SNF_CODE_ROOT"
exec $PYTHON -m suite.cli "\$@"
EOS
chmod +x "$SNF_DEPLOY_ROOT/snf.sh"
ok "实验入口已生成 $SNF_DEPLOY_ROOT/snf.sh"

step "5/7 下载并锁定模型权重"
if [ "$SNF_SKIP_WEIGHTS" = 1 ]; then
  warn "SNF_SKIP_WEIGHTS=1,跳过权重下载"
else
  LOCK=/weights/weights.lock.json
  [ -f "$LOCK" ] || echo '{}' > "$LOCK"
  # 串行执行:fetch 会对 weights.lock.json 做读-改-写,并发会丢条目
  for model in "${MODELS[@]}"; do
    echo "  · fetch $model"
    if ! snf fetch --models "$model" --weights /weights >"$LOG_DIR/fetch_$model.log" 2>&1; then
      echo "    fetch $model 失败,详见 $LOG_DIR/fetch_$model.log"; tail -3 "$LOG_DIR/fetch_$model.log" || true
    fi
  done
  python3 - "$LOCK" <<'CHECK'
import json, sys, os
lock = json.load(open(sys.argv[1]))
expect = {
    'MeissonFlow/Meissonic': None, 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K': None,
    'Wan-AI/Wan2.1-T2V-1.3B': None, 'gdhe17/Self-Forcing': None,
    'showlab/show-o': None, 'showlab/magvitv2': None, 'microsoft/phi-1_5': None,
    'stabilityai/stable-diffusion-3.5-medium': None, 'momask_archive_sha256': None,
}
missing = [k for k in expect if k not in lock]
print('  已锁定 %d/%d 个权重源' % (len(lock), len(expect)))
if missing:
    print('  缺失:', missing)
    sys.exit(1)
checks = {
    'MoMask 主模型': '/weights/momask/checkpoints/t2m/t2m_nlayer8_nhead6_ld384_ff1024_cdp0.1_rvq6ns/opt.txt',
    'MoMask 残差模型': '/weights/momask/checkpoints/t2m/tres_nlayer8_ld384_ff1024_rvq6ns_cdp0.2_sw/opt.txt',
    'Self-Forcing 学生权重': '/weights/self_forcing/checkpoints/self_forcing_dmd.pt',
    'Wan2.1 基础模型': '/weights/self_forcing/wan_models/Wan2.1-T2V-1.3B/config.json',
}
for name, path in checks.items():
    print('  %s %s' % ('✓' if os.path.exists(path) else '✗', name))
    if not os.path.exists(path):
        sys.exit(1)
CHECK
  [ $? -eq 0 ] || die "权重不完整,请查看 $LOG_DIR/fetch_*.log"
  ok "全部权重就绪(HF revision 已锁定在 /weights/weights.lock.json)"
  rm -rf /weights/huggingface/xet 2>/dev/null && ok "已清理 xet 分块缓存(约 9GB,不影响已下载权重)"
fi

step "6/7 验收:源码 / 五模型导入 / GPU"
snf source-check >/dev/null && ok "源码校验 397 文件通过"
for model in "${MODELS[@]}"; do
  OUT="$(/opt/envs/$model/bin/python -m suite.cli import-check --model "$model" 2>&1 | tail -1)"
  case "$OUT" in *'"imports": "passed"'*) ok "$model 导入通过";; *) warn "$model 导入失败: ${OUT:0:200}";; esac
done
if [ "$SNF_DOCTOR" = 1 ]; then
  if snf doctor >"$LOG_DIR/doctor.json" 2>&1; then
    $PYTHON - "$LOG_DIR/doctor.json" <<'DOC'
import json, sys
raw = open(sys.argv[1]).read()
d = json.loads(raw[raw.index('{'):])
print('  ✓ CUDA available=%s kernel_finite=%s' % (d['cuda_available'], d['kernel_finite']))
print('  ✓ GPU: %s' % ', '.join('%s %.0fGiB' % (g['name'], g['memory_gib']) for g in d['gpus']))
bad = [k for k, v in d['models'].items() if v['returncode']]
print('  ✓ 五模型环境导入: %s' % ('全部通过' if not bad else '失败 ' + str(bad)))
DOC
  else
    warn "doctor 未通过,详见 $LOG_DIR/doctor.json"
  fi
fi

if [ "$SNF_WITH_TASKS" = 1 ]; then
  step "7/7 生成默认任务清单(可选,属于实验输入)"
  snf create-tasks --output /data/tasks.curated.jsonl && ok "已生成 /data/tasks.curated.jsonl(120 test + 8 dev)"
else
  step "7/7 完成(未生成任务清单,SNF_WITH_TASKS=1 可开启)"
fi

cat <<SUMMARY

==================== 部署完成 ====================
源码      : $SNF_CODE_ROOT
存储      : $SNF_STORAGE/{weights,data,outputs}  (/weights /data /outputs 为软链)
模型环境  : /opt/envs/{meissonic,self_forcing,momask,sd35,showo}(各含 installed.freeze.txt)
权重锁    : /weights/weights.lock.json
实验入口  : $SNF_DEPLOY_ROOT/snf.sh
部署日志  : $LOG

跑实验(需内联传入 HF_TOKEN):
  cd $SNF_DEPLOY_ROOT
  HF_TOKEN=xxx bash snf.sh plan --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full-plan
  HF_TOKEN=xxx bash snf.sh run  --profile full --tasks /data/tasks.curated.jsonl --output /outputs/full \\
      --render-motion --resume --gpus 0,1,2
参考耗时(本机 3×H20 实测):Meissonic 4.2s/条、Show-o 2.2s/条、MoMask 0.15s/条、
SD3.5 11.6s/条、Self-Forcing 17.1s/条;full 主实验 27000 条 ≈ 12h,含 extras 49660 条 ≈ 24h。
==================================================
SUMMARY