Spaces:
Sleeping
Sleeping
Deploy Space
Browse files- .gitignore +26 -0
- MODEL_SCOPE_DEPLOY.md +299 -0
- README.md +68 -7
- app.py +282 -0
- blockdiffusion/__init__.py +9 -0
- blockdiffusion/data/__init__.py +18 -0
- blockdiffusion/data/captions.py +71 -0
- blockdiffusion/data/filename_captions.py +67 -0
- blockdiffusion/data/texture_caption_dataset.py +119 -0
- blockdiffusion/data/texture_folder.py +76 -0
- blockdiffusion/diffusion/__init__.py +5 -0
- blockdiffusion/diffusion/gaussian_diffusion.py +253 -0
- blockdiffusion/diffusion/schedules.py +43 -0
- blockdiffusion/models/README.md +143 -0
- blockdiffusion/models/__init__.py +15 -0
- blockdiffusion/models/text_unet16.py +67 -0
- blockdiffusion/models/unet16.py +272 -0
- blockdiffusion/text/__init__.py +6 -0
- blockdiffusion/text/simple_tokenizer.py +144 -0
- blockdiffusion/text/text_encoder.py +64 -0
- blockdiffusion/train/__init__.py +14 -0
- blockdiffusion/train/guidance.py +37 -0
- blockdiffusion/train/trainer.py +311 -0
- blockdiffusion/utils/__init__.py +5 -0
- blockdiffusion/utils/checkpoint.py +23 -0
- blockdiffusion/utils/ema.py +70 -0
- blockdiffusion/utils/image_io.py +39 -0
- blockdiffusion/utils/seed.py +36 -0
- modelscope_ui_app.py +235 -0
- requirements.txt +11 -0
- sample.py +138 -0
- sample_cpu.py +194 -0
- train.py +197 -0
- train_oneclick.py +311 -0
- webui.py +522 -0
.gitignore
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.idea/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
*.pyo
|
| 5 |
+
*.pyd
|
| 6 |
+
.Python
|
| 7 |
+
build/
|
| 8 |
+
dist/
|
| 9 |
+
*.egg-info/
|
| 10 |
+
.venv/
|
| 11 |
+
venv/
|
| 12 |
+
|
| 13 |
+
checkpoints/
|
| 14 |
+
samples/
|
| 15 |
+
outputs/
|
| 16 |
+
|
| 17 |
+
*.pt
|
| 18 |
+
*.pth
|
| 19 |
+
*.ckpt
|
| 20 |
+
|
| 21 |
+
*.png
|
| 22 |
+
*.jpg
|
| 23 |
+
*.jpeg
|
| 24 |
+
*.webp
|
| 25 |
+
|
| 26 |
+
|
MODEL_SCOPE_DEPLOY.md
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## ModelScope 快速部署指南(SwingDeploy / ModelScope Pipeline / 函数计算 FC)
|
| 2 |
+
|
| 3 |
+
本文面向“接手者/AI 代理”,目标是让你在不了解本仓库历史的前提下,也能把 **BlockDiffusion** 作为 **ModelScope Pipeline** 形态快速部署到 **SwingDeploy → FC**。
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
### 0. 目标与前置假设
|
| 8 |
+
|
| 9 |
+
#### 0.1 目标
|
| 10 |
+
|
| 11 |
+
- 让 ModelScope 能在云端自动:
|
| 12 |
+
- 安装依赖(`requirements.txt`)
|
| 13 |
+
- 拉取模型仓库文件(含权重)
|
| 14 |
+
- 通过 **ModelScope Pipeline** 标准接口加载并提供 HTTP 服务
|
| 15 |
+
- 最终实现:
|
| 16 |
+
- **SwingDeploy 一键部署**(选择 ModelScope Pipeline 框架)
|
| 17 |
+
- 部署目标可选:**函数计算 FC / PAI-EAS / ENS**
|
| 18 |
+
|
| 19 |
+
#### 0.2 前置假设
|
| 20 |
+
|
| 21 |
+
- 你已经有一个可用的 `.pt` checkpoint(训练脚本产物)。
|
| 22 |
+
- 该 checkpoint 遵循本仓库保存格式(推荐),即包含 `meta`(见下文),这样推理侧可以自动还原:
|
| 23 |
+
- 是否 text_cond
|
| 24 |
+
- UNet 配置、扩散调度、通道数等
|
| 25 |
+
|
| 26 |
+
---
|
| 27 |
+
|
| 28 |
+
### 1. 关键思路:allow_remote 外部组件
|
| 29 |
+
|
| 30 |
+
本仓库的模型并不属于 ModelScope 主库内置模型类型,因此要走 **外部组件注册**方案:
|
| 31 |
+
|
| 32 |
+
- 在 `configuration.json` 中启用:
|
| 33 |
+
- `"allow_remote": true`
|
| 34 |
+
- 在仓库根目录提供 `ms_wrapper.py`:
|
| 35 |
+
- 用 `MODELS.register_module(...)` 注册自定义 Model
|
| 36 |
+
- 用 `PIPELINES.register_module(...)` 注册自定义 Pipeline
|
| 37 |
+
|
| 38 |
+
ModelScope 在加载该模型时会根据 `configuration.json` 的 `task / model.type / pipeline.type` 找到并加载 `ms_wrapper.py` 里同名注册的组件,从而让 SwingDeploy 能“识别这是可部署的 Pipeline 模型”。
|
| 39 |
+
|
| 40 |
+
---
|
| 41 |
+
|
| 42 |
+
### 2. 仓库根目录最小文件结构(必需)
|
| 43 |
+
|
| 44 |
+
以下文件需位于 **模型仓库根目录**(不是子目录):
|
| 45 |
+
|
| 46 |
+
- `configuration.json`
|
| 47 |
+
- 声明 `framework / task / model.type / pipeline.type / allow_remote`
|
| 48 |
+
- `ms_wrapper.py`
|
| 49 |
+
- 实现并注册自定义 Model + Pipeline
|
| 50 |
+
- `requirements.txt`
|
| 51 |
+
- 推理运行依赖(必须包含 `modelscope`)
|
| 52 |
+
- `pytorch_model.bin`(强烈建议)
|
| 53 |
+
- 模型权重文件,建议统一命名,减少部署时的歧义
|
| 54 |
+
- `README.md`
|
| 55 |
+
- 模型卡片(建议包含 tasks 标签;可选配置在线 Demo widget)
|
| 56 |
+
|
| 57 |
+
> 本仓库已提供 `configuration.json` 与 `ms_wrapper.py` 的基础实现;你只需要按约定上传权重,并把这些文件推送到 ModelScope 仓库。
|
| 58 |
+
|
| 59 |
+
---
|
| 60 |
+
|
| 61 |
+
### 3. configuration.json 规范(与 ms_wrapper.py 强绑定)
|
| 62 |
+
|
| 63 |
+
本仓库当前配置采用:
|
| 64 |
+
|
| 65 |
+
- `task`: `"text-to-image-synthesis"`
|
| 66 |
+
- `model.type`: `"blockdiffusion-text2img-model"`
|
| 67 |
+
- `pipeline.type`: `"blockdiffusion-text2img-pipeline"`
|
| 68 |
+
- `allow_remote`: `true`
|
| 69 |
+
|
| 70 |
+
关键约束:
|
| 71 |
+
|
| 72 |
+
- `model.type` 必须与 `ms_wrapper.py` 里 `@MODELS.register_module(task, module_name=...)` 的 `module_name` 一致。
|
| 73 |
+
- `pipeline.type` 必须与 `ms_wrapper.py` 里 `@PIPELINES.register_module(task, module_name=...)` 的 `module_name` 一致。
|
| 74 |
+
- `task` 必须在两处一致(否则 Registry 维度不匹配)。
|
| 75 |
+
|
| 76 |
+
如果你想换 task 名称:
|
| 77 |
+
|
| 78 |
+
- 必须同时修改:
|
| 79 |
+
- `configuration.json` 的 `task`
|
| 80 |
+
- `ms_wrapper.py` 两个注册装饰器里的 task(当前写死为 `"text-to-image-synthesis"`)
|
| 81 |
+
|
| 82 |
+
---
|
| 83 |
+
|
| 84 |
+
### 4. 权重文件命名与 checkpoint 格式(强烈建议)
|
| 85 |
+
|
| 86 |
+
#### 4.1 权重命名
|
| 87 |
+
|
| 88 |
+
部署时建议把目标 checkpoint 重命名为:
|
| 89 |
+
|
| 90 |
+
- `pytorch_model.bin`
|
| 91 |
+
|
| 92 |
+
并放在模型仓库根目录。
|
| 93 |
+
|
| 94 |
+
原因:
|
| 95 |
+
|
| 96 |
+
- 这是 ModelScope/transformers 生态里常见约定;
|
| 97 |
+
- 能避免仓库内存在多个 `.pt` 时选错权重;
|
| 98 |
+
- 本仓库的 `ms_wrapper.py` 会优先寻找该文件。
|
| 99 |
+
|
| 100 |
+
#### 4.2 checkpoint 字段(本仓库推荐格式)
|
| 101 |
+
|
| 102 |
+
推荐的 checkpoint 是一个 dict,至少包含:
|
| 103 |
+
|
| 104 |
+
- `model`: `state_dict`
|
| 105 |
+
- `meta`: dict(用于推理时重建配置)
|
| 106 |
+
- 可选:`ema`(采样质量更稳,推理侧会优先使用)
|
| 107 |
+
|
| 108 |
+
`meta` 常见字段(训练脚本已按此保存):
|
| 109 |
+
|
| 110 |
+
- `model_type`: `"text_cond"` 或(其他/缺省)无条件
|
| 111 |
+
- `unet_cfg`: `UNet16Config` 的字典
|
| 112 |
+
- `text_cfg`: `TextCondConfig` 的字典(仅 text_cond)
|
| 113 |
+
- `tokenizer`: `SimpleVocab.to_dict()`(仅 text_cond)
|
| 114 |
+
- `diffusion`: `{"timesteps": ..., "beta_schedule": ...}`
|
| 115 |
+
- `data`: `{"image_size": 16, "channels": 3/4}`
|
| 116 |
+
|
| 117 |
+
如果 checkpoint 缺少 `meta/unet_cfg`:
|
| 118 |
+
|
| 119 |
+
- 推理侧会回退到默认 16×16 RGB 的无条件配置
|
| 120 |
+
- 这会导致“通道数/条件/调度”等信息丢失,部署能跑但结果可能不符合预期
|
| 121 |
+
|
| 122 |
+
---
|
| 123 |
+
|
| 124 |
+
### 5. ms_wrapper.py 的职责边界(为什么必须有它)
|
| 125 |
+
|
| 126 |
+
`ms_wrapper.py` 是“让 ModelScope 识别如何推理”的关键文件,它做了四件事:
|
| 127 |
+
|
| 128 |
+
1. **加载 checkpoint**
|
| 129 |
+
- 在 `model_dir` 里寻找 `pytorch_model.bin` 等候选
|
| 130 |
+
2. **重建网络**
|
| 131 |
+
- `model_type == text_cond` → `TextCondUNet16`
|
| 132 |
+
- 否则 → `UNet16`
|
| 133 |
+
3. **构建 diffusion**
|
| 134 |
+
- 根据 `meta.diffusion` 还原 `timesteps/beta_schedule`
|
| 135 |
+
4. **提供 Pipeline 标准输出**
|
| 136 |
+
- 输出字典里包含:
|
| 137 |
+
- `OutputKeys.OUTPUT_IMG`:单张图(HWC uint8 ndarray)
|
| 138 |
+
- `OutputKeys.OUTPUT_IMGS`:多张图(List[HWC uint8 ndarray])
|
| 139 |
+
|
| 140 |
+
这样 SwingDeploy 才能把服务做成统一的 HTTP 形式并返回可序列化结果。
|
| 141 |
+
|
| 142 |
+
---
|
| 143 |
+
|
| 144 |
+
### 6. Pipeline 的输入输出协议(服务调用层面)
|
| 145 |
+
|
| 146 |
+
#### 6.1 输入(prompt)
|
| 147 |
+
|
| 148 |
+
Pipeline 支持以下输入:
|
| 149 |
+
|
| 150 |
+
- 直接传字符串(等价 prompt)
|
| 151 |
+
- 传 dict,支持键:
|
| 152 |
+
- `{"text": "xxx"}`
|
| 153 |
+
- `{"prompt": "xxx"}`
|
| 154 |
+
|
| 155 |
+
#### 6.2 参数(parameters)
|
| 156 |
+
|
| 157 |
+
当前支持的常用参数(都走 Pipeline 的 `parameters`):
|
| 158 |
+
|
| 159 |
+
- `n`: 生成张数(默认 1)
|
| 160 |
+
- `batch`: 单次采样 batch(默认 1)
|
| 161 |
+
- `use_ddim`: 是否用 DDIM(默认 True)
|
| 162 |
+
- `ddim_steps`: DDIM 步数(默认 50)
|
| 163 |
+
- `ddim_eta`: DDIM eta(默认 0)
|
| 164 |
+
- `cfg_scale`: CFG 强度(默认 3,仅 text_cond + prompt 时生效)
|
| 165 |
+
- `seed`: 随机种子(可选)
|
| 166 |
+
|
| 167 |
+
建议:
|
| 168 |
+
|
| 169 |
+
- FC 场景默认以“低延迟”为目标:
|
| 170 |
+
- `n=1`
|
| 171 |
+
- `batch=1`
|
| 172 |
+
- `ddim_steps` 适当调小(例如 20~50)
|
| 173 |
+
|
| 174 |
+
#### 6.3 输出
|
| 175 |
+
|
| 176 |
+
输出是标准 dict:
|
| 177 |
+
|
| 178 |
+
- `OutputKeys.OUTPUT_IMG`:一张图片(HWC,uint8,RGB 或 RGBA)
|
| 179 |
+
- `OutputKeys.OUTPUT_IMGS`:多张图片(List[HWC uint8])
|
| 180 |
+
|
| 181 |
+
注意:
|
| 182 |
+
|
| 183 |
+
- RGBA 会输出 4 通道;
|
| 184 |
+
- 下游服务需要支持带 alpha 的 image array 序列化(ModelScope 服务链路通常可处理)。
|
| 185 |
+
|
| 186 |
+
---
|
| 187 |
+
|
| 188 |
+
### 7. 将本地仓库正确推到 ModelScope(操作要点)
|
| 189 |
+
|
| 190 |
+
#### 7.1 你必须确认的内容
|
| 191 |
+
|
| 192 |
+
- 仓库根目录存在:
|
| 193 |
+
- `configuration.json`
|
| 194 |
+
- `ms_wrapper.py`
|
| 195 |
+
- `requirements.txt`
|
| 196 |
+
- `pytorch_model.bin`
|
| 197 |
+
- `README.md`
|
| 198 |
+
- `requirements.txt` 里包含 `modelscope`,并补齐 `numpy` 等必需依赖
|
| 199 |
+
- `configuration.json` 的 `task/model.type/pipeline.type` 与 `ms_wrapper.py` 注册一致
|
| 200 |
+
|
| 201 |
+
#### 7.2 README.md(模型卡片)建议
|
| 202 |
+
|
| 203 |
+
建议在 README 的 YAML header 中声明 tasks(用于平台识别任务类型),例如:
|
| 204 |
+
|
| 205 |
+
- task 选择与 `configuration.json.task` 一致(本仓库是 text-to-image-synthesis)
|
| 206 |
+
|
| 207 |
+
并可选配置 widget,让网页侧出现“在线体验”。
|
| 208 |
+
|
| 209 |
+
---
|
| 210 |
+
|
| 211 |
+
### 8. SwingDeploy → FC 一键部署(平台侧操作要点)
|
| 212 |
+
|
| 213 |
+
#### 8.1 选择部署方式
|
| 214 |
+
|
| 215 |
+
在模型页面点击“部署”:
|
| 216 |
+
|
| 217 |
+
- 选择 **快速部署(SwingDeploy)**
|
| 218 |
+
- 框架选择 **ModelScope Pipeline**
|
| 219 |
+
- 目标资源选择 **函数计算(FC)**
|
| 220 |
+
|
| 221 |
+
#### 8.2 资源与性能建议
|
| 222 |
+
|
| 223 |
+
- **强烈建议 GPU**:扩散采样是循环推理,CPU 延迟通常不可接受
|
| 224 |
+
- **冷启动**:
|
| 225 |
+
- PyTorch 依赖体积大、初始化重
|
| 226 |
+
- 建议使用平台提供的“预留/常驻”能力减少冷启动
|
| 227 |
+
- **超时**:
|
| 228 |
+
- 降低 `ddim_steps`
|
| 229 |
+
- 限制 `n/batch`
|
| 230 |
+
|
| 231 |
+
---
|
| 232 |
+
|
| 233 |
+
### 9. 常见故障与定位(按症状)
|
| 234 |
+
|
| 235 |
+
#### 9.1 部署时导入失败(ImportError / ModuleNotFoundError)
|
| 236 |
+
|
| 237 |
+
优先检查:
|
| 238 |
+
|
| 239 |
+
- `requirements.txt` 是否包含:
|
| 240 |
+
- `modelscope`
|
| 241 |
+
- `numpy`
|
| 242 |
+
- `torch/torchvision/pillow/tqdm`
|
| 243 |
+
- 是否存在额外依赖(例如某些 CUDA/系统库)在 FC 环境不可用
|
| 244 |
+
|
| 245 |
+
#### 9.2 找不到权重文件
|
| 246 |
+
|
| 247 |
+
症状:
|
| 248 |
+
|
| 249 |
+
- 报错提示未找到 `pytorch_model.bin` 等
|
| 250 |
+
|
| 251 |
+
处理:
|
| 252 |
+
|
| 253 |
+
- 确保权重放在仓库根目录
|
| 254 |
+
- 建议统一命名为 `pytorch_model.bin`
|
| 255 |
+
|
| 256 |
+
#### 9.3 state_dict 不匹配(size mismatch / missing keys)
|
| 257 |
+
|
| 258 |
+
常见原因:
|
| 259 |
+
|
| 260 |
+
- 使用了与权重不匹配的 `UNet16Config`(例如通道数 3/4 不一致)
|
| 261 |
+
- 续训/导出时 tokenizer/vocab 改了导致 TextEncoder embedding 尺寸变化
|
| 262 |
+
|
| 263 |
+
处理:
|
| 264 |
+
|
| 265 |
+
- 确保 checkpoint 内 `meta` 完整,推理侧会复用 `meta.unet_cfg/tokenizer/text_cfg`
|
| 266 |
+
- 不要在续训时重建新词表(会破坏 embedding 尺寸)
|
| 267 |
+
|
| 268 |
+
#### 9.4 prompt 无效 / CFG 无效
|
| 269 |
+
|
| 270 |
+
常见原因:
|
| 271 |
+
|
| 272 |
+
- 部署的是无条件 checkpoint(`meta.model_type != text_cond`)
|
| 273 |
+
|
| 274 |
+
处理:
|
| 275 |
+
|
| 276 |
+
- 用带 captions 训练出来的 text_cond checkpoint
|
| 277 |
+
|
| 278 |
+
#### 9.5 FC 超时 / 资源不足
|
| 279 |
+
|
| 280 |
+
处理方向:
|
| 281 |
+
|
| 282 |
+
- 降低 `ddim_steps`(优先)
|
| 283 |
+
- 强制 `n=1 batch=1`
|
| 284 |
+
- 选择更高规格 GPU 或改为 PAI-EAS
|
| 285 |
+
|
| 286 |
+
---
|
| 287 |
+
|
| 288 |
+
### 10. 对接下一个 AI 的“最短执行清单”
|
| 289 |
+
|
| 290 |
+
1. 把最终 checkpoint 重命名为 `pytorch_model.bin`,放仓库根目录
|
| 291 |
+
2. 确认根目录存在 `configuration.json / ms_wrapper.py / requirements.txt`
|
| 292 |
+
3. 确认 `configuration.json` 的 `task/model.type/pipeline.type` 与 `ms_wrapper.py` 注册一致
|
| 293 |
+
4. 推送到 ModelScope 模型仓库
|
| 294 |
+
5. 在模型页面:SwingDeploy → ModelScope Pipeline → FC 一键部署
|
| 295 |
+
6. 调用时传:
|
| 296 |
+
- `input.text`(prompt)
|
| 297 |
+
- `parameters.ddim_steps / cfg_scale / n / batch / seed`
|
| 298 |
+
|
| 299 |
+
|
README.md
CHANGED
|
@@ -1,12 +1,73 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji: 📊
|
| 4 |
-
colorFrom: purple
|
| 5 |
-
colorTo: red
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.3.0
|
| 8 |
app_file: app.py
|
| 9 |
-
pinned: false
|
| 10 |
---
|
| 11 |
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: BlockDiffusion API
|
|
|
|
|
|
|
|
|
|
| 3 |
sdk: gradio
|
|
|
|
| 4 |
app_file: app.py
|
|
|
|
| 5 |
---
|
| 6 |
|
| 7 |
+
## BlockDiffusion
|
| 8 |
+
|
| 9 |
+
目标:训练一个专用于生成 **16×16《我的世界》纹理** 的小型 Diffusion(UNet)模型(PyTorch),支持 **提示词条件生成** 与 **RGBA 透明通道**。
|
| 10 |
+
|
| 11 |
+
### Hugging Face Spaces(在线API + 伪私有)
|
| 12 |
+
- **Space入口文件**:`app.py`
|
| 13 |
+
- **伪私有方式**:在 Space 的 `Settings -> Secrets` 配置 `SPACE_API_KEY`,所有推理请求必须携带该 Key 才会执行。
|
| 14 |
+
- **checkpoint来源(二选一)**:
|
| 15 |
+
- 配置 `BD_CKPT_PATH`(Space仓库内的文件路径)
|
| 16 |
+
- 或配置 `BD_CKPT_REPO` + `BD_CKPT_FILENAME`(从 Hugging Face Hub 下载;私有仓库可再配 `HF_TOKEN`)
|
| 17 |
+
|
| 18 |
+
### 当前能力(已实现)
|
| 19 |
+
- **16×16 专用 UNet 扩散**:避免过深/过度下采样(16→8→4)
|
| 20 |
+
- **噪声调度**:linear / cosine
|
| 21 |
+
- **loss**:基础为 MSE,支持可选 SNR 加权(提升稳定性/质量)
|
| 22 |
+
- **训练/采样解耦**:Diffusion 公式与采样逻辑独立于训练器
|
| 23 |
+
- **文本条件(提示词)**:
|
| 24 |
+
- 支持 `captions.jsonl/csv`(图文配对)
|
| 25 |
+
- 支持“从文件名自动提取提示词”的弱标注训练(见 `train_oneclick.py`)
|
| 26 |
+
- 支持 CFG(训练丢条件 + 采样 scale)
|
| 27 |
+
- **RGB/RGBA**:`channels=3/4`,RGBA 输出 PNG 透明通道
|
| 28 |
+
|
| 29 |
+
### 主要入口
|
| 30 |
+
- **一键训练(推荐)**:`train_oneclick.py`
|
| 31 |
+
- **无需命令行参数**,只修改脚本顶部常量
|
| 32 |
+
- 默认从 `DATA_DIR` 扫描图片,并按文件名生成提示词训练
|
| 33 |
+
- 默认支持 **自动续训**:扫描 `outputs/<RUN_NAME>/checkpoints/step_*.pt` 选择最大 step 继续
|
| 34 |
+
- **可参数化训练**:`train.py`(保留)
|
| 35 |
+
- **采样**:`sample.py`(支持 `--prompt` + CFG)
|
| 36 |
+
- **WebUI 采样预览**:`webui.py`(浏览器里快速对比 checkpoint)
|
| 37 |
+
|
| 38 |
+
### 数据与提示词规则
|
| 39 |
+
#### 方式 A:文件名弱标注(`train_oneclick.py`)
|
| 40 |
+
适配命名:`HASH-tag_tag_tag.png`、`HASH-side.png`、`HASH-top.png` 等。
|
| 41 |
+
|
| 42 |
+
提取规则(实现于 `blockdiffusion/data/filename_captions.py`):
|
| 43 |
+
- 去扩展名
|
| 44 |
+
- 取第一个 `-` 后的部分
|
| 45 |
+
- `_`/`-` 视为分隔符(转空格),转小写
|
| 46 |
+
|
| 47 |
+
#### 方式 B:显式 captions(`train.py --captions`)
|
| 48 |
+
- `captions.jsonl`:每行 `{"file":"相对data_dir的路径或绝对路径","text":"提示词"}`
|
| 49 |
+
- `captions.csv`:两列 `file,text`(含表头)
|
| 50 |
+
|
| 51 |
+
### 输出与预览
|
| 52 |
+
- checkpoints:`outputs/<RUN_NAME>/checkpoints/step_XXXX.pt`
|
| 53 |
+
- 训练中采样图:`outputs/<RUN_NAME>/samples/step_XXXX.png`
|
| 54 |
+
- 采样脚本输出:默认 `samples/samples.png`
|
| 55 |
+
|
| 56 |
+
### 核心模块导航(给接手者)
|
| 57 |
+
- **模型文档**:见 `blockdiffusion/models/README.md`(只讲模型:结构/输入输出/条件/ckpt 格式)
|
| 58 |
+
- `blockdiffusion/models/unet16.py`:16×16 UNet(支持条件向量注入)
|
| 59 |
+
- `blockdiffusion/models/text_unet16.py`:TextCondUNet16(tokens/mask→cond→UNet)
|
| 60 |
+
- `blockdiffusion/text/simple_tokenizer.py`:轻量 tokenizer/vocab(可随 checkpoint 保存)
|
| 61 |
+
- `blockdiffusion/text/text_encoder.py`:轻量 TextEncoder(masked mean pooling + MLP)
|
| 62 |
+
- `blockdiffusion/diffusion/schedules.py`:beta schedule(linear/cosine)
|
| 63 |
+
- `blockdiffusion/diffusion/gaussian_diffusion.py`:训练损失 + DDPM/DDIM 采样 + CFG 训练丢条件
|
| 64 |
+
- `blockdiffusion/train/trainer.py`:AMP/EMA/checkpoint/采样、梯度累积、warmup+余弦、channels_last
|
| 65 |
+
- `blockdiffusion/utils/ema.py`:EMA(续训时自动对齐 device)
|
| 66 |
+
- `blockdiffusion/utils/image_io.py`:保存 RGB/RGBA 网格 PNG
|
| 67 |
+
|
| 68 |
+
### 常用提速/提质开关(修改 `train_oneclick.py` 顶部常量)
|
| 69 |
+
- **提速**:`NUM_WORKERS`、`PREFETCH_FACTOR`、`PERSISTENT_WORKERS`、`CHANNELS_LAST`
|
| 70 |
+
- **显存/吞吐**:`BATCH_SIZE`、`GRAD_ACCUM_STEPS`
|
| 71 |
+
- **质量/稳定**:`SNR_GAMMA`、`LR_WARMUP_STEPS`、`LR_MIN_RATIO`、`COND_DROP_PROB`、`SAMPLE_CFG_SCALE`
|
| 72 |
+
|
| 73 |
+
|
app.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
Hugging Face Spaces 入口(Gradio)。
|
| 5 |
+
|
| 6 |
+
目标:
|
| 7 |
+
- 复用本项目现有“加载 checkpoint -> DDPM/DDIM 采样 -> 输出图片”的逻辑;
|
| 8 |
+
- 增加“伪私有”鉴权:所有推理请求必须提供正确的 API_KEY,否则拒绝;
|
| 9 |
+
- 支持从 Hugging Face Hub 下载 checkpoint(便于 Spaces 部署,不必把大文件提交到 Space 仓库)。
|
| 10 |
+
|
| 11 |
+
重要说明(伪私有的边界):
|
| 12 |
+
- Space 本身仍然是公开的(你选择 public 的情况下),只是把推理能力用 API_KEY 做访问控制;
|
| 13 |
+
- 拿到 API_KEY 的人仍可调用(这是预期)。
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import hmac
|
| 17 |
+
import os
|
| 18 |
+
from typing import Optional, Tuple, List
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
import gradio as gr
|
| 22 |
+
from huggingface_hub import hf_hub_download
|
| 23 |
+
from PIL import Image
|
| 24 |
+
|
| 25 |
+
# 复用现有逻辑(不重复造轮子)
|
| 26 |
+
from webui import _load_cached, _tensor_to_pil # type: ignore
|
| 27 |
+
from blockdiffusion.train.guidance import CFGuidanceModel
|
| 28 |
+
from blockdiffusion.text.simple_tokenizer import tokenize
|
| 29 |
+
from blockdiffusion.utils.seed import seed_everything, get_device
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _get_env(name: str, default: str = "") -> str:
|
| 33 |
+
v = os.getenv(name, default)
|
| 34 |
+
return (v or "").strip()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _require_space_api_key() -> str:
|
| 38 |
+
"""
|
| 39 |
+
Space 的“伪私有”Key:建议在 Hugging Face Space Settings -> Secrets 里配置。
|
| 40 |
+
"""
|
| 41 |
+
key = _get_env("SPACE_API_KEY", "")
|
| 42 |
+
if not key:
|
| 43 |
+
# 失败更安全:避免误部署成“无鉴权公开推理”
|
| 44 |
+
raise RuntimeError("缺少环境变量 SPACE_API_KEY(请在 Space Secrets 中配置)")
|
| 45 |
+
return key
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
_RESOLVED_CKPT_PATH: Optional[str] = None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _resolve_ckpt_path() -> str:
|
| 52 |
+
"""
|
| 53 |
+
获取 checkpoint 路径:
|
| 54 |
+
- 优先使用本地路径 BD_CKPT_PATH
|
| 55 |
+
- 否则从 Hugging Face Hub 下载:BD_CKPT_REPO + BD_CKPT_FILENAME(可选 BD_CKPT_REVISION)
|
| 56 |
+
|
| 57 |
+
说明:
|
| 58 |
+
- 若模型仓库是私有/受限,需要在 Space Secrets 设置 HF_TOKEN(或 HUGGINGFACE_HUB_TOKEN)
|
| 59 |
+
"""
|
| 60 |
+
global _RESOLVED_CKPT_PATH
|
| 61 |
+
if _RESOLVED_CKPT_PATH and os.path.isfile(_RESOLVED_CKPT_PATH):
|
| 62 |
+
return _RESOLVED_CKPT_PATH
|
| 63 |
+
|
| 64 |
+
local_path = _get_env("BD_CKPT_PATH", "")
|
| 65 |
+
if local_path:
|
| 66 |
+
if not os.path.isfile(local_path):
|
| 67 |
+
raise RuntimeError(f"BD_CKPT_PATH 指向的文件不存在:{local_path}")
|
| 68 |
+
_RESOLVED_CKPT_PATH = local_path
|
| 69 |
+
return _RESOLVED_CKPT_PATH
|
| 70 |
+
|
| 71 |
+
repo_id = _get_env("BD_CKPT_REPO", "")
|
| 72 |
+
# 约定优先使用生态常见的权重名,减少歧义
|
| 73 |
+
filename = _get_env("BD_CKPT_FILENAME", "pytorch_model.bin")
|
| 74 |
+
revision = _get_env("BD_CKPT_REVISION", "") or None
|
| 75 |
+
if not repo_id:
|
| 76 |
+
raise RuntimeError("未配置 checkpoint:请设置 BD_CKPT_PATH 或 BD_CKPT_REPO/BD_CKPT_FILENAME")
|
| 77 |
+
|
| 78 |
+
token = _get_env("HF_TOKEN", "") or _get_env("HUGGINGFACE_HUB_TOKEN", "") or None
|
| 79 |
+
try:
|
| 80 |
+
path = hf_hub_download(repo_id=repo_id, filename=filename, revision=revision, token=token)
|
| 81 |
+
except Exception as e:
|
| 82 |
+
raise RuntimeError(f"从Hub下载checkpoint失败:repo={repo_id} file={filename} rev={revision} err={e!r}") from e
|
| 83 |
+
|
| 84 |
+
if not os.path.isfile(path):
|
| 85 |
+
raise RuntimeError(f"下载完成但文件不存在:{path}")
|
| 86 |
+
_RESOLVED_CKPT_PATH = path
|
| 87 |
+
return path
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _check_api_key(user_key: str) -> None:
|
| 91 |
+
"""
|
| 92 |
+
伪私有鉴权:对外只暴露“成功/失败”,避免泄漏真实 Key。
|
| 93 |
+
"""
|
| 94 |
+
expected = _require_space_api_key()
|
| 95 |
+
user_key = (user_key or "").strip()
|
| 96 |
+
if not user_key or not hmac.compare_digest(user_key, expected):
|
| 97 |
+
raise gr.Error("未授权:API_KEY错误")
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
@torch.no_grad()
|
| 101 |
+
def _infer(
|
| 102 |
+
api_key: str,
|
| 103 |
+
prompt: str,
|
| 104 |
+
cfg_scale: float,
|
| 105 |
+
use_ema: bool,
|
| 106 |
+
use_ddim: bool,
|
| 107 |
+
ddim_steps: int,
|
| 108 |
+
ddim_eta: float,
|
| 109 |
+
n_images: int,
|
| 110 |
+
batch: int,
|
| 111 |
+
seed: int,
|
| 112 |
+
preview_upscale: int,
|
| 113 |
+
) -> Tuple[str, Image.Image, List[Image.Image]]:
|
| 114 |
+
# 1) 鉴权
|
| 115 |
+
_check_api_key(api_key)
|
| 116 |
+
|
| 117 |
+
# 2) 参数整理
|
| 118 |
+
n_images = int(max(1, n_images))
|
| 119 |
+
batch = int(max(1, batch))
|
| 120 |
+
preview_upscale = int(max(1, preview_upscale))
|
| 121 |
+
seed_everything(int(seed), deterministic=False)
|
| 122 |
+
|
| 123 |
+
# 3) 定位 ckpt + device
|
| 124 |
+
ckpt_path = _resolve_ckpt_path()
|
| 125 |
+
# Spaces 上尽量自动选择(有GPU则用GPU)
|
| 126 |
+
device = get_device(None)
|
| 127 |
+
device_text = str(device)
|
| 128 |
+
|
| 129 |
+
# 4) 复用 webui 的缓存加载(包含:读取meta、构建模型、EMA覆盖、构建diffusion)
|
| 130 |
+
loaded = _load_cached(ckpt_path=ckpt_path, device_text=device_text, use_ema=bool(use_ema))
|
| 131 |
+
|
| 132 |
+
# 5) 条件采样(prompt + CFG)
|
| 133 |
+
sampler_model: torch.nn.Module = loaded.model
|
| 134 |
+
prompt = (prompt or "").strip()
|
| 135 |
+
if prompt:
|
| 136 |
+
if loaded.model_type != "text_cond":
|
| 137 |
+
raise gr.Error("该checkpoint为无条件模型,不支持prompt/CFG")
|
| 138 |
+
if loaded.vocab is None or loaded.text_cfg is None:
|
| 139 |
+
raise gr.Error("条件模型缺少tokenizer/text_cfg")
|
| 140 |
+
|
| 141 |
+
tok = torch.tensor(
|
| 142 |
+
[loaded.vocab.encode(prompt, max_len=loaded.text_cfg.max_text_len)],
|
| 143 |
+
dtype=torch.long,
|
| 144 |
+
device=device,
|
| 145 |
+
)
|
| 146 |
+
m = tok != loaded.vocab.pad_id
|
| 147 |
+
cond_kwargs = {"tokens": tok, "mask": m}
|
| 148 |
+
un_tok = torch.full_like(tok, fill_value=loaded.vocab.pad_id)
|
| 149 |
+
un_m = torch.zeros_like(m)
|
| 150 |
+
uncond_kwargs = {"tokens": un_tok, "mask": un_m}
|
| 151 |
+
sampler_model = CFGuidanceModel(
|
| 152 |
+
base_model=loaded.model,
|
| 153 |
+
cond_kwargs=cond_kwargs,
|
| 154 |
+
uncond_kwargs=uncond_kwargs,
|
| 155 |
+
scale=float(cfg_scale),
|
| 156 |
+
).to(device)
|
| 157 |
+
sampler_model.eval()
|
| 158 |
+
|
| 159 |
+
# 6) 采样(分批)
|
| 160 |
+
all_imgs: List[torch.Tensor] = []
|
| 161 |
+
remaining = n_images
|
| 162 |
+
while remaining > 0:
|
| 163 |
+
b = min(batch, remaining)
|
| 164 |
+
shape = (b, loaded.channels, loaded.image_size, loaded.image_size)
|
| 165 |
+
if use_ddim:
|
| 166 |
+
x = loaded.diffusion.ddim_sample_loop(
|
| 167 |
+
sampler_model,
|
| 168 |
+
shape=shape,
|
| 169 |
+
device=device,
|
| 170 |
+
steps=int(ddim_steps),
|
| 171 |
+
eta=float(ddim_eta),
|
| 172 |
+
model_kwargs=None,
|
| 173 |
+
progress=False,
|
| 174 |
+
)
|
| 175 |
+
else:
|
| 176 |
+
x = loaded.diffusion.p_sample_loop(
|
| 177 |
+
sampler_model,
|
| 178 |
+
shape=shape,
|
| 179 |
+
device=device,
|
| 180 |
+
model_kwargs=None,
|
| 181 |
+
progress=False,
|
| 182 |
+
)
|
| 183 |
+
x = (x.clamp(-1, 1) + 1) * 0.5 # [0,1]
|
| 184 |
+
all_imgs.append(x.detach().cpu())
|
| 185 |
+
remaining -= b
|
| 186 |
+
|
| 187 |
+
imgs = torch.cat(all_imgs, dim=0) # (N,C,H,W) on CPU
|
| 188 |
+
|
| 189 |
+
# 7) 输出:网格图 + gallery(按像素纹理预览做NEAREST放大)
|
| 190 |
+
# 为了不重复引入 torchvision/make_grid,这里直接拼接成单行网格(N很小时足够)。
|
| 191 |
+
# 如果你需要更复杂网格,可直接复用 webui.py 的 make_grid 逻辑。
|
| 192 |
+
gallery: List[Image.Image] = []
|
| 193 |
+
for i in range(n_images):
|
| 194 |
+
im = _tensor_to_pil(imgs[i])
|
| 195 |
+
if preview_upscale > 1:
|
| 196 |
+
w, h = im.size
|
| 197 |
+
im = im.resize((w * preview_upscale, h * preview_upscale), resample=Image.NEAREST)
|
| 198 |
+
gallery.append(im)
|
| 199 |
+
|
| 200 |
+
# 简单拼图:横向拼接(避免引入额外依赖/逻辑)
|
| 201 |
+
total_w = sum(im.size[0] for im in gallery)
|
| 202 |
+
max_h = max(im.size[1] for im in gallery)
|
| 203 |
+
grid = Image.new("RGBA", (total_w, max_h))
|
| 204 |
+
x0 = 0
|
| 205 |
+
for im in gallery:
|
| 206 |
+
grid.paste(im, (x0, 0))
|
| 207 |
+
x0 += im.size[0]
|
| 208 |
+
|
| 209 |
+
# 提示词token诊断:是否大量<unk>(仅用于展示)
|
| 210 |
+
unk_info = ""
|
| 211 |
+
if prompt and loaded.vocab is not None:
|
| 212 |
+
toks = tokenize(prompt)
|
| 213 |
+
unk = [t for t in toks if int(loaded.vocab.token_to_id.get(t, loaded.vocab.unk_id)) == int(loaded.vocab.unk_id)]
|
| 214 |
+
if unk:
|
| 215 |
+
unk_info = f" unk_tokens={len(unk)}"
|
| 216 |
+
|
| 217 |
+
info = (
|
| 218 |
+
f"ckpt={os.path.basename(ckpt_path)} device={device_text} "
|
| 219 |
+
f"type={loaded.model_type} size={loaded.image_size} ch={loaded.channels} "
|
| 220 |
+
f"sampler={'DDIM' if use_ddim else 'DDPM'} n={n_images} batch={batch} seed={int(seed)} "
|
| 221 |
+
f"prompt={'(none)' if not prompt else prompt[:80]}{unk_info}"
|
| 222 |
+
)
|
| 223 |
+
return info, grid, gallery
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def build_demo() -> gr.Blocks:
|
| 227 |
+
"""
|
| 228 |
+
Gradio UI(同时提供 /api/predict 的在线调用能力)。
|
| 229 |
+
"""
|
| 230 |
+
with gr.Blocks(title="BlockDiffusion (Pseudo-Private)") as demo:
|
| 231 |
+
gr.Markdown("### BlockDiffusion Spaces(伪私有:需API_KEY)")
|
| 232 |
+
|
| 233 |
+
with gr.Row():
|
| 234 |
+
api_key = gr.Textbox(label="API_KEY(必填)", type="password")
|
| 235 |
+
with gr.Row():
|
| 236 |
+
prompt = gr.Textbox(label="Prompt(条件模型可用;留空=无条件)", value="", lines=2)
|
| 237 |
+
cfg_scale = gr.Slider(minimum=0.0, maximum=12.0, value=3.0, step=0.1, label="CFG scale")
|
| 238 |
+
|
| 239 |
+
with gr.Row():
|
| 240 |
+
use_ema = gr.Checkbox(value=True, label="使用EMA权重(推荐)")
|
| 241 |
+
use_ddim = gr.Checkbox(value=True, label="使用DDIM(更快)")
|
| 242 |
+
ddim_steps = gr.Slider(minimum=10, maximum=200, value=50, step=1, label="DDIM steps")
|
| 243 |
+
ddim_eta = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.01, label="DDIM eta")
|
| 244 |
+
|
| 245 |
+
with gr.Row():
|
| 246 |
+
n_images = gr.Slider(minimum=1, maximum=8, value=1, step=1, label="生成张数N(Spaces建议<=8)")
|
| 247 |
+
batch = gr.Slider(minimum=1, maximum=8, value=1, step=1, label="采样batch(Spaces建议<=8)")
|
| 248 |
+
seed = gr.Number(value=123, precision=0, label="seed")
|
| 249 |
+
preview_upscale = gr.Slider(minimum=1, maximum=64, value=16, step=1, label="预览放大倍数(NEAREST)")
|
| 250 |
+
|
| 251 |
+
run = gr.Button("生成")
|
| 252 |
+
info = gr.Textbox(label="信息", lines=2)
|
| 253 |
+
grid = gr.Image(label="网格图", type="pil")
|
| 254 |
+
gallery = gr.Gallery(label="单张结果")
|
| 255 |
+
|
| 256 |
+
def _run(*args):
|
| 257 |
+
return _infer(*args)
|
| 258 |
+
|
| 259 |
+
run.click(
|
| 260 |
+
_run,
|
| 261 |
+
inputs=[
|
| 262 |
+
api_key,
|
| 263 |
+
prompt,
|
| 264 |
+
cfg_scale,
|
| 265 |
+
use_ema,
|
| 266 |
+
use_ddim,
|
| 267 |
+
ddim_steps,
|
| 268 |
+
ddim_eta,
|
| 269 |
+
n_images,
|
| 270 |
+
batch,
|
| 271 |
+
seed,
|
| 272 |
+
preview_upscale,
|
| 273 |
+
],
|
| 274 |
+
outputs=[info, grid, gallery],
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
return demo
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
demo = build_demo()
|
| 281 |
+
|
| 282 |
+
|
blockdiffusion/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
BlockDiffusion:16×16 方块纹理专用扩散模型(PyTorch)。
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
__all__ = ["__version__"]
|
| 6 |
+
|
| 7 |
+
__version__ = "0.1.0"
|
| 8 |
+
|
| 9 |
+
|
blockdiffusion/data/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
数据集:从目录读取纹理图片并统一到 16×16。
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from .texture_folder import TextureFolderDataset
|
| 6 |
+
from .captions import CaptionItem, load_captions
|
| 7 |
+
from .texture_caption_dataset import TextureCaptionDataset
|
| 8 |
+
from .filename_captions import build_caption_items_from_filenames
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"TextureFolderDataset",
|
| 12 |
+
"CaptionItem",
|
| 13 |
+
"load_captions",
|
| 14 |
+
"TextureCaptionDataset",
|
| 15 |
+
"build_caption_items_from_filenames",
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
|
blockdiffusion/data/captions.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import csv
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
from typing import Iterable, List, Sequence, Tuple
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@dataclass(frozen=True)
|
| 11 |
+
class CaptionItem:
|
| 12 |
+
"""
|
| 13 |
+
一条图文配对样本:
|
| 14 |
+
- image_path:图片路径(绝对路径)
|
| 15 |
+
- text:文本描述/提示词
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
image_path: str
|
| 19 |
+
text: str
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def load_captions(captions_path: str, data_root: str) -> List[CaptionItem]:
|
| 23 |
+
"""
|
| 24 |
+
读取 caption 文件,支持:
|
| 25 |
+
- .jsonl:每行 {"file": "...", "text": "..."},file 可相对 data_root 或绝对路径
|
| 26 |
+
- .csv:两列 file,text(含表头),file 同上
|
| 27 |
+
"""
|
| 28 |
+
captions_path = os.path.abspath(captions_path)
|
| 29 |
+
data_root = os.path.abspath(data_root)
|
| 30 |
+
|
| 31 |
+
if not os.path.exists(captions_path):
|
| 32 |
+
raise FileNotFoundError(f"未找到 captions 文件:{captions_path}")
|
| 33 |
+
|
| 34 |
+
items: List[CaptionItem] = []
|
| 35 |
+
ext = os.path.splitext(captions_path)[1].lower()
|
| 36 |
+
|
| 37 |
+
def resolve_path(p: str) -> str:
|
| 38 |
+
p = (p or "").strip()
|
| 39 |
+
if not p:
|
| 40 |
+
return ""
|
| 41 |
+
if os.path.isabs(p):
|
| 42 |
+
return p
|
| 43 |
+
return os.path.join(data_root, p)
|
| 44 |
+
|
| 45 |
+
if ext == ".jsonl":
|
| 46 |
+
with open(captions_path, "r", encoding="utf-8") as f:
|
| 47 |
+
for line in f:
|
| 48 |
+
line = line.strip()
|
| 49 |
+
if not line:
|
| 50 |
+
continue
|
| 51 |
+
obj = json.loads(line)
|
| 52 |
+
img = resolve_path(str(obj.get("file", "")))
|
| 53 |
+
txt = str(obj.get("text", ""))
|
| 54 |
+
if img and os.path.exists(img):
|
| 55 |
+
items.append(CaptionItem(image_path=img, text=txt))
|
| 56 |
+
elif ext == ".csv":
|
| 57 |
+
with open(captions_path, "r", encoding="utf-8") as f:
|
| 58 |
+
reader = csv.DictReader(f)
|
| 59 |
+
for row in reader:
|
| 60 |
+
img = resolve_path(str(row.get("file", "")))
|
| 61 |
+
txt = str(row.get("text", ""))
|
| 62 |
+
if img and os.path.exists(img):
|
| 63 |
+
items.append(CaptionItem(image_path=img, text=txt))
|
| 64 |
+
else:
|
| 65 |
+
raise ValueError("captions 文件仅支持 .jsonl 或 .csv")
|
| 66 |
+
|
| 67 |
+
if len(items) == 0:
|
| 68 |
+
raise ValueError("captions 文件未读取到有效样本(请检查 file 路径与图片是否存在)")
|
| 69 |
+
return items
|
| 70 |
+
|
| 71 |
+
|
blockdiffusion/data/filename_captions.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import List, Optional
|
| 5 |
+
|
| 6 |
+
from blockdiffusion.data.captions import CaptionItem
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
_IMG_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".bmp")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _caption_from_filename(filename: str) -> str:
|
| 13 |
+
"""
|
| 14 |
+
从文件名提取“弱标注提示词”。
|
| 15 |
+
|
| 16 |
+
适配命名格式:
|
| 17 |
+
- HASH-tag_tag_tag.png
|
| 18 |
+
- HASH-side.png / HASH-top.png / HASH-bottom.png
|
| 19 |
+
|
| 20 |
+
提取规则:
|
| 21 |
+
- 去掉扩展名
|
| 22 |
+
- 以第一个 '-' 作为分隔,取其后部分(若不存在 '-' 则使用全名)
|
| 23 |
+
- '_' / '-' 统一当作空格
|
| 24 |
+
"""
|
| 25 |
+
name = os.path.splitext(os.path.basename(filename))[0]
|
| 26 |
+
if "-" in name:
|
| 27 |
+
# 只切一次,避免 tag 本身含 '-' 时被过度切分
|
| 28 |
+
_, tail = name.split("-", 1)
|
| 29 |
+
else:
|
| 30 |
+
tail = name
|
| 31 |
+
tail = tail.replace("_", " ").replace("-", " ").strip().lower()
|
| 32 |
+
# 多空格归一
|
| 33 |
+
tail = " ".join([t for t in tail.split(" ") if t])
|
| 34 |
+
return tail
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def build_caption_items_from_filenames(
|
| 38 |
+
root_dir: str,
|
| 39 |
+
verbose: bool = False,
|
| 40 |
+
log_every: int = 10_000,
|
| 41 |
+
) -> List[CaptionItem]:
|
| 42 |
+
"""
|
| 43 |
+
扫描目录下所有图片(含子目录),用文件名自动生成 captions。
|
| 44 |
+
返回 CaptionItem 列表,可直接用于 TextureCaptionDataset。
|
| 45 |
+
"""
|
| 46 |
+
root_dir = os.path.abspath(root_dir)
|
| 47 |
+
items: List[CaptionItem] = []
|
| 48 |
+
seen = 0
|
| 49 |
+
for dirpath, _, filenames in os.walk(root_dir):
|
| 50 |
+
for fn in filenames:
|
| 51 |
+
if not fn.lower().endswith(_IMG_EXTS):
|
| 52 |
+
continue
|
| 53 |
+
path = os.path.join(dirpath, fn)
|
| 54 |
+
text = _caption_from_filename(fn)
|
| 55 |
+
# 若提取失败就给空字符串;TextEncoder 会把空句子当作“无条件”
|
| 56 |
+
items.append(CaptionItem(image_path=path, text=text))
|
| 57 |
+
seen += 1
|
| 58 |
+
if verbose and log_every > 0 and (seen % int(log_every) == 0):
|
| 59 |
+
print(f"[scan] 已发现图片 {seen} 张...")
|
| 60 |
+
items.sort(key=lambda x: x.image_path)
|
| 61 |
+
if len(items) == 0:
|
| 62 |
+
raise ValueError(f"目录下未找到图片:{root_dir}")
|
| 63 |
+
if verbose:
|
| 64 |
+
print(f"[scan] 完成:共 {len(items)} 张图片")
|
| 65 |
+
return items
|
| 66 |
+
|
| 67 |
+
|
blockdiffusion/data/texture_caption_dataset.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import List, Optional, Tuple
|
| 5 |
+
|
| 6 |
+
from PIL import Image
|
| 7 |
+
import torch
|
| 8 |
+
from torch.utils.data import Dataset
|
| 9 |
+
from torchvision import transforms as T
|
| 10 |
+
|
| 11 |
+
from blockdiffusion.data.captions import CaptionItem
|
| 12 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class TextureCaptionDataset(Dataset):
|
| 16 |
+
"""
|
| 17 |
+
图文配对纹理数据集:
|
| 18 |
+
- 输入:CaptionItem 列表(包含 image_path 与 text)
|
| 19 |
+
- 输出:(image[-1,1], tokens, mask)
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
def __init__(
|
| 23 |
+
self,
|
| 24 |
+
items: List[CaptionItem],
|
| 25 |
+
vocab: SimpleVocab,
|
| 26 |
+
image_size: int = 16,
|
| 27 |
+
channels: int = 3,
|
| 28 |
+
max_text_len: int = 32,
|
| 29 |
+
random_flip: bool = True,
|
| 30 |
+
pretokenize: bool = False,
|
| 31 |
+
append_alpha_tag: bool = False,
|
| 32 |
+
alpha_tag_transparent: str = "transparent",
|
| 33 |
+
alpha_tag_opaque: str = "opaque",
|
| 34 |
+
verbose: bool = False,
|
| 35 |
+
log_every: int = 50_000,
|
| 36 |
+
):
|
| 37 |
+
if len(items) == 0:
|
| 38 |
+
raise ValueError("items 不能为空")
|
| 39 |
+
self.items = items
|
| 40 |
+
self.vocab = vocab
|
| 41 |
+
self.image_size = int(image_size)
|
| 42 |
+
self.channels = int(channels)
|
| 43 |
+
if self.channels not in (3, 4):
|
| 44 |
+
raise ValueError("channels 仅支持 3(RGB) 或 4(RGBA)")
|
| 45 |
+
self.max_text_len = int(max_text_len)
|
| 46 |
+
self.pretokenize = bool(pretokenize)
|
| 47 |
+
self.append_alpha_tag = bool(append_alpha_tag)
|
| 48 |
+
self.alpha_tag_transparent = str(alpha_tag_transparent or "").strip()
|
| 49 |
+
self.alpha_tag_opaque = str(alpha_tag_opaque or "").strip()
|
| 50 |
+
self.verbose = bool(verbose)
|
| 51 |
+
self.log_every = int(log_every)
|
| 52 |
+
|
| 53 |
+
# 透明/不透明标签需要在 __getitem__ 里基于图像 alpha 判定;
|
| 54 |
+
# pretokenize 不读取图像,因此两者不兼容。
|
| 55 |
+
if self.append_alpha_tag and self.pretokenize:
|
| 56 |
+
raise ValueError("append_alpha_tag 与 pretokenize 不兼容:请关闭 pretokenize")
|
| 57 |
+
|
| 58 |
+
tfms = [
|
| 59 |
+
T.Resize((self.image_size, self.image_size), interpolation=T.InterpolationMode.NEAREST),
|
| 60 |
+
]
|
| 61 |
+
if random_flip:
|
| 62 |
+
tfms.append(T.RandomHorizontalFlip(p=0.5))
|
| 63 |
+
tfms.extend(
|
| 64 |
+
[
|
| 65 |
+
T.ToTensor(),
|
| 66 |
+
# 用 Normalize 替代 lambda,保证 Windows 多进程 DataLoader 可 pickle
|
| 67 |
+
T.Normalize(mean=[0.5] * self.channels, std=[0.5] * self.channels),
|
| 68 |
+
]
|
| 69 |
+
)
|
| 70 |
+
self.transform = T.Compose(tfms)
|
| 71 |
+
|
| 72 |
+
# 可选:预编码 tokens/mask(大数据集会耗时/占内存,默认关闭)
|
| 73 |
+
self._tokens = None
|
| 74 |
+
self._mask = None
|
| 75 |
+
if self.pretokenize:
|
| 76 |
+
tokens_list: List[torch.Tensor] = []
|
| 77 |
+
mask_list: List[torch.Tensor] = []
|
| 78 |
+
for i, it in enumerate(items, start=1):
|
| 79 |
+
ids = vocab.encode(it.text, max_len=self.max_text_len)
|
| 80 |
+
tok = torch.tensor(ids, dtype=torch.long)
|
| 81 |
+
m = (tok != vocab.pad_id)
|
| 82 |
+
tokens_list.append(tok)
|
| 83 |
+
mask_list.append(m)
|
| 84 |
+
if self.verbose and self.log_every > 0 and (i % self.log_every == 0):
|
| 85 |
+
print(f"[pretok] 已预编码 {i} 条文本...")
|
| 86 |
+
self._tokens = tokens_list
|
| 87 |
+
self._mask = mask_list
|
| 88 |
+
if self.verbose:
|
| 89 |
+
print(f"[pretok] 完成:共 {len(items)} 条文本")
|
| 90 |
+
|
| 91 |
+
def __len__(self) -> int:
|
| 92 |
+
return len(self.items)
|
| 93 |
+
|
| 94 |
+
def __getitem__(self, idx: int):
|
| 95 |
+
it = self.items[idx]
|
| 96 |
+
with Image.open(it.image_path) as im:
|
| 97 |
+
im = im.convert("RGBA" if self.channels == 4 else "RGB")
|
| 98 |
+
x = self.transform(im)
|
| 99 |
+
if self._tokens is not None and self._mask is not None:
|
| 100 |
+
return x, self._tokens[idx], self._mask[idx]
|
| 101 |
+
# 未预编码:按需编码(更快启动、更省内存)
|
| 102 |
+
text = it.text
|
| 103 |
+
# 可选:自动追加透明/不透明标签,让模型学会“透明度受提示词影响”
|
| 104 |
+
if self.append_alpha_tag and self.channels == 4:
|
| 105 |
+
# im 已被 convert("RGBA")
|
| 106 |
+
alpha = im.getchannel("A")
|
| 107 |
+
mn, mx = alpha.getextrema()
|
| 108 |
+
# 只要出现 <255 就认为“含透明”
|
| 109 |
+
tag = self.alpha_tag_transparent if int(mn) < 255 else self.alpha_tag_opaque
|
| 110 |
+
tag = (tag or "").strip()
|
| 111 |
+
if tag:
|
| 112 |
+
text = f"{text} {tag}".strip() if text else tag
|
| 113 |
+
|
| 114 |
+
ids = self.vocab.encode(text, max_len=self.max_text_len)
|
| 115 |
+
tok = torch.tensor(ids, dtype=torch.long)
|
| 116 |
+
m = (tok != self.vocab.pad_id)
|
| 117 |
+
return x, tok, m
|
| 118 |
+
|
| 119 |
+
|
blockdiffusion/data/texture_folder.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import List, Sequence, Tuple
|
| 5 |
+
|
| 6 |
+
from PIL import Image
|
| 7 |
+
import torch
|
| 8 |
+
from torch.utils.data import Dataset
|
| 9 |
+
from torchvision import transforms as T
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
_IMG_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".bmp")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _list_images(root: str) -> List[str]:
|
| 16 |
+
paths: List[str] = []
|
| 17 |
+
for dirpath, _, filenames in os.walk(root):
|
| 18 |
+
for fn in filenames:
|
| 19 |
+
if fn.lower().endswith(_IMG_EXTS):
|
| 20 |
+
paths.append(os.path.join(dirpath, fn))
|
| 21 |
+
paths.sort()
|
| 22 |
+
return paths
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class TextureFolderDataset(Dataset):
|
| 26 |
+
"""
|
| 27 |
+
纹理图片数据集:
|
| 28 |
+
- 读取目录下所有图片(含子目录)
|
| 29 |
+
- 统一转换为 RGB
|
| 30 |
+
- 缩放到 16×16
|
| 31 |
+
- 输出范围为 [-1, 1] 的张量(C,H,W)
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
def __init__(
|
| 35 |
+
self,
|
| 36 |
+
root_dir: str,
|
| 37 |
+
image_size: int = 16,
|
| 38 |
+
channels: int = 3,
|
| 39 |
+
random_flip: bool = True,
|
| 40 |
+
):
|
| 41 |
+
self.root_dir = root_dir
|
| 42 |
+
self.image_size = int(image_size)
|
| 43 |
+
self.channels = int(channels)
|
| 44 |
+
if self.channels not in (3, 4):
|
| 45 |
+
raise ValueError("channels 仅支持 3(RGB) 或 4(RGBA)")
|
| 46 |
+
|
| 47 |
+
self.paths = _list_images(root_dir)
|
| 48 |
+
if len(self.paths) == 0:
|
| 49 |
+
raise ValueError(f"数据集目录为空或未找到图片:{root_dir}")
|
| 50 |
+
|
| 51 |
+
tfms = [
|
| 52 |
+
T.Resize((self.image_size, self.image_size), interpolation=T.InterpolationMode.NEAREST),
|
| 53 |
+
]
|
| 54 |
+
if random_flip:
|
| 55 |
+
tfms.append(T.RandomHorizontalFlip(p=0.5))
|
| 56 |
+
tfms.extend(
|
| 57 |
+
[
|
| 58 |
+
T.ToTensor(), # [0,1]
|
| 59 |
+
# 用 Normalize 替代 lambda,保证 Windows 多进程 DataLoader 可 pickle
|
| 60 |
+
T.Normalize(mean=[0.5] * self.channels, std=[0.5] * self.channels), # [-1,1]
|
| 61 |
+
]
|
| 62 |
+
)
|
| 63 |
+
self.transform = T.Compose(tfms)
|
| 64 |
+
|
| 65 |
+
def __len__(self) -> int:
|
| 66 |
+
return len(self.paths)
|
| 67 |
+
|
| 68 |
+
def __getitem__(self, idx: int) -> torch.Tensor:
|
| 69 |
+
path = self.paths[idx]
|
| 70 |
+
# 使用 nearest resize,尽量保留像素风格边缘
|
| 71 |
+
with Image.open(path) as im:
|
| 72 |
+
im = im.convert("RGBA" if self.channels == 4 else "RGB")
|
| 73 |
+
x = self.transform(im)
|
| 74 |
+
return x
|
| 75 |
+
|
| 76 |
+
|
blockdiffusion/diffusion/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
扩散核心:噪声调度(beta schedule)与 Gaussian Diffusion 公式实现。
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
|
blockdiffusion/diffusion/gaussian_diffusion.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Any, Dict, Optional, Tuple
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch import nn
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def _extract(a: torch.Tensor, t: torch.Tensor, x_shape: Tuple[int, ...]) -> torch.Tensor:
|
| 11 |
+
"""
|
| 12 |
+
从 [T] 的一维张量 a 中按 batch 的 t 抽取,并 reshape 到 x 的维度用于广播。
|
| 13 |
+
"""
|
| 14 |
+
b = t.shape[0]
|
| 15 |
+
out = a.gather(0, t)
|
| 16 |
+
return out.reshape(b, *((1,) * (len(x_shape) - 1)))
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass(frozen=True)
|
| 20 |
+
class DiffusionConfig:
|
| 21 |
+
"""
|
| 22 |
+
扩散配置。
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
timesteps: int = 1000
|
| 26 |
+
beta_schedule: str = "cosine" # "linear" 或 "cosine"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class GaussianDiffusion:
|
| 30 |
+
"""
|
| 31 |
+
标准 Gaussian Diffusion:
|
| 32 |
+
- 训练:epsilon-prediction,loss 为 MSE
|
| 33 |
+
- 采样:支持 DDPM 与(可选)DDIM
|
| 34 |
+
训练循环/保存等不在这里做,以保证训练与采样逻辑解耦。
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def __init__(self, betas: torch.Tensor):
|
| 38 |
+
if betas.ndim != 1:
|
| 39 |
+
raise ValueError("betas 必须是一维张量")
|
| 40 |
+
self.device = betas.device
|
| 41 |
+
self.betas = betas.float()
|
| 42 |
+
self.num_timesteps = int(self.betas.shape[0])
|
| 43 |
+
|
| 44 |
+
alphas = 1.0 - self.betas
|
| 45 |
+
alphas_cumprod = torch.cumprod(alphas, dim=0)
|
| 46 |
+
alphas_cumprod_prev = torch.cat([torch.tensor([1.0], device=betas.device), alphas_cumprod[:-1]], dim=0)
|
| 47 |
+
|
| 48 |
+
self.alphas = alphas
|
| 49 |
+
self.alphas_cumprod = alphas_cumprod
|
| 50 |
+
self.alphas_cumprod_prev = alphas_cumprod_prev
|
| 51 |
+
|
| 52 |
+
self.sqrt_alphas_cumprod = torch.sqrt(alphas_cumprod)
|
| 53 |
+
self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 - alphas_cumprod)
|
| 54 |
+
self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / alphas_cumprod)
|
| 55 |
+
self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / alphas_cumprod - 1.0)
|
| 56 |
+
|
| 57 |
+
# q(x_{t-1}|x_t,x_0) 的 posterior
|
| 58 |
+
self.posterior_variance = (
|
| 59 |
+
self.betas * (1.0 - alphas_cumprod_prev) / (1.0 - alphas_cumprod)
|
| 60 |
+
).clamp(min=1e-20)
|
| 61 |
+
self.posterior_log_variance_clipped = torch.log(self.posterior_variance)
|
| 62 |
+
self.posterior_mean_coef1 = self.betas * torch.sqrt(alphas_cumprod_prev) / (1.0 - alphas_cumprod)
|
| 63 |
+
self.posterior_mean_coef2 = (1.0 - alphas_cumprod_prev) * torch.sqrt(alphas) / (1.0 - alphas_cumprod)
|
| 64 |
+
|
| 65 |
+
def q_sample(self, x_start: torch.Tensor, t: torch.Tensor, noise: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 66 |
+
"""
|
| 67 |
+
前向加噪:q(x_t | x_0)
|
| 68 |
+
"""
|
| 69 |
+
if noise is None:
|
| 70 |
+
noise = torch.randn_like(x_start)
|
| 71 |
+
return _extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + _extract(
|
| 72 |
+
self.sqrt_one_minus_alphas_cumprod, t, x_start.shape
|
| 73 |
+
) * noise
|
| 74 |
+
|
| 75 |
+
def predict_x0_from_eps(self, x_t: torch.Tensor, t: torch.Tensor, eps: torch.Tensor) -> torch.Tensor:
|
| 76 |
+
return _extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - _extract(
|
| 77 |
+
self.sqrt_recipm1_alphas_cumprod, t, x_t.shape
|
| 78 |
+
) * eps
|
| 79 |
+
|
| 80 |
+
def q_posterior(self, x_start: torch.Tensor, x_t: torch.Tensor, t: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 81 |
+
"""
|
| 82 |
+
返回 q(x_{t-1}|x_t,x_0) 的 posterior mean 与 variance。
|
| 83 |
+
"""
|
| 84 |
+
posterior_mean = _extract(self.posterior_mean_coef1, t, x_t.shape) * x_start + _extract(
|
| 85 |
+
self.posterior_mean_coef2, t, x_t.shape
|
| 86 |
+
) * x_t
|
| 87 |
+
posterior_variance = _extract(self.posterior_variance, t, x_t.shape)
|
| 88 |
+
return posterior_mean, posterior_variance
|
| 89 |
+
|
| 90 |
+
def training_losses(
|
| 91 |
+
self,
|
| 92 |
+
model: nn.Module,
|
| 93 |
+
x_start: torch.Tensor,
|
| 94 |
+
t: torch.Tensor,
|
| 95 |
+
model_kwargs: Optional[Dict[str, Any]] = None,
|
| 96 |
+
cond_drop_prob: float = 0.0,
|
| 97 |
+
snr_gamma: float = 0.0,
|
| 98 |
+
) -> Dict[str, torch.Tensor]:
|
| 99 |
+
"""
|
| 100 |
+
训练损失:MSE(eps_pred, eps)。
|
| 101 |
+
可选:SNR 加权(snr_gamma>0),通常能提升采样质量/稳定性。
|
| 102 |
+
"""
|
| 103 |
+
model_kwargs = dict(model_kwargs or {})
|
| 104 |
+
noise = torch.randn_like(x_start)
|
| 105 |
+
x_t = self.q_sample(x_start, t, noise=noise)
|
| 106 |
+
|
| 107 |
+
# Classifier-Free Guidance 的训练方式:以一定概率丢弃条件(mask 全 0 => 等价无条件)
|
| 108 |
+
if cond_drop_prob and cond_drop_prob > 0 and ("mask" in model_kwargs):
|
| 109 |
+
mask = model_kwargs.get("mask", None)
|
| 110 |
+
if mask is not None:
|
| 111 |
+
# mask: (B,L);drop: (B,1) 广播到 (B,L)
|
| 112 |
+
drop = torch.rand((x_start.shape[0], 1), device=x_start.device) < float(cond_drop_prob)
|
| 113 |
+
if mask.dtype == torch.bool:
|
| 114 |
+
model_kwargs["mask"] = mask & (~drop)
|
| 115 |
+
else:
|
| 116 |
+
drop_f = drop.to(dtype=mask.dtype)
|
| 117 |
+
model_kwargs["mask"] = mask * (1 - drop_f)
|
| 118 |
+
|
| 119 |
+
eps_pred = model(x_t, t, **model_kwargs)
|
| 120 |
+
mse = (eps_pred - noise) ** 2 # (B,C,H,W)
|
| 121 |
+
loss_per = mse.mean(dim=(1, 2, 3)) # (B,)
|
| 122 |
+
|
| 123 |
+
if snr_gamma and snr_gamma > 0:
|
| 124 |
+
# snr = alpha_bar / (1-alpha_bar)
|
| 125 |
+
alpha_bar = _extract(self.alphas_cumprod, t, x_start.shape).reshape(x_start.shape[0])
|
| 126 |
+
snr = alpha_bar / (1.0 - alpha_bar).clamp(min=1e-12)
|
| 127 |
+
# 经典做法:w = min(snr, gamma) / snr
|
| 128 |
+
w = torch.minimum(snr, torch.full_like(snr, float(snr_gamma))) / snr.clamp(min=1e-12)
|
| 129 |
+
loss_per = loss_per * w
|
| 130 |
+
|
| 131 |
+
loss = loss_per.mean()
|
| 132 |
+
return {"loss": loss}
|
| 133 |
+
|
| 134 |
+
@torch.no_grad()
|
| 135 |
+
def p_mean_variance(
|
| 136 |
+
self,
|
| 137 |
+
model: nn.Module,
|
| 138 |
+
x_t: torch.Tensor,
|
| 139 |
+
t: torch.Tensor,
|
| 140 |
+
model_kwargs: Optional[Dict[str, Any]] = None,
|
| 141 |
+
) -> Dict[str, torch.Tensor]:
|
| 142 |
+
"""
|
| 143 |
+
DDPM 反向一步:根据 eps_pred 推出 x0_pred,再得到 posterior。
|
| 144 |
+
"""
|
| 145 |
+
model_kwargs = dict(model_kwargs or {})
|
| 146 |
+
eps_pred = model(x_t, t, **model_kwargs)
|
| 147 |
+
x0_pred = self.predict_x0_from_eps(x_t, t, eps_pred).clamp(-1.0, 1.0)
|
| 148 |
+
model_mean, model_var = self.q_posterior(x0_pred, x_t, t)
|
| 149 |
+
return {"mean": model_mean, "variance": model_var, "pred_x0": x0_pred, "pred_eps": eps_pred}
|
| 150 |
+
|
| 151 |
+
@torch.no_grad()
|
| 152 |
+
def p_sample(
|
| 153 |
+
self,
|
| 154 |
+
model: nn.Module,
|
| 155 |
+
x_t: torch.Tensor,
|
| 156 |
+
t: torch.Tensor,
|
| 157 |
+
model_kwargs: Optional[Dict[str, Any]] = None,
|
| 158 |
+
) -> torch.Tensor:
|
| 159 |
+
"""
|
| 160 |
+
采样一步:x_t -> x_{t-1}(DDPM)。
|
| 161 |
+
"""
|
| 162 |
+
out = self.p_mean_variance(model, x_t, t, model_kwargs=model_kwargs)
|
| 163 |
+
mean, var = out["mean"], out["variance"]
|
| 164 |
+
noise = torch.randn_like(x_t)
|
| 165 |
+
# t==0 时不再加噪声
|
| 166 |
+
nonzero_mask = (t != 0).float().reshape(x_t.shape[0], *((1,) * (x_t.ndim - 1)))
|
| 167 |
+
return mean + nonzero_mask * torch.sqrt(var) * noise
|
| 168 |
+
|
| 169 |
+
@torch.no_grad()
|
| 170 |
+
def p_sample_loop(
|
| 171 |
+
self,
|
| 172 |
+
model: nn.Module,
|
| 173 |
+
shape: Tuple[int, int, int, int],
|
| 174 |
+
device: torch.device,
|
| 175 |
+
model_kwargs: Optional[Dict[str, Any]] = None,
|
| 176 |
+
progress: bool = True,
|
| 177 |
+
) -> torch.Tensor:
|
| 178 |
+
"""
|
| 179 |
+
从纯噪声开始逐步采样到 x_0(DDPM)。
|
| 180 |
+
"""
|
| 181 |
+
img = torch.randn(shape, device=device)
|
| 182 |
+
timesteps = range(self.num_timesteps - 1, -1, -1)
|
| 183 |
+
if progress:
|
| 184 |
+
try:
|
| 185 |
+
from tqdm import tqdm # 延迟导入,避免强依赖
|
| 186 |
+
except Exception:
|
| 187 |
+
tqdm = None
|
| 188 |
+
if tqdm is not None:
|
| 189 |
+
timesteps = tqdm(timesteps, desc="DDPM Sampling", total=self.num_timesteps)
|
| 190 |
+
|
| 191 |
+
for i in timesteps:
|
| 192 |
+
t = torch.full((shape[0],), i, device=device, dtype=torch.long)
|
| 193 |
+
img = self.p_sample(model, img, t, model_kwargs=model_kwargs)
|
| 194 |
+
return img
|
| 195 |
+
|
| 196 |
+
@torch.no_grad()
|
| 197 |
+
def ddim_sample_loop(
|
| 198 |
+
self,
|
| 199 |
+
model: nn.Module,
|
| 200 |
+
shape: Tuple[int, int, int, int],
|
| 201 |
+
device: torch.device,
|
| 202 |
+
steps: int = 50,
|
| 203 |
+
eta: float = 0.0,
|
| 204 |
+
model_kwargs: Optional[Dict[str, Any]] = None,
|
| 205 |
+
progress: bool = True,
|
| 206 |
+
) -> torch.Tensor:
|
| 207 |
+
"""
|
| 208 |
+
DDIM 采样(更少步数、更快)。eta=0 为确定性采样。
|
| 209 |
+
"""
|
| 210 |
+
if steps <= 1:
|
| 211 |
+
raise ValueError("steps 必须 > 1")
|
| 212 |
+
img = torch.randn(shape, device=device)
|
| 213 |
+
|
| 214 |
+
# 选择等间隔的时间序列(从 T-1 到 0)
|
| 215 |
+
# 注意:这里用 Python int 列表,避免在 CUDA Tensor 上做 int() 转换导致报错。
|
| 216 |
+
times = torch.linspace(self.num_timesteps - 1, 0, steps, dtype=torch.float32).long().tolist()
|
| 217 |
+
|
| 218 |
+
if progress:
|
| 219 |
+
try:
|
| 220 |
+
from tqdm import tqdm
|
| 221 |
+
except Exception:
|
| 222 |
+
tqdm = None
|
| 223 |
+
if tqdm is not None:
|
| 224 |
+
times = tqdm(list(times), desc="DDIM Sampling", total=steps)
|
| 225 |
+
|
| 226 |
+
for idx, t_i in enumerate(times):
|
| 227 |
+
t = torch.full((shape[0],), int(t_i), device=device, dtype=torch.long)
|
| 228 |
+
out = self.p_mean_variance(model, img, t, model_kwargs=model_kwargs)
|
| 229 |
+
eps = out["pred_eps"]
|
| 230 |
+
x0 = out["pred_x0"]
|
| 231 |
+
|
| 232 |
+
if idx == len(times) - 1:
|
| 233 |
+
img = x0
|
| 234 |
+
break
|
| 235 |
+
|
| 236 |
+
t_next = torch.full((shape[0],), int(times[idx + 1]), device=device, dtype=torch.long)
|
| 237 |
+
|
| 238 |
+
a_t = _extract(self.alphas_cumprod, t, img.shape)
|
| 239 |
+
a_next = _extract(self.alphas_cumprod, t_next, img.shape)
|
| 240 |
+
|
| 241 |
+
# DDIM 更新公式
|
| 242 |
+
sigma = (
|
| 243 |
+
eta
|
| 244 |
+
* torch.sqrt((1.0 - a_next) / (1.0 - a_t))
|
| 245 |
+
* torch.sqrt(1.0 - a_t / a_next)
|
| 246 |
+
)
|
| 247 |
+
noise = torch.randn_like(img)
|
| 248 |
+
dir_xt = torch.sqrt(1.0 - a_next - sigma**2) * eps
|
| 249 |
+
img = torch.sqrt(a_next) * x0 + dir_xt + sigma * noise
|
| 250 |
+
|
| 251 |
+
return img
|
| 252 |
+
|
| 253 |
+
|
blockdiffusion/diffusion/schedules.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from typing import Literal
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def linear_beta_schedule(timesteps: int, beta_start: float = 1e-4, beta_end: float = 2e-2) -> torch.Tensor:
|
| 10 |
+
"""
|
| 11 |
+
经典线性 beta 调度(DDPM)。
|
| 12 |
+
"""
|
| 13 |
+
return torch.linspace(beta_start, beta_end, timesteps, dtype=torch.float32)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def cosine_beta_schedule(timesteps: int, s: float = 0.008) -> torch.Tensor:
|
| 17 |
+
"""
|
| 18 |
+
Cosine 调度(Nichol & Dhariwal, 2021)。
|
| 19 |
+
参考实现:alpha_bar(t) = cos^2((t+s)/(1+s)*pi/2)
|
| 20 |
+
"""
|
| 21 |
+
steps = timesteps + 1
|
| 22 |
+
x = torch.linspace(0, timesteps, steps, dtype=torch.float64)
|
| 23 |
+
t = x / timesteps
|
| 24 |
+
alpha_bar = torch.cos((t + s) / (1 + s) * math.pi / 2) ** 2
|
| 25 |
+
alpha_bar = alpha_bar / alpha_bar[0]
|
| 26 |
+
betas = 1 - (alpha_bar[1:] / alpha_bar[:-1])
|
| 27 |
+
return betas.clamp(1e-8, 0.999).float()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def get_beta_schedule(
|
| 31 |
+
schedule: Literal["linear", "cosine"],
|
| 32 |
+
timesteps: int,
|
| 33 |
+
) -> torch.Tensor:
|
| 34 |
+
"""
|
| 35 |
+
统一入口:返回长度为 timesteps 的 betas。
|
| 36 |
+
"""
|
| 37 |
+
if schedule == "linear":
|
| 38 |
+
return linear_beta_schedule(timesteps)
|
| 39 |
+
if schedule == "cosine":
|
| 40 |
+
return cosine_beta_schedule(timesteps)
|
| 41 |
+
raise ValueError(f"未知 schedule: {schedule}")
|
| 42 |
+
|
| 43 |
+
|
blockdiffusion/models/README.md
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## BlockDiffusion 模型说明(Model README)
|
| 2 |
+
|
| 3 |
+
本文件只描述**模型本身**(网络结构、输入输出、条件机制、checkpoint 格式),不描述训练脚本的使用方式。
|
| 4 |
+
|
| 5 |
+
### 1. 模型定位
|
| 6 |
+
|
| 7 |
+
- **任务**:生成/还原 **16×16** 的《我的世界》纹理(支持 **RGB 或 RGBA**)。
|
| 8 |
+
- **扩散建模**:epsilon-predictor(预测噪声 \(\epsilon\)),网络输出通道数与输入一致。
|
| 9 |
+
- **条件方式**:可选的**文本提示词条件**(Text Conditioning),并支持 CFG(Classifier-Free Guidance)的“条件丢弃训练 + 采样引导”范式。
|
| 10 |
+
|
| 11 |
+
### 2. 模型组件与代码位置
|
| 12 |
+
|
| 13 |
+
- **UNet 主体**:`blockdiffusion/models/unet16.py`
|
| 14 |
+
- `UNet16Config`:UNet 配置(通道、层数、注意力分辨率等)
|
| 15 |
+
- `UNet16`:16×16 专用 UNet(epsilon predictor)
|
| 16 |
+
- **文本条件封装**:`blockdiffusion/models/text_unet16.py`
|
| 17 |
+
- `TextCondConfig`:文本侧配置(长度、embedding 维度等)
|
| 18 |
+
- `TextCondUNet16`:TextEncoder + UNet16 的组合模型
|
| 19 |
+
- **文本编码器**:`blockdiffusion/text/text_encoder.py`
|
| 20 |
+
- `TextEncoder`:masked mean pooling + MLP 投影到条件向量
|
| 21 |
+
- **Tokenizer / 词表**:`blockdiffusion/text/simple_tokenizer.py`
|
| 22 |
+
- `tokenize()`:轻量分词(支持 CJK 单字切分)
|
| 23 |
+
- `SimpleVocab`:极简词表(含 `<pad>/<unk>`),可序列化到 checkpoint
|
| 24 |
+
- **CFG 采样包装器**:`blockdiffusion/train/guidance.py`
|
| 25 |
+
- `CFGuidanceModel`:把任意支持 `forward(x, t, **kwargs)` 的模型包装成 CFG 采样模型
|
| 26 |
+
|
| 27 |
+
### 3. 前向接口(输入/输出约定)
|
| 28 |
+
|
| 29 |
+
#### 3.1 `UNet16`
|
| 30 |
+
|
| 31 |
+
- **函数签名(语义)**:`eps = UNet16(x, t, cond=None)`
|
| 32 |
+
- **输入**
|
| 33 |
+
- `x`:形状 `(B, C, 16, 16)`,取值范围约定为 **[-1, 1]**
|
| 34 |
+
- `t`:形状 `(B,)`,dtype 通常为 `int64`,表示扩散 timestep
|
| 35 |
+
- `cond`(可选):形状 `(B, cond_dim)` 的条件向量;`None` 表示无条件
|
| 36 |
+
- **输出**
|
| 37 |
+
- `eps`:形状 `(B, C, 16, 16)`,预测噪声 \(\epsilon\)
|
| 38 |
+
|
| 39 |
+
#### 3.2 `TextCondUNet16`
|
| 40 |
+
|
| 41 |
+
- **函数签名(语义)**:`eps = TextCondUNet16(x, t, tokens=None, mask=None)`
|
| 42 |
+
- **输入**
|
| 43 |
+
- `x` / `t`:同 `UNet16`
|
| 44 |
+
- `tokens`(可选):形状 `(B, L)`,dtype 通常为 `int64`
|
| 45 |
+
- `mask`(可选):形状 `(B, L)`,bool 或 0/1
|
| 46 |
+
- `mask=None` 时会自动由 `tokens != pad_id` 推导
|
| 47 |
+
- 当 `tokens=None` 或 `mask` 全 0 时,模型**等价于无条件**
|
| 48 |
+
- **输出**
|
| 49 |
+
- `eps`:同 `UNet16`
|
| 50 |
+
|
| 51 |
+
### 4. 条件注入机制(Text Conditioning)
|
| 52 |
+
|
| 53 |
+
- `TextEncoder` 把 `tokens/mask` 编码为 `cond`(形状 `(B, time_emb_dim)`)。
|
| 54 |
+
- `cond` 通过“**加和**”注入到 UNet 的 time embedding:
|
| 55 |
+
- `t_emb = time_embed(t)`
|
| 56 |
+
- `t_emb = t_emb + cond_proj(cond)`(无 `cond` 时不加)
|
| 57 |
+
- 这是一种轻量、易训练、可随 checkpoint 保存的条件方案,不依赖外部大型文本模型。
|
| 58 |
+
|
| 59 |
+
### 5. CFG(Classifier-Free Guidance)说明
|
| 60 |
+
|
| 61 |
+
- **训练侧**:通过一定概率把条件 `mask` 置空(等价无条件),让模型同时学到 cond/uncond。
|
| 62 |
+
- **采样侧**:`CFGuidanceModel` 组合两次前向,按以下公式合成:
|
| 63 |
+
|
| 64 |
+
\[
|
| 65 |
+
\epsilon = \epsilon_{\text{uncond}} + s \cdot (\epsilon_{\text{cond}} - \epsilon_{\text{uncond}})
|
| 66 |
+
\]
|
| 67 |
+
|
| 68 |
+
其中 \(s\) 为 `cfg_scale`。
|
| 69 |
+
|
| 70 |
+
### 6. 关键配置项(建议对照 checkpoint 的 meta)
|
| 71 |
+
|
| 72 |
+
#### 6.1 `UNet16Config`
|
| 73 |
+
|
| 74 |
+
| 字段 | 含义 | 备注 |
|
| 75 |
+
|---|---|---|
|
| 76 |
+
| `in_channels` | 输入通道数 | 3=RGB,4=RGBA |
|
| 77 |
+
| `out_channels` | 输出通道数 | epsilon 预测,通常与输入一致 |
|
| 78 |
+
| `base_channels` | 基础通道宽度 | 模型容量主开关 |
|
| 79 |
+
| `channel_mults` | 各层通道倍率 | 默认 `(1,2,2)` 对应 16→8→4 |
|
| 80 |
+
| `num_res_blocks` | 每层 ResBlock 数 | 16×16 不建议太大 |
|
| 81 |
+
| `attn_resolutions` | 启用自注意力的分辨率 | 默认在 8/4 上更划算 |
|
| 82 |
+
| `dropout` | ResBlock dropout | 0 表示关闭 |
|
| 83 |
+
| `time_emb_dim` | time embedding 维度 | 也作为条件注入维度基准 |
|
| 84 |
+
| `cond_dim` | 条件向量维度 | 通过 `cond_proj` 对齐到 `time_emb_dim` |
|
| 85 |
+
|
| 86 |
+
#### 6.2 `TextCondConfig`
|
| 87 |
+
|
| 88 |
+
| 字段 | 含义 | 备注 |
|
| 89 |
+
|---|---|---|
|
| 90 |
+
| `max_text_len` | 最大 token 长度 \(L\) | 影响 `tokens` 形状与 TextEncoder pooling |
|
| 91 |
+
| `emb_dim` | token embedding 维度 | TextEncoder 内部使用 |
|
| 92 |
+
| `dropout` | TextEncoder dropout | 0 表示关闭 |
|
| 93 |
+
|
| 94 |
+
> `TextEncoderConfig.out_dim` 会被设置为 `UNet16Config.time_emb_dim`,从而保证条件向量可直接注入 UNet。
|
| 95 |
+
|
| 96 |
+
### 7. Tokenizer 与词表(`SimpleVocab`)
|
| 97 |
+
|
| 98 |
+
- **Tokenizer 特性**
|
| 99 |
+
- 统一转小写
|
| 100 |
+
- 英文/数字按“词”切分(把 `_`/`-` 视作词内字符)
|
| 101 |
+
- CJK(中日韩)字符按“单字”切分,适配无空格中文提示词
|
| 102 |
+
- **词表约定**
|
| 103 |
+
- `id=0`:`<pad>`
|
| 104 |
+
- `id=1`:`<unk>`
|
| 105 |
+
- **重要限制(续训/推理一致性)**
|
| 106 |
+
- 续训或推理时必须复用 checkpoint 内保存的词表;否则 `Embedding` 尺寸会不匹配。
|
| 107 |
+
- 新出现但不在旧词表中的词会被编码成 `<unk>`:可继续运行,但无法学习/表达新词语义。
|
| 108 |
+
|
| 109 |
+
### 8. Checkpoint(.pt)格式与推荐加载流程
|
| 110 |
+
|
| 111 |
+
#### 8.1 文件内容(关键键)
|
| 112 |
+
|
| 113 |
+
训练保存的 checkpoint 是一个 `dict`,关键字段如下:
|
| 114 |
+
|
| 115 |
+
- `model`:模型 `state_dict`
|
| 116 |
+
- `ema`:EMA `state_dict`(若存在,采样建议使用)
|
| 117 |
+
- `step`:保存时的全局 step
|
| 118 |
+
- `meta`:**重建模型所需的结构化信息**(强烈依赖)
|
| 119 |
+
- `trainer_cfg` / `optimizer` / `scaler`:与训练相关(推理通常不需要)
|
| 120 |
+
|
| 121 |
+
#### 8.2 `meta` 约定(常用字段)
|
| 122 |
+
|
| 123 |
+
- `model_type`:`"text_cond"` 或(缺省/其他)无条件
|
| 124 |
+
- `unet_cfg`:`UNet16Config` 的字典形式(`dataclasses.asdict`)
|
| 125 |
+
- `text_cfg`:`TextCondConfig` 的字典形式(仅 text_cond)
|
| 126 |
+
- `tokenizer`:`SimpleVocab.to_dict()` 的结果(仅 text_cond)
|
| 127 |
+
- `diffusion`:`{"timesteps": ..., "beta_schedule": ...}`(采样需一致)
|
| 128 |
+
- `data`:`{"image_size": 16, "channels": 3/4}`(采样 shape 需一致)
|
| 129 |
+
|
| 130 |
+
#### 8.3 推荐加载流程(文字版)
|
| 131 |
+
|
| 132 |
+
- **步骤**
|
| 133 |
+
- 读取 `.pt` 到内存(`torch.load`)
|
| 134 |
+
- 从 `meta` 判断 `model_type`
|
| 135 |
+
- 由 `meta["unet_cfg"]` 重建 `UNet16Config`
|
| 136 |
+
- 若为 `text_cond`:从 `meta["tokenizer"]` 重建 `SimpleVocab`,从 `meta["text_cfg"]` 重建 `TextCondConfig`
|
| 137 |
+
- 实例化 `UNet16` 或 `TextCondUNet16`
|
| 138 |
+
- `load_state_dict(ckpt["model"], strict=True)`
|
| 139 |
+
- 如需更稳的采样:把 `ckpt["ema"]` 应用到模型参数
|
| 140 |
+
|
| 141 |
+
> 兼容性提示:若旧 checkpoint 不含 `meta/unet_cfg`,项目内的采样入口会回落到默认的 16×16 RGB 无条件配置;但这会丢失“通道数/条件/调度”等信息,不建议用于长期维护。
|
| 142 |
+
|
| 143 |
+
|
blockdiffusion/models/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
模型定义:适配 16×16 的小型 UNet。
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from .unet16 import UNet16, UNet16Config
|
| 6 |
+
from .text_unet16 import TextCondUNet16, TextCondConfig
|
| 7 |
+
|
| 8 |
+
__all__ = [
|
| 9 |
+
"UNet16",
|
| 10 |
+
"UNet16Config",
|
| 11 |
+
"TextCondUNet16",
|
| 12 |
+
"TextCondConfig",
|
| 13 |
+
]
|
| 14 |
+
|
| 15 |
+
|
blockdiffusion/models/text_unet16.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch import nn
|
| 8 |
+
|
| 9 |
+
from blockdiffusion.models.unet16 import UNet16, UNet16Config
|
| 10 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab
|
| 11 |
+
from blockdiffusion.text.text_encoder import TextEncoder, TextEncoderConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass(frozen=True)
|
| 15 |
+
class TextCondConfig:
|
| 16 |
+
"""
|
| 17 |
+
文本条件配置(用于 checkpoint/meta 保存)。
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
max_text_len: int = 32
|
| 21 |
+
emb_dim: int = 128
|
| 22 |
+
dropout: float = 0.0
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class TextCondUNet16(nn.Module):
|
| 26 |
+
"""
|
| 27 |
+
文本条件 UNet:
|
| 28 |
+
- TextEncoder 将 tokens/mask 编码为 cond 向量
|
| 29 |
+
- cond 向量注入到 UNet 的 time embedding(加和)
|
| 30 |
+
- tokens/mask 为空或 mask 全 0 时等价于无条件
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
def __init__(
|
| 34 |
+
self,
|
| 35 |
+
unet_cfg: UNet16Config,
|
| 36 |
+
vocab: SimpleVocab,
|
| 37 |
+
text_cfg: TextCondConfig,
|
| 38 |
+
):
|
| 39 |
+
super().__init__()
|
| 40 |
+
self.unet_cfg = unet_cfg
|
| 41 |
+
self.text_cfg = text_cfg
|
| 42 |
+
|
| 43 |
+
self.vocab = vocab # 仅用于保存 meta;不注册为 buffer/module
|
| 44 |
+
|
| 45 |
+
te_cfg = TextEncoderConfig(
|
| 46 |
+
vocab_size=vocab.size,
|
| 47 |
+
max_len=text_cfg.max_text_len,
|
| 48 |
+
emb_dim=text_cfg.emb_dim,
|
| 49 |
+
out_dim=unet_cfg.time_emb_dim,
|
| 50 |
+
dropout=text_cfg.dropout,
|
| 51 |
+
)
|
| 52 |
+
self.text_encoder = TextEncoder(te_cfg, pad_id=vocab.pad_id)
|
| 53 |
+
self.unet = UNet16(unet_cfg)
|
| 54 |
+
|
| 55 |
+
def forward(
|
| 56 |
+
self,
|
| 57 |
+
x: torch.Tensor,
|
| 58 |
+
t: torch.Tensor,
|
| 59 |
+
tokens: Optional[torch.Tensor] = None,
|
| 60 |
+
mask: Optional[torch.Tensor] = None,
|
| 61 |
+
) -> torch.Tensor:
|
| 62 |
+
cond = None
|
| 63 |
+
if tokens is not None:
|
| 64 |
+
cond = self.text_encoder(tokens, mask=mask)
|
| 65 |
+
return self.unet(x, t, cond=cond)
|
| 66 |
+
|
| 67 |
+
|
blockdiffusion/models/unet16.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Iterable, List, Sequence, Tuple
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from torch import nn
|
| 9 |
+
from torch.nn import functional as F
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _num_groups(channels: int, max_groups: int = 32) -> int:
|
| 13 |
+
"""
|
| 14 |
+
GroupNorm 组数必须能整除 channels;这里自动选择合适组数。
|
| 15 |
+
"""
|
| 16 |
+
for g in range(min(max_groups, channels), 0, -1):
|
| 17 |
+
if channels % g == 0:
|
| 18 |
+
return g
|
| 19 |
+
return 1
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class SinusoidalTimeEmbedding(nn.Module):
|
| 23 |
+
"""
|
| 24 |
+
将离散 timestep 映射到连续 embedding(sin/cos)。
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
def __init__(self, dim: int):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.dim = dim
|
| 30 |
+
|
| 31 |
+
def forward(self, t: torch.Tensor) -> torch.Tensor:
|
| 32 |
+
half = self.dim // 2
|
| 33 |
+
# 频率:exp(-log(10000) * i/(half-1))
|
| 34 |
+
freqs = torch.exp(
|
| 35 |
+
-math.log(10000.0) * torch.arange(0, half, device=t.device, dtype=torch.float32) / max(half - 1, 1)
|
| 36 |
+
)
|
| 37 |
+
args = t.float().unsqueeze(1) * freqs.unsqueeze(0)
|
| 38 |
+
emb = torch.cat([torch.sin(args), torch.cos(args)], dim=1)
|
| 39 |
+
if self.dim % 2 == 1:
|
| 40 |
+
emb = F.pad(emb, (0, 1))
|
| 41 |
+
return emb
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class ResBlock(nn.Module):
|
| 45 |
+
"""
|
| 46 |
+
ResNet block(带 time embedding 注入),避免对 16×16 不友好的过深/过度下采样。
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(self, in_ch: int, out_ch: int, time_emb_dim: int, dropout: float = 0.0):
|
| 50 |
+
super().__init__()
|
| 51 |
+
self.in_ch = in_ch
|
| 52 |
+
self.out_ch = out_ch
|
| 53 |
+
|
| 54 |
+
self.norm1 = nn.GroupNorm(_num_groups(in_ch), in_ch)
|
| 55 |
+
self.conv1 = nn.Conv2d(in_ch, out_ch, kernel_size=3, padding=1)
|
| 56 |
+
|
| 57 |
+
self.time_proj = nn.Sequential(
|
| 58 |
+
nn.SiLU(),
|
| 59 |
+
nn.Linear(time_emb_dim, out_ch),
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
self.norm2 = nn.GroupNorm(_num_groups(out_ch), out_ch)
|
| 63 |
+
self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
|
| 64 |
+
self.conv2 = nn.Conv2d(out_ch, out_ch, kernel_size=3, padding=1)
|
| 65 |
+
|
| 66 |
+
self.skip = nn.Identity() if in_ch == out_ch else nn.Conv2d(in_ch, out_ch, kernel_size=1)
|
| 67 |
+
|
| 68 |
+
def forward(self, x: torch.Tensor, t_emb: torch.Tensor) -> torch.Tensor:
|
| 69 |
+
h = self.conv1(F.silu(self.norm1(x)))
|
| 70 |
+
# time embedding 以 bias 的形式注入到 feature map
|
| 71 |
+
h = h + self.time_proj(t_emb).unsqueeze(-1).unsqueeze(-1)
|
| 72 |
+
h = self.conv2(self.dropout(F.silu(self.norm2(h))))
|
| 73 |
+
return h + self.skip(x)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class AttentionBlock(nn.Module):
|
| 77 |
+
"""
|
| 78 |
+
轻量自注意力(用于 8×8 / 4×4 等小分辨率特征图)。
|
| 79 |
+
"""
|
| 80 |
+
|
| 81 |
+
def __init__(self, channels: int, num_heads: int = 4):
|
| 82 |
+
super().__init__()
|
| 83 |
+
if channels % num_heads != 0:
|
| 84 |
+
# 保守处理:无法整除就退化为 1 head,避免运行时报错
|
| 85 |
+
num_heads = 1
|
| 86 |
+
self.channels = channels
|
| 87 |
+
self.num_heads = num_heads
|
| 88 |
+
self.head_dim = channels // num_heads
|
| 89 |
+
|
| 90 |
+
self.norm = nn.GroupNorm(_num_groups(channels), channels)
|
| 91 |
+
self.qkv = nn.Conv2d(channels, channels * 3, kernel_size=1)
|
| 92 |
+
self.proj = nn.Conv2d(channels, channels, kernel_size=1)
|
| 93 |
+
|
| 94 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 95 |
+
b, c, h, w = x.shape
|
| 96 |
+
n = h * w
|
| 97 |
+
qkv = self.qkv(self.norm(x))
|
| 98 |
+
q, k, v = qkv.chunk(3, dim=1)
|
| 99 |
+
|
| 100 |
+
# (b, heads, head_dim, n)
|
| 101 |
+
q = q.reshape(b, self.num_heads, self.head_dim, n)
|
| 102 |
+
k = k.reshape(b, self.num_heads, self.head_dim, n)
|
| 103 |
+
v = v.reshape(b, self.num_heads, self.head_dim, n)
|
| 104 |
+
|
| 105 |
+
# 注意力:softmax(q^T k / sqrt(d))
|
| 106 |
+
scale = 1.0 / math.sqrt(self.head_dim)
|
| 107 |
+
attn = torch.einsum("bhdn,bhdm->bhnm", q * scale, k) # (b, heads, n, n)
|
| 108 |
+
attn = torch.softmax(attn, dim=-1)
|
| 109 |
+
out = torch.einsum("bhnm,bhdm->bhdn", attn, v) # (b, heads, head_dim, n)
|
| 110 |
+
out = out.reshape(b, c, h, w)
|
| 111 |
+
out = self.proj(out)
|
| 112 |
+
return x + out
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class Downsample(nn.Module):
|
| 116 |
+
"""
|
| 117 |
+
stride=2 卷积下采样(对小分辨率更友好、可学习)。
|
| 118 |
+
"""
|
| 119 |
+
|
| 120 |
+
def __init__(self, channels: int):
|
| 121 |
+
super().__init__()
|
| 122 |
+
self.conv = nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1)
|
| 123 |
+
|
| 124 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 125 |
+
return self.conv(x)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class Upsample(nn.Module):
|
| 129 |
+
"""
|
| 130 |
+
最近邻上采样 + 卷积。
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
def __init__(self, channels: int):
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.conv = nn.Conv2d(channels, channels, kernel_size=3, padding=1)
|
| 136 |
+
|
| 137 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 138 |
+
x = F.interpolate(x, scale_factor=2.0, mode="nearest")
|
| 139 |
+
return self.conv(x)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
@dataclass(frozen=True)
|
| 143 |
+
class UNet16Config:
|
| 144 |
+
"""
|
| 145 |
+
16×16 专用 UNet 配置。
|
| 146 |
+
"""
|
| 147 |
+
|
| 148 |
+
in_channels: int = 3
|
| 149 |
+
out_channels: int = 3 # 预测 eps,通道数与输入一致
|
| 150 |
+
base_channels: int = 64
|
| 151 |
+
channel_mults: Tuple[int, ...] = (1, 2, 2) # 16->8->4,避免过度下采样
|
| 152 |
+
num_res_blocks: int = 2
|
| 153 |
+
attn_resolutions: Tuple[int, ...] = (8, 4) # 小分辨率加注意力更划算
|
| 154 |
+
dropout: float = 0.0
|
| 155 |
+
time_emb_dim: int = 256
|
| 156 |
+
# 文本条件向量维度(与 time_emb_dim 对齐最简单)
|
| 157 |
+
cond_dim: int = 256
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class UNet16(nn.Module):
|
| 161 |
+
"""
|
| 162 |
+
小型 UNet(epsilon-predictor),适配 16×16 输入。
|
| 163 |
+
"""
|
| 164 |
+
|
| 165 |
+
def __init__(self, cfg: UNet16Config):
|
| 166 |
+
super().__init__()
|
| 167 |
+
self.cfg = cfg
|
| 168 |
+
|
| 169 |
+
self.time_embed = nn.Sequential(
|
| 170 |
+
SinusoidalTimeEmbedding(cfg.time_emb_dim),
|
| 171 |
+
nn.Linear(cfg.time_emb_dim, cfg.time_emb_dim * 4),
|
| 172 |
+
nn.SiLU(),
|
| 173 |
+
nn.Linear(cfg.time_emb_dim * 4, cfg.time_emb_dim),
|
| 174 |
+
)
|
| 175 |
+
self.cond_proj = nn.Identity() if cfg.cond_dim == cfg.time_emb_dim else nn.Linear(cfg.cond_dim, cfg.time_emb_dim)
|
| 176 |
+
|
| 177 |
+
self.in_conv = nn.Conv2d(cfg.in_channels, cfg.base_channels, kernel_size=3, padding=1)
|
| 178 |
+
|
| 179 |
+
# Down path(按 level 显式组织,避免小分辨率下采样/上采样位置出错)
|
| 180 |
+
ch = cfg.base_channels
|
| 181 |
+
self.down_resblocks = nn.ModuleList()
|
| 182 |
+
self.down_attn = nn.ModuleList()
|
| 183 |
+
self.downsamples = nn.ModuleList()
|
| 184 |
+
|
| 185 |
+
skip_chs: List[int] = []
|
| 186 |
+
resolution = 16 # 纹理固定为 16×16
|
| 187 |
+
for level, mult in enumerate(cfg.channel_mults):
|
| 188 |
+
out_ch = cfg.base_channels * mult
|
| 189 |
+
level_res = nn.ModuleList()
|
| 190 |
+
level_attn = nn.ModuleList()
|
| 191 |
+
for _ in range(cfg.num_res_blocks):
|
| 192 |
+
level_res.append(ResBlock(ch, out_ch, cfg.time_emb_dim, dropout=cfg.dropout))
|
| 193 |
+
ch = out_ch
|
| 194 |
+
level_attn.append(AttentionBlock(ch) if resolution in cfg.attn_resolutions else nn.Identity())
|
| 195 |
+
skip_chs.append(ch)
|
| 196 |
+
self.down_resblocks.append(level_res)
|
| 197 |
+
self.down_attn.append(level_attn)
|
| 198 |
+
if level != len(cfg.channel_mults) - 1:
|
| 199 |
+
self.downsamples.append(Downsample(ch))
|
| 200 |
+
resolution //= 2
|
| 201 |
+
|
| 202 |
+
# Mid
|
| 203 |
+
self.mid_block1 = ResBlock(ch, ch, cfg.time_emb_dim, dropout=cfg.dropout)
|
| 204 |
+
self.mid_attn = AttentionBlock(ch)
|
| 205 |
+
self.mid_block2 = ResBlock(ch, ch, cfg.time_emb_dim, dropout=cfg.dropout)
|
| 206 |
+
|
| 207 |
+
# Up path(从最深层开始逐级上采样)
|
| 208 |
+
self.up_resblocks = nn.ModuleList()
|
| 209 |
+
self.up_attn = nn.ModuleList()
|
| 210 |
+
self.upsamples = nn.ModuleList()
|
| 211 |
+
|
| 212 |
+
# 当前 resolution 对应最深层(例如 4×4)
|
| 213 |
+
# down 部分循环结束后,resolution 已被更新到最深层
|
| 214 |
+
for level, mult in reversed(list(enumerate(cfg.channel_mults))):
|
| 215 |
+
out_ch = cfg.base_channels * mult
|
| 216 |
+
level_res = nn.ModuleList()
|
| 217 |
+
level_attn = nn.ModuleList()
|
| 218 |
+
for _ in range(cfg.num_res_blocks):
|
| 219 |
+
skip_ch = skip_chs.pop()
|
| 220 |
+
level_res.append(ResBlock(ch + skip_ch, out_ch, cfg.time_emb_dim, dropout=cfg.dropout))
|
| 221 |
+
ch = out_ch
|
| 222 |
+
level_attn.append(AttentionBlock(ch) if resolution in cfg.attn_resolutions else nn.Identity())
|
| 223 |
+
self.up_resblocks.append(level_res)
|
| 224 |
+
self.up_attn.append(level_attn)
|
| 225 |
+
if level != 0:
|
| 226 |
+
self.upsamples.append(Upsample(ch))
|
| 227 |
+
resolution *= 2
|
| 228 |
+
|
| 229 |
+
self.out_norm = nn.GroupNorm(_num_groups(ch), ch)
|
| 230 |
+
self.out_conv = nn.Conv2d(ch, cfg.out_channels, kernel_size=3, padding=1)
|
| 231 |
+
|
| 232 |
+
def forward(self, x: torch.Tensor, t: torch.Tensor, cond: torch.Tensor | None = None) -> torch.Tensor:
|
| 233 |
+
"""
|
| 234 |
+
输入:
|
| 235 |
+
- x: (B, C, 16, 16) 的噪声图
|
| 236 |
+
- t: (B,) 的 timestep(int64)
|
| 237 |
+
- cond: (B, cond_dim) 的条件向量(可选;为空则无条件)
|
| 238 |
+
输出:
|
| 239 |
+
- eps_pred: (B, C, 16, 16)
|
| 240 |
+
"""
|
| 241 |
+
t_emb = self.time_embed(t)
|
| 242 |
+
if cond is not None:
|
| 243 |
+
t_emb = t_emb + self.cond_proj(cond)
|
| 244 |
+
h = self.in_conv(x)
|
| 245 |
+
|
| 246 |
+
skips: List[torch.Tensor] = []
|
| 247 |
+
for level in range(len(self.cfg.channel_mults)):
|
| 248 |
+
for i in range(self.cfg.num_res_blocks):
|
| 249 |
+
h = self.down_resblocks[level][i](h, t_emb)
|
| 250 |
+
h = self.down_attn[level][i](h)
|
| 251 |
+
skips.append(h)
|
| 252 |
+
if level < len(self.downsamples):
|
| 253 |
+
h = self.downsamples[level](h)
|
| 254 |
+
|
| 255 |
+
h = self.mid_block1(h, t_emb)
|
| 256 |
+
h = self.mid_attn(h)
|
| 257 |
+
h = self.mid_block2(h, t_emb)
|
| 258 |
+
|
| 259 |
+
# up_resblocks/up_attn 的顺序是从最深层到最浅层
|
| 260 |
+
for level in range(len(self.cfg.channel_mults)):
|
| 261 |
+
for i in range(self.cfg.num_res_blocks):
|
| 262 |
+
skip = skips.pop()
|
| 263 |
+
h = torch.cat([h, skip], dim=1)
|
| 264 |
+
h = self.up_resblocks[level][i](h, t_emb)
|
| 265 |
+
h = self.up_attn[level][i](h)
|
| 266 |
+
if level < len(self.upsamples):
|
| 267 |
+
h = self.upsamples[level](h)
|
| 268 |
+
|
| 269 |
+
h = self.out_conv(F.silu(self.out_norm(h)))
|
| 270 |
+
return h
|
| 271 |
+
|
| 272 |
+
|
blockdiffusion/text/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
文本条件相关:简易分词/词表、TextEncoder。
|
| 3 |
+
为避免引入大型依赖,这里提供轻量实现,适合 16×16 小模型/小数据集场景。
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
|
blockdiffusion/text/simple_tokenizer.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Dict, Iterable, List, Optional, Sequence
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _is_cjk(ch: str) -> bool:
|
| 9 |
+
"""
|
| 10 |
+
粗略判断 CJK 字符(覆盖常见中文/日文/韩文范围)。
|
| 11 |
+
"""
|
| 12 |
+
code = ord(ch)
|
| 13 |
+
return (
|
| 14 |
+
0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs
|
| 15 |
+
or 0x3400 <= code <= 0x4DBF # CJK Unified Ideographs Extension A
|
| 16 |
+
or 0x3040 <= code <= 0x30FF # Japanese Hiragana/Katakana
|
| 17 |
+
or 0xAC00 <= code <= 0xD7AF # Hangul Syllables
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def tokenize(text: str) -> List[str]:
|
| 22 |
+
"""
|
| 23 |
+
轻量 tokenizer:
|
| 24 |
+
- 英文/数字按“词”切分
|
| 25 |
+
- 中文(CJK)按“单字”切分(更适合无空格中文提示词)
|
| 26 |
+
"""
|
| 27 |
+
text = (text or "").strip().lower()
|
| 28 |
+
if not text:
|
| 29 |
+
return []
|
| 30 |
+
|
| 31 |
+
tokens: List[str] = []
|
| 32 |
+
buf: List[str] = []
|
| 33 |
+
|
| 34 |
+
def flush_buf() -> None:
|
| 35 |
+
if buf:
|
| 36 |
+
tokens.append("".join(buf))
|
| 37 |
+
buf.clear()
|
| 38 |
+
|
| 39 |
+
for ch in text:
|
| 40 |
+
if _is_cjk(ch):
|
| 41 |
+
flush_buf()
|
| 42 |
+
tokens.append(ch)
|
| 43 |
+
continue
|
| 44 |
+
# 英文/数字累计成 word,其他当作分隔符
|
| 45 |
+
if ch.isalnum() or ch in {"_", "-"}:
|
| 46 |
+
buf.append(ch)
|
| 47 |
+
else:
|
| 48 |
+
flush_buf()
|
| 49 |
+
flush_buf()
|
| 50 |
+
return tokens
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@dataclass
|
| 54 |
+
class VocabConfig:
|
| 55 |
+
max_vocab: int = 8000
|
| 56 |
+
min_freq: int = 1
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class SimpleVocab:
|
| 60 |
+
"""
|
| 61 |
+
极简词表:
|
| 62 |
+
- id=0: <pad>
|
| 63 |
+
- id=1: <unk>
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
PAD = "<pad>"
|
| 67 |
+
UNK = "<unk>"
|
| 68 |
+
|
| 69 |
+
def __init__(self, token_to_id: Dict[str, int]):
|
| 70 |
+
if self.PAD not in token_to_id or self.UNK not in token_to_id:
|
| 71 |
+
raise ValueError("token_to_id 必须包含 <pad>/<unk>")
|
| 72 |
+
self.token_to_id = token_to_id
|
| 73 |
+
self.id_to_token = {i: t for t, i in token_to_id.items()}
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def pad_id(self) -> int:
|
| 77 |
+
return int(self.token_to_id[self.PAD])
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def unk_id(self) -> int:
|
| 81 |
+
return int(self.token_to_id[self.UNK])
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def size(self) -> int:
|
| 85 |
+
return len(self.token_to_id)
|
| 86 |
+
|
| 87 |
+
@classmethod
|
| 88 |
+
def build(cls, texts: Iterable[str], cfg: VocabConfig, extra_tokens: Optional[Sequence[str]] = None) -> "SimpleVocab":
|
| 89 |
+
"""
|
| 90 |
+
构建词表。
|
| 91 |
+
- texts:训练语料
|
| 92 |
+
- extra_tokens:可选“强制加入”的 token 文本(会先 tokenize 再加入)
|
| 93 |
+
|
| 94 |
+
说明:
|
| 95 |
+
- 续训时 tokenizer/vocab 必须与 checkpoint 一致,否则 embedding 尺寸会不匹配;
|
| 96 |
+
因此 extra_tokens 仅适用于“新训练”阶段。
|
| 97 |
+
"""
|
| 98 |
+
freq: Dict[str, int] = {}
|
| 99 |
+
for t in texts:
|
| 100 |
+
for tok in tokenize(t):
|
| 101 |
+
freq[tok] = freq.get(tok, 0) + 1
|
| 102 |
+
|
| 103 |
+
# 按频率排序,截断到 max_vocab(保留 PAD/UNK)
|
| 104 |
+
items = sorted(freq.items(), key=lambda x: (-x[1], x[0]))
|
| 105 |
+
items = [it for it in items if it[1] >= cfg.min_freq]
|
| 106 |
+
|
| 107 |
+
token_to_id = {cls.PAD: 0, cls.UNK: 1}
|
| 108 |
+
|
| 109 |
+
# 先插入强制 token(优先级高于按频率截断)
|
| 110 |
+
if extra_tokens:
|
| 111 |
+
forced: List[str] = []
|
| 112 |
+
for t in extra_tokens:
|
| 113 |
+
forced.extend(tokenize(t))
|
| 114 |
+
for tok in forced:
|
| 115 |
+
if tok in token_to_id:
|
| 116 |
+
continue
|
| 117 |
+
if len(token_to_id) >= int(cfg.max_vocab):
|
| 118 |
+
break
|
| 119 |
+
token_to_id[tok] = len(token_to_id)
|
| 120 |
+
|
| 121 |
+
# 再按频率补齐到 max_vocab
|
| 122 |
+
for tok, _ in items:
|
| 123 |
+
if tok in token_to_id:
|
| 124 |
+
continue
|
| 125 |
+
if len(token_to_id) >= int(cfg.max_vocab):
|
| 126 |
+
break
|
| 127 |
+
token_to_id[tok] = len(token_to_id)
|
| 128 |
+
return cls(token_to_id)
|
| 129 |
+
|
| 130 |
+
def encode(self, text: str, max_len: int) -> List[int]:
|
| 131 |
+
toks = tokenize(text)
|
| 132 |
+
ids = [self.token_to_id.get(t, self.unk_id) for t in toks[:max_len]]
|
| 133 |
+
if len(ids) < max_len:
|
| 134 |
+
ids.extend([self.pad_id] * (max_len - len(ids)))
|
| 135 |
+
return ids
|
| 136 |
+
|
| 137 |
+
def to_dict(self) -> Dict:
|
| 138 |
+
return {"token_to_id": self.token_to_id}
|
| 139 |
+
|
| 140 |
+
@classmethod
|
| 141 |
+
def from_dict(cls, d: Dict) -> "SimpleVocab":
|
| 142 |
+
return cls(token_to_id=dict(d["token_to_id"]))
|
| 143 |
+
|
| 144 |
+
|
blockdiffusion/text/text_encoder.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch import nn
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@dataclass(frozen=True)
|
| 11 |
+
class TextEncoderConfig:
|
| 12 |
+
"""
|
| 13 |
+
文本编码器配置:
|
| 14 |
+
- 目标是轻量、可训练、可随 checkpoint 一起保存,不依赖外部大模型。
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
vocab_size: int
|
| 18 |
+
max_len: int = 32
|
| 19 |
+
emb_dim: int = 128
|
| 20 |
+
out_dim: int = 256
|
| 21 |
+
dropout: float = 0.0
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class TextEncoder(nn.Module):
|
| 25 |
+
"""
|
| 26 |
+
极简 TextEncoder:
|
| 27 |
+
- token embedding
|
| 28 |
+
- masked mean pooling
|
| 29 |
+
- MLP 投影到 out_dim(通常与 UNet time_emb_dim 对齐)
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
def __init__(self, cfg: TextEncoderConfig, pad_id: int = 0):
|
| 33 |
+
super().__init__()
|
| 34 |
+
self.cfg = cfg
|
| 35 |
+
self.pad_id = int(pad_id)
|
| 36 |
+
|
| 37 |
+
self.emb = nn.Embedding(cfg.vocab_size, cfg.emb_dim, padding_idx=self.pad_id)
|
| 38 |
+
self.proj = nn.Sequential(
|
| 39 |
+
nn.Dropout(cfg.dropout) if cfg.dropout > 0 else nn.Identity(),
|
| 40 |
+
nn.Linear(cfg.emb_dim, cfg.out_dim),
|
| 41 |
+
nn.SiLU(),
|
| 42 |
+
nn.Linear(cfg.out_dim, cfg.out_dim),
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
def forward(self, tokens: torch.Tensor, mask: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 46 |
+
"""
|
| 47 |
+
tokens: (B, L) int64
|
| 48 |
+
mask: (B, L) bool/0-1;为 None 时自动根据 pad_id 生成
|
| 49 |
+
返回:
|
| 50 |
+
- cond: (B, out_dim)
|
| 51 |
+
"""
|
| 52 |
+
if mask is None:
|
| 53 |
+
mask = (tokens != self.pad_id)
|
| 54 |
+
mask_f = mask.to(dtype=self.emb.weight.dtype)
|
| 55 |
+
|
| 56 |
+
x = self.emb(tokens) # (B,L,D)
|
| 57 |
+
# masked mean
|
| 58 |
+
denom = mask_f.sum(dim=1, keepdim=True).clamp(min=1.0)
|
| 59 |
+
pooled = (x * mask_f.unsqueeze(-1)).sum(dim=1) / denom
|
| 60 |
+
|
| 61 |
+
# 若整句为空(mask 全 0),pooled 会是全 0,等价于无条件
|
| 62 |
+
return self.proj(pooled)
|
| 63 |
+
|
| 64 |
+
|
blockdiffusion/train/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
训练模块:AMP、EMA、checkpoint。
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from .trainer import Trainer, TrainerConfig
|
| 6 |
+
from .guidance import CFGuidanceModel
|
| 7 |
+
|
| 8 |
+
__all__ = [
|
| 9 |
+
"Trainer",
|
| 10 |
+
"TrainerConfig",
|
| 11 |
+
"CFGuidanceModel",
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
|
blockdiffusion/train/guidance.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any, Dict, Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class CFGuidanceModel(nn.Module):
|
| 10 |
+
"""
|
| 11 |
+
Classifier-Free Guidance 采样包装器:
|
| 12 |
+
eps = eps_uncond + scale * (eps_cond - eps_uncond)
|
| 13 |
+
|
| 14 |
+
说明:
|
| 15 |
+
- base_model 必须支持 forward(x, t, **model_kwargs)
|
| 16 |
+
- cond_kwargs/uncond_kwargs 用于传 tokens/mask 等条件信息
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
def __init__(
|
| 20 |
+
self,
|
| 21 |
+
base_model: nn.Module,
|
| 22 |
+
cond_kwargs: Optional[Dict[str, Any]] = None,
|
| 23 |
+
uncond_kwargs: Optional[Dict[str, Any]] = None,
|
| 24 |
+
scale: float = 3.0,
|
| 25 |
+
):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.base_model = base_model
|
| 28 |
+
self.cond_kwargs = dict(cond_kwargs or {})
|
| 29 |
+
self.uncond_kwargs = dict(uncond_kwargs or {})
|
| 30 |
+
self.scale = float(scale)
|
| 31 |
+
|
| 32 |
+
def forward(self, x: torch.Tensor, t: torch.Tensor, **_: Any) -> torch.Tensor:
|
| 33 |
+
eps_uncond = self.base_model(x, t, **self.uncond_kwargs)
|
| 34 |
+
eps_cond = self.base_model(x, t, **self.cond_kwargs)
|
| 35 |
+
return eps_uncond + self.scale * (eps_cond - eps_uncond)
|
| 36 |
+
|
| 37 |
+
|
blockdiffusion/train/trainer.py
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Any, Dict, Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from torch import nn
|
| 9 |
+
from contextlib import contextmanager
|
| 10 |
+
from torch.utils.data import DataLoader
|
| 11 |
+
|
| 12 |
+
from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion
|
| 13 |
+
from blockdiffusion.train.guidance import CFGuidanceModel
|
| 14 |
+
from blockdiffusion.utils.checkpoint import load_checkpoint, save_checkpoint
|
| 15 |
+
from blockdiffusion.utils.ema import EMA
|
| 16 |
+
from blockdiffusion.utils.image_io import save_tensor_grid
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _make_grad_scaler(enabled: bool, device_type: str):
|
| 20 |
+
"""
|
| 21 |
+
兼容不同 PyTorch 版本的 AMP GradScaler:
|
| 22 |
+
- 新版推荐 torch.amp.GradScaler(device, ...)
|
| 23 |
+
- 旧版使用 torch.cuda.amp.GradScaler(...)
|
| 24 |
+
"""
|
| 25 |
+
enabled = bool(enabled and device_type == "cuda")
|
| 26 |
+
try:
|
| 27 |
+
from torch.amp import GradScaler as _GS # PyTorch 2.x
|
| 28 |
+
|
| 29 |
+
if not enabled:
|
| 30 |
+
return _GS(enabled=False)
|
| 31 |
+
# 有的版本是 GradScaler(device='cuda', ...),有的接受位置参数
|
| 32 |
+
try:
|
| 33 |
+
return _GS("cuda", enabled=True)
|
| 34 |
+
except TypeError:
|
| 35 |
+
return _GS(enabled=True)
|
| 36 |
+
except Exception:
|
| 37 |
+
from torch.cuda.amp import GradScaler as _GS # 兼容旧版本
|
| 38 |
+
|
| 39 |
+
return _GS(enabled=enabled)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@contextmanager
|
| 43 |
+
def _autocast(device_type: str, enabled: bool):
|
| 44 |
+
"""
|
| 45 |
+
兼容不同 PyTorch 版本的 autocast:
|
| 46 |
+
- torch.amp.autocast(device_type=..., enabled=...)
|
| 47 |
+
- torch.cuda.amp.autocast(enabled=...)
|
| 48 |
+
"""
|
| 49 |
+
enabled = bool(enabled and device_type == "cuda")
|
| 50 |
+
try:
|
| 51 |
+
from torch.amp import autocast as _AC
|
| 52 |
+
|
| 53 |
+
with _AC(device_type=device_type, enabled=enabled):
|
| 54 |
+
yield
|
| 55 |
+
except Exception:
|
| 56 |
+
from torch.cuda.amp import autocast as _AC
|
| 57 |
+
|
| 58 |
+
with _AC(enabled=enabled):
|
| 59 |
+
yield
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass
|
| 63 |
+
class TrainerConfig:
|
| 64 |
+
"""
|
| 65 |
+
训练器配置(尽量给出稳定且适合 16×16 的默认值)。
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
# 数据
|
| 69 |
+
image_size: int = 16
|
| 70 |
+
channels: int = 3
|
| 71 |
+
batch_size: int = 512
|
| 72 |
+
num_workers: int = 4
|
| 73 |
+
|
| 74 |
+
# 训练
|
| 75 |
+
lr: float = 2e-4
|
| 76 |
+
weight_decay: float = 1e-4
|
| 77 |
+
total_steps: int = 200_000
|
| 78 |
+
grad_clip: float = 1.0
|
| 79 |
+
amp: bool = True
|
| 80 |
+
cond_drop_prob: float = 0.1 # 仅在有条件训练时生效(CFG 训练)
|
| 81 |
+
grad_accum_steps: int = 1 # 梯度累积:提高有效 batch(质量更稳),也可用于控制显存
|
| 82 |
+
lr_warmup_steps: int = 2000 # 学习率 warmup 步数(0 表示关闭)
|
| 83 |
+
lr_min_ratio: float = 0.1 # 余弦退火最低 lr 比例(相对 lr)
|
| 84 |
+
snr_gamma: float = 5.0 # SNR 加权 MSE 的 gamma(<=0 表示关闭)
|
| 85 |
+
channels_last: bool = True # Conv 的内存布局优化,通常能略提速
|
| 86 |
+
|
| 87 |
+
# EMA
|
| 88 |
+
ema_decay: float = 0.9999
|
| 89 |
+
|
| 90 |
+
# 日志/保存
|
| 91 |
+
log_every: int = 100
|
| 92 |
+
save_every: int = 5_000
|
| 93 |
+
sample_every: int = 5_000
|
| 94 |
+
sample_batch: int = 64
|
| 95 |
+
sample_use_ddim: bool = True
|
| 96 |
+
sample_ddim_steps: int = 50
|
| 97 |
+
sample_ddim_eta: float = 0.0
|
| 98 |
+
sample_cfg_scale: float = 3.0
|
| 99 |
+
|
| 100 |
+
out_dir: str = "outputs"
|
| 101 |
+
run_name: str = "default"
|
| 102 |
+
resume_path: Optional[str] = None
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class Trainer:
|
| 106 |
+
"""
|
| 107 |
+
训练器只负责:
|
| 108 |
+
- dataloader
|
| 109 |
+
- 优化器/AMP/EMA
|
| 110 |
+
- checkpoint 与训练过程采样
|
| 111 |
+
扩散公式与采样过程由 GaussianDiffusion 负责,保证解耦。
|
| 112 |
+
"""
|
| 113 |
+
|
| 114 |
+
def __init__(
|
| 115 |
+
self,
|
| 116 |
+
model: nn.Module,
|
| 117 |
+
diffusion: GaussianDiffusion,
|
| 118 |
+
dataloader: DataLoader,
|
| 119 |
+
device: torch.device,
|
| 120 |
+
cfg: TrainerConfig,
|
| 121 |
+
meta: Optional[dict] = None,
|
| 122 |
+
sample_model_kwargs: Optional[Dict[str, Any]] = None,
|
| 123 |
+
sample_uncond_model_kwargs: Optional[Dict[str, Any]] = None,
|
| 124 |
+
):
|
| 125 |
+
self.model = model.to(device)
|
| 126 |
+
self.diffusion = diffusion
|
| 127 |
+
self.dataloader = dataloader
|
| 128 |
+
self.device = device
|
| 129 |
+
self.cfg = cfg
|
| 130 |
+
self.meta = meta or {}
|
| 131 |
+
self.sample_model_kwargs = sample_model_kwargs
|
| 132 |
+
self.sample_uncond_model_kwargs = sample_uncond_model_kwargs
|
| 133 |
+
|
| 134 |
+
# 可选:channels_last 提升卷积吞吐(对小图也常有帮助)
|
| 135 |
+
if cfg.channels_last and device.type == "cuda":
|
| 136 |
+
self.model = self.model.to(memory_format=torch.channels_last)
|
| 137 |
+
|
| 138 |
+
self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)
|
| 139 |
+
use_amp = bool(cfg.amp and device.type == "cuda")
|
| 140 |
+
self.scaler = _make_grad_scaler(enabled=use_amp, device_type=device.type)
|
| 141 |
+
self.ema = EMA(self.model, decay=cfg.ema_decay)
|
| 142 |
+
|
| 143 |
+
self.step = 0
|
| 144 |
+
self._prepare_dirs()
|
| 145 |
+
|
| 146 |
+
if cfg.resume_path:
|
| 147 |
+
self._resume(cfg.resume_path)
|
| 148 |
+
|
| 149 |
+
# 学习率调度(warmup + cosine decay),提升稳定性/质量
|
| 150 |
+
self.base_lr = float(cfg.lr)
|
| 151 |
+
self.min_lr = float(cfg.lr) * float(cfg.lr_min_ratio)
|
| 152 |
+
|
| 153 |
+
def _prepare_dirs(self) -> None:
|
| 154 |
+
self.run_dir = os.path.join(self.cfg.out_dir, self.cfg.run_name)
|
| 155 |
+
self.ckpt_dir = os.path.join(self.run_dir, "checkpoints")
|
| 156 |
+
self.sample_dir = os.path.join(self.run_dir, "samples")
|
| 157 |
+
os.makedirs(self.ckpt_dir, exist_ok=True)
|
| 158 |
+
os.makedirs(self.sample_dir, exist_ok=True)
|
| 159 |
+
|
| 160 |
+
def _resume(self, path: str) -> None:
|
| 161 |
+
ckpt = load_checkpoint(path, map_location="cpu")
|
| 162 |
+
self.model.load_state_dict(ckpt["model"], strict=True)
|
| 163 |
+
if "ema" in ckpt:
|
| 164 |
+
self.ema.load_state_dict(ckpt["ema"])
|
| 165 |
+
# 续训:确保 EMA shadow 跟随当前训练设备
|
| 166 |
+
self.ema.to(self.device)
|
| 167 |
+
if "optimizer" in ckpt:
|
| 168 |
+
self.optimizer.load_state_dict(ckpt["optimizer"])
|
| 169 |
+
# 续训:optimizer state 里的 tensor 默认仍在 CPU,需要搬到当前设备
|
| 170 |
+
for state in self.optimizer.state.values():
|
| 171 |
+
for kk, vv in list(state.items()):
|
| 172 |
+
if torch.is_tensor(vv):
|
| 173 |
+
state[kk] = vv.to(self.device)
|
| 174 |
+
if "scaler" in ckpt:
|
| 175 |
+
self.scaler.load_state_dict(ckpt["scaler"])
|
| 176 |
+
self.step = int(ckpt.get("step", 0))
|
| 177 |
+
|
| 178 |
+
def _save(self) -> None:
|
| 179 |
+
path = os.path.join(self.ckpt_dir, f"step_{self.step}.pt")
|
| 180 |
+
payload = {
|
| 181 |
+
"step": self.step,
|
| 182 |
+
"meta": self.meta,
|
| 183 |
+
"trainer_cfg": self.cfg.__dict__,
|
| 184 |
+
"model": self.model.state_dict(),
|
| 185 |
+
"ema": self.ema.state_dict(),
|
| 186 |
+
"optimizer": self.optimizer.state_dict(),
|
| 187 |
+
"scaler": self.scaler.state_dict(),
|
| 188 |
+
}
|
| 189 |
+
save_checkpoint(path, payload)
|
| 190 |
+
|
| 191 |
+
@torch.no_grad()
|
| 192 |
+
def _sample(self) -> None:
|
| 193 |
+
# 采样时使用 EMA 参数,避免训练噪声导致的质量抖动
|
| 194 |
+
tmp = {k: v.detach().clone() for k, v in self.model.state_dict().items()}
|
| 195 |
+
self.ema.copy_to(self.model)
|
| 196 |
+
self.model.eval()
|
| 197 |
+
|
| 198 |
+
shape = (self.cfg.sample_batch, self.cfg.channels, self.cfg.image_size, self.cfg.image_size)
|
| 199 |
+
sampler_model: nn.Module = self.model
|
| 200 |
+
if self.sample_model_kwargs is not None and self.sample_uncond_model_kwargs is not None:
|
| 201 |
+
sampler_model = CFGuidanceModel(
|
| 202 |
+
base_model=self.model,
|
| 203 |
+
cond_kwargs=self.sample_model_kwargs,
|
| 204 |
+
uncond_kwargs=self.sample_uncond_model_kwargs,
|
| 205 |
+
scale=self.cfg.sample_cfg_scale,
|
| 206 |
+
)
|
| 207 |
+
if self.cfg.sample_use_ddim:
|
| 208 |
+
x = self.diffusion.ddim_sample_loop(
|
| 209 |
+
sampler_model,
|
| 210 |
+
shape=shape,
|
| 211 |
+
device=self.device,
|
| 212 |
+
steps=self.cfg.sample_ddim_steps,
|
| 213 |
+
eta=self.cfg.sample_ddim_eta,
|
| 214 |
+
model_kwargs=None,
|
| 215 |
+
progress=False,
|
| 216 |
+
)
|
| 217 |
+
else:
|
| 218 |
+
x = self.diffusion.p_sample_loop(sampler_model, shape=shape, device=self.device, model_kwargs=None, progress=False)
|
| 219 |
+
|
| 220 |
+
# [-1,1] -> [0,1] 保存
|
| 221 |
+
x = (x.clamp(-1, 1) + 1) * 0.5
|
| 222 |
+
out_path = os.path.join(self.sample_dir, f"step_{self.step}.png")
|
| 223 |
+
save_tensor_grid(x, out_path, nrow=int(self.cfg.sample_batch**0.5), padding=1)
|
| 224 |
+
|
| 225 |
+
# 恢复训练参数
|
| 226 |
+
self.model.load_state_dict(tmp, strict=True)
|
| 227 |
+
self.model.train()
|
| 228 |
+
|
| 229 |
+
def train(self) -> None:
|
| 230 |
+
self.model.train()
|
| 231 |
+
data_iter = iter(self.dataloader)
|
| 232 |
+
|
| 233 |
+
while self.step < self.cfg.total_steps:
|
| 234 |
+
# 梯度累积:每 grad_accum_steps 次 forward 才做一次 optimizer step
|
| 235 |
+
accum_steps = max(1, int(self.cfg.grad_accum_steps))
|
| 236 |
+
self.optimizer.zero_grad(set_to_none=True)
|
| 237 |
+
|
| 238 |
+
total_loss = 0.0
|
| 239 |
+
for micro in range(accum_steps):
|
| 240 |
+
try:
|
| 241 |
+
batch = next(data_iter)
|
| 242 |
+
except StopIteration:
|
| 243 |
+
data_iter = iter(self.dataloader)
|
| 244 |
+
batch = next(data_iter)
|
| 245 |
+
|
| 246 |
+
model_kwargs: Dict[str, Any] = {}
|
| 247 |
+
if isinstance(batch, (tuple, list)) and len(batch) == 3:
|
| 248 |
+
x0, tokens, mask = batch
|
| 249 |
+
model_kwargs["tokens"] = tokens.to(self.device, non_blocking=True)
|
| 250 |
+
model_kwargs["mask"] = mask.to(self.device, non_blocking=True)
|
| 251 |
+
else:
|
| 252 |
+
x0 = batch
|
| 253 |
+
|
| 254 |
+
# channels_last:数据也转换同布局
|
| 255 |
+
x0 = x0.to(self.device, non_blocking=True)
|
| 256 |
+
if self.cfg.channels_last and self.device.type == "cuda":
|
| 257 |
+
x0 = x0.contiguous(memory_format=torch.channels_last)
|
| 258 |
+
|
| 259 |
+
b = x0.shape[0]
|
| 260 |
+
t = torch.randint(0, self.diffusion.num_timesteps, (b,), device=self.device, dtype=torch.long)
|
| 261 |
+
|
| 262 |
+
with _autocast(device_type=self.device.type, enabled=self.scaler.is_enabled()):
|
| 263 |
+
loss = self.diffusion.training_losses(
|
| 264 |
+
self.model,
|
| 265 |
+
x0,
|
| 266 |
+
t,
|
| 267 |
+
model_kwargs=model_kwargs,
|
| 268 |
+
cond_drop_prob=self.cfg.cond_drop_prob,
|
| 269 |
+
snr_gamma=float(self.cfg.snr_gamma),
|
| 270 |
+
)["loss"]
|
| 271 |
+
# 累积时按比例缩放,保持有效学习率不变
|
| 272 |
+
loss = loss / accum_steps
|
| 273 |
+
|
| 274 |
+
self.scaler.scale(loss).backward()
|
| 275 |
+
total_loss += float(loss.detach().item())
|
| 276 |
+
|
| 277 |
+
# 学习率 warmup + cosine decay(基于全局 step)
|
| 278 |
+
if self.cfg.lr_warmup_steps and self.cfg.lr_warmup_steps > 0 and self.step < self.cfg.lr_warmup_steps:
|
| 279 |
+
lr = self.min_lr + (self.base_lr - self.min_lr) * (float(self.step + 1) / float(self.cfg.lr_warmup_steps))
|
| 280 |
+
else:
|
| 281 |
+
# cosine 从 warmup 结束后开始
|
| 282 |
+
warm = max(0, int(self.cfg.lr_warmup_steps))
|
| 283 |
+
prog = (self.step - warm) / max(1.0, float(self.cfg.total_steps - warm))
|
| 284 |
+
prog = min(max(prog, 0.0), 1.0)
|
| 285 |
+
import math
|
| 286 |
+
|
| 287 |
+
lr = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1.0 + math.cos(math.pi * prog))
|
| 288 |
+
for pg in self.optimizer.param_groups:
|
| 289 |
+
pg["lr"] = lr
|
| 290 |
+
|
| 291 |
+
if self.cfg.grad_clip and self.cfg.grad_clip > 0:
|
| 292 |
+
self.scaler.unscale_(self.optimizer)
|
| 293 |
+
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.cfg.grad_clip)
|
| 294 |
+
|
| 295 |
+
self.scaler.step(self.optimizer)
|
| 296 |
+
self.scaler.update()
|
| 297 |
+
|
| 298 |
+
self.ema.update(self.model)
|
| 299 |
+
self.step += 1
|
| 300 |
+
|
| 301 |
+
if self.cfg.log_every > 0 and (self.step % self.cfg.log_every == 0):
|
| 302 |
+
# total_loss 已按 accum_steps 平均后的值(每 micro 先 /accum)
|
| 303 |
+
print(f"[step {self.step}] loss={total_loss:.6f} lr={lr:.2e}")
|
| 304 |
+
|
| 305 |
+
if self.cfg.save_every > 0 and (self.step % self.cfg.save_every == 0):
|
| 306 |
+
self._save()
|
| 307 |
+
|
| 308 |
+
if self.cfg.sample_every > 0 and (self.step % self.cfg.sample_every == 0):
|
| 309 |
+
self._sample()
|
| 310 |
+
|
| 311 |
+
|
blockdiffusion/utils/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
通用工具:EMA、随机种子、checkpoint 等。
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
|
blockdiffusion/utils/checkpoint.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Any, Dict, Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def save_checkpoint(path: str, payload: Dict[str, Any]) -> None:
|
| 10 |
+
"""
|
| 11 |
+
保存 checkpoint(会自动创建目录)。
|
| 12 |
+
"""
|
| 13 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 14 |
+
torch.save(payload, path)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def load_checkpoint(path: str, map_location: Optional[str] = None) -> Dict[str, Any]:
|
| 18 |
+
"""
|
| 19 |
+
读取 checkpoint。
|
| 20 |
+
"""
|
| 21 |
+
return torch.load(path, map_location=map_location)
|
| 22 |
+
|
| 23 |
+
|
blockdiffusion/utils/ema.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Dict
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class EMA:
|
| 10 |
+
"""
|
| 11 |
+
Exponential Moving Average:对模型参数做滑动平均,提升采样稳定性。
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
def __init__(self, model: nn.Module, decay: float = 0.9999):
|
| 15 |
+
if not (0.0 < decay < 1.0):
|
| 16 |
+
raise ValueError("decay 必须在 (0, 1) 之间")
|
| 17 |
+
self.decay = float(decay)
|
| 18 |
+
self.shadow: Dict[str, torch.Tensor] = {}
|
| 19 |
+
self._init_from(model)
|
| 20 |
+
|
| 21 |
+
def _init_from(self, model: nn.Module) -> None:
|
| 22 |
+
self.shadow = {
|
| 23 |
+
k: v.detach().clone().float()
|
| 24 |
+
for k, v in model.state_dict().items()
|
| 25 |
+
if v.dtype.is_floating_point
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
@torch.no_grad()
|
| 29 |
+
def update(self, model: nn.Module) -> None:
|
| 30 |
+
"""
|
| 31 |
+
用当前模型参数更新 EMA。
|
| 32 |
+
"""
|
| 33 |
+
msd = model.state_dict()
|
| 34 |
+
for k, v in msd.items():
|
| 35 |
+
if k not in self.shadow:
|
| 36 |
+
continue
|
| 37 |
+
if not v.dtype.is_floating_point:
|
| 38 |
+
continue
|
| 39 |
+
# 续训场景:shadow 可能从 checkpoint 载入在 CPU,需要对齐到参数所在 device
|
| 40 |
+
if self.shadow[k].device != v.device:
|
| 41 |
+
self.shadow[k] = self.shadow[k].to(device=v.device)
|
| 42 |
+
self.shadow[k].mul_(self.decay).add_(v.detach().float(), alpha=1.0 - self.decay)
|
| 43 |
+
|
| 44 |
+
def state_dict(self) -> Dict[str, torch.Tensor]:
|
| 45 |
+
return {"decay": torch.tensor(self.decay), "shadow": self.shadow}
|
| 46 |
+
|
| 47 |
+
def load_state_dict(self, state: Dict) -> None:
|
| 48 |
+
self.decay = float(state["decay"].item() if isinstance(state["decay"], torch.Tensor) else state["decay"])
|
| 49 |
+
self.shadow = {k: v.detach().clone() for k, v in state["shadow"].items()}
|
| 50 |
+
|
| 51 |
+
def to(self, device: torch.device) -> None:
|
| 52 |
+
"""
|
| 53 |
+
将 EMA 的 shadow 参数移动到指定 device(续训/切换设备时使用)。
|
| 54 |
+
"""
|
| 55 |
+
self.shadow = {k: v.to(device=device) for k, v in self.shadow.items()}
|
| 56 |
+
|
| 57 |
+
@torch.no_grad()
|
| 58 |
+
def copy_to(self, model: nn.Module) -> None:
|
| 59 |
+
"""
|
| 60 |
+
将 EMA 参数拷贝到目标模型(常用于采样时临时替换)。
|
| 61 |
+
"""
|
| 62 |
+
msd = model.state_dict()
|
| 63 |
+
for k in self.shadow.keys():
|
| 64 |
+
if k in msd:
|
| 65 |
+
src = self.shadow[k]
|
| 66 |
+
if src.device != msd[k].device:
|
| 67 |
+
src = src.to(msd[k].device)
|
| 68 |
+
msd[k].copy_(src.to(msd[k].dtype))
|
| 69 |
+
|
| 70 |
+
|
blockdiffusion/utils/image_io.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from PIL import Image
|
| 8 |
+
from torchvision.utils import make_grid
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def save_tensor_grid(
|
| 12 |
+
x: torch.Tensor,
|
| 13 |
+
path: str,
|
| 14 |
+
nrow: int,
|
| 15 |
+
padding: int = 1,
|
| 16 |
+
) -> None:
|
| 17 |
+
"""
|
| 18 |
+
保存图片网格,支持 RGB(3) / RGBA(4)。
|
| 19 |
+
|
| 20 |
+
约定:
|
| 21 |
+
- x: (B,C,H,W) 且值域在 [0,1]
|
| 22 |
+
- path: 输出文件路径(推荐 .png)
|
| 23 |
+
"""
|
| 24 |
+
if x.ndim != 4:
|
| 25 |
+
raise ValueError("x 必须是 (B,C,H,W)")
|
| 26 |
+
if x.shape[1] not in (3, 4):
|
| 27 |
+
raise ValueError("仅支持 3(RGB) 或 4(RGBA) 通道")
|
| 28 |
+
|
| 29 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 30 |
+
|
| 31 |
+
grid = make_grid(x, nrow=nrow, padding=padding) # (C,H,W)
|
| 32 |
+
grid = grid.clamp(0.0, 1.0)
|
| 33 |
+
arr = (grid.permute(1, 2, 0) * 255.0).to(torch.uint8).cpu().numpy()
|
| 34 |
+
|
| 35 |
+
mode = "RGBA" if arr.shape[2] == 4 else "RGB"
|
| 36 |
+
im = Image.fromarray(arr, mode=mode)
|
| 37 |
+
im.save(path)
|
| 38 |
+
|
| 39 |
+
|
blockdiffusion/utils/seed.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import random
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def seed_everything(seed: int, deterministic: bool = False) -> None:
|
| 11 |
+
"""
|
| 12 |
+
统一设置随机种子,便于复现实验。
|
| 13 |
+
注意:deterministic=True 可能会降低性能。
|
| 14 |
+
"""
|
| 15 |
+
os.environ["PYTHONHASHSEED"] = str(seed)
|
| 16 |
+
random.seed(seed)
|
| 17 |
+
torch.manual_seed(seed)
|
| 18 |
+
# 重要:CPU 推理/CPU 环境下不要触发 CUDA 初始化,避免无谓开销或在无 CUDA 编译时报错。
|
| 19 |
+
if torch.cuda.is_available():
|
| 20 |
+
torch.cuda.manual_seed_all(seed)
|
| 21 |
+
if deterministic:
|
| 22 |
+
torch.backends.cudnn.deterministic = True
|
| 23 |
+
torch.backends.cudnn.benchmark = False
|
| 24 |
+
else:
|
| 25 |
+
torch.backends.cudnn.benchmark = True
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def get_device(device: Optional[str] = None) -> torch.device:
|
| 29 |
+
"""
|
| 30 |
+
返回训练设备。默认优先 CUDA,其次 CPU。
|
| 31 |
+
"""
|
| 32 |
+
if device is not None:
|
| 33 |
+
return torch.device(device)
|
| 34 |
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 35 |
+
|
| 36 |
+
|
modelscope_ui_app.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
ModelScope SwingDeploy(FC)UI 函数 app.py(仅适配 text-to-image-synthesis)。
|
| 5 |
+
|
| 6 |
+
用法:
|
| 7 |
+
- 将本文件内容复制到 FC 的 model_ui_func 里作为 app.py
|
| 8 |
+
- 环境变量由部署自动注入:
|
| 9 |
+
- MODEL_ID / MODEL_VERSION / TASK / API_URL / MODEL_BACKEND
|
| 10 |
+
|
| 11 |
+
说明:
|
| 12 |
+
- UI 会把请求转发到 API_URL + "/invoke"(pipeline 模式默认);
|
| 13 |
+
- 兼容服务端返回 image 的两种形式:
|
| 14 |
+
- base64 字符串(可带 data:image/... 前缀)
|
| 15 |
+
- HWC 的嵌套数组(uint8)
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import base64
|
| 19 |
+
import io
|
| 20 |
+
import os
|
| 21 |
+
import json
|
| 22 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 23 |
+
|
| 24 |
+
import gradio as gr
|
| 25 |
+
import numpy as np
|
| 26 |
+
import requests
|
| 27 |
+
from PIL import Image
|
| 28 |
+
|
| 29 |
+
MODEL_ID = os.getenv("MODEL_ID", "") or ""
|
| 30 |
+
MODEL_VERSION = os.getenv("MODEL_VERSION", "") or ""
|
| 31 |
+
TASK = os.getenv("TASK", "") or ""
|
| 32 |
+
API_URL = (os.getenv("API_URL", "") or "").rstrip("/")
|
| 33 |
+
MODEL_BACKEND = os.getenv("MODEL_BACKEND", "pipeline") or "pipeline"
|
| 34 |
+
|
| 35 |
+
# pipeline 默认推理路径为 /invoke;ollama 会是 /v1(这里只做兼容,不建议用 UI 跑 ollama)
|
| 36 |
+
INFER_PATH = "v1" if MODEL_BACKEND == "ollama" else "invoke"
|
| 37 |
+
INFER_URL = (API_URL + "/" + INFER_PATH) if API_URL else ""
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _unwrap_response(resp_json: Any) -> Dict[str, Any]:
|
| 41 |
+
"""
|
| 42 |
+
SwingDeploy 常见返回:
|
| 43 |
+
- {"Code":200,"Data":{...},...}
|
| 44 |
+
也兼容直接返回 dict 的情况。
|
| 45 |
+
"""
|
| 46 |
+
if isinstance(resp_json, dict):
|
| 47 |
+
if "Code" in resp_json and resp_json.get("Code") != 200:
|
| 48 |
+
raise gr.Error(f"推理失败:Code={resp_json.get('Code')} Message={resp_json.get('Message')}")
|
| 49 |
+
data = resp_json.get("Data", None)
|
| 50 |
+
if isinstance(data, dict):
|
| 51 |
+
return data
|
| 52 |
+
# 有些情况下 Data 不是 dict,就当作整体返回
|
| 53 |
+
return resp_json
|
| 54 |
+
raise gr.Error("推理返回不是JSON对象")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _decode_base64_image(s: str) -> Optional[Image.Image]:
|
| 58 |
+
s = (s or "").strip()
|
| 59 |
+
if not s:
|
| 60 |
+
return None
|
| 61 |
+
if s.startswith("data:image"):
|
| 62 |
+
parts = s.split(",", 1)
|
| 63 |
+
if len(parts) == 2:
|
| 64 |
+
s = parts[1].strip()
|
| 65 |
+
try:
|
| 66 |
+
raw = base64.b64decode(s, validate=False)
|
| 67 |
+
return Image.open(io.BytesIO(raw)).convert("RGBA")
|
| 68 |
+
except Exception:
|
| 69 |
+
return None
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _decode_array_image(x: Any) -> Optional[Image.Image]:
|
| 73 |
+
try:
|
| 74 |
+
a = np.asarray(x)
|
| 75 |
+
except Exception:
|
| 76 |
+
return None
|
| 77 |
+
if a.ndim != 3 or a.shape[-1] not in (3, 4):
|
| 78 |
+
return None
|
| 79 |
+
if a.dtype != np.uint8:
|
| 80 |
+
try:
|
| 81 |
+
a = a.astype(np.uint8)
|
| 82 |
+
except Exception:
|
| 83 |
+
return None
|
| 84 |
+
mode = "RGBA" if a.shape[-1] == 4 else "RGB"
|
| 85 |
+
try:
|
| 86 |
+
return Image.fromarray(a, mode=mode)
|
| 87 |
+
except Exception:
|
| 88 |
+
return None
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _extract_images(data: Dict[str, Any]) -> List[Image.Image]:
|
| 92 |
+
"""
|
| 93 |
+
兼容 keys:
|
| 94 |
+
- output_img / output_imgs
|
| 95 |
+
- 其他上游包装层(递归查找)
|
| 96 |
+
"""
|
| 97 |
+
|
| 98 |
+
def pick(obj: Any) -> List[Any]:
|
| 99 |
+
found: List[Any] = []
|
| 100 |
+
if isinstance(obj, dict):
|
| 101 |
+
for k in ("output_imgs", "output_img", "images", "image", "imgs", "img"):
|
| 102 |
+
if k in obj and obj[k] is not None:
|
| 103 |
+
found.append(obj[k])
|
| 104 |
+
for v in obj.values():
|
| 105 |
+
if isinstance(v, (dict, list)):
|
| 106 |
+
found.extend(pick(v))
|
| 107 |
+
elif isinstance(obj, list):
|
| 108 |
+
for v in obj:
|
| 109 |
+
if isinstance(v, (dict, list)):
|
| 110 |
+
found.extend(pick(v))
|
| 111 |
+
return found
|
| 112 |
+
|
| 113 |
+
candidates = pick(data)
|
| 114 |
+
images: List[Image.Image] = []
|
| 115 |
+
|
| 116 |
+
def push_one(v: Any) -> None:
|
| 117 |
+
if v is None:
|
| 118 |
+
return
|
| 119 |
+
if isinstance(v, Image.Image):
|
| 120 |
+
images.append(v)
|
| 121 |
+
return
|
| 122 |
+
if isinstance(v, str):
|
| 123 |
+
im = _decode_base64_image(v)
|
| 124 |
+
if im is not None:
|
| 125 |
+
images.append(im)
|
| 126 |
+
return
|
| 127 |
+
# 数组(HWC)
|
| 128 |
+
im = _decode_array_image(v)
|
| 129 |
+
if im is not None:
|
| 130 |
+
images.append(im)
|
| 131 |
+
|
| 132 |
+
for c in candidates:
|
| 133 |
+
# output_imgs 可能是 list[img]
|
| 134 |
+
if isinstance(c, list):
|
| 135 |
+
# 也可能是单张 HWC 嵌套数组
|
| 136 |
+
im_single = _decode_array_image(c)
|
| 137 |
+
if im_single is not None:
|
| 138 |
+
images.append(im_single)
|
| 139 |
+
continue
|
| 140 |
+
for one in c:
|
| 141 |
+
push_one(one)
|
| 142 |
+
else:
|
| 143 |
+
push_one(c)
|
| 144 |
+
|
| 145 |
+
if not images:
|
| 146 |
+
raise gr.Error("推理成功但未在返回中找到图片(output_img/output_imgs)")
|
| 147 |
+
return images
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _post_infer(payload: Dict[str, Any]) -> Dict[str, Any]:
|
| 151 |
+
if not INFER_URL:
|
| 152 |
+
raise gr.Error("缺少 API_URL 环境变量,无法调用推理接口")
|
| 153 |
+
try:
|
| 154 |
+
r = requests.post(INFER_URL, json=payload, timeout=600)
|
| 155 |
+
except Exception as e:
|
| 156 |
+
raise gr.Error(f"请求失败:{e!r}")
|
| 157 |
+
try:
|
| 158 |
+
j = r.json()
|
| 159 |
+
except Exception:
|
| 160 |
+
raise gr.Error(f"返回非JSON,HTTP={r.status_code}")
|
| 161 |
+
return _unwrap_response(j)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def build_demo() -> gr.Blocks:
|
| 165 |
+
title = "ModelScope × FC:text-to-image-synthesis"
|
| 166 |
+
article = (
|
| 167 |
+
f"- 模型ID:{MODEL_ID}\n"
|
| 168 |
+
f"- 模型版本:{MODEL_VERSION}\n"
|
| 169 |
+
f"- TASK:{TASK}\n"
|
| 170 |
+
f"- 推理URL:{INFER_URL}\n"
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
with gr.Blocks(title=title) as demo:
|
| 174 |
+
gr.Markdown(f"## {title}")
|
| 175 |
+
gr.Markdown(article)
|
| 176 |
+
|
| 177 |
+
if TASK != "text-to-image-synthesis":
|
| 178 |
+
gr.Markdown("### 当前 UI 仅支持 TASK=text-to-image-synthesis")
|
| 179 |
+
return demo
|
| 180 |
+
|
| 181 |
+
with gr.Row():
|
| 182 |
+
prompt = gr.Textbox(label="Prompt", value="stone block texture", lines=2)
|
| 183 |
+
|
| 184 |
+
def build_seed_component():
|
| 185 |
+
# 兼容旧版 gradio:可能没有 gr.Number
|
| 186 |
+
if hasattr(gr, "Number"):
|
| 187 |
+
return gr.Number(label="seed(-1表示不传)", value=-1, precision=0)
|
| 188 |
+
return gr.Textbox(label="seed(-1表示不传)", value="-1")
|
| 189 |
+
|
| 190 |
+
def build_json_component():
|
| 191 |
+
# 兼容旧版 gradio:可能没有 gr.JSON
|
| 192 |
+
if hasattr(gr, "JSON"):
|
| 193 |
+
return gr.JSON(label="原始输出(Data)")
|
| 194 |
+
return gr.Textbox(label="原始输出(Data)", lines=12)
|
| 195 |
+
|
| 196 |
+
with gr.Row():
|
| 197 |
+
n = gr.Slider(label="n(生成张数)", minimum=1, maximum=8, step=1, value=1)
|
| 198 |
+
ddim_steps = gr.Slider(label="ddim_steps", minimum=5, maximum=100, step=1, value=50)
|
| 199 |
+
cfg_scale = gr.Slider(label="cfg_scale(仅 text_cond 生效)", minimum=0.0, maximum=20.0, step=0.5, value=3.0)
|
| 200 |
+
seed = build_seed_component()
|
| 201 |
+
|
| 202 |
+
run_btn = gr.Button("生成")
|
| 203 |
+
# 兼容旧版 gradio:Gallery 可能没有 .style()
|
| 204 |
+
gallery = gr.Gallery(label="结果")
|
| 205 |
+
raw_json = build_json_component()
|
| 206 |
+
|
| 207 |
+
def handler(p: str, n_v: int, steps_v: int, cfg_v: float, seed_v: Any):
|
| 208 |
+
payload: Dict[str, Any] = {
|
| 209 |
+
"input": {"text": p},
|
| 210 |
+
"parameters": {"n": int(n_v), "ddim_steps": int(steps_v), "cfg_scale": float(cfg_v)},
|
| 211 |
+
}
|
| 212 |
+
# seed 兼容:Number 会传 float/int,Textbox 会传 str
|
| 213 |
+
try:
|
| 214 |
+
seed_int = int(float(seed_v))
|
| 215 |
+
except Exception:
|
| 216 |
+
seed_int = -1
|
| 217 |
+
if seed_int >= 0:
|
| 218 |
+
payload["parameters"]["seed"] = seed_int
|
| 219 |
+
data = _post_infer(payload)
|
| 220 |
+
imgs = _extract_images(data)
|
| 221 |
+
# Gallery 支持 list[PIL.Image]
|
| 222 |
+
if hasattr(gr, "JSON"):
|
| 223 |
+
return imgs, data
|
| 224 |
+
return imgs, json.dumps(data, ensure_ascii=False, indent=2)
|
| 225 |
+
|
| 226 |
+
run_btn.click(handler, inputs=[prompt, n, ddim_steps, cfg_scale, seed], outputs=[gallery, raw_json])
|
| 227 |
+
|
| 228 |
+
return demo
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
if __name__ == "__main__":
|
| 232 |
+
demo = build_demo()
|
| 233 |
+
demo.launch(server_name="0.0.0.0")
|
| 234 |
+
|
| 235 |
+
|
requirements.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch
|
| 2 |
+
torchvision
|
| 3 |
+
pillow
|
| 4 |
+
tqdm
|
| 5 |
+
numpy
|
| 6 |
+
modelscope
|
| 7 |
+
gradio
|
| 8 |
+
|
| 9 |
+
# Hugging Face Hub:用于 Spaces 部署时从 Hub 下载 checkpoint
|
| 10 |
+
huggingface_hub
|
| 11 |
+
|
sample.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import os
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion
|
| 10 |
+
from blockdiffusion.diffusion.schedules import get_beta_schedule
|
| 11 |
+
from blockdiffusion.models.unet16 import UNet16, UNet16Config
|
| 12 |
+
from blockdiffusion.models.text_unet16 import TextCondConfig, TextCondUNet16
|
| 13 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab
|
| 14 |
+
from blockdiffusion.train.guidance import CFGuidanceModel
|
| 15 |
+
from blockdiffusion.utils.checkpoint import load_checkpoint
|
| 16 |
+
from blockdiffusion.utils.ema import EMA
|
| 17 |
+
from blockdiffusion.utils.image_io import save_tensor_grid
|
| 18 |
+
from blockdiffusion.utils.seed import get_device, seed_everything
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def parse_args() -> argparse.Namespace:
|
| 22 |
+
p = argparse.ArgumentParser(description="BlockDiffusion 16×16 纹理采样入口")
|
| 23 |
+
p.add_argument("--ckpt", type=str, required=True, help="训练生成的 checkpoint 路径(.pt)")
|
| 24 |
+
p.add_argument("--out_dir", type=str, default="samples", help="输出目录")
|
| 25 |
+
p.add_argument("--n", type=int, default=256, help="生成张数")
|
| 26 |
+
p.add_argument("--batch", type=int, default=256, help="单次采样 batch(受显存影响)")
|
| 27 |
+
|
| 28 |
+
p.add_argument("--use_ema", action="store_true", help="使用 EMA 权重采样(推荐)")
|
| 29 |
+
p.add_argument("--use_ddim", action="store_true", help="使用 DDIM 快速采样")
|
| 30 |
+
p.add_argument("--ddim_steps", type=int, default=50)
|
| 31 |
+
p.add_argument("--ddim_eta", type=float, default=0.0)
|
| 32 |
+
p.add_argument("--prompt", type=str, default=None, help="提示词(仅条件模型可用)")
|
| 33 |
+
p.add_argument("--cfg_scale", type=float, default=3.0, help="CFG scale(仅条件模型可用)")
|
| 34 |
+
|
| 35 |
+
p.add_argument("--seed", type=int, default=123)
|
| 36 |
+
p.add_argument("--device", type=str, default=None)
|
| 37 |
+
return p.parse_args()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@torch.no_grad()
|
| 41 |
+
def main() -> None:
|
| 42 |
+
args = parse_args()
|
| 43 |
+
seed_everything(args.seed, deterministic=False)
|
| 44 |
+
device = get_device(args.device)
|
| 45 |
+
|
| 46 |
+
ckpt = load_checkpoint(args.ckpt, map_location="cpu")
|
| 47 |
+
meta = ckpt.get("meta", {})
|
| 48 |
+
|
| 49 |
+
model_type = str(meta.get("model_type", "uncond"))
|
| 50 |
+
unet_cfg_dict = meta.get("unet_cfg", None)
|
| 51 |
+
if unet_cfg_dict is None:
|
| 52 |
+
# 兼容:若旧 checkpoint 未保存配置,则使用默认 16×16 RGB 配置
|
| 53 |
+
unet_cfg = UNet16Config()
|
| 54 |
+
timesteps = 1000
|
| 55 |
+
beta_schedule = "cosine"
|
| 56 |
+
image_size = 16
|
| 57 |
+
channels = 3
|
| 58 |
+
model_type = "uncond"
|
| 59 |
+
else:
|
| 60 |
+
unet_cfg = UNet16Config(**unet_cfg_dict)
|
| 61 |
+
diff_meta = meta.get("diffusion", {})
|
| 62 |
+
timesteps = int(diff_meta.get("timesteps", 1000))
|
| 63 |
+
beta_schedule = str(diff_meta.get("beta_schedule", "cosine"))
|
| 64 |
+
data_meta = meta.get("data", {})
|
| 65 |
+
image_size = int(data_meta.get("image_size", 16))
|
| 66 |
+
channels = int(data_meta.get("channels", unet_cfg.in_channels))
|
| 67 |
+
|
| 68 |
+
vocab = None
|
| 69 |
+
text_cfg = None
|
| 70 |
+
if model_type == "text_cond":
|
| 71 |
+
tok_dict = meta.get("tokenizer", None)
|
| 72 |
+
text_cfg_dict = meta.get("text_cfg", None)
|
| 73 |
+
if tok_dict is None or text_cfg_dict is None:
|
| 74 |
+
raise ValueError("该 checkpoint 标记为 text_cond,但缺少 tokenizer/text_cfg")
|
| 75 |
+
vocab = SimpleVocab.from_dict(tok_dict)
|
| 76 |
+
text_cfg = TextCondConfig(**text_cfg_dict)
|
| 77 |
+
model = TextCondUNet16(unet_cfg=unet_cfg, vocab=vocab, text_cfg=text_cfg).to(device)
|
| 78 |
+
else:
|
| 79 |
+
model = UNet16(unet_cfg).to(device)
|
| 80 |
+
model.load_state_dict(ckpt["model"], strict=True)
|
| 81 |
+
|
| 82 |
+
if args.use_ema and ("ema" in ckpt):
|
| 83 |
+
ema = EMA(model, decay=0.9999)
|
| 84 |
+
ema.load_state_dict(ckpt["ema"])
|
| 85 |
+
ema.copy_to(model)
|
| 86 |
+
|
| 87 |
+
betas = get_beta_schedule(beta_schedule, timesteps).to(device)
|
| 88 |
+
diffusion = GaussianDiffusion(betas)
|
| 89 |
+
|
| 90 |
+
os.makedirs(args.out_dir, exist_ok=True)
|
| 91 |
+
|
| 92 |
+
# 条件采样:如果提供 prompt 且模型支持,则启用 CFG
|
| 93 |
+
sampler_model = model
|
| 94 |
+
model_kwargs = None
|
| 95 |
+
if args.prompt and model_type == "text_cond":
|
| 96 |
+
if vocab is None or text_cfg is None:
|
| 97 |
+
raise ValueError("内部错误:条件模型缺少 vocab/text_cfg")
|
| 98 |
+
tok = torch.tensor([vocab.encode(args.prompt, max_len=text_cfg.max_text_len)], dtype=torch.long, device=device)
|
| 99 |
+
m = (tok != vocab.pad_id)
|
| 100 |
+
cond_kwargs = {"tokens": tok, "mask": m}
|
| 101 |
+
un_tok = torch.full_like(tok, fill_value=vocab.pad_id)
|
| 102 |
+
un_m = torch.zeros_like(m)
|
| 103 |
+
uncond_kwargs = {"tokens": un_tok, "mask": un_m}
|
| 104 |
+
sampler_model = CFGuidanceModel(model, cond_kwargs=cond_kwargs, uncond_kwargs=uncond_kwargs, scale=args.cfg_scale).to(device)
|
| 105 |
+
model_kwargs = None
|
| 106 |
+
elif args.prompt and model_type != "text_cond":
|
| 107 |
+
raise ValueError("当前 checkpoint 是无条件模型,不支持 --prompt;请用带 captions 训练的模型")
|
| 108 |
+
|
| 109 |
+
all_imgs = []
|
| 110 |
+
remaining = args.n
|
| 111 |
+
while remaining > 0:
|
| 112 |
+
b = min(args.batch, remaining)
|
| 113 |
+
shape = (b, channels, image_size, image_size)
|
| 114 |
+
if args.use_ddim:
|
| 115 |
+
x = diffusion.ddim_sample_loop(
|
| 116 |
+
sampler_model,
|
| 117 |
+
shape=shape,
|
| 118 |
+
device=device,
|
| 119 |
+
steps=args.ddim_steps,
|
| 120 |
+
eta=args.ddim_eta,
|
| 121 |
+
model_kwargs=model_kwargs,
|
| 122 |
+
progress=False,
|
| 123 |
+
)
|
| 124 |
+
else:
|
| 125 |
+
x = diffusion.p_sample_loop(sampler_model, shape=shape, device=device, model_kwargs=model_kwargs, progress=False)
|
| 126 |
+
x = (x.clamp(-1, 1) + 1) * 0.5
|
| 127 |
+
all_imgs.append(x.cpu())
|
| 128 |
+
remaining -= b
|
| 129 |
+
|
| 130 |
+
imgs = torch.cat(all_imgs, dim=0)
|
| 131 |
+
out_path = os.path.join(args.out_dir, "samples.png")
|
| 132 |
+
save_tensor_grid(imgs, out_path, nrow=int(max(1, args.n) ** 0.5), padding=1)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
if __name__ == "__main__":
|
| 136 |
+
main()
|
| 137 |
+
|
| 138 |
+
|
sample_cpu.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
BlockDiffusion:CPU 推理/采样脚本(强制使用 CPU)
|
| 5 |
+
|
| 6 |
+
设计目标:
|
| 7 |
+
- 即使机器有 GPU,也只使用 CPU(避免 CUDA 初始化与显存占用)
|
| 8 |
+
- 复用项目现有的 ckpt 元信息/模型构建/EMA/CFG/扩散采样逻辑
|
| 9 |
+
|
| 10 |
+
注意:
|
| 11 |
+
- CPU 采样会明显更慢,建议配合 --use_ddim 与较小的 --ddim_steps / --batch
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import os
|
| 16 |
+
from typing import Optional
|
| 17 |
+
|
| 18 |
+
# 必须在 import torch 之前设置,否则可能已经触发 CUDA 初始化
|
| 19 |
+
os.environ.setdefault("CUDA_VISIBLE_DEVICES", "")
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
|
| 23 |
+
from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion
|
| 24 |
+
from blockdiffusion.diffusion.schedules import get_beta_schedule
|
| 25 |
+
from blockdiffusion.models.text_unet16 import TextCondConfig, TextCondUNet16
|
| 26 |
+
from blockdiffusion.models.unet16 import UNet16, UNet16Config
|
| 27 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab
|
| 28 |
+
from blockdiffusion.train.guidance import CFGuidanceModel
|
| 29 |
+
from blockdiffusion.utils.checkpoint import load_checkpoint
|
| 30 |
+
from blockdiffusion.utils.ema import EMA
|
| 31 |
+
from blockdiffusion.utils.image_io import save_tensor_grid
|
| 32 |
+
from blockdiffusion.utils.seed import seed_everything
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def parse_args() -> argparse.Namespace:
|
| 36 |
+
p = argparse.ArgumentParser(description="BlockDiffusion CPU 采样入口(16×16 纹理)")
|
| 37 |
+
p.add_argument("--ckpt", type=str, required=True, help="训练生成的 checkpoint 路径(.pt)")
|
| 38 |
+
p.add_argument("--out_dir", type=str, default="samples_cpu", help="输出目录")
|
| 39 |
+
p.add_argument("--out_name", type=str, default="samples.png", help="输出文件名(建议 .png)")
|
| 40 |
+
|
| 41 |
+
p.add_argument("--n", type=int, default=16, help="生成张数(CPU 建议较小)")
|
| 42 |
+
p.add_argument("--batch", type=int, default=16, help="单次采样 batch(CPU 建议较小)")
|
| 43 |
+
p.add_argument("--threads", type=int, default=0, help="torch CPU 线程数(0 表示不设置)")
|
| 44 |
+
|
| 45 |
+
p.add_argument("--use_ema", action="store_true", help="使用 EMA 权重采样(推荐)")
|
| 46 |
+
p.add_argument("--use_ddim", action="store_true", help="使用 DDIM 快速采样")
|
| 47 |
+
p.add_argument("--ddim_steps", type=int, default=50, help="DDIM steps(越小越快)")
|
| 48 |
+
p.add_argument("--ddim_eta", type=float, default=0.0, help="DDIM eta(0=确定性)")
|
| 49 |
+
|
| 50 |
+
p.add_argument("--prompt", type=str, default=None, help="提示词(仅 text_cond 模型可用)")
|
| 51 |
+
p.add_argument("--cfg_scale", type=float, default=3.0, help="CFG scale(仅 text_cond 模型可用)")
|
| 52 |
+
|
| 53 |
+
p.add_argument("--seed", type=int, default=123, help="随机种子")
|
| 54 |
+
return p.parse_args()
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _set_cpu_threads(threads: int) -> None:
|
| 58 |
+
"""
|
| 59 |
+
可选设置 CPU 线程数,便于在不同机器上控制推理占用。
|
| 60 |
+
"""
|
| 61 |
+
threads = int(threads)
|
| 62 |
+
if threads <= 0:
|
| 63 |
+
return
|
| 64 |
+
try:
|
| 65 |
+
torch.set_num_threads(threads)
|
| 66 |
+
except Exception:
|
| 67 |
+
# 某些环境可能限制 set_num_threads;这里静默忽略即可
|
| 68 |
+
pass
|
| 69 |
+
try:
|
| 70 |
+
# inter-op 线程在部分版本/平台上可能不可设置
|
| 71 |
+
torch.set_num_interop_threads(threads)
|
| 72 |
+
except Exception:
|
| 73 |
+
pass
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@torch.no_grad()
|
| 77 |
+
def main() -> None:
|
| 78 |
+
args = parse_args()
|
| 79 |
+
_set_cpu_threads(args.threads)
|
| 80 |
+
seed_everything(args.seed, deterministic=False)
|
| 81 |
+
|
| 82 |
+
# 强制 CPU
|
| 83 |
+
device = torch.device("cpu")
|
| 84 |
+
|
| 85 |
+
ckpt = load_checkpoint(args.ckpt, map_location="cpu")
|
| 86 |
+
meta = ckpt.get("meta", {}) or {}
|
| 87 |
+
|
| 88 |
+
model_type = str(meta.get("model_type", "uncond"))
|
| 89 |
+
unet_cfg_dict = meta.get("unet_cfg", None)
|
| 90 |
+
if unet_cfg_dict is None:
|
| 91 |
+
# 兼容:旧 checkpoint 未保存配置
|
| 92 |
+
unet_cfg = UNet16Config()
|
| 93 |
+
timesteps = 1000
|
| 94 |
+
beta_schedule = "cosine"
|
| 95 |
+
image_size = 16
|
| 96 |
+
channels = 3
|
| 97 |
+
model_type = "uncond"
|
| 98 |
+
else:
|
| 99 |
+
unet_cfg = UNet16Config(**unet_cfg_dict)
|
| 100 |
+
diff_meta = meta.get("diffusion", {}) or {}
|
| 101 |
+
timesteps = int(diff_meta.get("timesteps", 1000))
|
| 102 |
+
beta_schedule = str(diff_meta.get("beta_schedule", "cosine"))
|
| 103 |
+
data_meta = meta.get("data", {}) or {}
|
| 104 |
+
image_size = int(data_meta.get("image_size", 16))
|
| 105 |
+
channels = int(data_meta.get("channels", unet_cfg.in_channels))
|
| 106 |
+
|
| 107 |
+
vocab: Optional[SimpleVocab] = None
|
| 108 |
+
text_cfg: Optional[TextCondConfig] = None
|
| 109 |
+
if model_type == "text_cond":
|
| 110 |
+
tok_dict = meta.get("tokenizer", None)
|
| 111 |
+
text_cfg_dict = meta.get("text_cfg", None)
|
| 112 |
+
if tok_dict is None or text_cfg_dict is None:
|
| 113 |
+
raise ValueError("该 checkpoint 标记为 text_cond,但缺少 tokenizer/text_cfg")
|
| 114 |
+
vocab = SimpleVocab.from_dict(tok_dict)
|
| 115 |
+
text_cfg = TextCondConfig(**text_cfg_dict)
|
| 116 |
+
model = TextCondUNet16(unet_cfg=unet_cfg, vocab=vocab, text_cfg=text_cfg).to(device)
|
| 117 |
+
else:
|
| 118 |
+
model = UNet16(unet_cfg).to(device)
|
| 119 |
+
|
| 120 |
+
model.load_state_dict(ckpt["model"], strict=True)
|
| 121 |
+
model.eval()
|
| 122 |
+
|
| 123 |
+
if args.use_ema and ("ema" in ckpt):
|
| 124 |
+
ema = EMA(model, decay=0.9999)
|
| 125 |
+
ema.load_state_dict(ckpt["ema"])
|
| 126 |
+
ema.copy_to(model)
|
| 127 |
+
model.eval()
|
| 128 |
+
|
| 129 |
+
betas = get_beta_schedule(beta_schedule, timesteps).to(device)
|
| 130 |
+
diffusion = GaussianDiffusion(betas)
|
| 131 |
+
|
| 132 |
+
# 条件采样:如果提供 prompt 且模型支持,则启用 CFG
|
| 133 |
+
sampler_model = model
|
| 134 |
+
model_kwargs = None
|
| 135 |
+
if args.prompt and model_type == "text_cond":
|
| 136 |
+
if vocab is None or text_cfg is None:
|
| 137 |
+
raise ValueError("内部错误:条件模型缺少 vocab/text_cfg")
|
| 138 |
+
tok = torch.tensor([vocab.encode(args.prompt, max_len=text_cfg.max_text_len)], dtype=torch.long, device=device)
|
| 139 |
+
m = tok != vocab.pad_id
|
| 140 |
+
cond_kwargs = {"tokens": tok, "mask": m}
|
| 141 |
+
|
| 142 |
+
un_tok = torch.full_like(tok, fill_value=vocab.pad_id)
|
| 143 |
+
un_m = torch.zeros_like(m)
|
| 144 |
+
uncond_kwargs = {"tokens": un_tok, "mask": un_m}
|
| 145 |
+
|
| 146 |
+
sampler_model = CFGuidanceModel(
|
| 147 |
+
model,
|
| 148 |
+
cond_kwargs=cond_kwargs,
|
| 149 |
+
uncond_kwargs=uncond_kwargs,
|
| 150 |
+
scale=float(args.cfg_scale),
|
| 151 |
+
).to(device)
|
| 152 |
+
model_kwargs = None
|
| 153 |
+
elif args.prompt and model_type != "text_cond":
|
| 154 |
+
raise ValueError("当前 checkpoint 是无条件模型,不支持 --prompt;请用带 captions 训练的模型")
|
| 155 |
+
|
| 156 |
+
os.makedirs(args.out_dir, exist_ok=True)
|
| 157 |
+
out_path = os.path.join(args.out_dir, args.out_name)
|
| 158 |
+
|
| 159 |
+
all_imgs = []
|
| 160 |
+
remaining = int(args.n)
|
| 161 |
+
while remaining > 0:
|
| 162 |
+
b = min(int(args.batch), remaining)
|
| 163 |
+
shape = (b, int(channels), int(image_size), int(image_size))
|
| 164 |
+
if args.use_ddim:
|
| 165 |
+
x = diffusion.ddim_sample_loop(
|
| 166 |
+
sampler_model,
|
| 167 |
+
shape=shape,
|
| 168 |
+
device=device,
|
| 169 |
+
steps=int(args.ddim_steps),
|
| 170 |
+
eta=float(args.ddim_eta),
|
| 171 |
+
model_kwargs=model_kwargs,
|
| 172 |
+
progress=True,
|
| 173 |
+
)
|
| 174 |
+
else:
|
| 175 |
+
x = diffusion.p_sample_loop(
|
| 176 |
+
sampler_model,
|
| 177 |
+
shape=shape,
|
| 178 |
+
device=device,
|
| 179 |
+
model_kwargs=model_kwargs,
|
| 180 |
+
progress=True,
|
| 181 |
+
)
|
| 182 |
+
x = (x.clamp(-1, 1) + 1) * 0.5
|
| 183 |
+
all_imgs.append(x.cpu())
|
| 184 |
+
remaining -= b
|
| 185 |
+
|
| 186 |
+
imgs = torch.cat(all_imgs, dim=0)
|
| 187 |
+
nrow = int(max(1, int(args.n)) ** 0.5)
|
| 188 |
+
save_tensor_grid(imgs, out_path, nrow=nrow, padding=1)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
if __name__ == "__main__":
|
| 192 |
+
main()
|
| 193 |
+
|
| 194 |
+
|
train.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import dataclasses
|
| 5 |
+
import os
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch.utils.data import DataLoader
|
| 10 |
+
|
| 11 |
+
from blockdiffusion.data.texture_folder import TextureFolderDataset
|
| 12 |
+
from blockdiffusion.data.captions import load_captions
|
| 13 |
+
from blockdiffusion.data.texture_caption_dataset import TextureCaptionDataset
|
| 14 |
+
from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion
|
| 15 |
+
from blockdiffusion.diffusion.schedules import get_beta_schedule
|
| 16 |
+
from blockdiffusion.models.unet16 import UNet16, UNet16Config
|
| 17 |
+
from blockdiffusion.models.text_unet16 import TextCondConfig, TextCondUNet16
|
| 18 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab, VocabConfig
|
| 19 |
+
from blockdiffusion.train.trainer import Trainer, TrainerConfig
|
| 20 |
+
from blockdiffusion.utils.seed import get_device, seed_everything
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def parse_args() -> argparse.Namespace:
|
| 24 |
+
p = argparse.ArgumentParser(description="BlockDiffusion 16×16 纹理训练入口")
|
| 25 |
+
|
| 26 |
+
# 数据
|
| 27 |
+
p.add_argument("--data_dir", type=str, required=True, help="纹理图片目录(可含子目录)")
|
| 28 |
+
p.add_argument("--image_size", type=int, default=16, help="纹理边长(默认 16)")
|
| 29 |
+
p.add_argument("--channels", type=int, default=3, help="纹理通道数(默认 RGB=3)")
|
| 30 |
+
p.add_argument("--captions", type=str, default=None, help="图文配对文件(.jsonl/.csv),提供则启用提示词条件训练")
|
| 31 |
+
p.add_argument("--max_text_len", type=int, default=32, help="文本最大长度(中文按字、英文按词)")
|
| 32 |
+
p.add_argument("--vocab_max", type=int, default=8000, help="词表最大大小")
|
| 33 |
+
|
| 34 |
+
# 扩散
|
| 35 |
+
p.add_argument("--timesteps", type=int, default=1000, help="扩散步数")
|
| 36 |
+
p.add_argument("--beta_schedule", type=str, default="cosine", choices=["linear", "cosine"], help="噪声调度")
|
| 37 |
+
|
| 38 |
+
# UNet
|
| 39 |
+
p.add_argument("--base_channels", type=int, default=64, help="UNet 基础通道数")
|
| 40 |
+
p.add_argument("--num_res_blocks", type=int, default=2, help="每个 level 的 ResBlock 数")
|
| 41 |
+
p.add_argument("--dropout", type=float, default=0.0, help="ResBlock dropout(一般可为 0)")
|
| 42 |
+
|
| 43 |
+
# 训练
|
| 44 |
+
p.add_argument("--batch_size", type=int, default=512)
|
| 45 |
+
# Windows 下多进程 DataLoader 可能更挑剔,默认给 0 更稳;需要更快可手动调大。
|
| 46 |
+
p.add_argument("--num_workers", type=int, default=(0 if os.name == "nt" else 4))
|
| 47 |
+
p.add_argument("--lr", type=float, default=2e-4)
|
| 48 |
+
p.add_argument("--weight_decay", type=float, default=1e-4)
|
| 49 |
+
p.add_argument("--total_steps", type=int, default=200_000)
|
| 50 |
+
p.add_argument("--grad_clip", type=float, default=1.0)
|
| 51 |
+
p.add_argument("--no_amp", action="store_true", help="关闭混合精度")
|
| 52 |
+
p.add_argument("--ema_decay", type=float, default=0.9999)
|
| 53 |
+
p.add_argument("--cond_drop_prob", type=float, default=0.1, help="CFG 训练:丢条件概率(0~1)")
|
| 54 |
+
p.add_argument("--sample_prompt", type=str, default=None, help="训练中采样使用的提示词(仅条件训练时生效)")
|
| 55 |
+
p.add_argument("--sample_cfg_scale", type=float, default=3.0, help="采样 CFG scale(仅条件训练时生效)")
|
| 56 |
+
|
| 57 |
+
# 输出
|
| 58 |
+
p.add_argument("--out_dir", type=str, default="outputs")
|
| 59 |
+
p.add_argument("--run_name", type=str, default="default")
|
| 60 |
+
p.add_argument("--resume", type=str, default=None, help="从 checkpoint 恢复训练")
|
| 61 |
+
|
| 62 |
+
# 其他
|
| 63 |
+
p.add_argument("--seed", type=int, default=42)
|
| 64 |
+
p.add_argument("--device", type=str, default=None, help='如 "cuda" / "cuda:0" / "cpu"')
|
| 65 |
+
|
| 66 |
+
return p.parse_args()
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def main() -> None:
|
| 70 |
+
args = parse_args()
|
| 71 |
+
|
| 72 |
+
seed_everything(args.seed, deterministic=False)
|
| 73 |
+
device = get_device(args.device)
|
| 74 |
+
|
| 75 |
+
# 性能优化:适用于 RTX 40 系列(不影响数值正确性)
|
| 76 |
+
if hasattr(torch, "set_float32_matmul_precision"):
|
| 77 |
+
torch.set_float32_matmul_precision("high")
|
| 78 |
+
if device.type == "cuda":
|
| 79 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 80 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 81 |
+
|
| 82 |
+
sample_model_kwargs = None
|
| 83 |
+
sample_uncond_model_kwargs = None
|
| 84 |
+
|
| 85 |
+
# 选择数据集:无条件/条件(captions)
|
| 86 |
+
if args.captions:
|
| 87 |
+
cap_items = load_captions(args.captions, data_root=args.data_dir)
|
| 88 |
+
# 可选:强制加入透明/不透明 token,方便学习 alpha 与提示词关联(需你在 captions 里实际使用这些词)
|
| 89 |
+
vocab = SimpleVocab.build(
|
| 90 |
+
(it.text for it in cap_items),
|
| 91 |
+
VocabConfig(max_vocab=args.vocab_max, min_freq=1),
|
| 92 |
+
extra_tokens=["transparent", "opaque"],
|
| 93 |
+
)
|
| 94 |
+
dataset = TextureCaptionDataset(
|
| 95 |
+
items=cap_items,
|
| 96 |
+
vocab=vocab,
|
| 97 |
+
image_size=args.image_size,
|
| 98 |
+
channels=args.channels,
|
| 99 |
+
max_text_len=args.max_text_len,
|
| 100 |
+
random_flip=True,
|
| 101 |
+
)
|
| 102 |
+
else:
|
| 103 |
+
vocab = None
|
| 104 |
+
dataset = TextureFolderDataset(
|
| 105 |
+
root_dir=args.data_dir,
|
| 106 |
+
image_size=args.image_size,
|
| 107 |
+
channels=args.channels,
|
| 108 |
+
random_flip=True,
|
| 109 |
+
)
|
| 110 |
+
dataloader = DataLoader(
|
| 111 |
+
dataset,
|
| 112 |
+
batch_size=args.batch_size,
|
| 113 |
+
shuffle=True,
|
| 114 |
+
num_workers=args.num_workers,
|
| 115 |
+
pin_memory=(device.type == "cuda"),
|
| 116 |
+
drop_last=True,
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
unet_cfg = UNet16Config(
|
| 120 |
+
in_channels=args.channels,
|
| 121 |
+
out_channels=args.channels,
|
| 122 |
+
base_channels=args.base_channels,
|
| 123 |
+
channel_mults=(1, 2, 2), # 16->8->4
|
| 124 |
+
num_res_blocks=args.num_res_blocks,
|
| 125 |
+
attn_resolutions=(8, 4),
|
| 126 |
+
dropout=args.dropout,
|
| 127 |
+
time_emb_dim=256,
|
| 128 |
+
cond_dim=256,
|
| 129 |
+
)
|
| 130 |
+
model_type = "uncond"
|
| 131 |
+
text_cfg = None
|
| 132 |
+
if args.captions:
|
| 133 |
+
# 条件模型:TextEncoder 输出维度与 time_emb_dim 对齐
|
| 134 |
+
text_cfg = TextCondConfig(max_text_len=args.max_text_len, emb_dim=128, dropout=0.0)
|
| 135 |
+
model = TextCondUNet16(unet_cfg=unet_cfg, vocab=vocab, text_cfg=text_cfg)
|
| 136 |
+
model_type = "text_cond"
|
| 137 |
+
|
| 138 |
+
# 训练中采样(可选):提供 sample_prompt 才启用 CFG 采样,否则走无条件采样
|
| 139 |
+
if args.sample_prompt:
|
| 140 |
+
tok = torch.tensor([vocab.encode(args.sample_prompt, max_len=args.max_text_len)], dtype=torch.long)
|
| 141 |
+
m = (tok != vocab.pad_id)
|
| 142 |
+
sample_model_kwargs = {"tokens": tok.to(device), "mask": m.to(device)}
|
| 143 |
+
# 无条件分支:mask 全 0(tokens 可以随意,统一用 pad)
|
| 144 |
+
un_tok = torch.full_like(tok, fill_value=vocab.pad_id)
|
| 145 |
+
un_m = torch.zeros_like(m)
|
| 146 |
+
sample_uncond_model_kwargs = {"tokens": un_tok.to(device), "mask": un_m.to(device)}
|
| 147 |
+
else:
|
| 148 |
+
model = UNet16(unet_cfg)
|
| 149 |
+
|
| 150 |
+
betas = get_beta_schedule(args.beta_schedule, args.timesteps).to(device)
|
| 151 |
+
diffusion = GaussianDiffusion(betas)
|
| 152 |
+
|
| 153 |
+
trainer_cfg = TrainerConfig(
|
| 154 |
+
image_size=args.image_size,
|
| 155 |
+
channels=args.channels,
|
| 156 |
+
batch_size=args.batch_size,
|
| 157 |
+
num_workers=args.num_workers,
|
| 158 |
+
lr=args.lr,
|
| 159 |
+
weight_decay=args.weight_decay,
|
| 160 |
+
total_steps=args.total_steps,
|
| 161 |
+
grad_clip=args.grad_clip,
|
| 162 |
+
amp=not args.no_amp,
|
| 163 |
+
ema_decay=args.ema_decay,
|
| 164 |
+
cond_drop_prob=args.cond_drop_prob,
|
| 165 |
+
sample_cfg_scale=args.sample_cfg_scale,
|
| 166 |
+
out_dir=args.out_dir,
|
| 167 |
+
run_name=args.run_name,
|
| 168 |
+
resume_path=args.resume,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
meta = {
|
| 172 |
+
"model_type": model_type,
|
| 173 |
+
"unet_cfg": dataclasses.asdict(unet_cfg),
|
| 174 |
+
"diffusion": {"timesteps": args.timesteps, "beta_schedule": args.beta_schedule},
|
| 175 |
+
"data": {"image_size": args.image_size, "channels": args.channels},
|
| 176 |
+
}
|
| 177 |
+
if args.captions and vocab is not None and text_cfg is not None:
|
| 178 |
+
meta["tokenizer"] = vocab.to_dict()
|
| 179 |
+
meta["text_cfg"] = dataclasses.asdict(text_cfg)
|
| 180 |
+
|
| 181 |
+
trainer = Trainer(
|
| 182 |
+
model=model,
|
| 183 |
+
diffusion=diffusion,
|
| 184 |
+
dataloader=dataloader,
|
| 185 |
+
device=device,
|
| 186 |
+
cfg=trainer_cfg,
|
| 187 |
+
meta=meta,
|
| 188 |
+
sample_model_kwargs=sample_model_kwargs,
|
| 189 |
+
sample_uncond_model_kwargs=sample_uncond_model_kwargs,
|
| 190 |
+
)
|
| 191 |
+
trainer.train()
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
if __name__ == "__main__":
|
| 195 |
+
main()
|
| 196 |
+
|
| 197 |
+
|
train_oneclick.py
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
一键训练脚本(无命令行参数):
|
| 5 |
+
- 直接修改本文件顶部常量即可
|
| 6 |
+
- 数据目录:DATA_DIR
|
| 7 |
+
- 提示词来自文件名(HASH-xxx_yyy.png => "xxx yyy")
|
| 8 |
+
|
| 9 |
+
运行方式(推荐 Python 3.10):
|
| 10 |
+
- py -3.10 train_oneclick.py
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import dataclasses
|
| 14 |
+
import os
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
from torch.utils.data import DataLoader
|
| 18 |
+
|
| 19 |
+
from blockdiffusion.data.filename_captions import build_caption_items_from_filenames
|
| 20 |
+
from blockdiffusion.data.texture_caption_dataset import TextureCaptionDataset
|
| 21 |
+
from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion
|
| 22 |
+
from blockdiffusion.diffusion.schedules import get_beta_schedule
|
| 23 |
+
from blockdiffusion.models.text_unet16 import TextCondConfig, TextCondUNet16
|
| 24 |
+
from blockdiffusion.models.unet16 import UNet16Config
|
| 25 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab, VocabConfig
|
| 26 |
+
from blockdiffusion.train.trainer import Trainer, TrainerConfig
|
| 27 |
+
from blockdiffusion.utils.checkpoint import load_checkpoint
|
| 28 |
+
from blockdiffusion.utils.seed import get_device, seed_everything
|
| 29 |
+
|
| 30 |
+
# =========================
|
| 31 |
+
# 可修改常量(按需改这里)
|
| 32 |
+
# =========================
|
| 33 |
+
|
| 34 |
+
# 数据目录(你提供的路径)
|
| 35 |
+
DATA_DIR = r"G:\PycharmProjects\modrinth_downloads\Training_v2"
|
| 36 |
+
|
| 37 |
+
# 纹理尺寸(本项目目标 16)
|
| 38 |
+
IMAGE_SIZE = 16
|
| 39 |
+
|
| 40 |
+
# 通道数:3=RGB,4=RGBA(需要保留透明就用 4)
|
| 41 |
+
CHANNELS = 4
|
| 42 |
+
|
| 43 |
+
# 扩散步数与调度
|
| 44 |
+
TIMESTEPS = 1000
|
| 45 |
+
BETA_SCHEDULE = "cosine" # "linear" / "cosine"
|
| 46 |
+
|
| 47 |
+
# UNet 规模(16×16 不要太深)
|
| 48 |
+
BASE_CHANNELS = 64
|
| 49 |
+
NUM_RES_BLOCKS = 2
|
| 50 |
+
DROPOUT = 0.0
|
| 51 |
+
|
| 52 |
+
# 文本(提示词)配置
|
| 53 |
+
MAX_TEXT_LEN = 32
|
| 54 |
+
VOCAB_MAX = 8000
|
| 55 |
+
COND_DROP_PROB = 0.1 # CFG 训练丢条件概率(0~1)
|
| 56 |
+
PRETOKENIZE = False # 大数据集建议 False:启动更快、占用更小
|
| 57 |
+
|
| 58 |
+
# 透明度提示词(可选):
|
| 59 |
+
# - 仅在 CHANNELS=4 且“新训练(非续训)”时建议开启
|
| 60 |
+
# - 作用:自动根据图片 alpha 是否含透明像素,为文本追加 "transparent"/"opaque"
|
| 61 |
+
# 这样模型才能学到“透明度与提示词相关”的监督信号
|
| 62 |
+
APPEND_ALPHA_TAG = False
|
| 63 |
+
ALPHA_TAG_TRANSPARENT = "transparent"
|
| 64 |
+
ALPHA_TAG_OPAQUE = "opaque"
|
| 65 |
+
|
| 66 |
+
# 训练配置(4070Ti 12G:RGBA 建议先从 128/256 试起)
|
| 67 |
+
BATCH_SIZE = 128
|
| 68 |
+
# 数据加载并行:Windows 若不稳定可改回 0;稳定时建议 4~8
|
| 69 |
+
NUM_WORKERS = 4
|
| 70 |
+
PREFETCH_FACTOR = 4 # NUM_WORKERS>0 时生效
|
| 71 |
+
PERSISTENT_WORKERS = True # NUM_WORKERS>0 时生效
|
| 72 |
+
LR = 2e-4
|
| 73 |
+
WEIGHT_DECAY = 1e-4
|
| 74 |
+
TOTAL_STEPS = 700_000
|
| 75 |
+
GRAD_CLIP = 1.0
|
| 76 |
+
AMP = True
|
| 77 |
+
EMA_DECAY = 0.9999
|
| 78 |
+
GRAD_ACCUM_STEPS = 1 # 不想OOM但想更大有效batch:例如设为 2/4
|
| 79 |
+
LR_WARMUP_STEPS = 2000
|
| 80 |
+
LR_MIN_RATIO = 0.1
|
| 81 |
+
SNR_GAMMA = 5.0 # <=0 关闭;5 通常比较稳
|
| 82 |
+
CHANNELS_LAST = True
|
| 83 |
+
LOG_EVERY = 10
|
| 84 |
+
SAVE_EVERY = 5000
|
| 85 |
+
SAMPLE_EVERY = 5000
|
| 86 |
+
|
| 87 |
+
# 输出
|
| 88 |
+
OUT_DIR = "outputs"
|
| 89 |
+
RUN_NAME = "oneclick_filename_caption"
|
| 90 |
+
# 断点续训:
|
| 91 |
+
# - 若你明确指定某个 checkpoint,就填 RESUME_PATH
|
| 92 |
+
# - 若 RESUME_PATH=None 且 AUTO_RESUME=True,会自动扫描 outputs/<RUN_NAME>/checkpoints 下最新 step_*.pt 继续训练
|
| 93 |
+
RESUME_PATH = None # 例如 r"outputs/oneclick_filename_caption/checkpoints/step_5000.pt"
|
| 94 |
+
AUTO_RESUME = True
|
| 95 |
+
|
| 96 |
+
# 训练中采样(可选):用这个提示词做固定可视化
|
| 97 |
+
SAMPLE_PROMPT = "block" # 改成你想看的关键词;不想要可设为 None
|
| 98 |
+
SAMPLE_CFG_SCALE = 3.0
|
| 99 |
+
|
| 100 |
+
# 随机种子
|
| 101 |
+
SEED = 42
|
| 102 |
+
DEVICE = None # 例如 "cuda:0";None 表示自动
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def main() -> None:
|
| 106 |
+
seed_everything(SEED, deterministic=False)
|
| 107 |
+
device = get_device(DEVICE)
|
| 108 |
+
print(f"[init] device={device} channels={CHANNELS} image_size={IMAGE_SIZE}")
|
| 109 |
+
|
| 110 |
+
# RTX 40 系列性能优化(不影响数值正确性)
|
| 111 |
+
if hasattr(torch, "set_float32_matmul_precision"):
|
| 112 |
+
torch.set_float32_matmul_precision("high")
|
| 113 |
+
if device.type == "cuda":
|
| 114 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 115 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 116 |
+
|
| 117 |
+
# =====
|
| 118 |
+
# 续训注意:
|
| 119 |
+
# - 如果设置了 RESUME_PATH,为保证词表/模型结构不变,本脚本会优先从 checkpoint 读取并复用配置;
|
| 120 |
+
# - 新增数据会自动被扫描并参与训练;新出现的词如果不在旧词表里,会被编码为 <unk>(可继续训练,但不会学到新词语义)。
|
| 121 |
+
# =====
|
| 122 |
+
|
| 123 |
+
def _auto_find_latest_ckpt() -> str | None:
|
| 124 |
+
ckpt_dir = os.path.join(OUT_DIR, RUN_NAME, "checkpoints")
|
| 125 |
+
if not os.path.isdir(ckpt_dir):
|
| 126 |
+
return None
|
| 127 |
+
best_step = -1
|
| 128 |
+
best_path = None
|
| 129 |
+
for fn in os.listdir(ckpt_dir):
|
| 130 |
+
if not (fn.startswith("step_") and fn.endswith(".pt")):
|
| 131 |
+
continue
|
| 132 |
+
# 形如 step_5000.pt
|
| 133 |
+
core = fn[len("step_") : -len(".pt")]
|
| 134 |
+
try:
|
| 135 |
+
step = int(core)
|
| 136 |
+
except Exception:
|
| 137 |
+
continue
|
| 138 |
+
if step > best_step:
|
| 139 |
+
best_step = step
|
| 140 |
+
best_path = os.path.join(ckpt_dir, fn)
|
| 141 |
+
return best_path
|
| 142 |
+
|
| 143 |
+
effective_resume_path = RESUME_PATH
|
| 144 |
+
if (effective_resume_path is None) and AUTO_RESUME:
|
| 145 |
+
effective_resume_path = _auto_find_latest_ckpt()
|
| 146 |
+
|
| 147 |
+
resume_meta = None
|
| 148 |
+
if effective_resume_path:
|
| 149 |
+
print(f"[resume] 使用 checkpoint: {effective_resume_path}")
|
| 150 |
+
ckpt = load_checkpoint(effective_resume_path, map_location="cpu")
|
| 151 |
+
resume_meta = ckpt.get("meta", {}) or {}
|
| 152 |
+
|
| 153 |
+
# 从 checkpoint 复用关键配置(避免“重建词表”导致继续训练变差/不可用)
|
| 154 |
+
if resume_meta:
|
| 155 |
+
data_meta = resume_meta.get("data", {}) or {}
|
| 156 |
+
diff_meta = resume_meta.get("diffusion", {}) or {}
|
| 157 |
+
unet_cfg_dict = resume_meta.get("unet_cfg", None)
|
| 158 |
+
text_cfg_dict = resume_meta.get("text_cfg", None)
|
| 159 |
+
tok_dict = resume_meta.get("tokenizer", None)
|
| 160 |
+
|
| 161 |
+
cfg_image_size = int(data_meta.get("image_size", IMAGE_SIZE))
|
| 162 |
+
cfg_channels = int(data_meta.get("channels", CHANNELS))
|
| 163 |
+
cfg_timesteps = int(diff_meta.get("timesteps", TIMESTEPS))
|
| 164 |
+
cfg_beta_schedule = str(diff_meta.get("beta_schedule", BETA_SCHEDULE))
|
| 165 |
+
|
| 166 |
+
if unet_cfg_dict is None or text_cfg_dict is None or tok_dict is None:
|
| 167 |
+
raise ValueError("RESUME_PATH 指向的 checkpoint 缺少 tokenizer/text_cfg/unet_cfg,无法安全续训")
|
| 168 |
+
vocab = SimpleVocab.from_dict(tok_dict)
|
| 169 |
+
text_cfg = TextCondConfig(**text_cfg_dict)
|
| 170 |
+
unet_cfg = UNet16Config(**unet_cfg_dict)
|
| 171 |
+
else:
|
| 172 |
+
cfg_image_size = IMAGE_SIZE
|
| 173 |
+
cfg_channels = CHANNELS
|
| 174 |
+
cfg_timesteps = TIMESTEPS
|
| 175 |
+
cfg_beta_schedule = BETA_SCHEDULE
|
| 176 |
+
text_cfg = TextCondConfig(max_text_len=MAX_TEXT_LEN, emb_dim=128, dropout=0.0)
|
| 177 |
+
unet_cfg = UNet16Config(
|
| 178 |
+
in_channels=cfg_channels,
|
| 179 |
+
out_channels=cfg_channels,
|
| 180 |
+
base_channels=BASE_CHANNELS,
|
| 181 |
+
channel_mults=(1, 2, 2), # 16->8->4
|
| 182 |
+
num_res_blocks=NUM_RES_BLOCKS,
|
| 183 |
+
attn_resolutions=(8, 4),
|
| 184 |
+
dropout=DROPOUT,
|
| 185 |
+
time_emb_dim=256,
|
| 186 |
+
cond_dim=256,
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
# 非续训:词表会基于当前数据构建
|
| 190 |
+
vocab = None
|
| 191 |
+
|
| 192 |
+
# 扫描目录(只扫描一次):用于训练集;非续训时也用于构建词表
|
| 193 |
+
print("[data] 扫描图片并从文件名提取提示词...")
|
| 194 |
+
cap_items = build_caption_items_from_filenames(DATA_DIR, verbose=True)
|
| 195 |
+
if vocab is None:
|
| 196 |
+
# 新训练:可强制加入额外 token,避免采样时提示词变成 <unk> 而“看起来不遵从”
|
| 197 |
+
extra_tokens = None
|
| 198 |
+
append_alpha_tag = bool(APPEND_ALPHA_TAG and (cfg_channels == 4))
|
| 199 |
+
if append_alpha_tag:
|
| 200 |
+
extra_tokens = [ALPHA_TAG_TRANSPARENT, ALPHA_TAG_OPAQUE]
|
| 201 |
+
vocab = SimpleVocab.build(
|
| 202 |
+
(it.text for it in cap_items),
|
| 203 |
+
VocabConfig(max_vocab=VOCAB_MAX, min_freq=1),
|
| 204 |
+
extra_tokens=extra_tokens,
|
| 205 |
+
)
|
| 206 |
+
else:
|
| 207 |
+
# 续训:vocab 固定,不能改变;否则 embedding 尺寸不匹配
|
| 208 |
+
append_alpha_tag = False
|
| 209 |
+
|
| 210 |
+
dataset = TextureCaptionDataset(
|
| 211 |
+
items=cap_items,
|
| 212 |
+
vocab=vocab,
|
| 213 |
+
image_size=cfg_image_size,
|
| 214 |
+
channels=cfg_channels,
|
| 215 |
+
max_text_len=text_cfg.max_text_len,
|
| 216 |
+
random_flip=True,
|
| 217 |
+
pretokenize=PRETOKENIZE,
|
| 218 |
+
append_alpha_tag=append_alpha_tag,
|
| 219 |
+
alpha_tag_transparent=ALPHA_TAG_TRANSPARENT,
|
| 220 |
+
alpha_tag_opaque=ALPHA_TAG_OPAQUE,
|
| 221 |
+
verbose=True,
|
| 222 |
+
)
|
| 223 |
+
print(f"[data] dataset_size={len(dataset)} vocab_size={vocab.size} pretokenize={PRETOKENIZE}")
|
| 224 |
+
dl_kwargs = dict(
|
| 225 |
+
batch_size=BATCH_SIZE,
|
| 226 |
+
shuffle=True,
|
| 227 |
+
num_workers=NUM_WORKERS,
|
| 228 |
+
pin_memory=(device.type == "cuda"),
|
| 229 |
+
drop_last=True,
|
| 230 |
+
)
|
| 231 |
+
if NUM_WORKERS > 0:
|
| 232 |
+
dl_kwargs["prefetch_factor"] = int(PREFETCH_FACTOR)
|
| 233 |
+
dl_kwargs["persistent_workers"] = bool(PERSISTENT_WORKERS)
|
| 234 |
+
dataloader = DataLoader(dataset, **dl_kwargs)
|
| 235 |
+
print(f"[data] dataloader_workers={NUM_WORKERS}")
|
| 236 |
+
|
| 237 |
+
# 2) 模型(文本条件)
|
| 238 |
+
model = TextCondUNet16(unet_cfg=unet_cfg, vocab=vocab, text_cfg=text_cfg)
|
| 239 |
+
|
| 240 |
+
# 3) Diffusion
|
| 241 |
+
betas = get_beta_schedule(cfg_beta_schedule, cfg_timesteps).to(device)
|
| 242 |
+
diffusion = GaussianDiffusion(betas)
|
| 243 |
+
|
| 244 |
+
# 4) 训练配置
|
| 245 |
+
trainer_cfg = TrainerConfig(
|
| 246 |
+
image_size=cfg_image_size,
|
| 247 |
+
channels=cfg_channels,
|
| 248 |
+
batch_size=BATCH_SIZE,
|
| 249 |
+
num_workers=NUM_WORKERS,
|
| 250 |
+
lr=LR,
|
| 251 |
+
weight_decay=WEIGHT_DECAY,
|
| 252 |
+
total_steps=TOTAL_STEPS,
|
| 253 |
+
grad_clip=GRAD_CLIP,
|
| 254 |
+
amp=AMP,
|
| 255 |
+
ema_decay=EMA_DECAY,
|
| 256 |
+
cond_drop_prob=COND_DROP_PROB,
|
| 257 |
+
grad_accum_steps=GRAD_ACCUM_STEPS,
|
| 258 |
+
lr_warmup_steps=LR_WARMUP_STEPS,
|
| 259 |
+
lr_min_ratio=LR_MIN_RATIO,
|
| 260 |
+
snr_gamma=SNR_GAMMA,
|
| 261 |
+
channels_last=CHANNELS_LAST,
|
| 262 |
+
log_every=LOG_EVERY,
|
| 263 |
+
save_every=SAVE_EVERY,
|
| 264 |
+
sample_every=SAMPLE_EVERY,
|
| 265 |
+
sample_cfg_scale=SAMPLE_CFG_SCALE,
|
| 266 |
+
out_dir=OUT_DIR,
|
| 267 |
+
run_name=RUN_NAME,
|
| 268 |
+
resume_path=effective_resume_path,
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
# 训练中采样 CFG(可选)
|
| 272 |
+
sample_model_kwargs = None
|
| 273 |
+
sample_uncond_model_kwargs = None
|
| 274 |
+
if SAMPLE_PROMPT:
|
| 275 |
+
tok = torch.tensor([vocab.encode(SAMPLE_PROMPT, max_len=text_cfg.max_text_len)], dtype=torch.long)
|
| 276 |
+
m = (tok != vocab.pad_id)
|
| 277 |
+
sample_model_kwargs = {"tokens": tok.to(device), "mask": m.to(device)}
|
| 278 |
+
un_tok = torch.full_like(tok, fill_value=vocab.pad_id)
|
| 279 |
+
un_m = torch.zeros_like(m)
|
| 280 |
+
sample_uncond_model_kwargs = {"tokens": un_tok.to(device), "mask": un_m.to(device)}
|
| 281 |
+
|
| 282 |
+
meta = {
|
| 283 |
+
"model_type": "text_cond",
|
| 284 |
+
"unet_cfg": dataclasses.asdict(unet_cfg),
|
| 285 |
+
"diffusion": {"timesteps": cfg_timesteps, "beta_schedule": cfg_beta_schedule},
|
| 286 |
+
"data": {"image_size": cfg_image_size, "channels": cfg_channels},
|
| 287 |
+
"tokenizer": vocab.to_dict(),
|
| 288 |
+
"text_cfg": dataclasses.asdict(text_cfg),
|
| 289 |
+
# 额外信息:方便回溯
|
| 290 |
+
"data_dir": os.path.abspath(DATA_DIR),
|
| 291 |
+
"caption_source": "filename",
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
trainer = Trainer(
|
| 295 |
+
model=model,
|
| 296 |
+
diffusion=diffusion,
|
| 297 |
+
dataloader=dataloader,
|
| 298 |
+
device=device,
|
| 299 |
+
cfg=trainer_cfg,
|
| 300 |
+
meta=meta,
|
| 301 |
+
sample_model_kwargs=sample_model_kwargs,
|
| 302 |
+
sample_uncond_model_kwargs=sample_uncond_model_kwargs,
|
| 303 |
+
)
|
| 304 |
+
print("[train] start...")
|
| 305 |
+
trainer.train()
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
if __name__ == "__main__":
|
| 309 |
+
main()
|
| 310 |
+
|
| 311 |
+
|
webui.py
ADDED
|
@@ -0,0 +1,522 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
BlockDiffusion WebUI(用于快速目测 checkpoint 的采样效果)
|
| 5 |
+
|
| 6 |
+
说明:
|
| 7 |
+
- 本文件只做“加载 ckpt -> 采样 -> 展示/保存”,复用项目现有扩散/CFG/EMA 逻辑。
|
| 8 |
+
- 依赖:gradio(见 requirements.txt)。
|
| 9 |
+
- 像素纹理预览:默认用 NEAREST 放大,方便看 16×16 的细节,不改变真实输出内容。
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import glob
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
import re
|
| 16 |
+
import time
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
from PIL import Image
|
| 22 |
+
from torchvision.utils import make_grid
|
| 23 |
+
|
| 24 |
+
from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion
|
| 25 |
+
from blockdiffusion.diffusion.schedules import get_beta_schedule
|
| 26 |
+
from blockdiffusion.models.text_unet16 import TextCondConfig, TextCondUNet16
|
| 27 |
+
from blockdiffusion.models.unet16 import UNet16, UNet16Config
|
| 28 |
+
from blockdiffusion.text.simple_tokenizer import SimpleVocab, tokenize
|
| 29 |
+
from blockdiffusion.train.guidance import CFGuidanceModel
|
| 30 |
+
from blockdiffusion.utils.checkpoint import load_checkpoint
|
| 31 |
+
from blockdiffusion.utils.ema import EMA
|
| 32 |
+
from blockdiffusion.utils.seed import get_device, seed_everything
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# ==============
|
| 36 |
+
# Gradio import
|
| 37 |
+
# ==============
|
| 38 |
+
try:
|
| 39 |
+
import gradio as gr
|
| 40 |
+
except Exception as e: # pragma: no cover
|
| 41 |
+
# 这里不要给“命令行安装指令”,只给明确缺依赖提示。
|
| 42 |
+
raise RuntimeError("缺少依赖:gradio。请先安装后再运行 WebUI。") from e
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class _Loaded:
|
| 47 |
+
"""
|
| 48 |
+
缓存后的模型与配置(避免每次点击都重新加载 ckpt)。
|
| 49 |
+
注意:
|
| 50 |
+
- device 变化时会重新加载
|
| 51 |
+
- use_ema 变化时也会重新加载(因为 EMA 会 copy_to 覆盖权重)
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
key: Tuple[str, str, bool] # (ckpt_path, device_str, use_ema)
|
| 55 |
+
model_type: str
|
| 56 |
+
image_size: int
|
| 57 |
+
channels: int
|
| 58 |
+
timesteps: int
|
| 59 |
+
beta_schedule: str
|
| 60 |
+
unet_cfg: UNet16Config
|
| 61 |
+
text_cfg: Optional[TextCondConfig]
|
| 62 |
+
vocab: Optional[SimpleVocab]
|
| 63 |
+
model: torch.nn.Module
|
| 64 |
+
diffusion: GaussianDiffusion
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
_CACHE: Optional[_Loaded] = None
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _ensure_localhost_no_proxy() -> None:
|
| 71 |
+
"""
|
| 72 |
+
规避 Windows/公司网络环境下的代理设置导致 localhost 访问被转发到代理,从而触发 Gradio 启动自检 502。
|
| 73 |
+
- gradio.launch() 内部会请求 `http://127.0.0.1:<port>/gradio_api/startup-events` 做自检
|
| 74 |
+
- 若 HTTP(S)_PROXY 存在且 NO_PROXY 未包含 localhost/127.0.0.1,可能出现 502
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
# 兼容大小写环境变量
|
| 78 |
+
existing = os.environ.get("NO_PROXY") or os.environ.get("no_proxy") or ""
|
| 79 |
+
parts = [p.strip() for p in existing.split(",") if p.strip()]
|
| 80 |
+
want = ["127.0.0.1", "localhost"]
|
| 81 |
+
for w in want:
|
| 82 |
+
if w not in parts:
|
| 83 |
+
parts.append(w)
|
| 84 |
+
val = ",".join(parts)
|
| 85 |
+
os.environ["NO_PROXY"] = val
|
| 86 |
+
os.environ["no_proxy"] = val
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _parse_step_from_filename(path: str) -> int:
|
| 90 |
+
"""
|
| 91 |
+
从 step_XXXX.pt 提取步数;提取失败则返回 -1,确保排序时落后。
|
| 92 |
+
"""
|
| 93 |
+
|
| 94 |
+
base = os.path.basename(path)
|
| 95 |
+
m = re.search(r"step_(\d+)\.pt$", base)
|
| 96 |
+
if not m:
|
| 97 |
+
return -1
|
| 98 |
+
try:
|
| 99 |
+
return int(m.group(1))
|
| 100 |
+
except Exception:
|
| 101 |
+
return -1
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def list_checkpoints(outputs_dir: str = "outputs") -> List[str]:
|
| 105 |
+
"""
|
| 106 |
+
扫描 outputs/**/checkpoints/step_*.pt,按 step 倒序排列(新的在前)。
|
| 107 |
+
"""
|
| 108 |
+
|
| 109 |
+
pattern = os.path.join(outputs_dir, "**", "checkpoints", "step_*.pt")
|
| 110 |
+
paths = glob.glob(pattern, recursive=True)
|
| 111 |
+
paths.sort(key=lambda p: (_parse_step_from_filename(p), p), reverse=True)
|
| 112 |
+
return paths
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _device_from_ui(device_text: str) -> torch.device:
|
| 116 |
+
"""
|
| 117 |
+
UI 设备选择:
|
| 118 |
+
- "自动" => get_device(None)
|
| 119 |
+
- 其他 => get_device(str)
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
device_text = (device_text or "").strip()
|
| 123 |
+
if device_text in ("", "自动"):
|
| 124 |
+
return get_device(None)
|
| 125 |
+
return get_device(device_text)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _tensor_to_pil(x_chw_01: torch.Tensor) -> Image.Image:
|
| 129 |
+
"""
|
| 130 |
+
(C,H,W) 且值域 [0,1] -> PIL(RGB/RGBA)
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
if x_chw_01.ndim != 3:
|
| 134 |
+
raise ValueError("x 必须是 (C,H,W)")
|
| 135 |
+
c = int(x_chw_01.shape[0])
|
| 136 |
+
if c not in (3, 4):
|
| 137 |
+
raise ValueError("仅支持 3(RGB) / 4(RGBA) 通道")
|
| 138 |
+
|
| 139 |
+
x = x_chw_01.clamp(0.0, 1.0)
|
| 140 |
+
arr = (x.permute(1, 2, 0) * 255.0).to(torch.uint8).cpu().numpy()
|
| 141 |
+
mode = "RGBA" if c == 4 else "RGB"
|
| 142 |
+
return Image.fromarray(arr, mode=mode)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _upscale_nearest(im: Image.Image, scale: int) -> Image.Image:
|
| 146 |
+
"""
|
| 147 |
+
像素纹理预览:NEAREST 放大,不改像素风。
|
| 148 |
+
"""
|
| 149 |
+
|
| 150 |
+
scale = int(scale)
|
| 151 |
+
if scale <= 1:
|
| 152 |
+
return im
|
| 153 |
+
w, h = im.size
|
| 154 |
+
return im.resize((w * scale, h * scale), resample=Image.NEAREST)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
@torch.no_grad()
|
| 158 |
+
def _load_cached(ckpt_path: str, device_text: str, use_ema: bool) -> _Loaded:
|
| 159 |
+
"""
|
| 160 |
+
加载/复用模型缓存。
|
| 161 |
+
"""
|
| 162 |
+
|
| 163 |
+
global _CACHE
|
| 164 |
+
ckpt_path = os.path.abspath(ckpt_path)
|
| 165 |
+
key = (ckpt_path, (device_text or "").strip() or "自动", bool(use_ema))
|
| 166 |
+
if _CACHE is not None and _CACHE.key == key:
|
| 167 |
+
return _CACHE
|
| 168 |
+
|
| 169 |
+
device = _device_from_ui(device_text)
|
| 170 |
+
ckpt = load_checkpoint(ckpt_path, map_location="cpu")
|
| 171 |
+
meta = ckpt.get("meta", {}) or {}
|
| 172 |
+
|
| 173 |
+
model_type = str(meta.get("model_type", "uncond"))
|
| 174 |
+
unet_cfg_dict = meta.get("unet_cfg", None)
|
| 175 |
+
if unet_cfg_dict is None:
|
| 176 |
+
# 兼容:旧 ckpt 未保存配置
|
| 177 |
+
unet_cfg = UNet16Config()
|
| 178 |
+
timesteps = 1000
|
| 179 |
+
beta_schedule = "cosine"
|
| 180 |
+
image_size = 16
|
| 181 |
+
channels = 3
|
| 182 |
+
model_type = "uncond"
|
| 183 |
+
else:
|
| 184 |
+
unet_cfg = UNet16Config(**unet_cfg_dict)
|
| 185 |
+
diff_meta = meta.get("diffusion", {}) or {}
|
| 186 |
+
timesteps = int(diff_meta.get("timesteps", 1000))
|
| 187 |
+
beta_schedule = str(diff_meta.get("beta_schedule", "cosine"))
|
| 188 |
+
data_meta = meta.get("data", {}) or {}
|
| 189 |
+
image_size = int(data_meta.get("image_size", 16))
|
| 190 |
+
channels = int(data_meta.get("channels", unet_cfg.in_channels))
|
| 191 |
+
|
| 192 |
+
vocab = None
|
| 193 |
+
text_cfg = None
|
| 194 |
+
if model_type == "text_cond":
|
| 195 |
+
tok_dict = meta.get("tokenizer", None)
|
| 196 |
+
text_cfg_dict = meta.get("text_cfg", None)
|
| 197 |
+
if tok_dict is None or text_cfg_dict is None:
|
| 198 |
+
raise ValueError("该 checkpoint 标记为 text_cond,但缺少 tokenizer/text_cfg")
|
| 199 |
+
vocab = SimpleVocab.from_dict(tok_dict)
|
| 200 |
+
text_cfg = TextCondConfig(**text_cfg_dict)
|
| 201 |
+
model = TextCondUNet16(unet_cfg=unet_cfg, vocab=vocab, text_cfg=text_cfg).to(device)
|
| 202 |
+
else:
|
| 203 |
+
model = UNet16(unet_cfg).to(device)
|
| 204 |
+
|
| 205 |
+
model.load_state_dict(ckpt["model"], strict=True)
|
| 206 |
+
model.eval()
|
| 207 |
+
|
| 208 |
+
if use_ema and ("ema" in ckpt):
|
| 209 |
+
# 使用 EMA 权重采样(推荐),会覆盖 model 当前参数
|
| 210 |
+
ema = EMA(model, decay=0.9999)
|
| 211 |
+
ema.load_state_dict(ckpt["ema"])
|
| 212 |
+
ema.copy_to(model)
|
| 213 |
+
model.eval()
|
| 214 |
+
|
| 215 |
+
betas = get_beta_schedule(beta_schedule, timesteps).to(device)
|
| 216 |
+
diffusion = GaussianDiffusion(betas)
|
| 217 |
+
|
| 218 |
+
_CACHE = _Loaded(
|
| 219 |
+
key=key,
|
| 220 |
+
model_type=model_type,
|
| 221 |
+
image_size=image_size,
|
| 222 |
+
channels=channels,
|
| 223 |
+
timesteps=timesteps,
|
| 224 |
+
beta_schedule=beta_schedule,
|
| 225 |
+
unet_cfg=unet_cfg,
|
| 226 |
+
text_cfg=text_cfg,
|
| 227 |
+
vocab=vocab,
|
| 228 |
+
model=model,
|
| 229 |
+
diffusion=diffusion,
|
| 230 |
+
)
|
| 231 |
+
return _CACHE
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
@torch.no_grad()
|
| 235 |
+
def _sample(
|
| 236 |
+
ckpt_path: str,
|
| 237 |
+
prompt: str,
|
| 238 |
+
cfg_scale: float,
|
| 239 |
+
use_ema: bool,
|
| 240 |
+
use_ddim: bool,
|
| 241 |
+
ddim_steps: int,
|
| 242 |
+
ddim_eta: float,
|
| 243 |
+
n_images: int,
|
| 244 |
+
batch: int,
|
| 245 |
+
seed: int,
|
| 246 |
+
device_text: str,
|
| 247 |
+
save_dir: str,
|
| 248 |
+
preview_upscale: int,
|
| 249 |
+
gallery_n: int,
|
| 250 |
+
) -> Tuple[str, Optional[str], Optional[List[Image.Image]], Optional[Image.Image]]:
|
| 251 |
+
"""
|
| 252 |
+
返回:
|
| 253 |
+
- info 文本(用于 UI 展示)
|
| 254 |
+
- 保存路径(或 None)
|
| 255 |
+
- gallery 图片列表(或 None)
|
| 256 |
+
- 网格图 PIL(或 None)
|
| 257 |
+
"""
|
| 258 |
+
|
| 259 |
+
ckpt_path = (ckpt_path or "").strip()
|
| 260 |
+
if not ckpt_path:
|
| 261 |
+
return "错误:请先选择/填写 checkpoint 路径。", None, None, None
|
| 262 |
+
if not os.path.isfile(ckpt_path):
|
| 263 |
+
return f"错误:checkpoint 不存在:{ckpt_path}", None, None, None
|
| 264 |
+
|
| 265 |
+
save_dir = (save_dir or "").strip() or "webui_samples"
|
| 266 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 267 |
+
|
| 268 |
+
# 采样随机性(只影响本次点击)
|
| 269 |
+
seed_everything(int(seed), deterministic=False)
|
| 270 |
+
|
| 271 |
+
loaded = _load_cached(ckpt_path=ckpt_path, device_text=device_text, use_ema=use_ema)
|
| 272 |
+
device = _device_from_ui(device_text)
|
| 273 |
+
|
| 274 |
+
# 条件采样:仅 text_cond 支持 prompt + CFG
|
| 275 |
+
sampler_model: torch.nn.Module = loaded.model
|
| 276 |
+
prompt_tokens: List[str] = []
|
| 277 |
+
prompt_unk: List[str] = []
|
| 278 |
+
if (prompt or "").strip():
|
| 279 |
+
if loaded.model_type != "text_cond":
|
| 280 |
+
return "错误:当前 checkpoint 是无条件模型,不支持 prompt/CFG。", None, None, None
|
| 281 |
+
if loaded.vocab is None or loaded.text_cfg is None:
|
| 282 |
+
return "错误:条件模型缺少 tokenizer/text_cfg。", None, None, None
|
| 283 |
+
|
| 284 |
+
# 用于诊断:提示词是否大量变成 <unk>(这会导致“看起来不遵从提示词”)
|
| 285 |
+
prompt_tokens = tokenize(prompt)
|
| 286 |
+
for t in prompt_tokens:
|
| 287 |
+
tid = loaded.vocab.token_to_id.get(t, loaded.vocab.unk_id)
|
| 288 |
+
if int(tid) == int(loaded.vocab.unk_id):
|
| 289 |
+
prompt_unk.append(t)
|
| 290 |
+
|
| 291 |
+
tok = torch.tensor(
|
| 292 |
+
[loaded.vocab.encode(prompt, max_len=loaded.text_cfg.max_text_len)],
|
| 293 |
+
dtype=torch.long,
|
| 294 |
+
device=device,
|
| 295 |
+
)
|
| 296 |
+
m = tok != loaded.vocab.pad_id
|
| 297 |
+
cond_kwargs = {"tokens": tok, "mask": m}
|
| 298 |
+
un_tok = torch.full_like(tok, fill_value=loaded.vocab.pad_id)
|
| 299 |
+
un_m = torch.zeros_like(m)
|
| 300 |
+
uncond_kwargs = {"tokens": un_tok, "mask": un_m}
|
| 301 |
+
sampler_model = CFGuidanceModel(
|
| 302 |
+
base_model=loaded.model,
|
| 303 |
+
cond_kwargs=cond_kwargs,
|
| 304 |
+
uncond_kwargs=uncond_kwargs,
|
| 305 |
+
scale=float(cfg_scale),
|
| 306 |
+
).to(device)
|
| 307 |
+
sampler_model.eval()
|
| 308 |
+
else:
|
| 309 |
+
# 不提供 prompt 时:对条件模型等价无条件采样
|
| 310 |
+
pass
|
| 311 |
+
|
| 312 |
+
n_images = int(max(1, n_images))
|
| 313 |
+
batch = int(max(1, batch))
|
| 314 |
+
gallery_n = int(max(0, min(gallery_n, n_images)))
|
| 315 |
+
|
| 316 |
+
# 分批采样
|
| 317 |
+
all_imgs: List[torch.Tensor] = []
|
| 318 |
+
remaining = n_images
|
| 319 |
+
t0 = time.time()
|
| 320 |
+
while remaining > 0:
|
| 321 |
+
b = min(batch, remaining)
|
| 322 |
+
shape = (b, loaded.channels, loaded.image_size, loaded.image_size)
|
| 323 |
+
if use_ddim:
|
| 324 |
+
x = loaded.diffusion.ddim_sample_loop(
|
| 325 |
+
sampler_model,
|
| 326 |
+
shape=shape,
|
| 327 |
+
device=device,
|
| 328 |
+
steps=int(ddim_steps),
|
| 329 |
+
eta=float(ddim_eta),
|
| 330 |
+
model_kwargs=None,
|
| 331 |
+
progress=False,
|
| 332 |
+
)
|
| 333 |
+
else:
|
| 334 |
+
x = loaded.diffusion.p_sample_loop(
|
| 335 |
+
sampler_model,
|
| 336 |
+
shape=shape,
|
| 337 |
+
device=device,
|
| 338 |
+
model_kwargs=None,
|
| 339 |
+
progress=False,
|
| 340 |
+
)
|
| 341 |
+
x = (x.clamp(-1, 1) + 1) * 0.5 # [0,1]
|
| 342 |
+
all_imgs.append(x.detach().cpu())
|
| 343 |
+
remaining -= b
|
| 344 |
+
dt = time.time() - t0
|
| 345 |
+
|
| 346 |
+
imgs = torch.cat(all_imgs, dim=0) # (N,C,H,W) on CPU
|
| 347 |
+
|
| 348 |
+
# 网格图(更适合一次看很多张)
|
| 349 |
+
nrow = int(max(1, math.isqrt(n_images)))
|
| 350 |
+
grid = make_grid(imgs, nrow=nrow, padding=1).clamp(0.0, 1.0) # (C,H,W)
|
| 351 |
+
grid_im_raw = _tensor_to_pil(grid) # 原始分辨率,用于保存
|
| 352 |
+
grid_im = _upscale_nearest(grid_im_raw, int(preview_upscale)) # 仅预览放大
|
| 353 |
+
|
| 354 |
+
# 单张 gallery(可选)
|
| 355 |
+
gallery: Optional[List[Image.Image]] = None
|
| 356 |
+
if gallery_n > 0:
|
| 357 |
+
gallery = []
|
| 358 |
+
for i in range(gallery_n):
|
| 359 |
+
im = _tensor_to_pil(imgs[i])
|
| 360 |
+
im = _upscale_nearest(im, int(preview_upscale))
|
| 361 |
+
gallery.append(im)
|
| 362 |
+
|
| 363 |
+
# 保存(默认保存网格图,文件名包含 step 与时间戳,方便对比)
|
| 364 |
+
step = _parse_step_from_filename(ckpt_path)
|
| 365 |
+
ts = time.strftime("%Y%m%d-%H%M%S")
|
| 366 |
+
out_name = f"grid_step_{step if step >= 0 else 'unk'}_{ts}.png"
|
| 367 |
+
out_path = os.path.join(save_dir, out_name)
|
| 368 |
+
grid_im_raw.save(out_path)
|
| 369 |
+
|
| 370 |
+
info = (
|
| 371 |
+
f"ckpt: {os.path.abspath(ckpt_path)}\n"
|
| 372 |
+
f"model_type: {loaded.model_type}\n"
|
| 373 |
+
f"size: {loaded.image_size}x{loaded.image_size} channels: {loaded.channels}\n"
|
| 374 |
+
f"timesteps: {loaded.timesteps} beta_schedule: {loaded.beta_schedule}\n"
|
| 375 |
+
f"device: {device}\n"
|
| 376 |
+
f"use_ema: {bool(use_ema)} sampler: {'DDIM' if use_ddim else 'DDPM'}\n"
|
| 377 |
+
f"n_images: {n_images} batch: {batch} seed: {int(seed)}\n"
|
| 378 |
+
f"prompt: {(prompt or '').strip() or '(none)'} cfg_scale: {float(cfg_scale):.2f}\n"
|
| 379 |
+
+ (
|
| 380 |
+
f"prompt_tokens: {' '.join(prompt_tokens[:64])}\n"
|
| 381 |
+
f"prompt_unk({len(prompt_unk)}): {' '.join(prompt_unk[:64])}\n"
|
| 382 |
+
if prompt_tokens
|
| 383 |
+
else ""
|
| 384 |
+
)
|
| 385 |
+
f"preview_upscale(NEAREST): {int(preview_upscale)}x\n"
|
| 386 |
+
f"save: {os.path.abspath(out_path)}\n"
|
| 387 |
+
f"time: {dt:.2f}s\n"
|
| 388 |
+
)
|
| 389 |
+
return info, os.path.abspath(out_path), gallery, grid_im
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def _build_ui() -> gr.Blocks:
|
| 393 |
+
ckpts = list_checkpoints("outputs")
|
| 394 |
+
default_ckpt = ckpts[0] if ckpts else ""
|
| 395 |
+
|
| 396 |
+
with gr.Blocks(title="BlockDiffusion WebUI") as demo:
|
| 397 |
+
gr.Markdown("### BlockDiffusion WebUI(采样预览)")
|
| 398 |
+
|
| 399 |
+
with gr.Row():
|
| 400 |
+
ckpt_quick = gr.Dropdown(
|
| 401 |
+
choices=ckpts,
|
| 402 |
+
value=default_ckpt or None,
|
| 403 |
+
label="快速选择 checkpoint(从 outputs 扫描)",
|
| 404 |
+
)
|
| 405 |
+
refresh = gr.Button("刷新列表")
|
| 406 |
+
|
| 407 |
+
ckpt_path = gr.Textbox(value=default_ckpt, label="Checkpoint 路径(可手填)")
|
| 408 |
+
|
| 409 |
+
with gr.Row():
|
| 410 |
+
prompt = gr.Textbox(value="", label="Prompt(条件模型可用;留空=无条件)")
|
| 411 |
+
cfg_scale = gr.Slider(minimum=0.0, maximum=12.0, value=3.0, step=0.1, label="CFG scale")
|
| 412 |
+
|
| 413 |
+
with gr.Row():
|
| 414 |
+
use_ema = gr.Checkbox(value=True, label="使用 EMA 权重(推荐)")
|
| 415 |
+
use_ddim = gr.Checkbox(value=True, label="使用 DDIM(更快)")
|
| 416 |
+
ddim_steps = gr.Slider(minimum=10, maximum=200, value=50, step=1, label="DDIM steps")
|
| 417 |
+
ddim_eta = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.01, label="DDIM eta")
|
| 418 |
+
|
| 419 |
+
with gr.Row():
|
| 420 |
+
n_images = gr.Slider(minimum=1, maximum=256, value=64, step=1, label="生成张数 N")
|
| 421 |
+
batch = gr.Slider(minimum=1, maximum=256, value=64, step=1, label="采样 batch(显存不够就调小)")
|
| 422 |
+
seed = gr.Number(value=123, precision=0, label="随机种子 seed")
|
| 423 |
+
|
| 424 |
+
with gr.Row():
|
| 425 |
+
device_text = gr.Dropdown(
|
| 426 |
+
choices=["自动", "cuda", "cuda:0", "cpu"],
|
| 427 |
+
value="自动",
|
| 428 |
+
label="设备",
|
| 429 |
+
)
|
| 430 |
+
save_dir = gr.Textbox(value="webui_samples", label="保存目录")
|
| 431 |
+
preview_upscale = gr.Slider(minimum=1, maximum=64, value=16, step=1, label="预览放大倍数(NEAREST)")
|
| 432 |
+
gallery_n = gr.Slider(minimum=0, maximum=64, value=16, step=1, label="单张预览数量(0=关闭)")
|
| 433 |
+
|
| 434 |
+
run = gr.Button("生成")
|
| 435 |
+
|
| 436 |
+
info = gr.Textbox(label="信息/日志", lines=10)
|
| 437 |
+
grid_out = gr.Image(label="网格图(已放大预览)", type="pil")
|
| 438 |
+
gallery_out = gr.Gallery(label="单张预览(已放大)", columns=8, rows=2)
|
| 439 |
+
saved_path = gr.Textbox(label="已保存路径", lines=1)
|
| 440 |
+
|
| 441 |
+
def _on_refresh():
|
| 442 |
+
new_ckpts = list_checkpoints("outputs")
|
| 443 |
+
return gr.update(choices=new_ckpts, value=(new_ckpts[0] if new_ckpts else None))
|
| 444 |
+
|
| 445 |
+
def _on_pick(p: Optional[str]) -> str:
|
| 446 |
+
return p or ""
|
| 447 |
+
|
| 448 |
+
def _on_run(
|
| 449 |
+
ckpt_path_v: str,
|
| 450 |
+
prompt_v: str,
|
| 451 |
+
cfg_scale_v: float,
|
| 452 |
+
use_ema_v: bool,
|
| 453 |
+
use_ddim_v: bool,
|
| 454 |
+
ddim_steps_v: int,
|
| 455 |
+
ddim_eta_v: float,
|
| 456 |
+
n_images_v: int,
|
| 457 |
+
batch_v: int,
|
| 458 |
+
seed_v: int,
|
| 459 |
+
device_text_v: str,
|
| 460 |
+
save_dir_v: str,
|
| 461 |
+
preview_upscale_v: int,
|
| 462 |
+
gallery_n_v: int,
|
| 463 |
+
):
|
| 464 |
+
try:
|
| 465 |
+
txt, out_path, gal, grid = _sample(
|
| 466 |
+
ckpt_path=ckpt_path_v,
|
| 467 |
+
prompt=prompt_v,
|
| 468 |
+
cfg_scale=float(cfg_scale_v),
|
| 469 |
+
use_ema=bool(use_ema_v),
|
| 470 |
+
use_ddim=bool(use_ddim_v),
|
| 471 |
+
ddim_steps=int(ddim_steps_v),
|
| 472 |
+
ddim_eta=float(ddim_eta_v),
|
| 473 |
+
n_images=int(n_images_v),
|
| 474 |
+
batch=int(batch_v),
|
| 475 |
+
seed=int(seed_v),
|
| 476 |
+
device_text=device_text_v,
|
| 477 |
+
save_dir=save_dir_v,
|
| 478 |
+
preview_upscale=int(preview_upscale_v),
|
| 479 |
+
gallery_n=int(gallery_n_v),
|
| 480 |
+
)
|
| 481 |
+
return txt, grid, (gal or []), (out_path or "")
|
| 482 |
+
except Exception as e:
|
| 483 |
+
# UI 侧直接给出错误信息,避免整个服务崩溃
|
| 484 |
+
return f"错误:{e}", None, [], ""
|
| 485 |
+
|
| 486 |
+
refresh.click(_on_refresh, inputs=[], outputs=[ckpt_quick])
|
| 487 |
+
ckpt_quick.change(_on_pick, inputs=[ckpt_quick], outputs=[ckpt_path])
|
| 488 |
+
run.click(
|
| 489 |
+
_on_run,
|
| 490 |
+
inputs=[
|
| 491 |
+
ckpt_path,
|
| 492 |
+
prompt,
|
| 493 |
+
cfg_scale,
|
| 494 |
+
use_ema,
|
| 495 |
+
use_ddim,
|
| 496 |
+
ddim_steps,
|
| 497 |
+
ddim_eta,
|
| 498 |
+
n_images,
|
| 499 |
+
batch,
|
| 500 |
+
seed,
|
| 501 |
+
device_text,
|
| 502 |
+
save_dir,
|
| 503 |
+
preview_upscale,
|
| 504 |
+
gallery_n,
|
| 505 |
+
],
|
| 506 |
+
outputs=[info, grid_out, gallery_out, saved_path],
|
| 507 |
+
)
|
| 508 |
+
|
| 509 |
+
return demo
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
def main() -> None:
|
| 513 |
+
demo = _build_ui()
|
| 514 |
+
# 默认监听 127.0.0.1:仅本机访问更安全
|
| 515 |
+
_ensure_localhost_no_proxy()
|
| 516 |
+
demo.launch(server_name="127.0.0.1", share=False)
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
if __name__ == "__main__":
|
| 520 |
+
main()
|
| 521 |
+
|
| 522 |
+
|