Spaces:
Sleeping
Sleeping
Download blockdiffusion/train/trainer.py from wkea/blockdiffusion-api: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/wkea/blockdiffusion-api/resolve/main/blockdiffusion/train/trainer.py
- Command line
-
hf download hf://spaces/wkea/blockdiffusion-api/blockdiffusion/train/trainer.py
-
curl -L -o trainer.py https://huggingface.co/spaces/wkea/blockdiffusion-api/resolve/main/blockdiffusion/train/trainer.py
11.9 kB
| from __future__ import annotations | |
| import os | |
| from dataclasses import dataclass | |
| from typing import Any, Dict, Optional | |
| import torch | |
| from torch import nn | |
| from contextlib import contextmanager | |
| from torch.utils.data import DataLoader | |
| from blockdiffusion.diffusion.gaussian_diffusion import GaussianDiffusion | |
| from blockdiffusion.train.guidance import CFGuidanceModel | |
| from blockdiffusion.utils.checkpoint import load_checkpoint, save_checkpoint | |
| from blockdiffusion.utils.ema import EMA | |
| from blockdiffusion.utils.image_io import save_tensor_grid | |
| def _make_grad_scaler(enabled: bool, device_type: str): | |
| """ | |
| 兼容不同 PyTorch 版本的 AMP GradScaler: | |
| - 新版推荐 torch.amp.GradScaler(device, ...) | |
| - 旧版使用 torch.cuda.amp.GradScaler(...) | |
| """ | |
| enabled = bool(enabled and device_type == "cuda") | |
| try: | |
| from torch.amp import GradScaler as _GS # PyTorch 2.x | |
| if not enabled: | |
| return _GS(enabled=False) | |
| # 有的版本是 GradScaler(device='cuda', ...),有的接受位置参数 | |
| try: | |
| return _GS("cuda", enabled=True) | |
| except TypeError: | |
| return _GS(enabled=True) | |
| except Exception: | |
| from torch.cuda.amp import GradScaler as _GS # 兼容旧版本 | |
| return _GS(enabled=enabled) | |
| def _autocast(device_type: str, enabled: bool): | |
| """ | |
| 兼容不同 PyTorch 版本的 autocast: | |
| - torch.amp.autocast(device_type=..., enabled=...) | |
| - torch.cuda.amp.autocast(enabled=...) | |
| """ | |
| enabled = bool(enabled and device_type == "cuda") | |
| try: | |
| from torch.amp import autocast as _AC | |
| with _AC(device_type=device_type, enabled=enabled): | |
| yield | |
| except Exception: | |
| from torch.cuda.amp import autocast as _AC | |
| with _AC(enabled=enabled): | |
| yield | |
| class TrainerConfig: | |
| """ | |
| 训练器配置(尽量给出稳定且适合 16×16 的默认值)。 | |
| """ | |
| # 数据 | |
| image_size: int = 16 | |
| channels: int = 3 | |
| batch_size: int = 512 | |
| num_workers: int = 4 | |
| # 训练 | |
| lr: float = 2e-4 | |
| weight_decay: float = 1e-4 | |
| total_steps: int = 200_000 | |
| grad_clip: float = 1.0 | |
| amp: bool = True | |
| cond_drop_prob: float = 0.1 # 仅在有条件训练时生效(CFG 训练) | |
| grad_accum_steps: int = 1 # 梯度累积:提高有效 batch(质量更稳),也可用于控制显存 | |
| lr_warmup_steps: int = 2000 # 学习率 warmup 步数(0 表示关闭) | |
| lr_min_ratio: float = 0.1 # 余弦退火最低 lr 比例(相对 lr) | |
| snr_gamma: float = 5.0 # SNR 加权 MSE 的 gamma(<=0 表示关闭) | |
| channels_last: bool = True # Conv 的内存布局优化,通常能略提速 | |
| # EMA | |
| ema_decay: float = 0.9999 | |
| # 日志/保存 | |
| log_every: int = 100 | |
| save_every: int = 5_000 | |
| sample_every: int = 5_000 | |
| sample_batch: int = 64 | |
| sample_use_ddim: bool = True | |
| sample_ddim_steps: int = 50 | |
| sample_ddim_eta: float = 0.0 | |
| sample_cfg_scale: float = 3.0 | |
| out_dir: str = "outputs" | |
| run_name: str = "default" | |
| resume_path: Optional[str] = None | |
| class Trainer: | |
| """ | |
| 训练器只负责: | |
| - dataloader | |
| - 优化器/AMP/EMA | |
| - checkpoint 与训练过程采样 | |
| 扩散公式与采样过程由 GaussianDiffusion 负责,保证解耦。 | |
| """ | |
| def __init__( | |
| self, | |
| model: nn.Module, | |
| diffusion: GaussianDiffusion, | |
| dataloader: DataLoader, | |
| device: torch.device, | |
| cfg: TrainerConfig, | |
| meta: Optional[dict] = None, | |
| sample_model_kwargs: Optional[Dict[str, Any]] = None, | |
| sample_uncond_model_kwargs: Optional[Dict[str, Any]] = None, | |
| ): | |
| self.model = model.to(device) | |
| self.diffusion = diffusion | |
| self.dataloader = dataloader | |
| self.device = device | |
| self.cfg = cfg | |
| self.meta = meta or {} | |
| self.sample_model_kwargs = sample_model_kwargs | |
| self.sample_uncond_model_kwargs = sample_uncond_model_kwargs | |
| # 可选:channels_last 提升卷积吞吐(对小图也常有帮助) | |
| if cfg.channels_last and device.type == "cuda": | |
| self.model = self.model.to(memory_format=torch.channels_last) | |
| self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay) | |
| use_amp = bool(cfg.amp and device.type == "cuda") | |
| self.scaler = _make_grad_scaler(enabled=use_amp, device_type=device.type) | |
| self.ema = EMA(self.model, decay=cfg.ema_decay) | |
| self.step = 0 | |
| self._prepare_dirs() | |
| if cfg.resume_path: | |
| self._resume(cfg.resume_path) | |
| # 学习率调度(warmup + cosine decay),提升稳定性/质量 | |
| self.base_lr = float(cfg.lr) | |
| self.min_lr = float(cfg.lr) * float(cfg.lr_min_ratio) | |
| def _prepare_dirs(self) -> None: | |
| self.run_dir = os.path.join(self.cfg.out_dir, self.cfg.run_name) | |
| self.ckpt_dir = os.path.join(self.run_dir, "checkpoints") | |
| self.sample_dir = os.path.join(self.run_dir, "samples") | |
| os.makedirs(self.ckpt_dir, exist_ok=True) | |
| os.makedirs(self.sample_dir, exist_ok=True) | |
| def _resume(self, path: str) -> None: | |
| ckpt = load_checkpoint(path, map_location="cpu") | |
| self.model.load_state_dict(ckpt["model"], strict=True) | |
| if "ema" in ckpt: | |
| self.ema.load_state_dict(ckpt["ema"]) | |
| # 续训:确保 EMA shadow 跟随当前训练设备 | |
| self.ema.to(self.device) | |
| if "optimizer" in ckpt: | |
| self.optimizer.load_state_dict(ckpt["optimizer"]) | |
| # 续训:optimizer state 里的 tensor 默认仍在 CPU,需要搬到当前设备 | |
| for state in self.optimizer.state.values(): | |
| for kk, vv in list(state.items()): | |
| if torch.is_tensor(vv): | |
| state[kk] = vv.to(self.device) | |
| if "scaler" in ckpt: | |
| self.scaler.load_state_dict(ckpt["scaler"]) | |
| self.step = int(ckpt.get("step", 0)) | |
| def _save(self) -> None: | |
| path = os.path.join(self.ckpt_dir, f"step_{self.step}.pt") | |
| payload = { | |
| "step": self.step, | |
| "meta": self.meta, | |
| "trainer_cfg": self.cfg.__dict__, | |
| "model": self.model.state_dict(), | |
| "ema": self.ema.state_dict(), | |
| "optimizer": self.optimizer.state_dict(), | |
| "scaler": self.scaler.state_dict(), | |
| } | |
| save_checkpoint(path, payload) | |
| def _sample(self) -> None: | |
| # 采样时使用 EMA 参数,避免训练噪声导致的质量抖动 | |
| tmp = {k: v.detach().clone() for k, v in self.model.state_dict().items()} | |
| self.ema.copy_to(self.model) | |
| self.model.eval() | |
| shape = (self.cfg.sample_batch, self.cfg.channels, self.cfg.image_size, self.cfg.image_size) | |
| sampler_model: nn.Module = self.model | |
| if self.sample_model_kwargs is not None and self.sample_uncond_model_kwargs is not None: | |
| sampler_model = CFGuidanceModel( | |
| base_model=self.model, | |
| cond_kwargs=self.sample_model_kwargs, | |
| uncond_kwargs=self.sample_uncond_model_kwargs, | |
| scale=self.cfg.sample_cfg_scale, | |
| ) | |
| if self.cfg.sample_use_ddim: | |
| x = self.diffusion.ddim_sample_loop( | |
| sampler_model, | |
| shape=shape, | |
| device=self.device, | |
| steps=self.cfg.sample_ddim_steps, | |
| eta=self.cfg.sample_ddim_eta, | |
| model_kwargs=None, | |
| progress=False, | |
| ) | |
| else: | |
| x = self.diffusion.p_sample_loop(sampler_model, shape=shape, device=self.device, model_kwargs=None, progress=False) | |
| # [-1,1] -> [0,1] 保存 | |
| x = (x.clamp(-1, 1) + 1) * 0.5 | |
| out_path = os.path.join(self.sample_dir, f"step_{self.step}.png") | |
| save_tensor_grid(x, out_path, nrow=int(self.cfg.sample_batch**0.5), padding=1) | |
| # 恢复训练参数 | |
| self.model.load_state_dict(tmp, strict=True) | |
| self.model.train() | |
| def train(self) -> None: | |
| self.model.train() | |
| data_iter = iter(self.dataloader) | |
| while self.step < self.cfg.total_steps: | |
| # 梯度累积:每 grad_accum_steps 次 forward 才做一次 optimizer step | |
| accum_steps = max(1, int(self.cfg.grad_accum_steps)) | |
| self.optimizer.zero_grad(set_to_none=True) | |
| total_loss = 0.0 | |
| for micro in range(accum_steps): | |
| try: | |
| batch = next(data_iter) | |
| except StopIteration: | |
| data_iter = iter(self.dataloader) | |
| batch = next(data_iter) | |
| model_kwargs: Dict[str, Any] = {} | |
| if isinstance(batch, (tuple, list)) and len(batch) == 3: | |
| x0, tokens, mask = batch | |
| model_kwargs["tokens"] = tokens.to(self.device, non_blocking=True) | |
| model_kwargs["mask"] = mask.to(self.device, non_blocking=True) | |
| else: | |
| x0 = batch | |
| # channels_last:数据也转换同布局 | |
| x0 = x0.to(self.device, non_blocking=True) | |
| if self.cfg.channels_last and self.device.type == "cuda": | |
| x0 = x0.contiguous(memory_format=torch.channels_last) | |
| b = x0.shape[0] | |
| t = torch.randint(0, self.diffusion.num_timesteps, (b,), device=self.device, dtype=torch.long) | |
| with _autocast(device_type=self.device.type, enabled=self.scaler.is_enabled()): | |
| loss = self.diffusion.training_losses( | |
| self.model, | |
| x0, | |
| t, | |
| model_kwargs=model_kwargs, | |
| cond_drop_prob=self.cfg.cond_drop_prob, | |
| snr_gamma=float(self.cfg.snr_gamma), | |
| )["loss"] | |
| # 累积时按比例缩放,保持有效学习率不变 | |
| loss = loss / accum_steps | |
| self.scaler.scale(loss).backward() | |
| total_loss += float(loss.detach().item()) | |
| # 学习率 warmup + cosine decay(基于全局 step) | |
| if self.cfg.lr_warmup_steps and self.cfg.lr_warmup_steps > 0 and self.step < self.cfg.lr_warmup_steps: | |
| lr = self.min_lr + (self.base_lr - self.min_lr) * (float(self.step + 1) / float(self.cfg.lr_warmup_steps)) | |
| else: | |
| # cosine 从 warmup 结束后开始 | |
| warm = max(0, int(self.cfg.lr_warmup_steps)) | |
| prog = (self.step - warm) / max(1.0, float(self.cfg.total_steps - warm)) | |
| prog = min(max(prog, 0.0), 1.0) | |
| import math | |
| lr = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1.0 + math.cos(math.pi * prog)) | |
| for pg in self.optimizer.param_groups: | |
| pg["lr"] = lr | |
| if self.cfg.grad_clip and self.cfg.grad_clip > 0: | |
| self.scaler.unscale_(self.optimizer) | |
| torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.cfg.grad_clip) | |
| self.scaler.step(self.optimizer) | |
| self.scaler.update() | |
| self.ema.update(self.model) | |
| self.step += 1 | |
| if self.cfg.log_every > 0 and (self.step % self.cfg.log_every == 0): | |
| # total_loss 已按 accum_steps 平均后的值(每 micro 先 /accum) | |
| print(f"[step {self.step}] loss={total_loss:.6f} lr={lr:.2e}") | |
| if self.cfg.save_every > 0 and (self.step % self.cfg.save_every == 0): | |
| self._save() | |
| if self.cfg.sample_every > 0 and (self.step % self.cfg.sample_every == 0): | |
| self._sample() | |