from __future__ import annotations from typing import Dict import torch from torch import nn class EMA: """ Exponential Moving Average:对模型参数做滑动平均,提升采样稳定性。 """ def __init__(self, model: nn.Module, decay: float = 0.9999): if not (0.0 < decay < 1.0): raise ValueError("decay 必须在 (0, 1) 之间") self.decay = float(decay) self.shadow: Dict[str, torch.Tensor] = {} self._init_from(model) def _init_from(self, model: nn.Module) -> None: self.shadow = { k: v.detach().clone().float() for k, v in model.state_dict().items() if v.dtype.is_floating_point } @torch.no_grad() def update(self, model: nn.Module) -> None: """ 用当前模型参数更新 EMA。 """ msd = model.state_dict() for k, v in msd.items(): if k not in self.shadow: continue if not v.dtype.is_floating_point: continue # 续训场景:shadow 可能从 checkpoint 载入在 CPU,需要对齐到参数所在 device if self.shadow[k].device != v.device: self.shadow[k] = self.shadow[k].to(device=v.device) self.shadow[k].mul_(self.decay).add_(v.detach().float(), alpha=1.0 - self.decay) def state_dict(self) -> Dict[str, torch.Tensor]: return {"decay": torch.tensor(self.decay), "shadow": self.shadow} def load_state_dict(self, state: Dict) -> None: self.decay = float(state["decay"].item() if isinstance(state["decay"], torch.Tensor) else state["decay"]) self.shadow = {k: v.detach().clone() for k, v in state["shadow"].items()} def to(self, device: torch.device) -> None: """ 将 EMA 的 shadow 参数移动到指定 device(续训/切换设备时使用)。 """ self.shadow = {k: v.to(device=device) for k, v in self.shadow.items()} @torch.no_grad() def copy_to(self, model: nn.Module) -> None: """ 将 EMA 参数拷贝到目标模型(常用于采样时临时替换)。 """ msd = model.state_dict() for k in self.shadow.keys(): if k in msd: src = self.shadow[k] if src.device != msd[k].device: src = src.to(msd[k].device) msd[k].copy_(src.to(msd[k].dtype))