wkea's picture
Deploy Space
c388140 verified
Raw History Blame Contribute Delete
2.52 kB
from __future__ import annotations
from typing import Dict
import torch
from torch import nn
class EMA:
"""
Exponential Moving Average:对模型参数做滑动平均,提升采样稳定性。
"""
def __init__(self, model: nn.Module, decay: float = 0.9999):
if not (0.0 < decay < 1.0):
raise ValueError("decay 必须在 (0, 1) 之间")
self.decay = float(decay)
self.shadow: Dict[str, torch.Tensor] = {}
self._init_from(model)
def _init_from(self, model: nn.Module) -> None:
self.shadow = {
k: v.detach().clone().float()
for k, v in model.state_dict().items()
if v.dtype.is_floating_point
}
@torch.no_grad()
def update(self, model: nn.Module) -> None:
"""
用当前模型参数更新 EMA。
"""
msd = model.state_dict()
for k, v in msd.items():
if k not in self.shadow:
continue
if not v.dtype.is_floating_point:
continue
# 续训场景:shadow 可能从 checkpoint 载入在 CPU,需要对齐到参数所在 device
if self.shadow[k].device != v.device:
self.shadow[k] = self.shadow[k].to(device=v.device)
self.shadow[k].mul_(self.decay).add_(v.detach().float(), alpha=1.0 - self.decay)
def state_dict(self) -> Dict[str, torch.Tensor]:
return {"decay": torch.tensor(self.decay), "shadow": self.shadow}
def load_state_dict(self, state: Dict) -> None:
self.decay = float(state["decay"].item() if isinstance(state["decay"], torch.Tensor) else state["decay"])
self.shadow = {k: v.detach().clone() for k, v in state["shadow"].items()}
def to(self, device: torch.device) -> None:
"""
将 EMA 的 shadow 参数移动到指定 device(续训/切换设备时使用)。
"""
self.shadow = {k: v.to(device=device) for k, v in self.shadow.items()}
@torch.no_grad()
def copy_to(self, model: nn.Module) -> None:
"""
将 EMA 参数拷贝到目标模型(常用于采样时临时替换)。
"""
msd = model.state_dict()
for k in self.shadow.keys():
if k in msd:
src = self.shadow[k]
if src.device != msd[k].device:
src = src.to(msd[k].device)
msd[k].copy_(src.to(msd[k].dtype))