File size: 2,521 Bytes
c388140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
from __future__ import annotations

from typing import Dict

import torch
from torch import nn


class EMA:
    """

    Exponential Moving Average:对模型参数做滑动平均,提升采样稳定性。

    """

    def __init__(self, model: nn.Module, decay: float = 0.9999):
        if not (0.0 < decay < 1.0):
            raise ValueError("decay 必须在 (0, 1) 之间")
        self.decay = float(decay)
        self.shadow: Dict[str, torch.Tensor] = {}
        self._init_from(model)

    def _init_from(self, model: nn.Module) -> None:
        self.shadow = {
            k: v.detach().clone().float()
            for k, v in model.state_dict().items()
            if v.dtype.is_floating_point
        }

    @torch.no_grad()
    def update(self, model: nn.Module) -> None:
        """

        用当前模型参数更新 EMA。

        """
        msd = model.state_dict()
        for k, v in msd.items():
            if k not in self.shadow:
                continue
            if not v.dtype.is_floating_point:
                continue
            # 续训场景:shadow 可能从 checkpoint 载入在 CPU,需要对齐到参数所在 device
            if self.shadow[k].device != v.device:
                self.shadow[k] = self.shadow[k].to(device=v.device)
            self.shadow[k].mul_(self.decay).add_(v.detach().float(), alpha=1.0 - self.decay)

    def state_dict(self) -> Dict[str, torch.Tensor]:
        return {"decay": torch.tensor(self.decay), "shadow": self.shadow}

    def load_state_dict(self, state: Dict) -> None:
        self.decay = float(state["decay"].item() if isinstance(state["decay"], torch.Tensor) else state["decay"])
        self.shadow = {k: v.detach().clone() for k, v in state["shadow"].items()}

    def to(self, device: torch.device) -> None:
        """

        将 EMA 的 shadow 参数移动到指定 device(续训/切换设备时使用)。

        """
        self.shadow = {k: v.to(device=device) for k, v in self.shadow.items()}

    @torch.no_grad()
    def copy_to(self, model: nn.Module) -> None:
        """

        将 EMA 参数拷贝到目标模型(常用于采样时临时替换)。

        """
        msd = model.state_dict()
        for k in self.shadow.keys():
            if k in msd:
                src = self.shadow[k]
                if src.device != msd[k].device:
                    src = src.to(msd[k].device)
                msd[k].copy_(src.to(msd[k].dtype))