Spaces:
Running on Zero
Running on Zero
Download src/utils/weight_ema.py from hugging-apps/rigidformer: direct link, hf CLI and curl.
- Browser
- Download file 2.87 kB
-
https://huggingface.co/spaces/hugging-apps/rigidformer/resolve/main/src/utils/weight_ema.py
- Command line
-
hf download hf://spaces/hugging-apps/rigidformer/src/utils/weight_ema.py
-
curl -L -o weight_ema.py https://huggingface.co/spaces/hugging-apps/rigidformer/resolve/main/src/utils/weight_ema.py
2.87 kB
| """Exponential moving average (EMA) of a model's weights. | |
| The average covers every floating-point entry of the module's state dict (parameters and | |
| floating buffers such as BatchNorm running statistics); integer buffers (e.g. BatchNorm's | |
| num_batches_tracked) are copied. The shadow copy is kept in fp32. The effective decay ramps as | |
| min(decay, (1 + n) / (10 + n)) over the first updates so the average is not biased toward the | |
| initial weights. | |
| """ | |
| from typing import Dict, Optional | |
| import torch | |
| class WeightEMA: | |
| def __init__(self, module: torch.nn.Module, decay: float): | |
| if not 0.0 < float(decay) < 1.0: | |
| raise ValueError(f"EMA decay must be in (0, 1); got {decay}") | |
| self.module = module | |
| self.decay = float(decay) | |
| self.num_updates = 0 | |
| self._keys = list(module.state_dict(keep_vars=True).keys()) | |
| self.shadow: Dict[str, torch.Tensor] = {} | |
| self.reset() | |
| def _state(self) -> Dict[str, torch.Tensor]: | |
| sd = self.module.state_dict(keep_vars=True) | |
| if list(sd.keys()) != self._keys: | |
| raise RuntimeError("WeightEMA: the module's state-dict keys changed after construction") | |
| return sd | |
| def reset(self) -> None: | |
| """Restart the average from the module's current weights (e.g. after loading a checkpoint).""" | |
| self.shadow = {k: (v.detach().float().clone() if v.is_floating_point() else v.detach().clone()) | |
| for k, v in self._state().items()} | |
| self.num_updates = 0 | |
| def update(self) -> None: | |
| self.num_updates += 1 | |
| d = min(self.decay, (1.0 + self.num_updates) / (10.0 + self.num_updates)) | |
| sh, src = [], [] | |
| for k, v in self._state().items(): | |
| if v.is_floating_point(): | |
| sh.append(self.shadow[k]) | |
| src.append(v.detach().float()) | |
| else: | |
| self.shadow[k].copy_(v.detach()) | |
| torch._foreach_mul_(sh, d) | |
| torch._foreach_add_(sh, src, alpha=1.0 - d) | |
| def state_dict(self) -> Dict[str, torch.Tensor]: | |
| """Averaged weights in each entry's own dtype (loadable with module.load_state_dict).""" | |
| return {k: (self.shadow[k].to(v.dtype) if v.is_floating_point() else self.shadow[k].clone()) | |
| for k, v in self._state().items()} | |
| def load_state_dict(self, sd: Dict[str, torch.Tensor], num_updates: Optional[int] = None) -> None: | |
| missing = [k for k in self._keys if k not in sd] | |
| if missing: | |
| raise KeyError(f"WeightEMA.load_state_dict: {len(missing)} keys missing, e.g. {missing[:3]}") | |
| for k in self._keys: | |
| self.shadow[k].copy_(sd[k].to(device=self.shadow[k].device, dtype=self.shadow[k].dtype)) | |
| if num_updates is not None: | |
| self.num_updates = int(num_updates) | |