| """Minimal vendored replacements for the two espnet symbols multiconv_cgmlp | |
| needs, so the model runs without installing espnet.""" | |
| import torch | |
| import torch.nn as nn | |
| class LayerNorm(nn.LayerNorm): | |
| """espnet LayerNorm: nn.LayerNorm over `nout`, optionally on a given dim.""" | |
| def __init__(self, nout, dim=-1): | |
| super().__init__(nout, eps=1e-12) | |
| self.dim = dim | |
| def forward(self, x): | |
| if self.dim == -1: | |
| return super().forward(x) | |
| return super().forward(x.transpose(self.dim, -1)).transpose(self.dim, -1) | |
| def get_activation(name): | |
| return {"silu": nn.SiLU(), "swish": nn.SiLU(), "relu": nn.ReLU(), | |
| "gelu": nn.GELU(), "identity": nn.Identity()}[name] | |