Download source/part1/utils/positionalencoding.py from walke007/anlp-assignment2-part1-moe_shared: direct link, hf CLI and curl.
- Browser
- Download file 888 Bytes
-
https://huggingface.co/walke007/anlp-assignment2-part1-moe_shared/resolve/main/source/part1/utils/positionalencoding.py
- Command line
-
hf download hf://walke007/anlp-assignment2-part1-moe_shared/source/part1/utils/positionalencoding.py
-
curl -L -o positionalencoding.py https://huggingface.co/walke007/anlp-assignment2-part1-moe_shared/resolve/main/source/part1/utils/positionalencoding.py
888 Bytes
| import torch | |
| import math | |
| from torch import nn | |
| class PositionalEncoding(nn.Module): | |
| def __init__(self, context_size, dimension_model): | |
| super().__init__() | |
| self.c = context_size | |
| self.d = dimension_model | |
| pe = torch.zeros(self.c, self.d) | |
| for i in range(pe.size(0)): | |
| for j in range(pe.size(1)): | |
| if j % 2 == 0: | |
| pe[i][j] = math.sin(i / (10000 ** (2 * (j // 2) / dimension_model))) | |
| else: | |
| pe[i][j] = math.cos(i / (10000 ** (2 * (j // 2) / dimension_model))) | |
| self.register_buffer("pe", pe) | |
| def forward(self, input): | |
| #input = batch, context, dim_model | |
| sequence_length = input.size(1) | |
| logits = input + self.pe[:sequence_length].to( | |
| device=input.device, | |
| dtype=input.dtype, | |
| ) | |
| return logits | |