File size: 888 Bytes
6a892c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | import torch
import math
from torch import nn
class PositionalEncoding(nn.Module):
def __init__(self, context_size, dimension_model):
super().__init__()
self.c = context_size
self.d = dimension_model
pe = torch.zeros(self.c, self.d)
for i in range(pe.size(0)):
for j in range(pe.size(1)):
if j % 2 == 0:
pe[i][j] = math.sin(i / (10000 ** (2 * (j // 2) / dimension_model)))
else:
pe[i][j] = math.cos(i / (10000 ** (2 * (j // 2) / dimension_model)))
self.register_buffer("pe", pe)
def forward(self, input):
#input = batch, context, dim_model
sequence_length = input.size(1)
logits = input + self.pe[:sequence_length].to(
device=input.device,
dtype=input.dtype,
)
return logits
|