import torch import math from torch import nn class PositionalEncoding(nn.Module): def __init__(self, context_size, dimension_model): super().__init__() self.c = context_size self.d = dimension_model pe = torch.zeros(self.c, self.d) for i in range(pe.size(0)): for j in range(pe.size(1)): if j % 2 == 0: pe[i][j] = math.sin(i / (10000 ** (2 * (j // 2) / dimension_model))) else: pe[i][j] = math.cos(i / (10000 ** (2 * (j // 2) / dimension_model))) self.register_buffer("pe", pe) def forward(self, input): #input = batch, context, dim_model sequence_length = input.size(1) logits = input + self.pe[:sequence_length].to( device=input.device, dtype=input.dtype, ) return logits