walke007's picture
Upload final moe_shared checkpoint and test evaluation
6a892c3 verified
Raw History Blame Contribute Delete
888 Bytes
import torch
import math
from torch import nn
class PositionalEncoding(nn.Module):
def __init__(self, context_size, dimension_model):
super().__init__()
self.c = context_size
self.d = dimension_model
pe = torch.zeros(self.c, self.d)
for i in range(pe.size(0)):
for j in range(pe.size(1)):
if j % 2 == 0:
pe[i][j] = math.sin(i / (10000 ** (2 * (j // 2) / dimension_model)))
else:
pe[i][j] = math.cos(i / (10000 ** (2 * (j // 2) / dimension_model)))
self.register_buffer("pe", pe)
def forward(self, input):
#input = batch, context, dim_model
sequence_length = input.size(1)
logits = input + self.pe[:sequence_length].to(
device=input.device,
dtype=input.dtype,
)
return logits