File size: 888 Bytes
a3baf17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import torch
import math
from torch import nn

class PositionalEncoding(nn.Module):
    def __init__(self, context_size, dimension_model):
        super().__init__()
        self.c = context_size
        self.d = dimension_model

        pe = torch.zeros(self.c, self.d)
        for i in range(pe.size(0)):
            for j in range(pe.size(1)):
                if j % 2 == 0: 
                    pe[i][j] = math.sin(i / (10000 ** (2 * (j // 2) / dimension_model)))
                else: 
                    pe[i][j] = math.cos(i / (10000 ** (2 * (j // 2) / dimension_model)))

        self.register_buffer("pe", pe)

    def forward(self, input):
        #input = batch, context, dim_model

        sequence_length = input.size(1)
        logits = input + self.pe[:sequence_length].to(
            device=input.device,
            dtype=input.dtype,
        )
        return logits