Download positional_encoding.py from iankur/img2tex: direct link, hf CLI and curl.
- Browser
- Download file 2.88 kB
-
https://huggingface.co/spaces/iankur/img2tex/resolve/main/positional_encoding.py
- Command line
-
hf download hf://spaces/iankur/img2tex/positional_encoding.py
-
curl -L -o positional_encoding.py https://huggingface.co/spaces/iankur/img2tex/resolve/main/positional_encoding.py
2.88 kB
| import math | |
| import torch | |
| import torch.nn as nn | |
| from torch import Tensor | |
| class PositionalEncoding2D(nn.Module): | |
| """2-D positional encodings for the feature maps produced by the encoder. | |
| Following https://arxiv.org/abs/2103.06450 by Sumeet Singh. | |
| Reference: | |
| https://github.com/full-stack-deep-learning/fsdl-text-recognizer-2021-labs/blob/main/lab9/text_recognizer/models/transformer_util.py | |
| """ | |
| def __init__(self, d_model: int, max_h: int = 2000, max_w: int = 2000) -> None: | |
| super().__init__() | |
| self.d_model = d_model | |
| assert d_model % 2 == 0, f"Embedding depth {d_model} is not even" | |
| pe = self.make_pe(d_model, max_h, max_w) # (d_model, max_h, max_w) | |
| self.register_buffer("pe", pe) | |
| def make_pe(d_model: int, max_h: int, max_w: int) -> Tensor: | |
| """Compute positional encoding.""" | |
| pe_h = PositionalEncoding1D.make_pe(d_model=d_model // 2, max_len=max_h) # (max_h, 1 d_model // 2) | |
| pe_h = pe_h.permute(2, 0, 1).expand(-1, -1, max_w) # (d_model // 2, max_h, max_w) | |
| pe_w = PositionalEncoding1D.make_pe(d_model=d_model // 2, max_len=max_w) # (max_w, 1, d_model // 2) | |
| pe_w = pe_w.permute(2, 1, 0).expand(-1, max_h, -1) # (d_model // 2, max_h, max_w) | |
| pe = torch.cat([pe_h, pe_w], dim=0) # (d_model, max_h, max_w) | |
| return pe | |
| def forward(self, x: Tensor) -> Tensor: | |
| """Forward pass. | |
| Args: | |
| x: (B, d_model, H, W) | |
| Returns: | |
| (B, d_model, H, W) | |
| """ | |
| assert x.shape[1] == self.pe.shape[0] # type: ignore | |
| x = x + self.pe[:, : x.size(2), : x.size(3)] # type: ignore | |
| return x | |
| class PositionalEncoding1D(nn.Module): | |
| """Classic Attention-is-all-you-need positional encoding.""" | |
| def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000) -> None: | |
| super().__init__() | |
| self.dropout = nn.Dropout(p=dropout) | |
| pe = self.make_pe(d_model, max_len) # (max_len, 1, d_model) | |
| self.register_buffer("pe", pe) | |
| def make_pe(d_model: int, max_len: int) -> Tensor: | |
| """Compute positional encoding.""" | |
| pe = torch.zeros(max_len, d_model) | |
| position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) | |
| div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) | |
| pe[:, 0::2] = torch.sin(position * div_term) | |
| pe[:, 1::2] = torch.cos(position * div_term) | |
| pe = pe.unsqueeze(1) | |
| return pe | |
| def forward(self, x: Tensor) -> Tensor: | |
| """Forward pass. | |
| Args: | |
| x: (S, B, d_model) | |
| Returns: | |
| (B, d_model, H, W) | |
| """ | |
| assert x.shape[2] == self.pe.shape[2] # type: ignore | |
| x = x + self.pe[: x.size(0)] # type: ignore | |
| return self.dropout(x) | |