File size: 1,741 Bytes
7ed0c3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | import math
import torch
from torch import nn
from .layers import Block
class Encoder(nn.Module):
def __init__(self):
super().__init__()
self.patch_embed = nn.Conv2d(3, 1024, 16, stride=16)
self.prefix_tokens = nn.Parameter(torch.empty(1, 5, 1024))
self.blocks = nn.ModuleList(
Block(1024, 16, epsilon=1e-5, scaled=True) for _ in range(18)
)
self.norm = nn.LayerNorm(1024, eps=1e-5)
self.projection = nn.Linear(2048, 1024)
self.register_buffer("periods", 100 ** (torch.arange(16).float() / 16))
def forward(self, images):
features = self.patch_embed(images)
batch_size, _, height, width = features.shape
features = torch.cat(
(
self.prefix_tokens.expand(batch_size, -1, -1),
features.flatten(2).transpose(1, 2),
),
dim=1,
)
coordinates = torch.meshgrid(
torch.arange(height, device=images.device).float() + 0.5,
torch.arange(width, device=images.device).float() + 0.5,
indexing="ij",
)
coordinates = (
torch.stack(coordinates, dim=-1).flatten(0, 1) / max(height, width) * 2 - 1
)
angles = (
(2 * math.pi * coordinates[:, :, None] / self.periods.float())
.flatten(1)
.repeat(1, 2)
)
rotary = angles.sin(), angles.cos()
intermediate = []
for index, block in enumerate(self.blocks):
features = block(features, rotary=rotary)
if index in (11, 17):
intermediate.append(self.norm(features)[:, 5:])
return self.projection(torch.cat(intermediate, dim=-1))
|