File size: 1,741 Bytes
7ed0c3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import math

import torch
from torch import nn

from .layers import Block


class Encoder(nn.Module):
    def __init__(self):
        super().__init__()
        self.patch_embed = nn.Conv2d(3, 1024, 16, stride=16)
        self.prefix_tokens = nn.Parameter(torch.empty(1, 5, 1024))
        self.blocks = nn.ModuleList(
            Block(1024, 16, epsilon=1e-5, scaled=True) for _ in range(18)
        )
        self.norm = nn.LayerNorm(1024, eps=1e-5)
        self.projection = nn.Linear(2048, 1024)
        self.register_buffer("periods", 100 ** (torch.arange(16).float() / 16))

    def forward(self, images):
        features = self.patch_embed(images)
        batch_size, _, height, width = features.shape
        features = torch.cat(
            (
                self.prefix_tokens.expand(batch_size, -1, -1),
                features.flatten(2).transpose(1, 2),
            ),
            dim=1,
        )
        coordinates = torch.meshgrid(
            torch.arange(height, device=images.device).float() + 0.5,
            torch.arange(width, device=images.device).float() + 0.5,
            indexing="ij",
        )
        coordinates = (
            torch.stack(coordinates, dim=-1).flatten(0, 1) / max(height, width) * 2 - 1
        )
        angles = (
            (2 * math.pi * coordinates[:, :, None] / self.periods.float())
            .flatten(1)
            .repeat(1, 2)
        )
        rotary = angles.sin(), angles.cos()
        intermediate = []
        for index, block in enumerate(self.blocks):
            features = block(features, rotary=rotary)
            if index in (11, 17):
                intermediate.append(self.norm(features)[:, 5:])
        return self.projection(torch.cat(intermediate, dim=-1))