File size: 6,338 Bytes
e66463a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
import torch
from torch import nn
from torch.nn import functional as F
import math

class SelfAttention(nn.Module):
    def __init__(self, n_heads, d_embed, in_proj_bias=True, out_proj_bias=True):
        super().__init__()
        self.in_proj = nn.Linear(d_embed, 3 * d_embed, bias=in_proj_bias)
        self.out_proj = nn.Linear(d_embed, d_embed, bias=out_proj_bias)
        self.n_heads = n_heads
        self.d_head = d_embed // n_heads

    def forward(self, x, causal_mask=False):
        input_shape = x.shape
        batch_size, sequence_length, d_embed = input_shape
        interim_shape = (batch_size, sequence_length, self.n_heads, self.d_head)
        q, k, v = self.in_proj(x).chunk(3, dim=-1)
        q = q.view(interim_shape).transpose(1, 2)
        k = k.view(interim_shape).transpose(1, 2)
        v = v.view(interim_shape).transpose(1, 2)
        weight = q @ k.transpose(-1, -2)
        if causal_mask:
            mask = torch.ones_like(weight, dtype=torch.bool).triu(1)
            weight.masked_fill_(mask, -torch.inf)
        weight /= math.sqrt(self.d_head)
        weight = F.softmax(weight, dim=-1)
        output = weight @ v
        output = output.transpose(1, 2)
        output = output.reshape(input_shape)
        output = self.out_proj(output)
        return output

class VAE_AttentionBlock(nn.Module):
    def __init__(self, channels):
        super().__init__()
        self.groupnorm = nn.GroupNorm(32, channels)
        self.attention = SelfAttention(1, channels)

    def forward(self, x):
        residue = x
        x = self.groupnorm(x)
        n, c, h, w = x.shape
        x = x.view((n, c, h * w)).transpose(1, 2)
        x = self.attention(x)
        x = x.transpose(1, 2).reshape((n, c, h, w))
        x += residue
        return x

class VAE_ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.groupnorm_1 = nn.GroupNorm(32, in_channels)
        self.conv_1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1)
        self.groupnorm_2 = nn.GroupNorm(32, out_channels)
        self.conv_2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)
        if in_channels == out_channels:
            self.residual_layer = nn.Identity()
        else:
            self.residual_layer = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding=0)

    def forward(self, x):
        residue = x
        x = self.groupnorm_1(x)
        x = F.silu(x)
        x = self.conv_1(x)
        x = self.groupnorm_2(x)
        x = F.silu(x)
        x = self.conv_2(x)
        return x + self.residual_layer(residue)

class VAE_Encoder(nn.Module):
    def __init__(self, in_channels=3, base_channels=128, channel_multipliers=[1, 2, 4, 4],
                 num_residual_blocks_per_level=[2, 2, 2, 2], z_channels=4):
        super().__init__()
        self.channel_multipliers = channel_multipliers
        self.conv_in = nn.Conv2d(in_channels, base_channels, kernel_size=3, padding=1)
        layers = []
        current_channels = base_channels
        for i, multiplier in enumerate(self.channel_multipliers):
            out_channels = base_channels * multiplier
            for _ in range(num_residual_blocks_per_level[i]):
                layers.append(VAE_ResidualBlock(current_channels, out_channels))
                current_channels = out_channels
            if i < len(self.channel_multipliers) - 1:
                layers.append(nn.Conv2d(current_channels, current_channels, kernel_size=3, stride=2, padding=1))
        layers.extend([
            VAE_ResidualBlock(current_channels, current_channels),
            VAE_AttentionBlock(current_channels),
            VAE_ResidualBlock(current_channels, current_channels),
            nn.GroupNorm(32, current_channels),
            nn.SiLU(),
            nn.Conv2d(current_channels, 2 * z_channels, kernel_size=3, padding=1)
        ])
        self.model = nn.ModuleList(layers)

    def forward(self, x):
        x = self.conv_in(x)
        for module in self.model:
            x = module(x)
        mean, log_variance = torch.chunk(x, 2, dim=1)
        return mean, log_variance

class VAE_Decoder(nn.Module):
    def __init__(self, out_channels=3, base_channels=128, channel_multipliers=[1, 2, 4, 4],
                 num_residual_blocks_per_level=[2, 2, 2, 2], z_channels=4):
        super().__init__()
        current_channels = base_channels * channel_multipliers[-1]
        self.conv_in = nn.Conv2d(z_channels, current_channels, kernel_size=3, padding=1)
        layers = [
            VAE_ResidualBlock(current_channels, current_channels),
            VAE_AttentionBlock(current_channels),
            VAE_ResidualBlock(current_channels, current_channels)
        ]
        for i in reversed(range(len(channel_multipliers))):
            out_channels_level = base_channels * channel_multipliers[i]
            for _ in range(num_residual_blocks_per_level[i] + 1):
                layers.append(VAE_ResidualBlock(current_channels, out_channels_level))
                current_channels = out_channels_level
            if i > 0:
                layers.extend([
                    nn.Upsample(scale_factor=2, mode='bicubic'),
                    nn.Conv2d(current_channels, current_channels, kernel_size=3, padding=1)
                ])
        
        layers.extend([
            nn.GroupNorm(32, current_channels),
            nn.SiLU(),
            nn.Conv2d(current_channels, out_channels, kernel_size=3, padding=1),
            nn.Tanh()
        ])
        self.model = nn.ModuleList(layers)

    def forward(self, x):
        x = self.conv_in(x)
        for module in self.model:
            x = module(x)
        return x

class VAE(nn.Module):
    def __init__(self, encoder_config, decoder_config):
        super().__init__()
        self.encoder = VAE_Encoder(**encoder_config)
        self.decoder = VAE_Decoder(**decoder_config)
    
    def reparameterize(self, mu, logvar):
        logvar = torch.clamp(logvar, min=-30.0, max=20.0)
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        return mu + eps * std
    
    def forward(self, x):
        mu, logvar = self.encoder(x)
        z = self.reparameterize(mu, logvar)
        reconstructed_x = self.decoder(z)
        return reconstructed_x, mu, logvar, z