Pi4 / pi4 /models /encoder.py
yyfz233's picture
Replicate Pi4 ZeroGPU Space with optimized viewer and 5px motion filtering
7ed0c3d
Raw History Blame Contribute Delete
1.74 kB
import math
import torch
from torch import nn
from .layers import Block
class Encoder(nn.Module):
def __init__(self):
super().__init__()
self.patch_embed = nn.Conv2d(3, 1024, 16, stride=16)
self.prefix_tokens = nn.Parameter(torch.empty(1, 5, 1024))
self.blocks = nn.ModuleList(
Block(1024, 16, epsilon=1e-5, scaled=True) for _ in range(18)
)
self.norm = nn.LayerNorm(1024, eps=1e-5)
self.projection = nn.Linear(2048, 1024)
self.register_buffer("periods", 100 ** (torch.arange(16).float() / 16))
def forward(self, images):
features = self.patch_embed(images)
batch_size, _, height, width = features.shape
features = torch.cat(
(
self.prefix_tokens.expand(batch_size, -1, -1),
features.flatten(2).transpose(1, 2),
),
dim=1,
)
coordinates = torch.meshgrid(
torch.arange(height, device=images.device).float() + 0.5,
torch.arange(width, device=images.device).float() + 0.5,
indexing="ij",
)
coordinates = (
torch.stack(coordinates, dim=-1).flatten(0, 1) / max(height, width) * 2 - 1
)
angles = (
(2 * math.pi * coordinates[:, :, None] / self.periods.float())
.flatten(1)
.repeat(1, 2)
)
rotary = angles.sin(), angles.cos()
intermediate = []
for index, block in enumerate(self.blocks):
features = block(features, rotary=rotary)
if index in (11, 17):
intermediate.append(self.norm(features)[:, 5:])
return self.projection(torch.cat(intermediate, dim=-1))