Pi4 / pi4 /models /decoder.py
yyfz233's picture
Replicate Pi4 ZeroGPU Space with optimized viewer and 5px motion filtering
7ed0c3d
Raw History Blame Contribute Delete
4.05 kB
import torch
from torch import nn
from .layers import Block
class TimeGates(nn.Module):
def __init__(self, steps):
super().__init__()
self.register_buffer("encoding", torch.empty(steps, 128))
self.proj = nn.Sequential(nn.Linear(128, 64), nn.SiLU(), nn.Linear(64, 3072))
def forward(self, step, batch_size):
encoding = self.encoding[step].expand(batch_size, -1).contiguous()
gates = self.proj(encoding) + 1
return gates.reshape(batch_size, 3, 1024).permute(1, 0, 2).unsqueeze(2)
class GeometryDecoder(nn.Module):
def __init__(self):
super().__init__()
self.register_tokens = nn.Parameter(torch.empty(1, 1, 5, 1024))
self.frame_block = Block(1024, 16, normalize=True, scaled=True)
self.global_block = Block(1024, 16, normalize=True, scaled=True)
self.readout = Block(1024, 16, normalize=True, scaled=True)
self.frame_gates = TimeGates(16)
self.global_gates = TimeGates(16)
def forward(self, tokens, positions, batch_size, frames):
initial = torch.cat(
(
self.register_tokens.expand(batch_size, frames, -1, -1).reshape(
batch_size * frames, 5, 1024
),
tokens,
),
dim=1,
)
features = initial
token_count = features.shape[1]
for index in range(16):
features = self.frame_block(
features.reshape(batch_size * frames, token_count, 1024),
positions,
gates=self.frame_gates(index, batch_size * frames),
)
features = self.global_block(
features.reshape(batch_size, frames * token_count, 1024),
gates=self.global_gates(index, batch_size),
)
features = self.readout(
features.reshape(batch_size * frames, token_count, 1024), positions
)
return features, initial
class SpatialCondition(nn.Module):
def __init__(self):
super().__init__()
self.geometry = nn.Sequential(nn.LayerNorm(1024), nn.Linear(1024, 256))
self.time = nn.Linear(64, 256)
self.output = nn.Sequential(nn.SiLU(), nn.Linear(256, 4096))
self.register_buffer("time_encoding", torch.empty(8, 64))
def forward(self, geometry, step):
encoding = self.time_encoding[step].expand(geometry.shape[0], -1).contiguous()
return self.output(
self.geometry(geometry.float()) + self.time(encoding)[:, None]
).chunk(4, dim=-1)
class MotionDecoder(nn.Module):
def __init__(self):
super().__init__()
self.frame_block = Block(1024, 16, normalize=True, scaled=True)
self.global_block = Block(1024, 16, normalize=True, scaled=True)
self.readout = Block(1024, 16, normalize=True, scaled=True)
self.frame_condition = SpatialCondition()
self.global_condition = SpatialCondition()
self.frame_gates = TimeGates(8)
self.global_gates = TimeGates(8)
def forward(self, initial, geometry, positions, batch_size, frames):
token_count = initial.shape[1]
features = initial
for index in range(8):
frame_geometry = geometry.reshape(batch_size * frames, token_count, 1024)
features = self.frame_block(
features.reshape_as(frame_geometry),
positions,
gates=self.frame_gates(index, batch_size * frames),
modulation=self.frame_condition(frame_geometry, index),
)
global_geometry = geometry.reshape(batch_size, frames * token_count, 1024)
features = self.global_block(
features.reshape_as(global_geometry),
gates=self.global_gates(index, batch_size),
modulation=self.global_condition(global_geometry, index),
)
return self.readout(
features.reshape(batch_size * frames, token_count, 1024), positions
).float()