File size: 4,054 Bytes
7ed0c3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | import torch
from torch import nn
from .layers import Block
class TimeGates(nn.Module):
def __init__(self, steps):
super().__init__()
self.register_buffer("encoding", torch.empty(steps, 128))
self.proj = nn.Sequential(nn.Linear(128, 64), nn.SiLU(), nn.Linear(64, 3072))
def forward(self, step, batch_size):
encoding = self.encoding[step].expand(batch_size, -1).contiguous()
gates = self.proj(encoding) + 1
return gates.reshape(batch_size, 3, 1024).permute(1, 0, 2).unsqueeze(2)
class GeometryDecoder(nn.Module):
def __init__(self):
super().__init__()
self.register_tokens = nn.Parameter(torch.empty(1, 1, 5, 1024))
self.frame_block = Block(1024, 16, normalize=True, scaled=True)
self.global_block = Block(1024, 16, normalize=True, scaled=True)
self.readout = Block(1024, 16, normalize=True, scaled=True)
self.frame_gates = TimeGates(16)
self.global_gates = TimeGates(16)
def forward(self, tokens, positions, batch_size, frames):
initial = torch.cat(
(
self.register_tokens.expand(batch_size, frames, -1, -1).reshape(
batch_size * frames, 5, 1024
),
tokens,
),
dim=1,
)
features = initial
token_count = features.shape[1]
for index in range(16):
features = self.frame_block(
features.reshape(batch_size * frames, token_count, 1024),
positions,
gates=self.frame_gates(index, batch_size * frames),
)
features = self.global_block(
features.reshape(batch_size, frames * token_count, 1024),
gates=self.global_gates(index, batch_size),
)
features = self.readout(
features.reshape(batch_size * frames, token_count, 1024), positions
)
return features, initial
class SpatialCondition(nn.Module):
def __init__(self):
super().__init__()
self.geometry = nn.Sequential(nn.LayerNorm(1024), nn.Linear(1024, 256))
self.time = nn.Linear(64, 256)
self.output = nn.Sequential(nn.SiLU(), nn.Linear(256, 4096))
self.register_buffer("time_encoding", torch.empty(8, 64))
def forward(self, geometry, step):
encoding = self.time_encoding[step].expand(geometry.shape[0], -1).contiguous()
return self.output(
self.geometry(geometry.float()) + self.time(encoding)[:, None]
).chunk(4, dim=-1)
class MotionDecoder(nn.Module):
def __init__(self):
super().__init__()
self.frame_block = Block(1024, 16, normalize=True, scaled=True)
self.global_block = Block(1024, 16, normalize=True, scaled=True)
self.readout = Block(1024, 16, normalize=True, scaled=True)
self.frame_condition = SpatialCondition()
self.global_condition = SpatialCondition()
self.frame_gates = TimeGates(8)
self.global_gates = TimeGates(8)
def forward(self, initial, geometry, positions, batch_size, frames):
token_count = initial.shape[1]
features = initial
for index in range(8):
frame_geometry = geometry.reshape(batch_size * frames, token_count, 1024)
features = self.frame_block(
features.reshape_as(frame_geometry),
positions,
gates=self.frame_gates(index, batch_size * frames),
modulation=self.frame_condition(frame_geometry, index),
)
global_geometry = geometry.reshape(batch_size, frames * token_count, 1024)
features = self.global_block(
features.reshape_as(global_geometry),
gates=self.global_gates(index, batch_size),
modulation=self.global_condition(global_geometry, index),
)
return self.readout(
features.reshape(batch_size * frames, token_count, 1024), positions
).float()
|