Download pi4/models/decoder.py from yyfz233/Pi4: direct link, hf CLI and curl.
- Browser
- Download file 4.05 kB
-
https://huggingface.co/spaces/yyfz233/Pi4/resolve/main/pi4/models/decoder.py
- Command line
-
hf download hf://spaces/yyfz233/Pi4/pi4/models/decoder.py
-
curl -L -o decoder.py https://huggingface.co/spaces/yyfz233/Pi4/resolve/main/pi4/models/decoder.py
4.05 kB
| import torch | |
| from torch import nn | |
| from .layers import Block | |
| class TimeGates(nn.Module): | |
| def __init__(self, steps): | |
| super().__init__() | |
| self.register_buffer("encoding", torch.empty(steps, 128)) | |
| self.proj = nn.Sequential(nn.Linear(128, 64), nn.SiLU(), nn.Linear(64, 3072)) | |
| def forward(self, step, batch_size): | |
| encoding = self.encoding[step].expand(batch_size, -1).contiguous() | |
| gates = self.proj(encoding) + 1 | |
| return gates.reshape(batch_size, 3, 1024).permute(1, 0, 2).unsqueeze(2) | |
| class GeometryDecoder(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.register_tokens = nn.Parameter(torch.empty(1, 1, 5, 1024)) | |
| self.frame_block = Block(1024, 16, normalize=True, scaled=True) | |
| self.global_block = Block(1024, 16, normalize=True, scaled=True) | |
| self.readout = Block(1024, 16, normalize=True, scaled=True) | |
| self.frame_gates = TimeGates(16) | |
| self.global_gates = TimeGates(16) | |
| def forward(self, tokens, positions, batch_size, frames): | |
| initial = torch.cat( | |
| ( | |
| self.register_tokens.expand(batch_size, frames, -1, -1).reshape( | |
| batch_size * frames, 5, 1024 | |
| ), | |
| tokens, | |
| ), | |
| dim=1, | |
| ) | |
| features = initial | |
| token_count = features.shape[1] | |
| for index in range(16): | |
| features = self.frame_block( | |
| features.reshape(batch_size * frames, token_count, 1024), | |
| positions, | |
| gates=self.frame_gates(index, batch_size * frames), | |
| ) | |
| features = self.global_block( | |
| features.reshape(batch_size, frames * token_count, 1024), | |
| gates=self.global_gates(index, batch_size), | |
| ) | |
| features = self.readout( | |
| features.reshape(batch_size * frames, token_count, 1024), positions | |
| ) | |
| return features, initial | |
| class SpatialCondition(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.geometry = nn.Sequential(nn.LayerNorm(1024), nn.Linear(1024, 256)) | |
| self.time = nn.Linear(64, 256) | |
| self.output = nn.Sequential(nn.SiLU(), nn.Linear(256, 4096)) | |
| self.register_buffer("time_encoding", torch.empty(8, 64)) | |
| def forward(self, geometry, step): | |
| encoding = self.time_encoding[step].expand(geometry.shape[0], -1).contiguous() | |
| return self.output( | |
| self.geometry(geometry.float()) + self.time(encoding)[:, None] | |
| ).chunk(4, dim=-1) | |
| class MotionDecoder(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.frame_block = Block(1024, 16, normalize=True, scaled=True) | |
| self.global_block = Block(1024, 16, normalize=True, scaled=True) | |
| self.readout = Block(1024, 16, normalize=True, scaled=True) | |
| self.frame_condition = SpatialCondition() | |
| self.global_condition = SpatialCondition() | |
| self.frame_gates = TimeGates(8) | |
| self.global_gates = TimeGates(8) | |
| def forward(self, initial, geometry, positions, batch_size, frames): | |
| token_count = initial.shape[1] | |
| features = initial | |
| for index in range(8): | |
| frame_geometry = geometry.reshape(batch_size * frames, token_count, 1024) | |
| features = self.frame_block( | |
| features.reshape_as(frame_geometry), | |
| positions, | |
| gates=self.frame_gates(index, batch_size * frames), | |
| modulation=self.frame_condition(frame_geometry, index), | |
| ) | |
| global_geometry = geometry.reshape(batch_size, frames * token_count, 1024) | |
| features = self.global_block( | |
| features.reshape_as(global_geometry), | |
| gates=self.global_gates(index, batch_size), | |
| modulation=self.global_condition(global_geometry, index), | |
| ) | |
| return self.readout( | |
| features.reshape(batch_size * frames, token_count, 1024), positions | |
| ).float() | |