make-it-animatable / mia /models_ae.py
luca115's picture
Deploy from 3dvalley spaces/make-it-animatable
7d46f0e verified
Raw History Blame Contribute Delete
15.1 kB
from functools import wraps
import numpy as np
import torch
from torch import nn, einsum
import torch.nn.functional as F
from einops import rearrange, repeat
from torch_cluster import fps
from timm.layers import DropPath
def exists(val):
return val is not None
def default(val, d):
return val if exists(val) else d
def cache_fn(f):
cache = None
@wraps(f)
def cached_fn(*args, _cache = True, **kwargs):
if not _cache:
return f(*args, **kwargs)
nonlocal cache
if cache is not None:
return cache
cache = f(*args, **kwargs)
return cache
return cached_fn
class PreNorm(nn.Module):
def __init__(self, dim, fn, context_dim = None):
super().__init__()
self.fn = fn
self.norm = nn.LayerNorm(dim)
self.norm_context = nn.LayerNorm(context_dim) if exists(context_dim) else None
def forward(self, x, **kwargs):
x = self.norm(x)
if exists(self.norm_context):
context = kwargs['context']
normed_context = self.norm_context(context)
kwargs.update(context = normed_context)
return self.fn(x, **kwargs)
class GEGLU(nn.Module):
def forward(self, x):
x, gates = x.chunk(2, dim = -1)
return x * F.gelu(gates)
class FeedForward(nn.Module):
def __init__(self, dim, mult = 4, drop_path_rate = 0.0):
super().__init__()
self.net = nn.Sequential(
nn.Linear(dim, dim * mult * 2),
GEGLU(),
nn.Linear(dim * mult, dim)
)
self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
def forward(self, x):
return self.drop_path(self.net(x))
class Attention(nn.Module):
def __init__(self, query_dim, context_dim = None, heads = 8, dim_head = 64, drop_path_rate = 0.0):
super().__init__()
inner_dim = dim_head * heads
context_dim = default(context_dim, query_dim)
self.scale = dim_head ** -0.5
self.heads = heads
self.to_q = nn.Linear(query_dim, inner_dim, bias = False)
self.to_kv = nn.Linear(context_dim, inner_dim * 2, bias = False)
self.to_out = nn.Linear(inner_dim, query_dim)
self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
def forward(self, x, context = None, mask = None, return_score=False):
h = self.heads
q = self.to_q(x)
context = default(context, x)
k, v = self.to_kv(context).chunk(2, dim = -1)
q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h = h), (q, k, v))
sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
sim = sim.nan_to_num()
if exists(mask):
# mask = rearrange(mask, 'b ... -> b (...)')
if len(mask.shape) == 2:
mask = repeat(mask, 'b j -> (b h) () j', h = h)
else:
mask = repeat(mask, 'b ... -> (b h) ...', h = h)
max_neg_value = -torch.finfo(sim.dtype).max
sim.masked_fill_(~mask, max_neg_value)
# attention, what we cannot get enough of
attn = sim.softmax(dim = -1)
out = einsum('b i j, b j d -> b i d', attn, v)
out = rearrange(out, '(b h) n d -> b n (h d)', h = h)
out = self.drop_path(self.to_out(out))
if return_score:
return out, rearrange(attn, '(b h) i j -> b h i j', h = h)
return out
class PointEmbed(nn.Module):
def __init__(self, hidden_dim=48, dim=128):
super().__init__()
assert hidden_dim % 6 == 0
self.embedding_dim = hidden_dim
e = torch.pow(2, torch.arange(self.embedding_dim // 6)).float() * np.pi
e = torch.stack([
torch.cat([e, torch.zeros(self.embedding_dim // 6),
torch.zeros(self.embedding_dim // 6)]),
torch.cat([torch.zeros(self.embedding_dim // 6), e,
torch.zeros(self.embedding_dim // 6)]),
torch.cat([torch.zeros(self.embedding_dim // 6),
torch.zeros(self.embedding_dim // 6), e]),
])
self.register_buffer('basis', e) # 3 x 16
self.mlp = nn.Linear(self.embedding_dim+3, dim)
@staticmethod
def embed(input, basis):
projections = torch.einsum(
'bnd,de->bne', input, basis)
embeddings = torch.cat([projections.sin(), projections.cos()], dim=2)
return embeddings
def forward(self, input):
# input: B x N x 3
embed = self.mlp(torch.cat([self.embed(input, self.basis), input], dim=2)) # B x N x C
return embed
class DiagonalGaussianDistribution(object):
def __init__(self, mean, logvar, deterministic=False):
self.mean = mean
self.logvar = logvar
self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
self.deterministic = deterministic
self.std = torch.exp(0.5 * self.logvar)
self.var = torch.exp(self.logvar)
if self.deterministic:
self.var = self.std = torch.zeros_like(self.mean).to(device=self.mean.device)
def sample(self):
x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.mean.device)
return x
def kl(self, other=None):
if self.deterministic:
return torch.Tensor([0.])
else:
if other is None:
return 0.5 * torch.mean(torch.pow(self.mean, 2)
+ self.var - 1.0 - self.logvar,
dim=[1, 2])
else:
return 0.5 * torch.mean(
torch.pow(self.mean - other.mean, 2) / other.var
+ self.var / other.var - 1.0 - self.logvar + other.logvar,
dim=[1, 2, 3])
def nll(self, sample, dims=[1,2,3]):
if self.deterministic:
return torch.Tensor([0.])
logtwopi = np.log(2.0 * np.pi)
return 0.5 * torch.sum(
logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
dim=dims)
def mode(self):
return self.mean
class AutoEncoder(nn.Module):
def __init__(
self,
*,
depth=24,
dim=512,
queries_dim=512,
output_dim = 1,
num_inputs = 2048,
num_latents = 512,
heads = 8,
dim_head = 64,
weight_tie_layers = False,
decoder_ff = False
):
super().__init__()
self.depth = depth
self.num_inputs = num_inputs
self.num_latents = num_latents
self.cross_attend_blocks = nn.ModuleList([
PreNorm(dim, Attention(dim, dim, heads = 1, dim_head = dim), context_dim = dim),
PreNorm(dim, FeedForward(dim))
])
self.point_embed = PointEmbed(dim=dim)
get_latent_attn = lambda: PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, drop_path_rate=0.1))
get_latent_ff = lambda: PreNorm(dim, FeedForward(dim, drop_path_rate=0.1))
get_latent_attn, get_latent_ff = map(cache_fn, (get_latent_attn, get_latent_ff))
self.layers = nn.ModuleList([])
cache_args = {'_cache': weight_tie_layers}
for i in range(depth):
self.layers.append(nn.ModuleList([
get_latent_attn(**cache_args),
get_latent_ff(**cache_args)
]))
self.decoder_cross_attn = PreNorm(queries_dim, Attention(queries_dim, dim, heads = 1, dim_head = dim), context_dim = dim)
self.decoder_ff = PreNorm(queries_dim, FeedForward(queries_dim)) if decoder_ff else None
self.to_outputs = nn.Linear(queries_dim, output_dim) if exists(output_dim) else nn.Identity()
def encode(self, pc):
# pc: B x N x 3
B, N, D = pc.shape
assert N == self.num_inputs
###### fps
flattened = pc.view(B*N, D)
batch = torch.arange(B).to(pc.device)
batch = torch.repeat_interleave(batch, N)
pos = flattened
ratio = 1.0 * self.num_latents / self.num_inputs
idx = fps(pos, batch, ratio=ratio)
sampled_pc = pos[idx]
sampled_pc = sampled_pc.view(B, -1, 3)
######
sampled_pc_embeddings = self.point_embed(sampled_pc)
pc_embeddings = self.point_embed(pc)
cross_attn, cross_ff = self.cross_attend_blocks
x = cross_attn(sampled_pc_embeddings, context = pc_embeddings, mask = None) + sampled_pc_embeddings
x = cross_ff(x) + x
return x
def decode(self, x, queries):
for self_attn, self_ff in self.layers:
x = self_attn(x) + x
x = self_ff(x) + x
# cross attend from decoder queries to latents
queries_embeddings = self.point_embed(queries)
latents = self.decoder_cross_attn(queries_embeddings, context = x)
# optional decoder feedforward
if exists(self.decoder_ff):
latents = latents + self.decoder_ff(latents)
return self.to_outputs(latents)
def forward(self, pc, queries):
x = self.encode(pc)
o = self.decode(x, queries).squeeze(-1)
return {'logits': o}
class KLAutoEncoder(nn.Module):
def __init__(
self,
*,
depth=24,
dim=512,
queries_dim=512,
output_dim = 1,
num_inputs = 2048,
num_latents = 512,
latent_dim = 64,
heads = 8,
dim_head = 64,
weight_tie_layers = False,
decoder_ff = False
):
super().__init__()
self.depth = depth
self.num_inputs = num_inputs
self.num_latents = num_latents
self.cross_attend_blocks = nn.ModuleList([
PreNorm(dim, Attention(dim, dim, heads = 1, dim_head = dim), context_dim = dim),
PreNorm(dim, FeedForward(dim))
])
self.point_embed = PointEmbed(dim=dim)
get_latent_attn = lambda: PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, drop_path_rate=0.1))
get_latent_ff = lambda: PreNorm(dim, FeedForward(dim, drop_path_rate=0.1))
get_latent_attn, get_latent_ff = map(cache_fn, (get_latent_attn, get_latent_ff))
self.layers = nn.ModuleList([])
cache_args = {'_cache': weight_tie_layers}
for i in range(depth):
self.layers.append(nn.ModuleList([
get_latent_attn(**cache_args),
get_latent_ff(**cache_args)
]))
self.decoder_cross_attn = PreNorm(queries_dim, Attention(queries_dim, dim, heads = 1, dim_head = dim), context_dim = dim)
self.decoder_ff = PreNorm(queries_dim, FeedForward(queries_dim)) if decoder_ff else None
self.to_outputs = nn.Linear(queries_dim, output_dim) if exists(output_dim) else nn.Identity()
self.proj = nn.Linear(latent_dim, dim)
self.mean_fc = nn.Linear(dim, latent_dim)
self.logvar_fc = nn.Linear(dim, latent_dim)
def encode(self, pc):
# pc: B x N x 3
B, N, D = pc.shape
assert N == self.num_inputs
###### fps
flattened = pc.view(B*N, D)
batch = torch.arange(B).to(pc.device)
batch = torch.repeat_interleave(batch, N)
pos = flattened
ratio = 1.0 * self.num_latents / self.num_inputs
idx = fps(pos, batch, ratio=ratio)
sampled_pc = pos[idx]
sampled_pc = sampled_pc.view(B, -1, 3)
######
sampled_pc_embeddings = self.point_embed(sampled_pc)
pc_embeddings = self.point_embed(pc)
cross_attn, cross_ff = self.cross_attend_blocks
x = cross_attn(sampled_pc_embeddings, context = pc_embeddings, mask = None) + sampled_pc_embeddings
x = cross_ff(x) + x
mean = self.mean_fc(x)
logvar = self.logvar_fc(x)
posterior = DiagonalGaussianDistribution(mean, logvar)
x = posterior.sample()
kl = posterior.kl()
return kl, x
def decode(self, x, queries):
x = self.proj(x)
for self_attn, self_ff in self.layers:
x = self_attn(x) + x
x = self_ff(x) + x
# cross attend from decoder queries to latents
queries_embeddings = self.point_embed(queries)
latents = self.decoder_cross_attn(queries_embeddings, context = x)
# optional decoder feedforward
if exists(self.decoder_ff):
latents = latents + self.decoder_ff(latents)
return self.to_outputs(latents)
def forward(self, pc, queries):
kl, x = self.encode(pc)
o = self.decode(x, queries).squeeze(-1)
# return o.squeeze(-1), kl
return {'logits': o, 'kl': kl}
def create_autoencoder(dim=512, M=512, latent_dim=64, N=2048, deterministic=False):
if deterministic:
model = AutoEncoder(
depth=24,
dim=dim,
queries_dim=dim,
output_dim = 1,
num_inputs = N,
num_latents = M,
heads = 8,
dim_head = 64,
)
else:
model = KLAutoEncoder(
depth=24,
dim=dim,
queries_dim=dim,
output_dim = 1,
num_inputs = N,
num_latents = M,
latent_dim = latent_dim,
heads = 8,
dim_head = 64,
)
return model
def kl_d512_m512_l512(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=512, N=N, deterministic=False)
def kl_d512_m512_l64(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=64, N=N, deterministic=False)
def kl_d512_m512_l32(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=32, N=N, deterministic=False)
def kl_d512_m512_l16(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=16, N=N, deterministic=False)
def kl_d512_m512_l8(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=8, N=N, deterministic=False)
def kl_d512_m512_l4(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=4, N=N, deterministic=False)
def kl_d512_m512_l2(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=2, N=N, deterministic=False)
def kl_d512_m512_l1(N=2048):
return create_autoencoder(dim=512, M=512, latent_dim=1, N=N, deterministic=False)
###
def ae_d512_m512(N=2048):
return create_autoencoder(dim=512, M=512, N=N, deterministic=True)
def ae_d512_m256(N=2048):
return create_autoencoder(dim=512, M=256, N=N, deterministic=True)
def ae_d512_m128(N=2048):
return create_autoencoder(dim=512, M=128, N=N, deterministic=True)
def ae_d512_m64(N=2048):
return create_autoencoder(dim=512, M=64, N=N, deterministic=True)
###
def ae_d256_m512(N=2048):
return create_autoencoder(dim=256, M=512, N=N, deterministic=True)
def ae_d128_m512(N=2048):
return create_autoencoder(dim=128, M=512, N=N, deterministic=True)
def ae_d64_m512(N=2048):
return create_autoencoder(dim=64, M=512, N=N, deterministic=True)