import math import torch import torch.nn as nn import torch.nn.functional as F import numpy as np def drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True): if drop_prob == 0. or not training: return x keep_prob = 1 - drop_prob shape = (x.shape[0],) + (1,) * (x.ndim - 1) random_tensor = x.new_empty(shape).bernoulli_(keep_prob) if keep_prob > 0.0 and scale_by_keep: random_tensor.div_(keep_prob) return x * random_tensor class DropPath(nn.Module): def __init__(self, drop_prob=None, scale_by_keep=True): super(DropPath, self).__init__() self.drop_prob = drop_prob self.scale_by_keep = scale_by_keep def forward(self, x): return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) def _no_grad_trunc_normal_(tensor, mean, std, a, b): def norm_cdf(x): return (1. + math.erf(x / math.sqrt(2.))) / 2. if (mean < a - 2 * std) or (mean > b + 2 * std): warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " "The distribution of values may be incorrect.", stacklevel=2) with torch.no_grad(): l = norm_cdf((a - mean) / std) u = norm_cdf((b - mean) / std) tensor.uniform_(2 * l - 1, 2 * u - 1) tensor.erfinv_() tensor.mul_(std * math.sqrt(2.)) tensor.add_(mean) tensor.clamp_(min=a, max=b) return tensor def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): return _no_grad_trunc_normal_(tensor, mean, std, a, b) def conv_branch_init(conv, branches): weight = conv.weight n = weight.size(0) k1 = weight.size(1) k2 = weight.size(2) nn.init.normal_(weight, 0, math.sqrt(2. / (n * k1 * k2 * branches))) if conv.bias is not None: nn.init.constant_(conv.bias, 0) def conv_init(conv): if conv.weight is not None: nn.init.kaiming_normal_(conv.weight, mode='fan_out') if conv.bias is not None: nn.init.constant_(conv.bias, 0) def bn_init(bn, scale): nn.init.constant_(bn.weight, scale) nn.init.constant_(bn.bias, 0) def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv') != -1: if hasattr(m, 'weight'): nn.init.kaiming_normal_(m.weight, mode='fan_out') if hasattr(m, 'bias') and m.bias is not None and isinstance(m.bias, torch.Tensor): nn.init.constant_(m.bias, 0) elif classname.find('BatchNorm') != -1: if hasattr(m, 'weight') and m.weight is not None: m.weight.data.normal_(1.0, 0.02) if hasattr(m, 'bias') and m.bias is not None: m.bias.data.fill_(0) class TemporalConv(nn.Module): def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1): super(TemporalConv, self).__init__() pad = (kernel_size + (kernel_size-1) * (dilation-1) - 1) // 2 self.conv = nn.Conv2d( in_channels, out_channels, kernel_size=(kernel_size, 1), padding=(pad, 0), stride=(stride, 1), dilation=(dilation, 1), ) self.bn = nn.BatchNorm2d(out_channels) def forward(self, x): x = self.conv(x) x = self.bn(x) return x class MultiScale_TemporalConv(nn.Module): def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, dilations=[1,2,3,4], residual=False, residual_kernel_size=1): super().__init__() assert out_channels % (len(dilations) + 2) == 0, '# out channels should be multiples of # branches' self.num_branches = len(dilations) + 2 branch_channels = out_channels // self.num_branches if type(kernel_size) == list: assert len(kernel_size) == len(dilations) else: kernel_size = [kernel_size]*len(dilations) self.branches = nn.ModuleList([ nn.Sequential( nn.Conv2d( in_channels, branch_channels, kernel_size=1, padding=0 ), nn.BatchNorm2d(branch_channels), nn.ReLU(inplace=True), TemporalConv( branch_channels, branch_channels, kernel_size=ks, stride=stride, dilation=dilation ), ) for ks, dilation in zip(kernel_size, dilations) ]) self.branches.append(nn.Sequential( nn.Conv2d(in_channels, branch_channels, kernel_size=1, padding=0), nn.BatchNorm2d(branch_channels), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=(3,1), stride=(stride,1), padding=(1,0)), nn.BatchNorm2d(branch_channels) )) self.branches.append(nn.Sequential( nn.Conv2d(in_channels, branch_channels, kernel_size=1, padding=0, stride=(stride,1)), nn.BatchNorm2d(branch_channels) )) if not residual: self.residual = lambda x: 0 elif (in_channels == out_channels) and (stride == 1): self.residual = lambda x: x else: self.residual = TemporalConv(in_channels, out_channels, kernel_size=residual_kernel_size, stride=stride) self.apply(weights_init) def forward(self, x): res = self.residual(x) branch_outs = [] for tempconv in self.branches: out = tempconv(x) branch_outs.append(out) out = torch.cat(branch_outs, dim=1) out += res return out class unit_tcn(nn.Module): def __init__(self, in_channels, out_channels, kernel_size=5, stride=1): super(unit_tcn, self).__init__() pad = int((kernel_size - 1) / 2) self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=(kernel_size, 1), padding=(pad, 0), stride=(stride, 1), groups=1) self.bn = nn.BatchNorm2d(out_channels) self.relu = nn.ReLU(inplace=True) conv_init(self.conv) bn_init(self.bn, 1) def forward(self, x): x = self.bn(self.conv(x)) return x class MHSA(nn.Module): def __init__(self, dim_in, dim, A, num_heads=6, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., insert_cls_layer=0, pe=False, num_point=25, outer=True, layer=0, **kwargs): super().__init__() self.num_heads = num_heads self.dim = dim head_dim = dim // num_heads self.scale = qk_scale or head_dim ** -0.5 self.num_point = num_point self.layer = layer h1 = A.sum(0) h1[h1 != 0] = 1 h = [None for _ in range(num_point)] h[0] = np.eye(num_point) h[1] = h1 self.hops = 0*h[0] for i in range(2, num_point): h[i] = h[i-1] @ h1.transpose(0, 1) h[i][h[i] != 0] = 1 for i in range(num_point-1, 0, -1): if np.any(h[i]-h[i-1]): h[i] = h[i] - h[i - 1] self.hops += i*h[i] else: continue self.hops = torch.tensor(self.hops).long() self.rpe = nn.Parameter(torch.zeros((self.hops.max()+1, dim))) self.w1 = nn.Parameter(torch.zeros(num_heads, head_dim)) A = A.sum(0) A[:, :] = 0 self.outer = nn.Parameter(torch.stack([torch.eye(A.shape[-1]) for _ in range(num_heads)], dim=0), requires_grad=True) self.alpha = nn.Parameter(torch.zeros(1), requires_grad=True) self.kv = nn.Conv2d(dim_in, dim * 2, 1, bias=qkv_bias) self.q = nn.Conv2d(dim_in, dim, 1, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Conv2d(dim, dim, 1, groups=6) self.proj_drop = nn.Dropout(proj_drop) self.apply(self._init_weights) self.insert_cls_layer = insert_cls_layer def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.LayerNorm): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def forward(self, x, e): N, C, T, V = x.shape kv = self.kv(x).reshape(N, 2, self.num_heads, self.dim // self.num_heads, T, V).permute(1, 0, 4, 2, 5, 3) k, v = kv[0], kv[1] q = self.q(x).reshape(N, self.num_heads, self.dim // self.num_heads, T, V).permute(0, 3, 1, 4, 2) e_k = e.reshape(N, self.num_heads, self.dim // self.num_heads, T, V).permute(0, 3, 1, 4, 2) pos_emb = self.rpe[self.hops] pos_emb = pos_emb.to(torch.float16) k_r = pos_emb.view(V, V, self.num_heads, self.dim // self.num_heads) q = q.to(torch.float16) e_k = e_k.to(torch.float16) b = torch.einsum("bthnc, nmhc->bthnm", q, k_r) c = torch.einsum("bthnc, bthmc->bthnm", q, e_k) w1 = self.w1.to(torch.float16) d = torch.einsum("hc, bthmc->bthm", w1, e_k).unsqueeze(-2) a = q @ k.transpose(-2, -1) attn = a + b + c + d attn = attn * self.scale attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) x = (self.alpha * attn + self.outer) @ v x = x.transpose(3, 4).reshape(N, T, -1, V).transpose(1, 2) x = self.proj(x) x = self.proj_drop(x) return x class Mlp(nn.Module): def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., num_heads=None): super().__init__() out_features = out_features or in_features hidden_features = hidden_features or in_features self.fc1 = nn.Conv2d(in_features, hidden_features, 1) self.act = act_layer() self.fc2 = nn.Conv2d(hidden_features, out_features, 1) self.drop = nn.Dropout(drop) self.apply(self._init_weights) def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.LayerNorm): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def forward(self, x): x = self.fc1(x) x = self.act(x) x = self.drop(x) x = self.fc2(x) x = self.drop(x) return x class unit_vit(nn.Module): def __init__(self, dim_in, dim, A, num_of_heads, add_skip_connection=True, qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., drop_path=0, act_layer=nn.GELU, norm_layer=nn.LayerNorm, layer=0, insert_cls_layer=0, pe=False, num_point=25, **kwargs): super().__init__() self.norm1 = norm_layer(dim_in) self.dim_in = dim_in self.dim = dim self.add_skip_connection = add_skip_connection self.num_point = num_point self.attn = MHSA(dim_in, dim, A, num_heads=num_of_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, insert_cls_layer=insert_cls_layer, pe=pe, num_point=num_point, layer=layer, **kwargs) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.dim_in != self.dim: self.skip_proj = nn.Conv2d(dim_in, dim, (1, 1), padding=(0, 0), bias=False) self.pe_proj = nn.Conv2d(dim_in, dim, 1, bias=False) self.pe = pe def forward(self, x, joint_label, groups): label = F.one_hot(torch.tensor(joint_label)).float().to(x.device) z = x @ (label / label.sum(dim=0, keepdim=True)) z = self.pe_proj(z).permute(3, 0, 1, 2) e = z[joint_label].permute(1, 2, 3, 0) if self.add_skip_connection: if self.dim_in != self.dim: x = self.skip_proj(x) + self.drop_path(self.attn(self.norm1(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2), e)) else: x = x + self.drop_path(self.attn(self.norm1(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2), e)) else: x = self.drop_path(self.attn(self.norm1(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2), e)) return x class TCN_ViT_unit(nn.Module): def __init__(self, in_channels, out_channels, A, stride=1, num_of_heads=6, residual=True, kernel_size=5, dilations=[1,2], pe=False, num_point=25, layer=0): super(TCN_ViT_unit, self).__init__() self.vit1 = unit_vit(in_channels, out_channels, A, add_skip_connection=residual, num_of_heads=num_of_heads, pe=pe, num_point=num_point, layer=layer) self.tcn1 = MultiScale_TemporalConv(out_channels, out_channels, kernel_size=kernel_size, stride=stride, dilations=dilations, residual=False) self.act = nn.ReLU(inplace=True) self.in_channels = in_channels self.out_channels = out_channels self.stride = stride if not residual: self.residual = lambda x: 0 elif (in_channels == out_channels) and (stride == 1): self.residual = lambda x: x else: self.residual = unit_tcn(in_channels, out_channels, kernel_size=1, stride=stride) def forward(self, x, joint_label, groups): y = self.act(self.tcn1(self.vit1(x, joint_label, groups)) + self.residual(x)) return y class Hyperformer_Model(nn.Module): def __init__(self, A, joint_label, num_point, in_channels=3, num_person=1, drop_out=0, num_of_heads=9, out_dim=512): super(Hyperformer_Model, self).__init__() A = np.asarray(A, dtype=np.float32) assert A.shape[-1] == num_point and len(joint_label) == num_point self.num_of_heads = num_of_heads self.num_point = num_point self.num_person = num_person self.feat_dim = 24 * num_of_heads self.out_dim = out_dim self.data_bn = nn.BatchNorm1d(num_person * in_channels * num_point) self.joint_label = list(joint_label) self.l1 = TCN_ViT_unit(in_channels, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=1) self.l2 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=2) self.l3 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=3) self.l4 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=4) self.l5 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, stride=2, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=5) self.l6 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=6) self.l7 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=7) self.l8 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, stride=2, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=8) self.l9 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=9) self.l10 = TCN_ViT_unit(24*num_of_heads, 24*num_of_heads, A, residual=True, num_of_heads=num_of_heads, pe=True, num_point=num_point, layer=10) self.fc2 = nn.Linear(24*num_of_heads, out_dim) nn.init.normal_(self.fc2.weight, 0, math.sqrt(2. / out_dim)) bn_init(self.data_bn, 1) if drop_out: self.drop_out = nn.Dropout(drop_out) else: self.drop_out = lambda x: x def forward(self, x): groups = [] for num in range(max(self.joint_label)+1): groups.append([ind for ind, element in enumerate(self.joint_label) if element==num]) N, C, T, V, M = x.size() x = x.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V * C, T) x = self.data_bn(x) x = x.view(N, M, V, C, T).contiguous().view(N * M, V, C, T).permute(0, 2, 3, 1) x = self.l1(x, self.joint_label, groups) x = self.l2(x, self.joint_label, groups) x = self.l3(x, self.joint_label, groups) x = self.l4(x, self.joint_label, groups) x = self.l5(x, self.joint_label, groups) x = self.l6(x, self.joint_label, groups) x = self.l7(x, self.joint_label, groups) x = self.l8(x, self.joint_label, groups) x = self.l9(x, self.joint_label, groups) x = self.l10(x, self.joint_label, groups) _ , C, T, V = x.size() z = x.view(N, M, C, T, V) z1 = z.mean(4).mean(1) z2 = z1.view(N,T,C) x = x.view(N, M, C, -1) x = x.mean(3).mean(1) x2 = self.fc2(x) return x, x2