Spaces:
Sleeping
Sleeping
Download models/content_encoder/vision_encoder.py from Spider-8251/UniFlow-Audio: direct link, hf CLI and curl.
- Browser
- Download file 6.51 kB
-
https://huggingface.co/spaces/Spider-8251/UniFlow-Audio/resolve/main/models/content_encoder/vision_encoder.py
- Command line
-
hf download hf://spaces/Spider-8251/UniFlow-Audio/models/content_encoder/vision_encoder.py
-
curl -L -o vision_encoder.py https://huggingface.co/spaces/Spider-8251/UniFlow-Audio/resolve/main/models/content_encoder/vision_encoder.py
6.51 kB
| import random | |
| from typing import Literal, Sequence | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from accel_hydra.utils.torch import create_mask_from_length | |
| from x_transformers.x_transformers import Encoder | |
| from ..mmdit.modules import ChannelLastConv1d, ConvMLP, MLP | |
| class CLIPEncoder(nn.Module): | |
| def __init__( | |
| self, | |
| video_feat_dim: int, | |
| embed_dim: int, | |
| ): | |
| super().__init__() | |
| self.mlp = nn.Linear(video_feat_dim, embed_dim) | |
| self.init_weights() | |
| def init_weights(self): | |
| def _init_weights(module): | |
| if isinstance(module, nn.Linear): | |
| nn.init.xavier_uniform_(module.weight) | |
| if module.bias is not None: | |
| nn.init.constant_(module.bias, 0.) | |
| self.apply(_init_weights) | |
| def forward( | |
| self, clip: torch.Tensor, clip_lengths: Sequence[int], **kwargs | |
| ): | |
| device = clip.device | |
| x = F.normalize(clip, p=2, dim=-1) | |
| x = self.mlp(x) | |
| mask = create_mask_from_length(clip_lengths).to(device) | |
| return { | |
| "ta_output": x, | |
| "ta_mask": mask, | |
| "nta_output": None, | |
| "nta_mask": None | |
| } | |
| class CLIPInterpolateVideoEncoder(nn.Module): | |
| def __init__( | |
| self, | |
| video_feat_dim: int, | |
| embed_dim: int, | |
| latent_token_rate: int, | |
| transformer_cfg: dict | None = None, | |
| ): | |
| super().__init__() | |
| self.mlp = nn.Linear(video_feat_dim, embed_dim) | |
| self.latent_token_rate = latent_token_rate | |
| self.transformer_enhancer = None | |
| if transformer_cfg is not None: | |
| self.transformer_enhancer = Encoder( | |
| dim=embed_dim, | |
| **transformer_cfg, | |
| ) | |
| self.init_weights() | |
| def init_weights(self): | |
| def _init_weights(module): | |
| if isinstance(module, nn.Linear): | |
| nn.init.xavier_uniform_(module.weight) | |
| if module.bias is not None: | |
| nn.init.constant_(module.bias, 0.) | |
| self.apply(_init_weights) | |
| def forward(self, clip: torch.Tensor, duration: Sequence[float], **kwargs): | |
| device = clip.device | |
| x = F.normalize(clip, p=2, dim=-1) | |
| x = self.mlp(x) | |
| target_len = int(max(duration) * self.latent_token_rate) | |
| x = x.transpose(1, 2) | |
| x = F.interpolate(x, target_len, mode='nearest-exact') | |
| x = x.transpose(1, 2) | |
| latent_length = torch.as_tensor(duration) * self.latent_token_rate | |
| mask = create_mask_from_length(latent_length).to(device) | |
| if self.transformer_enhancer is not None: | |
| x = self.transformer_enhancer(x, mask=mask) | |
| return { | |
| "ta_output": x, | |
| "ta_mask": mask, | |
| "nta_output": None, | |
| "nta_mask": None | |
| } | |
| class CLIPSyncEncoder(nn.Module): | |
| def __init__( | |
| self, | |
| clip_dim: int, | |
| sync_dim: int, | |
| embed_dim: int, | |
| latent_token_rate: int, | |
| fusion: Literal["add", "concat"] = "add", | |
| sync_drop_ratio: float = 0.0, | |
| ) -> None: | |
| super().__init__() | |
| self.fusion = fusion | |
| self.sync_drop_ratio = sync_drop_ratio | |
| self.clip_proj = nn.Linear(clip_dim, embed_dim) | |
| self.sync_proj = nn.Linear(sync_dim, embed_dim) | |
| self.sync_pos_emb = nn.Parameter(torch.zeros((1, 1, 8, sync_dim))) | |
| if fusion == "concat": | |
| self.fusion_proj = nn.Linear(embed_dim * 2, embed_dim) | |
| self.sync_frame_rate = 25 | |
| self.sync_num_frames_per_segment = 16 | |
| self.sync_step_size = 8 | |
| self.sync_downsample_rate = 2 | |
| self.latent_token_rate = latent_token_rate | |
| self.initialize_weights() | |
| def initialize_weights(self): | |
| def _basic_init(module): | |
| if isinstance(module, nn.Linear): | |
| torch.nn.init.xavier_uniform_(module.weight) | |
| if module.bias is not None: | |
| nn.init.constant_(module.bias, 0) | |
| self.apply(_basic_init) | |
| def get_sync_seq_len(self, duration): | |
| num_frames = duration * self.sync_frame_rate | |
| num_segments = ( | |
| num_frames - self.sync_num_frames_per_segment | |
| ) // self.sync_step_size + 1 | |
| sync_seq_len = int( | |
| num_segments * self.sync_num_frames_per_segment / | |
| self.sync_downsample_rate | |
| ) | |
| return sync_seq_len | |
| def forward( | |
| self, clip: torch.Tensor, sync: torch.Tensor, | |
| duration: Sequence[float], clip_lengths: Sequence[int], | |
| sync_lengths: Sequence[int], **kwargs | |
| ): | |
| clip = F.normalize(clip, p=2, dim=-1) | |
| sync = F.normalize(sync, p=2, dim=-1) | |
| latent_length = (torch.as_tensor(duration) * | |
| self.latent_token_rate).int() | |
| ta_mask = create_mask_from_length(latent_length).to(sync.device) | |
| bs = clip.shape[0] | |
| # target_len = int(max(duration) * self.latent_token_rate) | |
| target_len = latent_length.max() | |
| sync_seq_len = self.get_sync_seq_len(max(duration)) | |
| # B * num_segments (24) * 8 * 768 | |
| num_sync_segments = sync_seq_len // 8 | |
| sync = sync.view(bs, num_sync_segments, 8, -1) + self.sync_pos_emb | |
| sync = sync.flatten(1, 2) # (B, VN, D) | |
| # extend vf to match x | |
| clip = self.clip_proj(clip) | |
| sync = self.sync_proj(sync) # (B, D, VN) | |
| # upsample the sync features to match the audio | |
| sync = sync.transpose(1, 2) # (B, D, VN) | |
| sync = F.interpolate(sync, size=target_len, mode='nearest-exact') | |
| sync = sync.transpose(1, 2) # (B, N, D) | |
| if self.training: | |
| if self.sync_drop_ratio > 0: | |
| if random.random() < self.sync_drop_ratio: | |
| sync = torch.zeros_like(sync) | |
| # upsample the clip features to match the audio | |
| clip = clip.transpose(1, 2) # (B, D, VN) | |
| clip = F.interpolate(clip, size=target_len, mode='nearest-exact') | |
| clip = clip.transpose(1, 2) # (B, N, D) | |
| if self.fusion == "add": | |
| ta_output = clip + sync | |
| elif self.fusion == "concat": | |
| ta_output = self.fusion_proj(torch.cat([clip, sync], dim=-1)) | |
| else: | |
| raise ValueError(f"Invalid fusion method: {self.fusion}") | |
| return { | |
| "ta_output": ta_output, | |
| "ta_mask": ta_mask, | |
| "nta_output": None, | |
| "nta_mask": None | |
| } | |