UniFlow-Audio / models /common.py
wsntxxn's picture
Update to v1.1
6e6f7d1
Raw History Blame Contribute Delete
3.32 kB
from typing import Sequence
from accel_hydra.utils.torch import create_mask_from_length, loss_with_mask
import torch
import torch.nn.functional as F
from utils.torch_utilities import create_alignment_path
class DurationAdapterMixin:
def __init__(
self,
latent_token_rate: int,
offset: float = 1.0,
frame_resolution: float | None = None
):
self.latent_token_rate = latent_token_rate
self.offset = offset
self.frame_resolution = frame_resolution
def get_global_duration_loss(
self,
pred: torch.Tensor,
latent_mask: torch.Tensor,
reduce: bool = True,
):
target = torch.log(
latent_mask.sum(1) / self.latent_token_rate + self.offset
)
loss = F.mse_loss(target, pred, reduction="mean" if reduce else "none")
return loss
def get_local_duration_loss(
self, ground_truth: torch.Tensor, pred: torch.Tensor,
mask: torch.Tensor, is_time_aligned: Sequence[bool], reduce: bool
):
n_frames = torch.round(ground_truth / self.frame_resolution)
target = torch.log(n_frames + self.offset)
loss = loss_with_mask(
(target - pred)**2,
mask,
reduce=False,
)
loss *= is_time_aligned
if reduce:
if is_time_aligned.sum().item() == 0:
loss *= 0.0
loss = loss.mean()
else:
loss = loss.sum() / is_time_aligned.sum()
return loss
def prepare_local_duration(self, pred: torch.Tensor, mask: torch.Tensor):
pred = torch.exp(pred) * mask
pred = torch.ceil(pred) - self.offset
pred *= self.frame_resolution
pred = torch.round(pred * self.latent_token_rate)
return pred
def prepare_global_duration(
self,
global_pred: torch.Tensor,
local_pred: torch.Tensor,
is_time_aligned: Sequence[bool],
use_local: bool = True,
):
"""
global_pred: predicted duration value, processed by logarithmic and offset
local_pred: predicted latent length
"""
global_pred = torch.exp(global_pred) - self.offset
result = torch.round(global_pred * self.latent_token_rate)
# avoid error accumulation for each frame
if use_local:
pred_from_local = local_pred.sum(1)
result[is_time_aligned] = pred_from_local[is_time_aligned]
result = result.long()
return result
def expand_by_duration(
self,
x: torch.Tensor,
content_mask: torch.Tensor,
local_duration: torch.Tensor,
global_duration: torch.Tensor,
):
training = getattr(self, 'training', False)
if not training: # inference mode
latent_length = global_duration
else: # training mode
latent_length = local_duration.sum(1)
latent_mask = create_mask_from_length(latent_length).to(
content_mask.device
)
attn_mask = content_mask.unsqueeze(-1) * latent_mask.unsqueeze(1)
align_path = create_alignment_path(local_duration, attn_mask)
expanded_x = torch.matmul(align_path.transpose(1, 2).to(x.dtype), x)
return expanded_x, latent_mask