""" ein notation: b - batch n - sequence nt - text sequence nw - raw wave length d - dimension """ # ruff: noqa: F722 F821 from __future__ import annotations from random import random import math import torch import torch.nn.functional as F from torch import nn from torch.nn.utils.rnn import pad_sequence from torchdiffeq import odeint from wavegen_v_spec import normalize_inference_method from wavegen_v_spec.model.mel_spectrogram import TorchMel from wavegen_v_spec.model.modules import MelSpectrogramLoss from wavegen_v_spec.model.utils import ( exists, get_epss_timesteps, lens_to_mask, list_str_to_idx, list_str_to_tensor, mask_from_frac_lengths, ) class BaseCFM(nn.Module): def __init__( self, transformer: nn.Module, sigma=0.0, odeint_kwargs: dict = dict( # atol = 1e-5, # rtol = 1e-5, method="euler" # 'midpoint' ), audio_drop_prob=0.3, cond_drop_prob=0.2, joint_cond_drop_prob=0.0, waveform_kwargs: dict = dict(), frac_lengths_mask: tuple[float, float] = (0.7, 1.0), vocab_char_map: dict[str:int] | None = None, prediction: str = "flow", # "flow" | "x_pred" loss_space: str = "flow", # "flow" | "v" | "x" t_sampling: str = "uniform", # "uniform" | "logistic_normal" P_mean: float = 0.0, P_std: float = 1.0, time_shift: float = 1.0, t_eps: float = 1e-4, use_aux_mel_loss: bool = False, aux_mel_loss_weight: float = 0.0, aux_mel_loss_masked: bool = True, sample_rate: int = 16000, latents_scale: float = 1.0, sf_loss_weight: float = 0.1, mel_xt_drop_prob: float = 0.0, inference_bottleneck_dropout: float = 0.0, inference_bottleneck_dropout_seed: int = 0, ): super().__init__() self.frac_lengths_mask = frac_lengths_mask # waveform geometry waveform_kwargs = dict(waveform_kwargs) self.wav_frame_len = int(waveform_kwargs.pop("wav_frame_len", 480)) self.num_channels = self.wav_frame_len # classifier-free guidance self.audio_drop_prob = audio_drop_prob self.cond_drop_prob = cond_drop_prob self.joint_cond_drop_prob = joint_cond_drop_prob # transformer self.transformer = transformer self.dim = transformer.dim if hasattr(self.transformer, "set_wav_frame_len"): self.transformer.set_wav_frame_len(self.wav_frame_len) # conditional flow related self.sigma = sigma # sampling related self.odeint_kwargs = odeint_kwargs # vocab map for tokenization self.vocab_char_map = vocab_char_map # enhanced flow model settings self.prediction = prediction self.loss_space = loss_space self.t_sampling = t_sampling self.P_mean = P_mean self.P_std = P_std self.time_shift = time_shift self.t_eps = t_eps self.latents_scale = latents_scale self.target_sample_rate = sample_rate self.sf_loss_weight = float(sf_loss_weight) self.mel_xt_drop_prob = float(mel_xt_drop_prob) if not 0.0 <= self.mel_xt_drop_prob <= 1.0: raise ValueError(f"mel_xt_drop_prob must be in [0, 1], got {self.mel_xt_drop_prob}") self.inference_bottleneck_dropout = float(inference_bottleneck_dropout) if ( not math.isfinite(self.inference_bottleneck_dropout) or not 0.0 <= self.inference_bottleneck_dropout < 1.0 ): raise ValueError( "inference_bottleneck_dropout must be finite and in [0, 1), " f"got {inference_bottleneck_dropout}" ) self.inference_bottleneck_dropout_seed = int( inference_bottleneck_dropout_seed ) if self.inference_bottleneck_dropout > 0.0 and not bool( getattr(transformer, "use_mel_information_bottleneck", False) ): raise ValueError( "inference_bottleneck_dropout > 0 requires " "use_mel_information_bottleneck=True" ) self.mel_channels = int( getattr(transformer, "mel_channels", getattr(transformer, "mel_proj_out_dim", 64)) ) if self.mel_channels <= 0: raise ValueError(f"mel_channels must be positive, got {self.mel_channels}") self.mel_dim = self.mel_channels # backward-compatible internal alias # aux mel loss self.use_aux_mel_loss = use_aux_mel_loss self.aux_mel_loss_masked = aux_mel_loss_masked if self.use_aux_mel_loss: self.aux_mel_loss = MelSpectrogramLoss( sample_rate=sample_rate, n_mels=[5, 10, 20, 40, 80, 160, 320], window_lengths=[32, 64, 128, 256, 512, 1024, 2048], mel_fmin=[0, 0, 0, 0, 0, 0, 0], mel_fmax=[None] * 7, pow=1.0, clamp_eps=1e-5, weight=aux_mel_loss_weight, ) else: self.aux_mel_loss = None self.mask_align_to = 1 alignments = [] if self.aux_mel_loss is not None and hasattr(self.aux_mel_loss, "mel_transforms"): hop_lengths = [int(m.hop_length) for m in self.aux_mel_loss.mel_transforms] if len(hop_lengths) > 0: lcm_hop = hop_lengths[0] for h in hop_lengths[1:]: lcm_hop = math.lcm(lcm_hop, h) alignments.append(lcm_hop) if len(alignments) > 0: lcm_align = alignments[0] for a in alignments[1:]: lcm_align = math.lcm(lcm_align, a) self.mask_align_to = max(1, lcm_align) self.mel_f = TorchMel( n_fft=self.wav_frame_len * 4, n_mel_channels=self.mel_channels, target_sample_rate=sample_rate, hop_length=self.wav_frame_len, win_length=self.wav_frame_len * 4, ) @property def device(self): return next(self.parameters()).device def _sample_time(self, batch: int, dtype, device): if self.t_sampling == "uniform": t = torch.rand((batch,), dtype=dtype, device=device) elif self.t_sampling == "logistic_normal": # JiT: t = sigmoid(N(P_mean, P_std)) z = torch.randn((batch,), device=device, dtype=dtype) * self.P_std + self.P_mean t = torch.sigmoid(z) else: raise ValueError(f"Unknown t_sampling: {self.t_sampling}") if self.time_shift != 1.0: t = t / (t + self.time_shift * (1 - t)) return t def _x_to_v(self, x_pred, z, t): # v_pred = (x_pred - z) / (1 - t) denom = (1.0 - t).clamp_min(self.t_eps) while denom.ndim < z.ndim: denom = denom.unsqueeze(-1) return (x_pred - z) / denom def _ceil_to_frame_multiple(self, lengths: torch.Tensor) -> torch.Tensor: lengths = lengths.to(dtype=torch.long) return ((lengths + self.wav_frame_len - 1) // self.wav_frame_len) * self.wav_frame_len def _pad_wav_to_frame_multiple(self, wav: torch.Tensor) -> torch.Tensor: pad_len = (self.wav_frame_len - (wav.shape[-1] % self.wav_frame_len)) % self.wav_frame_len if pad_len == 0: return wav return F.pad(wav, (0, pad_len), value=0.0) def _wav_mask_to_mel_mask(self, wav_mask: torch.Tensor, mel_len: int | None = None) -> torch.Tensor: pad_len = (self.wav_frame_len - (wav_mask.shape[-1] % self.wav_frame_len)) % self.wav_frame_len if pad_len > 0: wav_mask = F.pad(wav_mask, (0, pad_len), value=False) mel_mask = wav_mask.view(wav_mask.shape[0], -1, self.wav_frame_len).any(dim=-1) if mel_len is None: return mel_mask if mel_mask.shape[1] < mel_len: mel_mask = F.pad(mel_mask, (0, mel_len - mel_mask.shape[1]), value=False) elif mel_mask.shape[1] > mel_len: mel_mask = mel_mask[:, :mel_len] return mel_mask def _mel_from_wav(self, wav: torch.Tensor, target_frames: int | None = None) -> torch.Tensor: mel = self.mel_f(wav.to(dtype=torch.float32)).to(dtype=wav.dtype) if target_frames is None: return mel if mel.shape[1] < target_frames: mel = F.pad(mel, (0, 0, 0, target_frames - mel.shape[1]), value=0.0) elif mel.shape[1] > target_frames: mel = mel[:, :target_frames] return mel def _unpack_transformer_pred( self, raw_pred: torch.Tensor | tuple[torch.Tensor, torch.Tensor | None], ) -> tuple[torch.Tensor, torch.Tensor | None]: if isinstance(raw_pred, (tuple, list)): if len(raw_pred) != 2: raise ValueError(f"Expected transformer to return (wav_pred, mel_pred), got {len(raw_pred)} outputs.") return raw_pred[0], raw_pred[1] raise ValueError("CFM expects transformer to return (wav_pred, mel_pred).") @torch.no_grad() def sample( self, cond: float["b n d"] | float["b nw"], text: int["b nt"] | list[str], duration: int | int["b"], *, lens: int["b"] | None = None, steps=32, cfg_strength=1.0, sway_sampling_coef=None, timestep_mapping="sway_sampling", timestep_power=None, shift=1.0, seed: int | None = None, mel_temperature: float = 1.0, cfg_schedule: str = "constant", inference_method: str = "Prompt_xt_Anchor", inference_bottleneck_dropout: float | None = None, inference_bottleneck_dropout_seed: int | None = None, max_duration=4096, use_epss=True, no_ref_audio=False, duplicate_test=False, t_inter=0.1, edit_mask=None, compute_final_mel: bool = False, ): self.eval() if not isinstance(compute_final_mel, bool): raise ValueError("compute_final_mel must be a boolean") inference_method = normalize_inference_method(inference_method) if inference_method == "Prompt_xt_Anchor" and (no_ref_audio or duplicate_test): raise ValueError("Prompt_xt_Anchor inference requires reference audio and does not support duplicate_test") if not math.isfinite(mel_temperature) or mel_temperature < 0: raise ValueError("mel_temperature must be finite and nonnegative") if cfg_schedule not in {"constant", "scheduled_k3"}: raise ValueError("cfg_schedule must be constant or scheduled_k3") if cfg_schedule == "scheduled_k3" and (steps < 3 or cfg_strength < 1e-5): raise ValueError("scheduled_k3 requires at least three steps and positive CFG") bottleneck_dropout = float( self.inference_bottleneck_dropout if inference_bottleneck_dropout is None else inference_bottleneck_dropout ) if ( not math.isfinite(bottleneck_dropout) or not 0.0 <= bottleneck_dropout < 1.0 ): raise ValueError( "inference_bottleneck_dropout must be finite and in [0, 1), " f"got {bottleneck_dropout}" ) bottleneck_dropout_seed = int( self.inference_bottleneck_dropout_seed if inference_bottleneck_dropout_seed is None else inference_bottleneck_dropout_seed ) if bottleneck_dropout > 0.0 and not bool( getattr(self.transformer, "use_mel_information_bottleneck", False) ): raise ValueError( "inference_bottleneck_dropout > 0 requires " "use_mel_information_bottleneck=True" ) active_bottleneck_dropout = ( bottleneck_dropout if bottleneck_dropout > 0.0 else None ) # raw wave if cond.ndim != 2: raise ValueError(f"WaveGen expects raw waveform conditioning [B, N], got {tuple(cond.shape)}") cond = cond.to(next(self.parameters()).dtype) cond_unscaled = cond cond = cond_unscaled * self.latents_scale batch, cond_seq_len, device = *cond.shape[:2], cond.device bottleneck_generator = None if active_bottleneck_dropout is not None: bottleneck_generator = torch.Generator(device=device) bottleneck_generator.manual_seed(bottleneck_dropout_seed) if not exists(lens): lens = torch.full((batch,), cond_seq_len, device=device, dtype=torch.long) else: lens = lens.to(device=device, dtype=torch.long) if inference_method == "Prompt_xt_Anchor": if lens.shape != (batch,) or not bool(((lens > 0) & (lens <= cond_seq_len)).all()): raise ValueError("Prompt_xt_Anchor inference requires a nonempty valid reference length for each sample") # text if isinstance(text, list): if exists(self.vocab_char_map): text = list_str_to_idx(text, self.vocab_char_map).to(device) else: text = list_str_to_tensor(text).to(device) assert text.shape[0] == batch elif torch.is_tensor(text): text = text.to(device) # duration cond_mask = lens_to_mask(lens, length=cond_seq_len) if edit_mask is not None: if inference_method == "Prompt_xt_Anchor" and ( not torch.is_tensor(edit_mask) or edit_mask.dtype != torch.bool or edit_mask.shape != cond_mask.shape or edit_mask.device != device ): raise ValueError("Prompt_xt_Anchor edit_mask must be a boolean reference mask on the conditioning device") cond_mask = cond_mask & edit_mask if inference_method == "Prompt_xt_Anchor" and not bool(cond_mask.any(dim=-1).all()): raise ValueError("Prompt_xt_Anchor inference requires at least one reference sample per item") if isinstance(duration, int): duration = torch.full((batch,), duration, device=device, dtype=torch.long) else: duration = duration.to(device=device, dtype=torch.long) # keep legacy max_duration semantics for wav path: 4096 means 4096 frame-tokens. max_duration_limit = max_duration * self.wav_frame_len duration = torch.maximum( torch.maximum((text != -1).sum(dim=-1), lens) + 1, duration ) # duration at least text/audio prompt length plus one token, so something is generated duration = torch.maximum(duration, torch.full_like(duration, cond_seq_len)) duration = self._ceil_to_frame_multiple(duration) duration = duration.clamp(max=max_duration_limit) if inference_method == "Prompt_xt_Anchor" and ( cond_seq_len > int(duration.amax().item()) or bool((duration <= lens).any()) ): raise ValueError("Prompt_xt_Anchor duration must leave room after the reference within max_duration") max_duration = int(duration.amax().item()) mel_duration = duration // self.wav_frame_len max_mel_duration = max_duration // self.wav_frame_len # duplicate test corner for inner time step oberservation test_mel_cond = None if duplicate_test: test_cond = F.pad(cond, (cond_seq_len, max_duration - 2 * cond_seq_len), value=0.0) test_mel_cond = self._mel_from_wav( test_cond / self.latents_scale, target_frames=max_mel_duration, ).to(dtype=cond.dtype) cond = F.pad(cond, (0, max_duration - cond_seq_len), value=0.0) cond_unscaled = F.pad(cond_unscaled, (0, max_duration - cond_seq_len), value=0.0) if no_ref_audio: cond = torch.zeros_like(cond) cond_unscaled = torch.zeros_like(cond_unscaled) cond_mask = F.pad(cond_mask, (0, max_duration - cond_mask.shape[-1]), value=False) step_cond = torch.where(cond_mask, cond, torch.zeros_like(cond)) mel_cond_mask = self._wav_mask_to_mel_mask(cond_mask, mel_len=max_mel_duration) if no_ref_audio: mel_cond_mask = torch.zeros_like(mel_cond_mask) step_mel_cond = self._mel_from_wav(cond_unscaled, target_frames=max_mel_duration) step_mel_cond = step_mel_cond.to(dtype=step_cond.dtype) step_mel_cond = torch.where( mel_cond_mask.unsqueeze(-1), step_mel_cond, torch.zeros_like(step_mel_cond), ) if batch > 1: mask = lens_to_mask(duration, length=max_duration) else: # save memory and speed up, as single inference need no mask currently mask = None # neural ode cfg_call_index = 0 rhs_call_index = 0 # This is the original length/edit mask, including silent reference samples. prompt_kwargs = {"prompt_mask": cond_mask} if inference_method == "Prompt_xt_Anchor" else {} def fn(t, state): nonlocal cfg_call_index, rhs_call_index x, mel = state predict_mel = not (skip_final_mel and rhs_call_index == final_rhs_index) rhs_call_index += 1 # at each step, conditioning is fixed # step_cond = torch.where(cond_mask, cond, torch.zeros_like(cond)) def to_v(raw_pred): pred_x_or_v, pred_mel_x_or_v = self._unpack_transformer_pred(raw_pred) # The final Mel state is never consumed. Keep its velocity zero # when the head is skipped, including in x-prediction models. if pred_mel_x_or_v is None: wav_v = pred_x_or_v if self.prediction == "flow" else self._x_to_v(pred_x_or_v, x, t) return wav_v, torch.zeros_like(mel) if self.prediction == "flow": return pred_x_or_v, pred_mel_x_or_v else: # x_pred return self._x_to_v(pred_x_or_v, x, t), self._x_to_v(pred_mel_x_or_v, mel, t) # predict flow (cond) if cfg_strength < 1e-5: raw_pred, *_ = self.transformer( x=x, cond=step_cond, mel=mel, mel_cond=step_mel_cond, text=text, time=t, mask=mask, drop_audio_cond=False, drop_text=False, cache=True, predict_mel=predict_mel, inference_bottleneck_dropout=active_bottleneck_dropout, inference_bottleneck_generator=bottleneck_generator, **prompt_kwargs, ) return to_v(raw_pred) # predict flow (cond and uncond), for classifier-free guidance scheduled_kwargs = {} if cfg_schedule == "scheduled_k3": scheduled_kwargs["cfg_null_audio"] = cfg_call_index >= 3 raw_pred_cfg, *_ = self.transformer( x=x, cond=step_cond, mel=mel, mel_cond=step_mel_cond, text=text, time=t, mask=mask, cfg_infer=True, cache=True, predict_mel=predict_mel, inference_bottleneck_dropout=active_bottleneck_dropout, inference_bottleneck_generator=bottleneck_generator, **scheduled_kwargs, **prompt_kwargs, ) cfg_call_index += 1 pred_cfg, mel_pred_cfg = self._unpack_transformer_pred(raw_pred_cfg) pred, null_pred = torch.chunk(pred_cfg, 2, dim=0) mel_pred, mel_null_pred = (None, None) if mel_pred_cfg is None else torch.chunk(mel_pred_cfg, 2, dim=0) if cfg_schedule == "scheduled_k3" and float(t.item()) < 0.5: # Preserve the registered FP32 null-proxy arithmetic followed # by the head's original dtype rounding (BF16 on CUDA). Direct # doubling of cfg_strength is NOT numerically equivalent. null_pred = (pred.float() - 2.0 * (pred.float() - null_pred.float())).to(pred.dtype) if mel_pred is not None: mel_null_pred = (mel_pred.float() - 2.0 * (mel_pred.float() - mel_null_pred.float())).to(mel_pred.dtype) v_cond, mel_v_cond = to_v((pred, mel_pred)) v_uncond, mel_v_uncond = to_v((null_pred, mel_null_pred)) return ( v_cond + (v_cond - v_uncond) * cfg_strength, mel_v_cond + (mel_v_cond - mel_v_uncond) * cfg_strength, ) # noise input # to make sure batch inference result is same with different batch size, and for sure single inference # still some difference maybe due to convolutional layers y0 = [] y0_mel = [] for dur, mel_dur in zip(duration, mel_duration): if exists(seed): torch.manual_seed(seed) y0.append(torch.randn(int(dur.item()), device=self.device, dtype=step_cond.dtype)) mel_noise = torch.randn( int(mel_dur.item()), self.mel_dim, device=self.device, dtype=step_mel_cond.dtype, ) # Always consume the same Gaussian draw, including at T=0. y0_mel.append(mel_noise if mel_temperature == 1.0 else mel_noise.mul(mel_temperature)) y0 = pad_sequence(y0, padding_value=0, batch_first=True) y0_mel = pad_sequence(y0_mel, padding_value=0, batch_first=True) # Keep the spectral trajectory prior Gaussian over the full sequence, # matching training where phi_mel is noised at every frame. The clean # prompt Mel is already supplied separately through step_mel_cond. t_start = 0 # duplicate test corner for inner time step oberservation if duplicate_test: t_start = t_inter y0 = (1 - t_start) * y0 + t_start * test_cond y0_mel = (1 - t_start) * y0_mel + t_start * test_mel_cond steps = int(steps * (1 - t_start)) use_epss = use_epss and timestep_mapping == "sway_sampling" if t_start == 0 and use_epss: # use Empirically Pruned Step Sampling for low NFE t = get_epss_timesteps(steps, device=self.device, dtype=torch.float32) else: t = torch.linspace(t_start, 1, steps + 1, device=self.device, dtype=torch.float32) if timestep_mapping == "uniform": pass elif timestep_mapping == "sway_sampling": if sway_sampling_coef is not None: t = t + sway_sampling_coef * (torch.cos(torch.pi / 2 * t) - 1 + t) elif timestep_mapping == "power": if timestep_power is None: raise ValueError("timestep_power must be provided when timestep_mapping='power'") t = t.pow(timestep_power) else: raise ValueError(f"Unknown timestep_mapping: {timestep_mapping}") effective_shift = shift if effective_shift != 1.0: t = t / (t + effective_shift * (1 - t)) # WaveGen inference uses Euler ODE sampling. odeint_kwargs = dict(self.odeint_kwargs) odeint_kwargs["method"] = "euler" # With the supplied grid and linear interpolation Euler evaluates once # per interval. Other grids/interpolants can need additional RHS calls, # so preserve full Mel evaluation for those custom solver settings. solver_options = odeint_kwargs.get("options") or {} skip_final_mel = ( not compute_final_mel and solver_options.get("step_size") is None and solver_options.get("grid_constructor") is None and solver_options.get("interp", "linear") == "linear" and odeint_kwargs.get("event_fn") is None ) final_rhs_index = t.numel() - 2 rhs = fn if inference_method == "Prompt_xt_Anchor": initial = (y0.clone(), y0_mel.clone()) clean = (step_cond, step_mel_cond) prompt = (cond_mask, mel_cond_mask.unsqueeze(-1).expand_as(y0_mel)) def rhs(t, state): # Keep the known prompt on its training interpolation at every # model call; use the original post-temperature Gaussian draws. corrected = tuple( torch.where(prompt[h], (1.0 - t) * initial[h] + t * clean[h], state[h]) for h in range(2) ) velocity = fn(t, corrected) return tuple( torch.where(prompt[h], clean[h] - initial[h], velocity[h]) for h in range(2) ) trajectory = odeint(rhs, (y0, y0_mel), t, **odeint_kwargs) self.transformer.clear_cache() wav_trajectory, _ = trajectory sampled = wav_trajectory[-1] out = sampled out = out / self.latents_scale cond_unscaled = cond / self.latents_scale out = torch.where(cond_mask, cond_unscaled, out) # ---- wav-only: trim to generated waveform duration ---- wav_list = [] for b in range(batch): wav_list.append(out[b, : duration[b].item()]) out = pad_sequence(wav_list, batch_first=True, padding_value=0.0) # [B, N] return out, wav_trajectory def forward( self, inp: float["b nw"], # raw waveform text: int["b nt"] | list[str], *, lens: int["b"] | None = None, noise_scheduler: str | None = None, ): # handle raw waveform if inp.ndim != 2: raise ValueError(f"WaveGen expects raw waveform input [B, N], got {tuple(inp.shape)}") inp = self._pad_wav_to_frame_multiple(inp) batch, seq_len, dtype, device, _sigma1 = *inp.shape[:2], inp.dtype, self.device, self.sigma # handle text as string if isinstance(text, list): if exists(self.vocab_char_map): text = list_str_to_idx(text, self.vocab_char_map).to(device) else: text = list_str_to_tensor(text).to(device) assert text.shape[0] == batch elif torch.is_tensor(text): text = text.to(device) # lens and mask if not exists(lens): # if lens not acquired by trainer from collate_fn lens = torch.full((batch,), seq_len, device=device) else: lens = lens.to(device=device, dtype=torch.long) mask = lens_to_mask(lens, length=seq_len) # get a random span to mask out for training conditionally frac_lengths = torch.zeros((batch,), device=self.device).float().uniform_(*self.frac_lengths_mask) if self.mask_align_to > 1: rand_span_mask = MelSpectrogramLoss._aligned_random_span_mask( lengths=lens, frac_lengths=frac_lengths, align_to=self.mask_align_to, max_length=seq_len, ) else: rand_span_mask = mask_from_frac_lengths(lens, frac_lengths) if exists(mask): rand_span_mask &= mask # x1 is raw waveform x1 = inp mel_frames = seq_len // self.wav_frame_len x1_mel = self._mel_from_wav(x1, target_frames=mel_frames) x1 = x1 * self.latents_scale # x0 is gaussian noise x0 = torch.randn_like(x1) x0_mel = torch.randn_like(x1_mel) # time step time = self._sample_time(batch, dtype=dtype, device=self.device) # sample xt (φ_t(x) in the paper) t = time.unsqueeze(-1) φ = (1 - t) * x0 + t * x1 φ_mel = (1 - t.unsqueeze(-1)) * x0_mel + t.unsqueeze(-1) * x1_mel flow = x1 - x0 mel_flow = x1_mel - x0_mel # only predict what is within the random mask span for infilling cond = torch.where(rand_span_mask, torch.zeros_like(x1), x1) mel_rand_span_mask = self._wav_mask_to_mel_mask(rand_span_mask, mel_len=mel_frames) mel_cond = torch.where(mel_rand_span_mask.unsqueeze(-1), torch.zeros_like(x1_mel), x1_mel) # transformer and cfg training with a drop rate if self.joint_cond_drop_prob > 0.0: joint_drop = random() < self.joint_cond_drop_prob drop_audio_cond = joint_drop drop_text = joint_drop else: drop_audio_cond = random() < self.audio_drop_prob # p_drop in voicebox paper if random() < self.cond_drop_prob: # p_uncond in voicebox paper drop_audio_cond = True drop_text = True else: drop_text = False # Modality dropout prevents the shared trunk from always taking a shortcut # through the noised mel trajectory. Keep the original φ_mel for prediction # interpretation and targets; only its transformer input copy is dropped. mel_xt_drop_mask = torch.zeros(batch, dtype=torch.bool, device=device) transformer_mel = φ_mel if self.mel_xt_drop_prob > 0.0: mel_xt_drop_mask = torch.rand(batch, device=device) < self.mel_xt_drop_prob transformer_mel = φ_mel.masked_fill(mel_xt_drop_mask[:, None, None], 0.0) # apply mask will use more memory; might adjust batchsize or batchsampler long sequence threshold raw_pred, *_ = self.transformer( x=φ, cond=cond, mel=transformer_mel, mel_cond=mel_cond, text=text, time=time, drop_audio_cond=drop_audio_cond, drop_text=drop_text, mask=mask, lens=lens, ) pred_wav_raw, pred_mel_raw = self._unpack_transformer_pred(raw_pred) # interpret prediction if self.prediction == "flow": v_pred, mel_v_pred = pred_wav_raw, pred_mel_raw x_pred = φ + (1.0 - t) * v_pred mel_x_pred = φ_mel + (1.0 - t.unsqueeze(-1)) * mel_v_pred elif self.prediction == "x_pred": x_pred, mel_x_pred = pred_wav_raw, pred_mel_raw v_pred = self._x_to_v(x_pred, φ, time) mel_v_pred = self._x_to_v(mel_x_pred, φ_mel, time.unsqueeze(-1)) else: raise ValueError(f"Unknown prediction: {self.prediction}") # loss space if self.loss_space == "flow": loss = F.mse_loss(v_pred, flow, reduction="none") sf_loss = F.mse_loss(mel_v_pred, mel_flow, reduction="none") elif self.loss_space == "v": # v-loss (same target flow, but v_pred computed from x_pred) & use clamp_min denom = (1.0 - time).clamp_min(self.t_eps) while denom.ndim < φ.ndim: denom = denom.unsqueeze(-1) target = (x1 - φ) / denom mel_target = (x1_mel - φ_mel) / denom.unsqueeze(-1) loss = F.mse_loss(v_pred, target, reduction="none") sf_loss = F.mse_loss(mel_v_pred, mel_target, reduction="none") elif self.loss_space == "x": loss = F.mse_loss(x_pred, x1, reduction="none") sf_loss = F.mse_loss(mel_x_pred, x1_mel, reduction="none") else: raise ValueError(f"Unknown loss_space: {self.loss_space}") loss = loss[rand_span_mask] flow_loss = loss.mean() total_loss = flow_loss # A dropped sample trains the waveform path without spectral forcing. # Excluding it here also keeps flow-prediction modes from being asked to # recover the independently sampled x0_mel without observing φ_mel. sf_loss_mask = mel_rand_span_mask & ~mel_xt_drop_mask[:, None] sf_loss = sf_loss[sf_loss_mask] sf_loss = sf_loss.mean() if sf_loss.numel() > 0 else flow_loss.new_zeros(()) total_loss = total_loss + sf_loss * self.sf_loss_weight aux_mel_loss = torch.tensor(0.0, device=device) if self.use_aux_mel_loss and self.aux_mel_loss is not None: x1_flat_unscaled = x1 / self.latents_scale x1_pred_flat_unscaled = x_pred / self.latents_scale aux_mel_kwargs = {} if self.aux_mel_loss_masked: aux_mel_kwargs.update( frame_mask=rand_span_mask, frame_lengths=lens, ) aux_mel_loss = self.aux_mel_loss( x1_pred_flat_unscaled, x1_flat_unscaled, **aux_mel_kwargs, ) total_loss = total_loss + aux_mel_loss loss_dict = { "total_loss": total_loss, "flow_loss": flow_loss, "aux_mel_loss": aux_mel_loss, "sf_loss": sf_loss, "mel_xt_drop_fraction": mel_xt_drop_mask.float().mean(), } return total_loss, cond, v_pred, loss_dict