diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..26c743c4513fd7c8e970697a9bc297f359a72b57 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.pyc +.DS_Store +*.pyo +*.egg-info/ +.eggs/ +dist/ +build/ +*.log +outputs/ +.hydra/ +wandb/ diff --git a/README.md b/README.md index 178f8422f7143ba28930e3e2f4b3bf6d27540883..d1c11f4c72ac1c2a9aaec25a1a1f5c434db13652 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,28 @@ --- title: MEGAMI -emoji: 🦀 +emoji: 🎚 colorFrom: yellow colorTo: gray sdk: gradio -sdk_version: 6.19.0 -python_version: '3.13' +sdk_version: 5.28.0 +python_version: "3.10" app_file: app.py pinned: false license: cc-by-nc-sa-4.0 short_description: Automatic Mixing via Generative Model of Effect Embeddings --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# MEGAMI — Automatic Music Mixing + +Upload 2–6 **dry (unprocessed) mono/stereo instrument stems** (at least ~12 s each) and MEGAMI will generate a professionally mixed stereo output. + +**Requirements:** +- Each track must be at least 11.9 s long (525 312 samples @ 44 100 Hz) +- Supported formats: WAV, FLAC, MP3 (auto-resampled to 44 100 Hz) +- At least 2 tracks required + +**How it works:** +1. MEGAMI’s diffusion model (FxGenerator) generates audio-effect embeddings from the stems +2. The effect processor (FxProcessor) applies those embeddings to produce a polished stereo mix + +Paper: [arxiv.org/abs/2511.08040](https://arxiv.org/abs/2511.08040) | License: CC BY-NC-SA 4.0 diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e3c20909f75265e97b818b5df43082722e17d77b --- /dev/null +++ b/app.py @@ -0,0 +1,209 @@ +import os +import shutil +import tempfile +import threading + +import numpy as np +import torch +import torchaudio +import soundfile as sf +import gradio as gr +from pyharp import ModelCard, build_endpoint + +# ---- Model card ---- + +model_card = ModelCard( + name="MEGAMI", + description=( + "Automatic music mixing via a generative model of audio effect embeddings." + "Upload 2–6 dry (unprocessed) instrument stems and MEGAMI will generate " + "a professionally mixed stereo output." + ), + author="Eloi Moliner, Marco A. Martínez-Ramírez, Junghyun Koo, Wei-Hsiang Liao, Kin Wai Cheuk, Joan Serrà, Vesa Välimäki, Yuki Mitsufuji", + tags=["mixing", "music-production", "generative", "diffusion"], +) + +# ---- Background model loading ---- + +_inference = None +_load_error = None +_model_ready = threading.Event() + +MIN_AUDIO_LEN = 525312 # samples @ 44100 Hz ≈ 11.9 s +SAMPLE_RATE = 44100 + + +def _download_checkpoints(): + import urllib.request + from huggingface_hub import hf_hub_download + + os.makedirs("checkpoints", exist_ok=True) + + github_base = "https://github.com/SonyResearch/MEGAMI/releases/download/v0/" + for fname in ["FxGenerator_public.pt", "FxProcessor_public.pt", "CLAP_DA_public.pt"]: + dest = os.path.join("checkpoints", fname) + if not os.path.exists(dest): + print(f"[download] {fname} …") + urllib.request.urlretrieve(github_base + fname, dest) + print(f"[download] {fname} done.") + + # The MEGAMI README cites the original LAION-CLAP file as the source for this checkpoint; + # no patching script is documented — we download the original and save under the expected filename + clap_dest = "checkpoints/music_audioset_epoch_15_esc_90.14.patched.pt" + if not os.path.exists(clap_dest): + print("[download] music_audioset_epoch_15_esc_90.14.patched.pt …") + cached = hf_hub_download( + repo_id="lukewys/laion_clap", + filename="music_audioset_epoch_15_esc_90.14.pt", + ) + shutil.copy(cached, clap_dest) + print("[download] music_audioset_epoch_15_esc_90.14.patched.pt done.") + + fxenc_path = "checkpoints/fxenc_plusplus_default.pt" + if not os.path.exists(fxenc_path): + print("[download] fxenc_plusplus_default.pt …") + cached = hf_hub_download( + repo_id="yytung/fxencoder-plusplus", + filename="fxenc_plusplus_default.pt", + ) + shutil.copy(cached, fxenc_path) + print("[download] fxenc_plusplus_default.pt done.") + + print("[download] All checkpoints ready.") + + +def _load_model(): + global _inference, _load_error + try: + _download_checkpoints() + + import omegaconf + from inference.inference import Inference + + method_args = omegaconf.OmegaConf.create( + { + "FxGenerator_code": "public", + "FxProcessor_code": "public", + "T": 30, + "cfg_scale": 1.0, + "Schurn": 0, + } + ) + _inference = Inference(method_args=method_args) + print("[model] Inference object ready.") + except Exception as exc: + import traceback + _load_error = str(exc) + print(f"[model] Load FAILED: {exc}") + traceback.print_exc() + finally: + _model_ready.set() + + +threading.Thread(target=_load_model, daemon=True).start() + +# ---- Process function ---- + +def process_fn(track1, track2, track3, track4, track5, track6, steps): + _model_ready.wait() + + if _load_error: + raise gr.Error(f"Model failed to load: {_load_error}") + + track_paths = [t for t in [track1, track2, track3, track4, track5, track6] if t is not None] + if len(track_paths) < 2: + raise gr.Error("Please upload at least 2 tracks.") + + from utils.feature_extractors.dsp_features import compute_log_rms_gated_max + + dry_tracks = [] + dry_segments = [] + + for path in track_paths: + audio, file_sr = sf.read(path, dtype="float32") + if audio.ndim == 1: + audio = np.stack([audio, audio], axis=-1) + elif audio.shape[1] == 1: + audio = np.concatenate([audio, audio], axis=-1) + + audio_tensor = torch.from_numpy(audio).T # (channels, samples) + if file_sr != SAMPLE_RATE: + audio_tensor = torchaudio.functional.resample(audio_tensor, file_sr, SAMPLE_RATE) + + x_mono = audio_tensor.mean(dim=0, keepdim=True) # (1, samples) + + if x_mono.shape[-1] < MIN_AUDIO_LEN: + needed_seconds = MIN_AUDIO_LEN / SAMPLE_RATE + got_seconds = x_mono.shape[-1] / SAMPLE_RATE + raise gr.Error( + f"Each track must be at least {needed_seconds:.1f} s long " + f"(got {got_seconds:.1f} s)." + ) + + x_mono = x_mono.to(_inference.device) + segment = _inference.select_high_energy_segment(x_mono) + + dry_tracks.append(x_mono) + dry_segments.append(segment) + + x_dry = torch.stack(dry_tracks) # (N, 1, L) + x_dry_segments = torch.stack(dry_segments) # (N, 1, MIN_AUDIO_LEN) + + rms = compute_log_rms_gated_max(x_dry_segments) + silent = (rms < -60).squeeze() + if silent.ndim == 0: + silent = silent.unsqueeze(0) + if silent.all(): + raise gr.Error("All uploaded tracks appear to be silent.") + if silent.any(): + x_dry = x_dry[~silent] + x_dry_segments = x_dry_segments[~silent] + + _inference.method_args.T = int(steps) # slider returns float, but T is used as a loop bound so casting to int + + embedding_preds = _inference.generate_Fx(x_dry_segments, num_samples=1) + fx_embeddings = _inference.embedding_post_processing(embedding_preds) + fx_embedding = fx_embeddings[0] + + y_final = _inference.apply_effects(x_dry.clone(), fx_embedding) # (N, 2, L) + mix = y_final.sum(dim=0) # (2, L) + + peak = torch.max(torch.abs(mix)).clamp(min=1e-8) # avoiding div by 0 for silent output + mix = mix / peak + + output_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + sf.write( + output_file.name, + mix.cpu().clamp(-1, 1).numpy().T, + SAMPLE_RATE, + subtype="PCM_16", + ) + return output_file.name + + +# ---- Gradio interface ---- + +inputs = [ + gr.Audio(type="filepath", label="Track 1 (required)").harp_required(True), + gr.Audio(type="filepath", label="Track 2 (required)").harp_required(True), + gr.Audio(type="filepath", label="Track 3"), + gr.Audio(type="filepath", label="Track 4"), + gr.Audio(type="filepath", label="Track 5"), + gr.Audio(type="filepath", label="Track 6"), + gr.Slider( + minimum=10, + maximum=100, + step=10, + value=30, + label="Diffusion Steps (higher = better quality, slower)", + ), +] + +outputs = [ + gr.Audio(type="filepath", label="MEGAMI Mix"), +] + +demo = build_endpoint(inputs, outputs, process_fn, model_card) + +if __name__ == "__main__": + demo.queue().launch(pwa=True) diff --git a/conf/conf_FxGenerator_Public.yaml b/conf/conf_FxGenerator_Public.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ea4a3fcea7ff650909682d0fa9f6eaf3967dae7 --- /dev/null +++ b/conf/conf_FxGenerator_Public.yaml @@ -0,0 +1,71 @@ +model_dir: "checkpoints" + +diff_params: + _target_: "diff_params.edm_multitrack_embs.EDM_Multitrack_Embeddings" + type: "ve_karras" + content_encoder_type: "CLAP" + style_encoder_type: "FxEncoder++_DynamicFeatures" + sample_rate: 44100 + cfg_dropout_prob: 0.2 + sde_hp: + sigma_data: 0.025 + sigma_min: 5e-4 + sigma_max: 5 + max_sigma: 5 + rho: 10 + CLAP_args: + ckpt_path: "checkpoints/music_audioset_epoch_15_esc_90.14.patched.pt" + distance_type: "cosine" + normalize: True + use_adaptor: True + adaptor_checkpoint: "checkpoints/CLAP_DA_public.pt" + adaptor_type: "MLP_CLAP_regressor" + add_noise: True + noise_sigma: 0.011 + fx_encoder_plusplus_args: + distance_type: "cosine" + ckpt_path: "checkpoints/fxenc_plusplus_default.pt" + context_signal: "wet" + apply_fxnormaug: False + fxnormaug_train: null + fxnormaug_inference: null + default_shape: [1, 3, 64, 33] + +network: + _target_: "networks.dit_multitrack.DiffusionTransformer" + io_channels: 64 + embed_dim: 512 + depth: 16 + num_heads: 16 + cond_token_dim: 64 + global_cond_dim: 0 + input_concat_dim: 0 + project_cond_tokens: false + transformer_type: "continuous_transformer" + pos_emb_strategy: "concatenation" + pos_emb_dim: 64 + pos_emb_type: "one-hot" + pos_emb_crossattn_strategy: "concatenation" + pos_emb_crossattn_dim: 64 + pos_emb_crossattn_type: "one-hot" + use_taxonomy_in_pos_emb: False + +tester: + checkpoint: null + sampler: + _target_: "inference.sampler_euler_heun_multitrack.SamplerEulerHeun" + sampling_params: + same_as_training: False + sde_hp: + sigma_data: 0.025 + sigma_min: 5e-4 + sigma_max: 5 + rho: 10 + Schurn: 0 + Snoise: 1 + Stmin: 0 + Stmax: 10 + order: 1 + T: 51 + schedule: "edm" + cfg_scale: 1.0 diff --git a/conf/conf_FxProcessor_Public.yaml b/conf/conf_FxProcessor_Public.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0520125b490d1144ee3b0192fb7a28f18763f4c0 --- /dev/null +++ b/conf/conf_FxProcessor_Public.yaml @@ -0,0 +1,40 @@ +model_dir: "checkpoints" + +exp: + apply_fxnorm: True + fxnorm: + _target_: "utils.fx_normalization.fxnorm_v2_public.FxNormAug" + sample_rate: 44100 + device: "cpu" + features_path: "utils/fx_normalization/features.npy" + use_gated_RMSnorm: True + RMS_norm: -16 + +network: + _target_: "networks.blackbox_TCN.TCNModel" + ninputs: 1 + noutputs: 2 + cond_dim: 2112 + nblocks: 14 + kernel_size: 15 + stride: 1 + dilation_growth: 2 + channel_growth: 1 + channel_width: 128 + stack_size: 15 + grouped: False + causal: False + skip_connections: False + use_CLAP: True + CLAP_args: + ckpt_path: "checkpoints/music_audioset_epoch_15_esc_90.14.patched.pt" + distance_type: "cosine" + normalize: True + use_adaptor: False + adaptor_checkpoint: null + adaptor_type: null + add_noise: False + noise_sigma: null + +tester: + checkpoint: null diff --git a/diff_params/__init__.py b/diff_params/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/diff_params/edm_multitrack_embs.py b/diff_params/edm_multitrack_embs.py new file mode 100644 index 0000000000000000000000000000000000000000..a71fc887585a0bb46d204306b666998fee6f4ed2 --- /dev/null +++ b/diff_params/edm_multitrack_embs.py @@ -0,0 +1,432 @@ +# Copyright (c) 2025 Sony Research +# Licensed under CC BY-NC-SA 4.0 +# See LICENSE file for details + +import torch +import torch.nn.functional as F +import sys +import math +import time +import os +import einops +import numpy as np + +from utils.multitrack_utils import multitrack_batched_processing +from utils.data_utils import apply_RMS_normalization + +class EDM_Multitrack_Embeddings: + """ + This implements the time-frequency domain diffusion + Definition of the diffusion parameterization, following ( Karras et al., "Elucidating...", 2022). + This includes only the utilities needed for training, not for sampling. + """ + + def __init__(self, + type, + sde_hp, + default_shape, + cfg_dropout_prob=0.2, + sample_rate=44100, + context_signal="dry", + content_encoder_type="CLAP", + style_encoder_type="fx_encoder2048+AFv6", + *args, + **kwargs + ): + + self.type = type + self.sde_hp = sde_hp + + self.sigma_data = self.sde_hp.sigma_data # depends on the training data!! precalculated variance of the dataset + self.sigma_min = self.sde_hp.sigma_min + self.sigma_max = self.sde_hp.sigma_max + self.rho = self.sde_hp.rho + + self.default_shape = torch.Size(default_shape) + + try: + self.max_t = self.sde_hp.max_sigma + except Exception as e: + print(e) + print("max_sigma not defined, please add it. It should be the highest sigma value seen during training") + + device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.device=device + + self.cfg_dropout_prob = cfg_dropout_prob + + self.context_signal=context_signal + + self.sample_rate=sample_rate + + self.prepare_content_encoder(content_encoder_type, sample_rate, *args, **kwargs) + self.prepare_style_encoder(style_encoder_type, *args, **kwargs) + + def prepare_content_encoder(self, type, sample_rate, *args, **kwargs): + + if type=="CLAP": + CLAP_args= kwargs.get("CLAP_args", None) + assert CLAP_args is not None, "CLAP_args must be provided for CLAP AE" + + # Save original path + from utils.feature_extractors.load_features import load_CLAP + CLAP_encoder= load_CLAP(CLAP_args, device=self.device) + + def encode_fn(x, *args, **kwargs): + x=x.to(self.device) + + type= kwargs.get("type", None) + z=CLAP_encoder(x, type) #shape (B, C) + + z=z.view(z.shape[0], 64, -1) #shape (B, 64, N) + + z=z.permute(0, 2, 1) #shape (B, N, 64) + + return z + + self.content_encode_fn=encode_fn + + else: + raise NotImplementedError(f"AE type {AE_type} not implemented") + + + def prepare_style_encoder(self, type, *args, **kwargs): + + if type=="FxEncoder++_DynamicFeatures": + + Fxencoder_plusplus_args=kwargs.get("fx_encoder_plusplus_args", None) + + from utils.feature_extractors.load_features import load_fx_encoder_plusplus_2048 + feat_extractor = load_fx_encoder_plusplus_2048(Fxencoder_plusplus_args, self.device) + + from utils.feature_extractors.AF_features_embedding import AF_fourier_embedding + AFembedding= AF_fourier_embedding(device=self.device) + + def fxencode_fn(x): + """ + x: tensor of shape [B, C, L] where B is the batch size, C is the number of channels and L is the length of the audio + """ + z=feat_extractor(x) + z=torch.nn.functional.normalize(z, dim=-1, p=2) + z=z*math.sqrt(z.shape[-1]) # rescale to keep the same scale + + z_af,_=AFembedding.encode(x) + z_af=z_af* math.sqrt(z_af.shape[-1]) # rescale to keep the same scale + + + z_all= torch.cat([z, z_af], dim=-1) + + #now L2 normalize + + norm_z= z_all/ math.sqrt(z_all.shape[-1]) # normalize by dividing by sqrt(dim) to keep the same scale + + norm_z=norm_z.view(norm_z.shape[0], 64, -1) # Reshape to [B, 64, L//64] where L//64 is the number of frames + + return norm_z + + def reshape_fn(embed): + """ + embed: tensor of shape [B, 64, L//64] where B is the batch size + """ + embed=embed.view(embed.shape[0], -1) + + return embed + + + self.style_encode_fn=fxencode_fn + self.style_reshape=reshape_fn + + else: + raise NotImplementedError(f"FX encoder type {type} not implemented") + + def sample_time_training(self, N): + """ + For training, getting t according to a similar criteria as sampling. Simpler and safer to what Karras et al. did + Args: + N (int): batch size + """ + a = torch.rand(N) + t = (self.sigma_max**(1/self.rho) +a *(self.sigma_min**(1/self.rho) - self.sigma_max**(1/self.rho)))**self.rho + + return t + + def sample_prior(self, shape=None, t=None, dtype=None): + """ + Just sample some gaussian noise, nothing more + Args: + shape (tuple): shape of the noise to sample, something like (B,T) + """ + assert shape is not None + if t is not None: + n = torch.randn(shape).to(t.device) * t + else: + n = torch.randn(shape) + return n + + def cskip(self, sigma): + """ + Just one of the preconditioning parameters + Args: + sigma (float): noise level (equal to timestep is sigma=t, which is our default) + + """ + return self.sigma_data**2 *(sigma**2+self.sigma_data**2)**-1 + + def cout(self, sigma): + """ + Just one of the preconditioning parameters + Args: + sigma (float): noise level (equal to timestep is sigma=t, which is our default) + """ + return sigma*self.sigma_data* (self.sigma_data**2+sigma**2)**(-0.5) + + def cin(self, sigma): + """ + Just one of the preconditioning parameters + Args: + sigma (float): noise level (equal to timestep is sigma=t, which is our default) + """ + return (self.sigma_data**2+sigma**2)**(-0.5) + + def cnoise(self, sigma): + """ + preconditioning of the noise embedding + Args: + sigma (float): noise level (equal to timestep is sigma=t, which is our default) + """ + return (1/4)*torch.log(sigma) + + def lambda_w(self, sigma): + """ + Score matching loss weighting + """ + return (sigma*self.sigma_data)**(-2) * (self.sigma_data**2+sigma**2) + + def prepare_train_preconditioning(self, x, t, n=None, *args, **kwargs): + + mu, sigma = self._mean(x, t), self._std(t).unsqueeze(-1) + sigma = sigma.view(*sigma.size(), *(1,)*(x.ndim - sigma.ndim)) + if n is None: + n=self.sample_prior(shape=x.shape).to(x.device) + x_perturbed = mu + sigma *n + #self.sample_prior(x.shape).to(x.device) + + cskip = self.cskip(sigma) + cout = self.cout(sigma) + cin = self.cin(sigma) + cnoise = self.cnoise(sigma.squeeze()) + + #check if cnoise is a scalar, if so, repeat it + if len(cnoise.shape) == 0: + cnoise = cnoise.repeat(x.shape[0],) + else: + cnoise = cnoise.view(x.shape[0],) + + target = 1/cout * (x - cskip * x_perturbed) + + return cin * x_perturbed, target, cnoise + + def loss_fn(self, net, sample=None, sample_aug=None, context=None, clusters=None, taxonomy=None, masks=None, *args, **kwargs): + """ + Loss function, which is the mean squared error between the denoised latent and the clean latent + Args: + net (nn.Module): Model of the denoiser + x (Tensor): shape: (B,T) Intermediate noisy latent to denoise + sigma (float): noise level (equal to timestep is sigma=t, which is our default) + """ + + start=time.time() + y=sample + + t = self.sample_time_training(y.shape[0]).to(y.device) + + if self.context_signal == "wet": + if sample_aug is not None: + context = sample_aug + else: + context = y.clone() # use the wet signal as context + + else: + assert context is not None, "Context must be provided if context_signal is not 'wet'" + + a=time.time + + with torch.no_grad(): + + y_style=self.style_encode(y, masks=masks, taxonomy=taxonomy) + + if context is not None: + z, x=self.transform_forward(context, is_condition=True, clusters=clusters, masks=masks, taxonomy=taxonomy, is_wet=(self.context_signal == "wet")) + if self.cfg_dropout_prob > 0.0: + null_embed = torch.zeros_like(z, device=z.device) + #dropout context with probability cfg_dropout_prob + mask = torch.rand(z.shape[0], device=z.device) < self.cfg_dropout_prob + z = torch.where(mask.view(-1,1,1,1), null_embed, z) + + + input, target, cnoise = self.prepare_train_preconditioning(y_style, t ) + + + if len(cnoise.shape)==1: + cnoise=cnoise.unsqueeze(-1) + if input.ndim==2: + input=input.unsqueeze(1) + + estimate = net(input, cnoise, cross_attn_cond=z, taxonomy=taxonomy, mask=masks, cross_attn_cond_mask=masks) + + if target.ndim==2 and estimate.ndim==3: + estimate=estimate.squeeze(1) + + error=torch.square(torch.abs(estimate-target)) + + # do not propagate the error of the padded tracks + error= error*masks.view(masks.shape[0], masks.shape[1], 1, 1) + + compensating_scalar= torch.numel(masks)/ torch.sum(masks, dim=(0,1), keepdim=False).clamp(min=1.0) + error= error * compensating_scalar.view(-1, 1, 1, 1) + + return error, self._std(t), x, y + + + def get_null_embed(self, context): + null_embed = torch.zeros_like(context, device=context.device) + return null_embed + + + def denoiser(self, xn , net, t, cond=None,cfg_scale=1.0, masks=None, taxonomy=None, **kwargs): + """ + This method does the whole denoising step, which implies applying the model and the preconditioning + Args: + x (Tensor): shape: (CQT shape?) Intermediate noisy latent to denoise + model (nn.Module): Model of the denoiser + sigma (float): noise level (equal to timestep is sigma=t, which is our default) + """ + sigma = self._std(t).unsqueeze(-1) + sigma = sigma.view(*sigma.size(), *(1,)*(xn.ndim - sigma.ndim)) + + cskip = self.cskip(sigma) + cout = self.cout(sigma) + cin = self.cin(sigma) + cnoise = self.cnoise(sigma.squeeze()) + + #check if cnoise is a scalar, if so, repeat it + if len(cnoise.shape) == 0: + cnoise = cnoise.repeat(xn.shape[0],).unsqueeze(-1) + else: + cnoise = cnoise.view(xn.shape[0],).unsqueeze(-1) + + + x_in=cin*xn + + if cfg_scale == 1.0: + net_out=net(x_in, cnoise.to(torch.float32), cross_attn_cond=cond, mask=masks, taxonomy=taxonomy, cross_attn_cond_mask=masks) #this will crash because of broadcasting problems, debug later! + else: + null_embed = self.get_null_embed(cond) + + inputs_cond= torch.cat([cond, null_embed], dim=0) + + x_in_cat= torch.cat([x_in, x_in], dim=0) + + cnoise= torch.cat([cnoise, cnoise], dim=0) + + masks_in= torch.cat([masks, masks], dim=0) if masks is not None else None + + + net_out_batch=net(x_in_cat, cnoise.to(torch.float32), cross_attn_cond=inputs_cond , mask=masks_in, cross_attn_cond_mask=masks_in) #this will crash because of broadcasting problems, debug later!0 + + cond_output, uncond_output = torch.chunk(net_out_batch, 2, dim=0) + + net_out = uncond_output + (cond_output - uncond_output) * cfg_scale + + + x_hat=cskip*xn + cout*net_out + x_hat=x_hat* masks.view(masks.shape[0], masks.shape[1], 1, 1) + + return x_hat + + def style_encode(self, x, masks=None, taxonomy=None, use_adaptor=False): + """ + Encode the input audio using the style encoder + Args: + x (Tensor): shape: (B,N, C, T) Audio to encode + masks (Tensor): shape: (B, N) Mask indicating which tracks are present in the batch + """ + def apply_fxenc(x_masked, taxonommy=None): + x_emb=self.style_encode_fn(x_masked) + + return x_emb + + assert masks is not None, "masks must be provided for style encoding" + + output_emb=multitrack_batched_processing( x, taxonomy=taxonomy ,function=apply_fxenc, class_dependent=False, masks=masks) + + return output_emb + + def _mean(self, x, t): + return x + + def _std(self, t): + return t + + def _ode_integrand(self, x, t, score): + return -t * score + + def transform_inverse(self, z): + #shape is (B, N, C, T) + B, N, C, T = z.shape + # Reshape z to (B*N, C, T) + z_reshaped = einops.rearrange(z, "b n c t -> (b n) c t") + z_reshaped= self.style_reshape(z_reshaped) + + # Reshape back to (B, N, C) + + z_out = einops.rearrange(z_reshaped, "(b n) c -> b n c", b=B, n=N) + + return z_out + + def preprocessor(self, x, is_test=False, taxonomy=None): + """ + x: tensor of shape (BxN, C, T) where B is the batch size, N is the number of tracks, C is the number of channels and T is the number of time steps + taxonomy: list of lists of strings, where each string is the taxonomy of the track with length BxN. It may be useful if we want to apply different augmentations depending on the taxonomy of the track. + """ + if x.shape[1] == 2: + x = torch.mean(x, dim=1, keepdim=True).expand(-1, 2, -1) # convert to stereo if it is mono + elif x.shape[1] == 1: # if context is mono, we expand it to stereo + x = x.expand(-1, 2, -1) + + if not is_test: + #random flip + if np.random.rand() > 0.5: + x = -x + + #rms normalize context to -25 dB + x= apply_RMS_normalization(x, -25, device=self.device) + + return x + + def Tweedie2score(self, tweedie, xt, t, *args, **kwargs): + return (tweedie - self._mean(xt, t)) / self._std(t)**2 + + def score2Tweedie(self, score, xt, t, *args, **kwargs): + return self._std(t)**2 * score + self._mean(xt, t) + + def transform_forward(self, x, y=None, is_condition=False, is_test=False, clusters=None, masks=None, taxonomy=None, is_wet=False): + + assert masks is not None + + #convert y to mono and rms normalize itdd + + def prerprocess_and_encode(x_masked, taxonomy=None): + if is_condition: + + x_masked=self.preprocessor(x_masked, is_test=is_test, taxonomy=taxonomy) + + with torch.no_grad(): + x_emb=self.content_encode_fn(x_masked, type="wet" if is_wet else "dry") + + return x_emb, x_masked + + z, x_out=multitrack_batched_processing( + x, taxonomy=taxonomy, function=prerprocess_and_encode, class_dependent=False, masks=masks, number_outputs=2 + ) + return z, x_out diff --git a/inference/__init__.py b/inference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/inference/inference.py b/inference/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..078c221c3f953dbd06e4bf2a3d756b6fc1fbe524 --- /dev/null +++ b/inference/inference.py @@ -0,0 +1,517 @@ +# Copyright (c) 2025 Sony Research +# Licensed under CC BY-NC-SA 4.0 +# See LICENSE file for details + +import sys +import os +from utils.feature_extractors.dsp_features import compute_log_rms_gated_max + +import pyloudnorm as pyln + +import hydra +import torch +import torchaudio +from hydra import initialize, compose +import utils.training_utils as tr_utils + +import torch +import omegaconf +import math + +import soundfile as sf +from utils.data_utils import apply_RMS_normalization + +import numpy as np +import glob + +from utils.data_utils import read_wav_segment + + +def load_audio( file, start=None, end=None, stereo=True): + + x, fs=read_wav_segment(file, start, end) + if stereo: + if len(x.shape)==1: + #print( "dry not stereo , doubling channels", x_dry.shape) + x=x[:,np.newaxis] + x= np.concatenate((x, x), axis=-1) + elif len(x.shape)==2 and x.shape[-1]==1: + #print( "dry not stereo , doubling channels", x_dry.shape) + x = np.concatenate((x, x), axis=-1) + + x=torch.from_numpy(x).permute(1,0) + + return x, fs + + +class Inference: + def __init__( + self, + method_args=None, + path_benchmark="/add/the/path/to/the/benchmark/data/here", + load_segment_length=525312, #segment length used for loading and extract CLAP embeddings + processor_segment_length=525312, #segment length used for loading and extract CLAP embeddings + processor_overlap=8192, + ): + + self.method_args = method_args + self.path_benchmark = path_benchmark + self.load_segment_length=load_segment_length + self.processor_segment_length=processor_segment_length + self.processor_overlap=processor_overlap + + self.FxGenerator_code = method_args.FxGenerator_code + self.FxProcessor_code = method_args.FxProcessor_code + + self.config_file_rel = "../conf" + # self.config_path="/home/eloi/projects/project_mfm_eloi/src/conf" + + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + self.load_FxGenerator() # Load the S1 model + self.load_FxProcessor() # Load the S2 model + self.prepare_feature_extractors() # Prepare the feature extractors + + def load_FxGenerator(self): + if self.FxGenerator_code == "public": + config_name = "conf_FxGenerator_Public.yaml" + model_dir = "checkpoints" + ckpt = "FxGenerator_public.pt" + else: + raise ValueError(f"Unknown FxGenerator_code: {self.FxGenerator_code}") + + overrides = [ + f"model_dir={model_dir}", + f"tester.checkpoint={ckpt}", + ] + + with initialize(version_base=None, config_path=self.config_file_rel): + args = compose(config_name=config_name, overrides=overrides) + + if not os.path.exists(args.model_dir): + raise Exception(f"Model directory {args.model_dir} does not exist") + + diff_params = hydra.utils.instantiate(args.diff_params) + + network = hydra.utils.instantiate(args.network) + network = network.to(self.device) + state_dict = torch.load( + os.path.join(args.model_dir, args.tester.checkpoint), + map_location=self.device, + weights_only=False, + ) + + tr_utils.load_state_dict(state_dict, ema=network) + + self.sampler = hydra.utils.instantiate( + args.tester.sampler, + network, + diff_params, + args, + ) + + def load_FxProcessor(self): + + ### Loading effects model ### + + if self.FxProcessor_code == "public": + config_name = "conf_FxProcessor_Public.yaml" + model_dir = "checkpoints" + ckpt = "FxProcessor_public.pt" + else: + raise ValueError(f"Unknown FxProcessor_code: {self.FxProcessor_code}") + + overrides = [ + f"model_dir={model_dir}", + f"tester.checkpoint={ckpt}", + ] + + with initialize(version_base=None, config_path=self.config_file_rel): + args = compose(config_name=config_name, overrides=overrides) + + if not os.path.exists(args.model_dir): + raise Exception(f"Model directory {args.model_dir} does not exist") + + fx_model = hydra.utils.instantiate(args.network) + self.fx_model = fx_model.to(self.device) + state_dict = torch.load( + os.path.join(args.model_dir, args.tester.checkpoint), + map_location=self.device, + weights_only=False, + ) + + tr_utils.load_state_dict(state_dict, network=fx_model) + + if args.exp.apply_fxnorm: + print("Applying fx_normalizer") + if "public" in self.FxProcessor_code: + fx_normalizer = hydra.utils.instantiate(args.exp.fxnorm, device=str(self.device)) + self.fx_normalizer = lambda x: fx_normalizer( + x, use_gate=args.exp.use_gated_RMSnorm, RMS=args.exp.RMS_norm + ) + else: + self.fx_normalizer = hydra.utils.instantiate(args.exp.fxnorm) + + else: + print("No fx_normalizer specified, using identity function") + self.fx_normalizer = ( + lambda x: x + ) # identity function if no fx_normalizer is specified + + + def prepare_feature_extractors(self): + + ### preparing feature extractor ### + + Fxencoder_kwargs = omegaconf.OmegaConf.create( + { + "ckpt_path": "checkpoints/fxenc_plusplus_default.pt" + } + ) + + from utils.feature_extractors.load_features import load_fx_encoder_plusplus_2048 + + feat_extractor = load_fx_encoder_plusplus_2048(Fxencoder_kwargs, self.device) + + from utils.feature_extractors.AF_features_embedding import AF_fourier_embedding + + AFembedding = AF_fourier_embedding(device=self.device) + + def FxEnc(x): + """ + x: tensor of shape [B, C, L] where B is the batch size, C is the number of channels and L is the length of the audio + """ + z = feat_extractor(x) + z = torch.nn.functional.normalize( + z, dim=-1, p=2 + ) # normalize to unit variance + z = z * math.sqrt(z.shape[-1]) # rescale to keep the same scale + + z_af, _ = AFembedding.encode(x) + z_af = z_af * math.sqrt(z_af.shape[-1]) # rescale to keep the same scale + + z_all = torch.cat([z, z_af], dim=-1) + + # now L2 normalize + + norm_z = z_all / math.sqrt( + z_all.shape[-1] + ) # normalize by dividing by sqrt(dim) to keep the same scale + + return norm_z + + self.FxEnc = FxEnc + + def embedding_post_processing(z): + """ + L2 normalize each of the features in z + """ + z_fxenc = z[ + ..., :2048 + ] # assuming the FxEncoder features are the first 2048 dimensions + z_af = z[ + ..., 2048: + ] # assuming the AF features are the last 2048 dimensions + + z_fxenc = torch.nn.functional.normalize( + z_fxenc, dim=-1, p=2 + ) # normalize to unit variance + z_af = torch.nn.functional.normalize(z_af, dim=-1, p=2) + + z_fxenc = z_fxenc * math.sqrt( + z_fxenc.shape[-1] + ) # rescale to keep the same scale + z_af = z_af * math.sqrt(z_af.shape[-1]) # rescale to + + z_all = torch.cat([z_fxenc, z_af], dim=-1) + + return z_all / math.sqrt( + z_all.shape[-1] + ) # normalize by dividing by sqrt(dim) to keep the same scale + + self.embedding_post_processing = embedding_post_processing + + def get_log_rms_from_z(z): + + z = z * math.sqrt(z.shape[-1]) # rescale to keep the same scale + AF = z[..., 2048:] # assuming the AF features are the last 2048 dimensions + AF = AF / math.sqrt(AF.shape[-1]) # normalize to unit variance + + features = AFembedding.decode(AF) + log_rms = features[0] + + return log_rms + + def generate_Fx( + x, input_type="dry", num_samples=1, T=30, cfg_scale=1.0, Schurn=10 + ): + N, C, L = ( + x.shape + ) # B is the batch size, N is the number of tracks, C is the number of channels and L is the length of the audio + B = 1 + + shape = self.sampler.diff_params.default_shape + shape = [ + num_samples, + N, + *shape[2:], + ] # B is the batch size, we want to sample B samples + + masks_fwd = torch.ones( + (B, N), dtype=torch.bool, device=self.device + ) # Create masks for all tracks, assuming all tracks are present + masks_diff = torch.ones( + (num_samples, N), dtype=torch.bool, device=self.device + ) # Create masks for all tracks, assuming all tracks are present + + self.sampler.T = T + self.sampler.Schurn = Schurn # Set the Schurn parameter for the sampler + + with torch.no_grad(): + is_wet = "wet" in input_type + cond, x_preprocessed = self.sampler.diff_params.transform_forward( + x.unsqueeze(0), + is_condition=True, + is_test=True, + masks=masks_fwd, + is_wet=is_wet, + ) + cond = cond.expand( + shape[0], -1, -1, -1 + ) # Expand the condition to match the batch size + preds, noise_init = self.sampler.predict_conditional( + shape, + cond=cond.contiguous(), + cfg_scale=cfg_scale, + device=self.device, + masks=masks_diff, + ) + + return preds + + self.generate_Fx = lambda x, num_samples: generate_Fx( + x, + input_type="wet", + num_samples=num_samples, + T=self.method_args.T, + cfg_scale=self.method_args.cfg_scale, + Schurn=self.method_args.Schurn, + ) + + + def apply_rms(y_hat, z_pred): + """ + Apply RMS normalization to the generated audio y_hat based on the predicted features z_pred. + """ + pred_logrms = get_log_rms_from_z( + z_pred + ) # get the log RMS from the generated features + pred_rms = 10 ** (pred_logrms / 20) # convert log RMS to linear scale + + log_rms_y_hat = compute_log_rms_gated_max( + y_hat, sample_rate=44100, threshold=-60 + ) + + rms_y_hat = 10 ** (log_rms_y_hat / 20) # convert log RMS to linear scale + + gain = pred_rms / ( + rms_y_hat + 1e-6 + ) # Compute the gain to apply to the generated audio + + y_final = y_hat * gain.unsqueeze(-1) + + return y_final + + self.apply_rms = apply_rms + + def apply_effects(x, z_pred): + segment_length = self.processor_segment_length + overlap = self.processor_overlap + total_length = x.shape[-1] + batch_size = x.shape[0] + + # Normalize input and conditioning outside block loop + x_norm = x.mean(dim=1, keepdim=True) + + if total_length > segment_length: + y_final = torch.zeros((batch_size, 2, total_length), device=x.device, dtype=x.dtype) + + hann = torch.hann_window(overlap * 2, device=x.device, dtype=x.dtype) + hann_left = hann[:overlap].view(1, 1, -1) + hann_right = hann[overlap:].view(1, 1, -1) + + step = segment_length - overlap + positions = list(range(0, total_length - overlap, step)) + for i, start in enumerate(positions): + end = min(start + segment_length, total_length) + + seg_x_norm = x_norm[..., start:end] + + #check activity in seg_x_norm + + rms_dry_segment=compute_log_rms_gated_max(seg_x_norm, sample_rate=44100) # Compute the log RMS of the dry audio + indices_non_silent = torch.where(rms_dry_segment > -45)[0] # Identify silent tracks + + + seg_x_norm_non_silent = seg_x_norm[indices_non_silent] + z_pred_non_silent = z_pred[indices_non_silent] + + if "public" in self.FxProcessor_code: + seg_x_norm_non_silent = self.fx_normalizer(seg_x_norm_non_silent) + else: + seg_x_norm_non_silent = apply_RMS_normalization(seg_x_norm_non_silent, -25.0, device=self.device, use_gate=True) + seg_x_norm_non_silent = self.fx_normalizer(seg_x_norm_non_silent, use_gate=True) + + with torch.no_grad(): + seg_y_hat_non_silent=torch.zeros((seg_x_norm_non_silent.shape[0],2, seg_x_norm_non_silent.shape[2]), device=x.device, dtype=x.dtype) + #I thought it may be better (but less efficient) to run it like this instead of in parallel. To avoid OOM issues. + for i in range(seg_x_norm_non_silent.shape[0]): + seg_y_hat_non_silent[i] = self.fx_model(seg_x_norm_non_silent[i].unsqueeze(0), z_pred_non_silent[i].unsqueeze(0)).squeeze(0) + + seg_y_hat_non_silent = apply_rms(seg_y_hat_non_silent, z_pred_non_silent) + + #fill with zeros the silent segments + seg_y_hat= torch.zeros((seg_x_norm.shape[0], seg_y_hat_non_silent.shape[1], seg_y_hat_non_silent.shape[2]), device=x.device, dtype=x.dtype) + seg_y_hat[indices_non_silent]=seg_y_hat_non_silent + + + seg_len = end - start + + if i == 0: + # First segment + y_final[..., start:end-overlap] += seg_y_hat[..., :seg_len-overlap] + y_final[..., end-overlap:end] += seg_y_hat[..., seg_len-overlap:] * hann_right + elif end == total_length: + # Last segment + y_final[..., start:start+overlap] += seg_y_hat[..., :overlap] * hann_left + y_final[..., start+overlap:end] += seg_y_hat[..., overlap:] + else: + # Middle segments + y_final[..., start:start+overlap] += seg_y_hat[..., :overlap] * hann_left + y_final[..., start+overlap:end-overlap] += seg_y_hat[..., overlap:seg_len-overlap] + y_final[..., end-overlap:end] += seg_y_hat[..., seg_len-overlap:] * hann_right + + return y_final + + else: + with torch.no_grad(): + y_hat=torch.zeros((x_norm.shape[0], 2, x_norm.shape[2]), device=x.device, dtype=x.dtype) + for i in range(x_norm.shape[0]): + y_hat[i] = self.fx_model(x_norm[i].unsqueeze(0), z_pred[i].unsqueeze(0)).squeeze(0) + y_final = apply_rms(y_hat, z_pred) + return y_final + + self.apply_effects = apply_effects + + def select_high_energy_segment(self, x_dry, seq_length=525312): + C, L = x_dry.shape + + track = x_dry + + # Calculate energy for windows of size seq_length + num_windows = L - seq_length + 1 + max_energy = 0 + max_energy_start = 0 + + for i in range(0, num_windows, 1000): # Step by 1000 for efficiency + segment = track[..., i:i+seq_length] + energy = (segment ** 2).sum() + + if energy > max_energy: + max_energy = energy + max_energy_start = i + + # Fine-tune search around the best region + fine_start = max(0, max_energy_start - 1000) + fine_end = min(L - seq_length + 1, max_energy_start + 1000) + + for i in range(fine_start, fine_end): + segment = track[..., i:i+seq_length] + energy = (segment ** 2).sum() + + if energy > max_energy: + max_energy = energy + max_energy_start = i + + return x_dry[:, max_energy_start:max_energy_start+seq_length] + + + def run_inference_single_song(self, exp_name="test_Sep28", directory=None, num_samples=1): + """ + Run the inference on a single example + """ + + dry_files=glob.glob(os.path.join(directory, "*.wav")) + + assert len(dry_files) > 0, f"No .wav files found in {directory}" + print(f"Found {len(dry_files)} dry files in {directory}") + print(dry_files) + + dry_tracks=[] + dry_tracks_segments=[] + + for f in dry_files: + x_dry_i, fs=load_audio(str(f), stereo=True) + x_dry_i=x_dry_i.to(self.device) + + if fs!=self.sampler.diff_params.sample_rate: + x_dry_i=torchaudio.functional.resample(x_dry_i, orig_sr=fs, target_sr=self.sampler.diff_params.sample_rate) + #stereo to mono + x_dry_i = x_dry_i.mean(dim=0, keepdim=True) + dry_tracks.append(x_dry_i ) + + if x_dry_i.shape[-1] >= self.load_segment_length: + # search for each track the segment of seq_length size that has highest energy + # x_dry_i shape is (N, C, L) where N is the number of tracks, C is the number of channels and L is the length of the audio + x_dry_i_segment = self.select_high_energy_segment(x_dry_i, seq_length=self.load_segment_length) + else: + raise ValueError(f"Input audio {f} is too short, needs to be at least {self.load_segment_length/self.sampler.diff_params.sample_rate:.2f} seconds.") + + dry_tracks_segments.append(x_dry_i_segment) + + x_dry=torch.stack(dry_tracks, dim=0) # shape (N, C, L) where N is the number of tracks, C is the number of channels and L is the length of the audio + x_dry_segments= torch.stack(dry_tracks_segments, dim=0) # shape (N, C, L) where N is the number of tracks, C is the number of channels and L is the length of the audio + + #first check if all tracks in x_dry have activity (RMS > -60 dBFS) + rms_dry=compute_log_rms_gated_max(x_dry_segments, sample_rate=44100) # Compute the log RMS of the dry audio + silent_tracks = rms_dry < -60 # Identify silent tracks + silent_tracks = silent_tracks.squeeze() # Remove singleton dimensions + + if silent_tracks.any(): + print(f"Removing {silent_tracks.sum()} silent tracks from the input audio.") + #shape before removing silent tracks is (N, C, L) + x_dry = x_dry[~silent_tracks] # Remove silent tracks + x_dry_segments = x_dry_segments[~silent_tracks] # Remove silent tracks + + preds = self.generate_Fx(x_dry_segments, num_samples) + + z_pred = self.embedding_post_processing( + preds + ) # post-process the generated features + + del self.sampler + + for i in range(num_samples): + z_i = z_pred[ + i + ] # Randomly sample 100 features from the generated features + + y_final = self.apply_effects( + x_dry.clone(), z_i + ) # Apply the effects to the input audio + y_hat_mixture = y_final.sum(dim=0, keepdim=False) + + # peak normalization of y_hat_mixture + peak = torch.max(torch.abs(y_hat_mixture)) + y_hat_mixture /= peak # Normalize the audio to [-1, 1] + + filename="MEGAMI_inference"+f"_sample{i}.wav" + os.makedirs(f"{directory}/{exp_name}", exist_ok=True) + sf.write( + f"{directory}/{exp_name}/{filename}", + y_hat_mixture.cpu().clamp(-1, 1).numpy().T, + 44100, + subtype="PCM_16", + ) + + diff --git a/inference/sampler.py b/inference/sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..b02eeaf5555f41cc12f314be62e71300d9046dcf --- /dev/null +++ b/inference/sampler.py @@ -0,0 +1,29 @@ + +import omegaconf +import os + + +class Sampler(): + + def __init__(self, model, diff_params, args): + + self.model = model.eval() #is it ok to do this here? + self.diff_params = diff_params #same as training, useful if we need to apply a wrapper or something + self.args=args + if self.args.tester.sampling_params.same_as_training: + self.sde_hp = diff_params.sde_hp + else: + self.sde_hp = self.args.tester.sampling_params.sde_hp + + self.T = self.args.tester.sampling_params.T + self.step_counter = 0 + + + #def setup_wandb(self): + # config=omegaconf.OmegaConf.to_container( + # self.args, resolve=True, throw_on_missing=True + # ) + # self.wandb_run=wandb.init(project=self.args.logging.wandb.project, entity=self.args.logging.wandb.entity, config=config) + # self.wandb_run.name=self.args.tester.wandb.run_name +os.path.basename(self.args.model_dir)+"_"+self.args.exp.exp_name+"_"+self.wandb_run.id + + diff --git a/inference/sampler_euler_heun_multitrack.py b/inference/sampler_euler_heun_multitrack.py new file mode 100644 index 0000000000000000000000000000000000000000..de6bc9235bd1297be000116813f970db15e60546 --- /dev/null +++ b/inference/sampler_euler_heun_multitrack.py @@ -0,0 +1,302 @@ +import torch +from inference.sampler import Sampler + + +class SamplerEulerHeun(Sampler): + + def __init__(self, model, diff_params, args): + super().__init__(model, diff_params, args) + + # stochasticity parameters + self.Schurn = self.args.tester.sampling_params.Schurn + self.Snoise = self.args.tester.sampling_params.Snoise + self.Stmin = self.args.tester.sampling_params.Stmin + self.Stmax = self.args.tester.sampling_params.Stmax + + # order of the sampler + self.order = self.args.tester.sampling_params.order + self.cond=None + self.cfg_scale = 1.0 + + def predict_DPS( + self, + shape, # observations (lowpssed signal) Tensor with shape ?? + cond=None, + cfg_scale=1.0, + device=None, # device + apply_inverse_transform=True, # whether to apply inverse transform + taxonomy=None, # taxonomy for the conditional input + masks=None, # masks for the conditional input + fwd_operator=None, + zeta=None, + dtype=torch.float32, # data type + ): + self.cond = cond + assert self.cond is not None, "Conditional input is None" + + self.taxonomy = taxonomy + self.masks = masks + + self.cfg_scale = cfg_scale + + # get the noise schedule + t = self.create_schedule().to(device).to(torch.float32) + + # sample prior + x = self.diff_params.sample_prior(t=t[0], shape=shape, dtype=dtype) + + # parameter for langevin stochasticity, if Schurn is 0, gamma will be 0 to, so the sampler will be deterministic + gamma = self.get_gamma(t).to(device) + + for i in range(0, self.T-1, 1): + self.step_counter = i + x, x_den = self.step_DPS(x, t[i], t[i + 1], gamma[i], fwd_operator, zeta) + + + if apply_inverse_transform: + with torch.no_grad(): + x_den_wave=self.diff_params.transform_inverse(x_den.detach()) + + return x_den_wave.detach(), None + else: + return x_den.detach(), None + + + def predict_conditional( + self, + shape, # observations (lowpssed signal) Tensor with shape ?? + cond=None, + cfg_scale=1.0, + device=None, # device + apply_inverse_transform=True, # whether to apply inverse transform + taxonomy=None, # taxonomy for the conditional input + masks=None, # masks for the conditional input + dtype=torch.float32, # data type + ): + self.cond = cond + assert self.cond is not None, "Conditional input is None" + + self.taxonomy = taxonomy + self.masks = masks + + self.cfg_scale = cfg_scale + + # get the noise schedule + t = self.create_schedule().to(device).to(torch.float32) + + # sample prior + x = self.diff_params.sample_prior(t=t[0], shape=shape, dtype=dtype) + + # parameter for langevin stochasticity, if Schurn is 0, gamma will be 0 to, so the sampler will be deterministic + gamma = self.get_gamma(t).to(device) + + for i in range(0, self.T-1, 1): + self.step_counter = i + x, x_den = self.step(x, t[i], t[i + 1], gamma[i]) + + + if apply_inverse_transform: + with torch.no_grad(): + x_den_wave=self.diff_params.transform_inverse(x_den.detach()) + + return x_den_wave.detach(), None + else: + return x_den.detach(), None + + + def predict_unconditional( + self, + shape, # observations (lowpssed signal) Tensor with shape ?? + device + ): + self.y = None + self.degradation = None + + return self.predict(shape, device) + + def get_gamma(self, t): + """ + Get the parameter gamma that defines the stochasticity of the sampler + Args + t (Tensor): shape: (N_steps, ) Tensor of timesteps, from which we will compute gamma + """ + N = t.shape[0] + gamma = torch.zeros(t.shape).to(t.device) + + # If desired, only apply stochasticity between a certain range of noises Stmin is 0 by default and Stmax is a huge number by default. (Unless these parameters are specified, this does nothing) + indexes = torch.logical_and(t > self.Stmin, t < self.Stmax) + + # We use Schurn=5 as the default in our experiments + gamma[indexes] = gamma[indexes] + torch.min(torch.Tensor([self.Schurn / N, 2 ** (1 / 2) - 1])) + + return gamma + + def get_Tweedie_estimate(self, x, t_i): + + if x.ndim==2: + x_=x.unsqueeze(1) + elif x.ndim==3: + pass + + if self.cond is not None: + x_hat = self.diff_params.denoiser(x, self.model, t_i, cond=self.cond, cfg_scale=self.cfg_scale, taxonomy=self.taxonomy, masks=self.masks) + else: + x_hat = self.diff_params.denoiser(x, self.model, t_i, taxonomy=self.taxonomy, masks=self.masks) + + return x_hat + + def Tweedie2score(self, tweedie, xt, t): + return self.diff_params.Tweedie2score(tweedie, xt, t) + + def score2Tweedie(self, score, xt, t): + return self.diff_params.score2Tweedie(score, xt, t) + + def stochastic_timestep(self, x, t, gamma, Snoise=1): + t_hat = t + gamma * t # if gamma_sig[i]==0 this is a deterministic step, make sure it doed not crash + t_hat=torch.clamp(t_hat, 0, self.diff_params.max_t) + epsilon = torch.randn(x.shape).to(x.device) * Snoise # sample Gaussiannoise, Snoise is 1 by default + if t_hat <= t: + x_hat = x + #print(f"t_hat<=t, gamma {gamma}") + else: + #print(t_hat, t) + x_hat = x + ((t_hat ** 2 - t ** 2) ** (1 / 2)) * epsilon # Perturb data + return x_hat, t_hat + + def step_DPS(self, x_i, t_i, t_iplus1, gamma_i , fwd_operator=None, zeta=None): + + #with torch.no_grad(): + + x_hat, t_hat=self.stochastic_timestep(x_i, t_i, gamma_i) + + x_hat.requires_grad=True + + x_den = self.get_Tweedie_estimate(x_hat, t_hat) + + #optionally L2 normalize here... + + #compute likelihood score + loss=fwd_operator(x_den) + + loss.backward(retain_graph=False) + + grads= x_hat.grad + + #lets normalize the grads + norm_factor= torch.sqrt(torch.tensor(x_hat.view(-1).shape[0])).to(x_hat.device) + normguide=torch.norm(grads)/ norm_factor + zeta= zeta/(normguide+1e-8) + lh_score=-zeta*grads/t_hat + + x_hat.detach_() # detach x_hat to avoid accumulating gradients + x_den.detach_() # detach x_den to avoid accumulating gradients + + #compute normal score + score = self.Tweedie2score(x_den, x_hat, t_hat) + + ode_integrand = self.diff_params._ode_integrand(x_hat, t_hat, score+ lh_score) + + dt = t_iplus1 - t_hat + + x_iplus1 = x_hat + dt * ode_integrand + + return x_iplus1, x_den + + def step(self, x_i, t_i, t_iplus1, gamma_i ): + + with torch.no_grad(): + x_hat, t_hat = self.stochastic_timestep(x_i, t_i, gamma_i) + x_den = self.get_Tweedie_estimate(x_hat, t_hat) + score = self.Tweedie2score(x_den, x_hat, t_hat) + ode_integrand = self.diff_params._ode_integrand(x_hat, t_hat, score) + dt = t_iplus1 - t_hat + + if t_iplus1 != 0 and self.order == 2: # second order correction + t_prime = t_iplus1 + x_prime = x_hat + dt * ode_integrand + x_den = self.get_Tweedie_estimate(x_prime, t_prime) + score = self.Tweedie2score(x_den, x_prime, t_prime) + ode_integrand_next = self.diff_params._ode_integrand(x_prime, t_prime, score) + ode_integrand_midpoint = .5 * (ode_integrand + ode_integrand_next) + x_iplus1 = x_hat + dt * ode_integrand_midpoint + + else: + x_iplus1 = x_hat + dt * ode_integrand + + return x_iplus1, x_den + + def get_domain_shape(self, shape, device): + + x=torch.zeros(shape, dtype=torch.float32).to(device) + X=self.diff_params.transform_forward(x) + + return X.shape, X.dtype + + def predict( + self, + shape, # observations (lowpssed signal) Tensor with shape ?? + device, # lambda function + dtype=torch.float32, # data type + apply_inverse_transform=True # whether to apply inverse transform + ): + + # get the noise schedule + t = self.create_schedule().to(device).to(torch.float32) + + # sample prior + x = self.diff_params.sample_prior(t=t[0], shape=shape, dtype=dtype) + + # parameter for langevin stochasticity, if Schurn is 0, gamma will be 0 to, so the sampler will be deterministic + gamma = self.get_gamma(t).to(device) + + for i in range(0, self.T-1, 1): + self.step_counter = i + x, x_den = self.step(x, t[i], t[i + 1], gamma[i]) + + + if apply_inverse_transform: + with torch.no_grad(): + x_den_wave=self.diff_params.transform_inverse(x_den.detach()) + + return x_den_wave.detach(), None + else: + return x_den.detach(), None + + def create_schedule(self, sigma_min=None, sigma_max=None, rho=None, T=None): + """ + EDM schedule by default + """ + if T is None: + T=self.T + + if self.args.tester.sampling_params.schedule == "edm": + if sigma_min is None: + sigma_min = self.sde_hp.sigma_min + if sigma_max is None: + sigma_max = self.sde_hp.sigma_max + if rho is None: + rho = self.sde_hp.rho + a = torch.arange(0, T) + t = (sigma_max**(1/rho) + a/(T-1) *(sigma_min**(1/rho) - sigma_max**(1/rho)))**rho + t[-1] = 0 + return t + + elif self.args.tester.sampling_params.schedule == "song": + if sigma_min is None: + sigma_min = self.sde_hp.sigma_min + if sigma_max is None: + sigma_max = self.sde_hp.sigma_max + if rho is None: + rho = self.sde_hp.rho + eps = 0. if not "t_eps" in self.args.tester.diff_params.keys() else self.args.tester.diff_params.t_eps + a = torch.arange(eps, T+1) + t = sigma_min**2 * (sigma_max / sigma_min)**(2*a) + t[-1] = 0 + return t + elif self.args.tester.sampling_params.schedule == "FM": + t = torch.linspace(1, 0, T+1) + return t + + else: + raise NotImplementedError(f"schedule {self.args.tester.sampling_params.schedule} not implemented") + diff --git a/inference/validator_FxProcessor.py b/inference/validator_FxProcessor.py new file mode 100644 index 0000000000000000000000000000000000000000..ddbcd3eb19a96edcb759c9811d36356f3b5f856f --- /dev/null +++ b/inference/validator_FxProcessor.py @@ -0,0 +1,436 @@ +# Copyright (c) 2025 Sony Research +# Licensed under CC BY-NC-SA 4.0 +# See LICENSE file for details + +from datetime import date +import math +import io +import matplotlib.pyplot as plt +from functools import partial +import re +import torch +import os +import wandb +import copy +from glob import glob +from tqdm import tqdm +import omegaconf +import hydra +import utils.log as utils_logging +import utils.training_utils as tr_utils + +import soundfile as sf +import numpy as np +import torchaudio + +from utils.data_utils import apply_RMS_normalization + + +class ValidatorFxProcessor: + def __init__( + self, + args, + network, + test_set_dict=None, + device=None, + in_training=False, + ): + self.args = args + self.network = network + self.device = device + self.test_set_dict = test_set_dict + + self.use_wandb = False # hardcoded for now + self.in_training = in_training + + if in_training: + self.use_wandb = True + # Will inherit wandb_run from Trainer + else: # If we use the tester in training, we will log in WandB in the Trainer() class, no need to create all these paths + torch.backends.cudnn.benchmark = True + if self.device is None: + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) + + self.setup_wandb() + + if self.args.tester.compute_metrics: + self.metrics_dict = self.prepare_metrics(self.args.tester.metrics) + else: + self.metrics_dict = {} + + + self.RMS_norm = ( + self.args.exp.RMS_norm + ) # Use fixed RMS for evaluation, hardcoded for now + + if self.args.exp.style_encoder_type == "FxEncoder++_DynamicFeatures": + + Fxencoder_kwargs = self.args.exp.fx_encoder_plusplus_args + + from utils.feature_extractors.load_features import ( + load_fx_encoder_plusplus_2048, + ) + + feat_extractor = load_fx_encoder_plusplus_2048( + Fxencoder_kwargs, self.device + ) + + from utils.feature_extractors.AF_features_embedding import ( + AF_fourier_embedding, + ) + + AFembedding = AF_fourier_embedding(device=self.device) + + def fxencode_fn(x): + """ + x: tensor of shape [B, C, L] where B is the batch size, C is the number of channels and L is the length of the audio + """ + z = feat_extractor(x) + z = torch.nn.functional.normalize( + z, dim=-1, p=2 + ) # normalize to unit variance + z = z * math.sqrt(z.shape[-1]) # rescale to keep the same scale + + z_af, _ = AFembedding.encode(x) + # embedding is l2 normalized, normalize to unit variance + z_af = z_af * math.sqrt( + z_af.shape[-1] + ) # rescale to keep the same scale + + # concatenate z and z_af (rescaling with sqrt(dim) to keep the same scale) + z_all = torch.cat([z, z_af], dim=-1) + + norm_z = z_all / math.sqrt( + z_all.shape[-1] + ) # normalize by dividing by sqrt(dim) to keep the same scale + + return norm_z + + self.style_encode = fxencode_fn + else: + raise NotImplementedError( + "Only FxEncoder++_DynamicFeatures is implemented for now" + ) + + if self.args.exp.apply_fxnorm: + self.fx_normalizer = hydra.utils.instantiate(self.args.exp.fxnorm) + + def setup_wandb(self): + """ + Configure wandb, open a new run and log the configuration. + """ + config = omegaconf.OmegaConf.to_container( + self.args, resolve=True, throw_on_missing=True + ) + self.wandb_run = wandb.init( + project="testing" + self.args.tester.wandb.project, + entity=self.args.tester.wandb.entity, + config=config, + tags=self.args.tester.wandb.tags, + ) + # wandb.watch(self.network, + # log_freq=self.args.logging.heavy_log_interval) + + self.wandb_run.name = self.args.tester.wandb.run_name + self.use_wandb = True + + def setup_wandb_run(self, run): + # get the wandb run object from outside (in trainer.py or somewhere else) + self.wandb_run = run + self.use_wandb = True + + def load_latest_checkpoint(self): + # load the latest checkpoint from self.args.model_dir + try: + # find latest checkpoint_id + save_basename = f"{self.args.exp.exp_name}-*.pt" + save_name = f"{self.args.model_dir}/{save_basename}" + list_weights = glob(save_name) + id_regex = re.compile(f"{self.args.exp.exp_name}-(\d*)\.pt") + list_ids = [ + int(id_regex.search(weight_path).groups()[0]) + for weight_path in list_weights + ] + checkpoint_id = max(list_ids) + + state_dict = torch.load( + f"{self.args.model_dir}/{self.args.exp.exp_name}-{checkpoint_id}.pt", + map_location=self.device, + ) + try: + self.network.load_state_dict(state_dict["network"]) + except Exception as e: + print(e) + print("Failed to load in strict mode, trying again without strict mode") + self.network.load_state_dict(state_dict["model"], strict=False) + + print(f"Loaded checkpoint {checkpoint_id}") + return True + except (FileNotFoundError, ValueError): + raise ValueError("No checkpoint found") + + def load_checkpoint(self, path): + state_dict = torch.load(path, map_location=self.device, weights_only=False) + print("state_dict keys:", state_dict.keys()) + try: + self.it = state_dict["it"] + except: + self.it = 0 + + print(f"loading checkpoint {self.it}") + return tr_utils.load_state_dict(state_dict, network=self.network) + + def log_figure(self, fig, name: str, step=None): + # Save the figure to a buffer + + self.wandb_run.log( + {name: wandb.Image(fig)}, step=step if step is not None else self.it + ) + + def log_metric(self, value, name: str, step=None): + # print("logging metric it:", self.it, "name:", name) + self.wandb_run.log({name: value}, step=step if step is not None else self.it) + + def log_audio(self, pred, name: str, it=None): + if it is None: + it = self.it + if self.use_wandb: + pred = pred.permute(1, 0) + self.wandb_run.log( + { + name: wandb.Audio( + pred.detach().cpu().numpy(), + sample_rate=self.args.exp.sample_rate, + ) + }, + step=it, + ) + + # ------------- UNCONDITIONAL SAMPLING ---------------# + + ############################## + ### UNCONDITIONAL SAMPLING ### + ############################## + + def prepare_metrics(self, metrics): + metrics_dict = {} + for metric in metrics: + print(f"Preparing metric {metric}") + if "pairwise" in metric: + from utils.evaluation.pairwise_metrics import metric_factory + + metrics_dict[metric] = metric_factory( + metric, self.args.exp.sample_rate, **self.args.tester + ) + + return metrics_dict + + def test_paired(self, mode, exp_description=""): + + # self.it = 0 + for k, test_set in self.test_set_dict.items(): + + print(f"Testing on {k} set", k) + + assert len(test_set) != 0, "No samples found in test set" + + dict_y = {} + dict_x = {} + dict_y_hat = {} + + i = 0 + + for x, y in tqdm(test_set): + + B, C, T = y.shape + + x = x.to(self.device).float() + if x.shape[-1] > self.args.exp.audio_len: + x = x[:, :, : self.args.exp.audio_len] + elif x.shape[-1] < self.args.exp.audio_len: + raise ValueError( + f"Sample length {x.shape[-1]} is less than expected {self.args.exp.audio_len}" + ) + + if mode == "paired": + y = y.to(self.device).float() + + if y.shape[-1] > self.args.exp.audio_len: + y = y[:, :, : self.args.exp.audio_len] + elif y.shape[-1] < self.args.exp.audio_len: + raise ValueError( + f"Sample length {y.shape[-1]} is less than expected {self.args.exp.audio_len}" + ) + + if x.shape[1] == 2: + x = x.mean( + dim=1, keepdim=True + ) # expand to [B*N, 1, L] to keep the shape consistent + + # RMS normalization of x and y + + if self.args.exp.apply_fxnorm: + x = self.fx_normalizer( + x, + RMS=self.args.exp.RMS_norm, + use_gate=self.args.exp.use_gated_RMSnorm, + ) + else: + x = apply_RMS_normalization( + x, + self.args.exp.RMS_norm, + use_gate=self.args.exp.use_gated_RMSnorm, + ) + + if "baseline" in mode: + if mode == "baseline_dry": + preds = x # Just return the dry vocals as baseline + elif mode == "baseline_autoencoder": + preds = self.autoencoder_reconstruction( + y + ) # Just return the dry vocals as baseline + elif mode == "baseline_random": + raise NotImplementedError( + "Baseline random sampling not implemented yet" + ) + pass + else: + with torch.no_grad(): + z = self.style_encode(y) + try: + preds = self.network( + x, z + ) # Get the predictions from the network + except Exception as e: + print(f"Error during inference: {e}") + continue + print( + "y_pred", + preds.shape, + preds.std(), + preds.mean(), + preds.min(), + preds.max(), + ) + + is_nan = torch.isnan(preds).any() + if is_nan: + num_nan = torch.sum(torch.isnan(x)).item() + print( + f"Number of NaN values in sample_x: {num_nan} of {x.numel()}" + ) + + y = apply_RMS_normalization( + y, self.args.exp.RMS_norm, use_gate=self.args.exp.use_gated_RMSnorm + ) + + for b in range(B): + if self.use_wandb: + if ( + i < self.args.tester.wandb.num_examples_to_log + ): # Log only first 10 samples + self.log_audio( + preds[b], f"pred_wet_{k}_{mode}_{i}", it=self.it + ) # Just log first sample + self.log_audio( + y[b], f"original_wet_{k}_{mode}_{i}", it=self.it + ) # Just log first sample + self.log_audio( + x[b], f"original_dry_{k}_{mode}_{i}", it=self.it + ) # Just log first sample + + dict_y[i] = y[b].detach().cpu().numpy() + dict_x[i] = x[b].detach().cpu().numpy() + dict_y_hat[i] = preds[b].detach().cpu().numpy() + + i += 1 + + if self.args.tester.compute_metrics: + for metric in self.metrics_dict.keys(): + try: + print(f"Computing metric {metric}") + result, result_dict = self.metrics_dict[metric].compute( + dict_y, dict_y_hat, dict_x + ) + + if self.use_wandb: + if result is not None: + self.log_metric( + result, metric + "_" + k + "_" + mode, step=self.it + ) + + for key, value in result_dict.items(): + if "figure" in key: + # log figure as an image + self.log_figure( + value, key + "_" + k + "_" + mode, step=self.it + ) + else: + self.log_metric( + value, key + "_" + k + "_" + mode, step=self.it + ) + + except Exception as e: + print(f"Error computing metric {metric}: {e}") + continue + + def prepare_directories(self, mode, unconditional=False, string=None): + + today = date.today() + self.paths = {} + if ( + "overriden_name" in self.args.tester.keys() + and self.args.tester.overriden_name is not None + ): + self.path_sampling = os.path.join( + self.args.model_dir, self.args.tester.overriden_name + ) + else: + self.path_sampling = os.path.join( + self.args.model_dir, "test" + today.strftime("%d_%m_%Y") + ) + if not os.path.exists(self.path_sampling): + os.makedirs(self.path_sampling) + + self.paths[mode] = os.path.join( + self.path_sampling, mode, self.args.exp.exp_name + ) + + if not os.path.exists(self.paths[mode]): + os.makedirs(self.paths[mode]) + if string is None: + string = "" + + self.paths[mode + "wet_original"] = os.path.join( + self.paths[mode], string + "wet_original" + ) + if not os.path.exists(self.paths[mode + "wet_original"]): + os.makedirs(self.paths[mode + "wet_original"]) + self.paths[mode + "dry"] = os.path.join(self.paths[mode], string + "dry") + if not os.path.exists(self.paths[mode + "dry"]): + os.makedirs(self.paths[mode + "dry"]) + self.paths[mode + "wet_estimate"] = os.path.join( + self.paths[mode], string + "wet_estimate" + ) + if not os.path.exists(self.paths[mode + "wet_estimate"]): + os.makedirs(self.paths[mode + "wet_estimate"]) + + def save_experiment_args(self, mode): + with open( + os.path.join(self.paths[mode], ".argv"), "w" + ) as f: # Keep track of the arguments we used for this experiment + omegaconf.OmegaConf.save(config=self.args, f=f.name) + + def do_test(self, it=0): + + self.it = it + for m in self.args.tester.modes: + if m == "paired": + if not self.in_training: + self.prepare_directories(m, unconditional=True) + self.save_experiment_args(m) + self.test_paired(m) + else: + print("Warning: unknown mode: ", m) diff --git a/networks/MLP_CLAP_regressor.py b/networks/MLP_CLAP_regressor.py new file mode 100644 index 0000000000000000000000000000000000000000..655b53ec95e8a76c263e6f11766b7d8547f505cf --- /dev/null +++ b/networks/MLP_CLAP_regressor.py @@ -0,0 +1,23 @@ +import torch +import torch.nn as nn + +class MLP_CLAP_regressor(nn.Module): + """ + A simple MLP regressor that uses CLAP features as input. + """ + + def __init__(self, dim=512, hidden_dim=512): + super(MLP_CLAP_regressor, self).__init__() + + self.model = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, dim) + ) + + def forward(self, x): + + emb= self.model(x) + #l2 normalization + return nn.functional.normalize(emb, p=2, dim=-1) + diff --git a/networks/__init__.py b/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/networks/blackbox_TCN.py b/networks/blackbox_TCN.py new file mode 100755 index 0000000000000000000000000000000000000000..24da4e20acd4f2bf9de0d6ede7d05d2c54615828 --- /dev/null +++ b/networks/blackbox_TCN.py @@ -0,0 +1,346 @@ +""" + Adapted from: https://github.com/SonyResearch/ITO-Master +""" +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.init as init + +import os +import sys + +import omegaconf +import torchaudio +from utils.feature_extractors.load_features import load_CLAP + +# 1-dimensional convolutional layer +# in the order of conv -> norm -> activation +class Conv1d_layer(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, \ + stride=1, \ + padding="SAME", dilation=1, bias=True, \ + norm="batch", activation="relu", \ + mode="conv"): + super(Conv1d_layer, self).__init__() + + self.conv1d = nn.Sequential() + + ''' padding ''' + if mode=="deconv": + padding = int(dilation * (kernel_size-1) / 2) + out_padding = 0 if stride==1 else 1 + elif mode=="conv" or "alias_free" in mode: + if padding == "SAME": + pad = int((kernel_size-1) * dilation) + l_pad = int(pad//2) + r_pad = pad - l_pad + padding_area = (l_pad, r_pad) + elif padding == "VALID": + padding_area = (0, 0) + else: + pass + + ''' convolutional layer ''' + if mode=="deconv": + self.conv1d.add_module("deconv1d", nn.ConvTranspose1d(in_channels, out_channels, kernel_size, \ + stride=stride, padding=padding, output_padding=out_padding, \ + dilation=dilation, \ + bias=bias)) + elif mode=="conv": + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module(f"{mode}1d", nn.Conv1d(in_channels, out_channels, kernel_size, \ + stride=stride, padding=0, \ + dilation=dilation, \ + bias=bias)) + elif "alias_free" in mode: + if "up" in mode: + up_factor = stride * 2 + down_factor = 2 + elif "down" in mode: + up_factor = 2 + down_factor = stride * 2 + else: + raise ValueError("choose alias-free method : 'up' or 'down'") + # procedure : conv -> upsample -> lrelu -> low-pass filter -> downsample + # the torchaudio.transforms.Resample's default resampling_method is 'sinc_interpolation' which performs low-pass filter during the process + # details at https://pytorch.org/audio/stable/transforms.html + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module(f"{mode}1d", nn.Conv1d(in_channels, out_channels, kernel_size, \ + stride=1, padding=0, \ + dilation=dilation, \ + bias=bias)) + self.conv1d.add_module(f"{mode}upsample", torchaudio.transforms.Resample(orig_freq=1, new_freq=up_factor)) + self.conv1d.add_module(f"{mode}lrelu", nn.LeakyReLU()) + self.conv1d.add_module(f"{mode}downsample", torchaudio.transforms.Resample(orig_freq=down_factor, new_freq=1)) + + ''' normalization ''' + if norm=="batch": + self.conv1d.add_module("batch_norm", nn.BatchNorm1d(out_channels)) + # self.conv1d.add_module("batch_norm", nn.SyncBatchNorm(out_channels)) + + ''' activation ''' + if 'alias_free' not in mode: + if activation=="relu": + self.conv1d.add_module("relu", nn.ReLU()) + elif activation=="lrelu": + self.conv1d.add_module("lrelu", nn.LeakyReLU()) + + + def forward(self, input): + # input shape should be : batch x channel x height x width + output = self.conv1d(input) + return output + + +# compute receptive field +def compute_receptive_field(kernels, strides, dilations): + rf = 0 + for i in range(len(kernels)): + rf += rf * strides[i] + (kernels[i]-strides[i]) * dilations[i] + return rf + +# Feature-wise Linear Modulation +class FiLM(nn.Module): + def __init__(self, condition_len=2048, feature_len=1024): + super(FiLM, self).__init__() + self.film_fc = nn.Linear(condition_len, feature_len*2) + self.feat_len = feature_len + + def forward(self, feature, condition): + film_factor = self.film_fc(condition).unsqueeze(-1) + r, b = torch.split(film_factor, self.feat_len, dim=1) + return r*feature + b + + +class ConvBlock(nn.Module): + def __init__(self, dimension, layer_num, \ + in_channels, out_channels, \ + kernel_size, \ + stride=1, padding="SAME", \ + dilation=1, \ + bias=True, \ + norm="batch", \ + activation="relu", last_activation="relu", \ + mode="conv"): + super(ConvBlock, self).__init__() + + conv_block = [] + if dimension==1: + for i in range(layer_num-1): + conv_block.append(Conv1d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation)) + conv_block.append(Conv1d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode)) + elif dimension==2: + for i in range(layer_num-1): + conv_block.append(Conv2d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation)) + conv_block.append(Conv2d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode)) + self.conv_block = nn.Sequential(*conv_block) + + def forward(self, input): + return self.conv_block(input) + + +class TCNBlock(torch.nn.Module): + def __init__(self, + in_ch, + out_ch, + kernel_size=3, + stride=1, + dilation=1, + cond_dim=2048, + grouped=False, + causal=False, + conditional=False, + **kwargs): + super(TCNBlock, self).__init__() + + self.in_ch = in_ch + self.out_ch = out_ch + self.kernel_size = kernel_size + self.dilation = dilation + self.grouped = grouped + self.causal = causal + self.conditional = conditional + + groups = out_ch if grouped and (in_ch % out_ch == 0) else 1 + + self.pad_length = ((kernel_size-1)*dilation) if self.causal else ((kernel_size-1)*dilation)//2 + self.conv1 = torch.nn.Conv1d(in_ch, + out_ch, + kernel_size=kernel_size, + stride=stride, + padding=self.pad_length, + dilation=dilation, + groups=groups, + bias=False) + if grouped: + self.conv1b = torch.nn.Conv1d(out_ch, out_ch, kernel_size=1) + + if conditional: + self.film = FiLM(cond_dim, out_ch) + self.bn = torch.nn.BatchNorm1d(out_ch) + + self.relu = torch.nn.LeakyReLU() + + if out_ch % in_ch == 0: + self.res = torch.nn.Conv1d(in_ch, + out_ch, + kernel_size=1, + stride=stride, + groups=in_ch, + bias=False) + else: + self.res = torch.nn.Conv1d(in_ch, + out_ch, + kernel_size=1, + stride=stride, + groups=1, + bias=False) + + def forward(self, x, p): + x_in = x + + x = self.relu(self.bn(self.conv1(x))) + #print("p", p.shape) + x = self.film(x, p) + + x_res = self.res(x_in) + + if self.causal: + x = x[..., :-self.pad_length] + x += x_res + + return x + +class FourierFeatures(nn.Module): + def __init__(self, in_features, out_features, std=1.): + super().__init__() + assert out_features % 2 == 0 + self.weight = nn.Parameter(torch.randn( + [out_features // 2, in_features]) * std) + + def forward(self, input): + f = 2 * math.pi * input @ self.weight.T + return torch.cat([f.cos(), f.sin()], dim=-1) + +class TCNModel(nn.Module): + """ Temporal convolutional network with conditioning module. + Args: + nparams (int): Number of conditioning parameters. + ninputs (int): Number of input channels (mono = 1, stereo 2). Default: 1 + noutputs (int): Number of output channels (mono = 1, stereo 2). Default: 1 + nblocks (int): Number of total TCN blocks. Default: 10 + kernel_size (int): Width of the convolutional kernels. Default: 3 + dialation_growth (int): Compute the dilation factor at each block as dilation_growth ** (n % stack_size). Default: 1 + channel_growth (int): Compute the output channels at each black as in_ch * channel_growth. Default: 2 + channel_width (int): When channel_growth = 1 all blocks use convolutions with this many channels. Default: 64 + stack_size (int): Number of blocks that constitute a single stack of blocks. Default: 10 + grouped (bool): Use grouped convolutions to reduce the total number of parameters. Default: False + causal (bool): Causal TCN configuration does not consider future input values. Default: False + skip_connections (bool): Skip connections from each block to the output. Default: False + """ + def __init__(self, + ninputs=1, + noutputs=2, + nblocks=14, + kernel_size=15, + stride=1, + dilation_growth=2, + channel_growth=1, + channel_width=128, + stack_size=15, + cond_dim=2048, + grouped=False, + causal=False, + skip_connections=False, + use_CLAP=False, + CLAP_args=None, + ): + super(TCNModel, self).__init__() + + self.use_CLAP = use_CLAP + if self.use_CLAP: + assert CLAP_args is not None, "CLAP_args must be provided for CLAP AE" + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + CLAP_encoder= load_CLAP(CLAP_args, device=device) + cond_dim+= 512 + def merge_CLAP_embeddings(x, emb): + + clap_embedding = CLAP_encoder(x, type="dry") + #l2 normalize the clap embedding + clap_embedding = F.normalize(clap_embedding, p=2, dim=-1) + + return torch.cat((emb, clap_embedding), dim=-1) + + self.merge_CLAP_embeddings = merge_CLAP_embeddings + + + self.hparams = { + "ninputs": ninputs, + "noutputs": noutputs, + "nblocks": nblocks, + "kernel_size": kernel_size, + "stride": stride, + "dilation_growth": dilation_growth, + "channel_growth": channel_growth, + "channel_width": channel_width, + "stack_size": stack_size, + "cond_dim": cond_dim, + "grouped": grouped, + "causal": causal, + "skip_connections": skip_connections, + } + self.hparams= omegaconf.OmegaConf.create(self.hparams) + + self.blocks = torch.nn.ModuleList() + for n in range(nblocks): + in_ch = out_ch if n > 0 else ninputs + + if self.hparams.channel_growth > 1: + out_ch = in_ch * self.hparams.channel_growth + else: + out_ch = self.hparams.channel_width + + dilation = self.hparams.dilation_growth ** (n % self.hparams.stack_size) + cur_stride = stride[n] if isinstance(stride, list) else stride + self.blocks.append(TCNBlock(in_ch, + out_ch, + kernel_size=self.hparams.kernel_size, + stride=cur_stride, + dilation=dilation, + padding="same" if self.hparams.causal else "valid", + causal=self.hparams.causal, + cond_dim=cond_dim, + grouped=self.hparams.grouped, + conditional=True )) + + self.output = torch.nn.Conv1d(out_ch, noutputs, kernel_size=1) + + def forward(self, x, cond): + # iterate over blocks passing conditioning + if self.use_CLAP: + with torch.no_grad(): + cond = self.merge_CLAP_embeddings(x, cond) + + for idx, block in enumerate(self.blocks): + # for SeFa + if isinstance(cond, list): + x = block(x, cond[idx]) + else: + x = block(x, cond) + skips = 0 + + # out = torch.tanh(self.output(x + skips)) + out = torch.clamp(self.output(x + skips), min=-1, max=1) + + return out + + def compute_receptive_field(self): + """ Compute the receptive field in samples.""" + rf = self.hparams.kernel_size + for n in range(1,self.hparams.nblocks): + dilation = self.hparams.dilation_growth ** (n % self.hparams.stack_size) + rf = rf + ((self.hparams.kernel_size-1) * dilation) + return rf + + diff --git a/networks/dit_multitrack.py b/networks/dit_multitrack.py new file mode 100644 index 0000000000000000000000000000000000000000..1b86045728eee48d68d8afd0c2c6474a2957a960 --- /dev/null +++ b/networks/dit_multitrack.py @@ -0,0 +1,488 @@ + +# adapted from https://github.com/Stability-AI/stable-audio-tools/blob/main/stable_audio_tools/models/dit.py + +import typing as tp +import math +import torch + +from einops import rearrange +from torch import nn +from torch.nn import functional as F + +class FourierFeatures(nn.Module): + def __init__(self, in_features, out_features, std=1.): + super().__init__() + assert out_features % 2 == 0 + self.weight = nn.Parameter(torch.randn( + [out_features // 2, in_features]) * std) + + def forward(self, input): + f = 2 * math.pi * input @ self.weight.T + return torch.cat([f.cos(), f.sin()], dim=-1) + +class OneHotPositionalEmbedding(nn.Module): + def __init__(self, max_seq_len): + super().__init__() + self.max_seq_len = max_seq_len + + def forward(self, x, pos=None, seq_start_pos=None): + seq_len, device = x.shape[1], x.device + assert seq_len <= self.max_seq_len, f'you are passing in a sequence length of {seq_len} but your one-hot positional embedding has a max sequence length of {self.max_seq_len}' + + if pos is None: + pos = torch.arange(seq_len, device=device) + + if seq_start_pos is not None: + pos = (pos - seq_start_pos[..., None]).clamp(min=0) + + pos_emb = F.one_hot(pos, num_classes=self.max_seq_len).to(x.dtype) + return pos_emb + + + +class DiffusionTransformer(nn.Module): + def __init__(self, + io_channels=32, + patch_size=1, + embed_dim=768, + cond_token_dim=0, + cond_token_proj_dim=64, + project_cond_tokens=False, + global_cond_dim=0, + project_global_cond=True, + input_concat_dim=0, + prepend_cond_dim=0, + depth=12, + num_heads=8, + transformer_type: tp.Literal["x-transformers", "continuous_transformer"] = "x-transformers", + global_cond_type: tp.Literal["prepend", "adaLN"] = "prepend", + timestep_cond_type: tp.Literal["global", "input_concat"] = "global", + timestep_embed_dim=None, + pos_emb_strategy="concatenation", + pos_emb_dim=None, + pos_emb_type="one-hot", + pos_emb_crossattn_strategy="concatenation", + pos_emb_crossattn_dim=None, + pos_emb_crossattn_type="one-hot", + use_taxonomy_in_pos_emb=True, + max_num_tracks=14,#used for one-hot positional embeddings + **kwargs): + + super().__init__() + + self.cond_token_dim = cond_token_dim + + # Timestep embeddings + self.timestep_cond_type = timestep_cond_type + + timestep_features_dim = 256 + + self.timestep_features = FourierFeatures(1, timestep_features_dim) + + if timestep_cond_type == "global": + timestep_embed_dim = embed_dim + elif timestep_cond_type == "input_concat": + assert timestep_embed_dim is not None, "timestep_embed_dim must be specified if timestep_cond_type is input_concat" + input_concat_dim += timestep_embed_dim + + self.to_timestep_embed = nn.Sequential( + nn.Linear(timestep_features_dim, timestep_embed_dim, bias=True), + nn.SiLU(), + nn.Linear(timestep_embed_dim, timestep_embed_dim, bias=True), + ) + + + self.project_cond_tokens = project_cond_tokens + if cond_token_dim > 0: + # Conditioning tokens + + if self.project_cond_tokens: + self.to_cond_embed = nn.Sequential( + nn.Linear(cond_token_dim, cond_token_proj_dim, bias=False), + nn.SiLU(), + nn.Linear(cond_token_proj_dim, cond_token_proj_dim, bias=False) + ) + cond_embed_dim = cond_token_proj_dim + else: + cond_embed_dim = cond_token_dim + else: + cond_embed_dim = 0 + + if global_cond_dim > 0: + # Global conditioning + global_embed_dim = global_cond_dim if not project_global_cond else embed_dim + self.to_global_embed = nn.Sequential( + nn.Linear(global_cond_dim, global_embed_dim, bias=False), + nn.SiLU(), + nn.Linear(global_embed_dim, global_embed_dim, bias=False) + ) + + if prepend_cond_dim > 0: + # Prepend conditioning + self.to_prepend_embed = nn.Sequential( + nn.Linear(prepend_cond_dim, embed_dim, bias=False), + nn.SiLU(), + nn.Linear(embed_dim, embed_dim, bias=False) + ) + + self.input_concat_dim = input_concat_dim + + dim_in = io_channels + self.input_concat_dim + + self.patch_size = patch_size + + # Transformer + + self.transformer_type = transformer_type + + self.global_cond_type = global_cond_type + + + if pos_emb_strategy == "concatenation": + + assert pos_emb_dim is not None, "pos_emb_dim must be specified if pos_emb_strategy is concatenation" + if pos_emb_type == "one-hot": + # One-hot positional embeddings + self.pos_emb = OneHotPositionalEmbedding(pos_emb_dim-max_num_tracks) + self.extra_dim = pos_emb_dim + + def concat_pos_emb(x, pos=None, seq_start_pos=None, taxonomy=None): + B, N, T, C = x.shape + pos_emb = self.pos_emb(x.view(-1,x.shape[-2], x.shape[-1]), pos=pos, seq_start_pos=seq_start_pos) + assert pos_emb.shape[-1] == pos_emb_dim-max_num_tracks, f"pos_emb shape mismatch: {pos_emb.shape[-1]} != {pos_emb_dim}" + assert pos_emb.shape[-2] == T, f"pos_emb sequence length mismatch: {pos_emb.shape[-2]} != {x.shape[-2]}" + + pos_emb=pos_emb.unsqueeze(0).unsqueeze(0).expand(B,N,T,-1) + + + assert pos_emb.ndim == 4, f"pos_emb must be 2D or 3D, got {pos_emb.ndim}" + assert pos_emb.shape[0] == x.shape[0], f"pos_emb batch size mismatch: {pos_emb.shape[0]} != {x.shape[0]}" + + pos_emb_track=torch.zeros((B, N, T, max_num_tracks), device=x.device, dtype=x.dtype) + for i in range(B): + for j in range(N): + if use_taxonomy_in_pos_emb: + raise NotImplementedError("use_taxonomy_in_pos_emb is not implemented for pos_emb_type 'one-hot'") + assert taxonomy is not None, "taxonomy must be provided if use_taxonomy_in_pos_emb is True" + if taxonomy[i][j]=="92": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([0], device=x.device), num_classes=3).to(x.dtype).expand(T, -1) + elif taxonomy[i][j]=="2": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([1], device=x.device), num_classes=3).to(x.dtype).expand(T, -1) + elif taxonomy[i][j]=="11": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([2], device=x.device), num_classes=3).to(x.dtype).expand(T, -1) + else: + if j >= max_num_tracks: + j= j% max_num_tracks + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([j], device=x.device), num_classes=max_num_tracks).to(x.dtype).expand(T, -1) + + return torch.cat((x, pos_emb, pos_emb_track), dim=-1) + + self.pos_emb_fn = concat_pos_emb + self.remove_pos_emb = lambda x: x[..., :-pos_emb_dim] if pos_emb_dim > 0 else x + + else: + raise ValueError(f"Unknown pos_emb_type: {pos_emb_type}") + else: + raise ValueError(f"Unknown pos_emb_strategy: {pos_emb_strategy}") + + + if pos_emb_crossattn_strategy == "concatenation": + assert pos_emb_crossattn_dim is not None, "pos_emb_crossattn_dim must be specified if pos_emb_crossattn_strategy is concatenation" + if pos_emb_crossattn_type == "one-hot": + # One-hot positional embeddings for cross-attention + self.pos_emb_crossattn = OneHotPositionalEmbedding(pos_emb_crossattn_dim-max_num_tracks) + self.crossattn_extra_dim = pos_emb_crossattn_dim + + if not self.project_cond_tokens: + def concat_pos_emb_crossattn(x, pos=None, seq_start_pos=None, taxonomy=None): + B, N, T, C = x.shape + + pos_emb = self.pos_emb_crossattn(x.view(-1, T,C), pos=pos, seq_start_pos=seq_start_pos) + assert pos_emb.shape[-1] == pos_emb_crossattn_dim-max_num_tracks, f"pos_emb shape mismatch: {pos_emb.shape[-1]} != {pos_emb_crossattn_dim}" + assert pos_emb.shape[-2] == T, f"pos_emb sequence length mismatch: {pos_emb.shape[-2]} != {x.shape[-2]}" + + pos_emb=pos_emb.unsqueeze(0).unsqueeze(0).expand(B,N,T,-1) + + #if pos_emb.ndim == 3: + assert pos_emb.ndim == 4, f"pos_emb must be 2D or 3D, got {pos_emb.ndim}" + assert pos_emb.shape[0] == x.shape[0], f"pos_emb batch size mismatch: {pos_emb.shape[0]} != {x.shape[0]}" + + pos_emb_track=torch.zeros((B, N, T, max_num_tracks), device=x.device, dtype=x.dtype) + for i in range(B): + for j in range(N): + if use_taxonomy_in_pos_emb: + raise NotImplementedError("use_taxonomy_in_pos_emb is not implemented for pos_emb_crossattn_type 'one-hot'") + assert taxonomy is not None, "taxonomy must be provided if use_taxonomy_in_pos_emb is True" + if taxonomy[i][j]=="92": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([0], device=x.device), num_classes=3).to(x.dtype).expand(T, -1) + elif taxonomy[i][j]=="2": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([1], device=x.device), num_classes=3).to(x.dtype).expand(T, -1) + elif taxonomy[i][j]=="11": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([2], device=x.device), num_classes=3).to(x.dtype).expand(T, -1) + else: + if j >= max_num_tracks: + j= j% max_num_tracks + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([j], device=x.device), num_classes=max_num_tracks).to(x.dtype).expand(T, -1) + + return torch.cat((x, pos_emb, pos_emb_track), dim=-1) + else: + def concat_pos_emb_crossattn(x, pos=None, seq_start_pos=None, taxonomy=None): + #print("x shape",x.shape) + + B, N, T, C = x.shape + + assert T*C== cond_token_dim, f"cond_token_proj_dim must match T*C, got {cond_token_dim} != {T*C}" + + #rehape to B, N, 1, T*C + x= rearrange(x, "b n t c -> b n 1 (t c)") + + #pos_emb = self.pos_emb_crossattn(x.view(-1, T,C), pos=pos, seq_start_pos=seq_start_pos) + #assert pos_emb.shape[-1] == pos_emb_crossattn_dim-3, f"pos_emb shape mismatch: {pos_emb.shape[-1]} != {pos_emb_crossattn_dim}" + #assert pos_emb.shape[-2] == T, f"pos_emb sequence length mismatch: {pos_emb.shape[-2]} != {x.shape[-2]}" + + #pos_emb=pos_emb.unsqueeze(0).unsqueeze(0).expand(B,N,T,-1) + + #if pos_emb.ndim == 3: + #assert pos_emb.ndim == 4, f"pos_emb must be 2D or 3D, got {pos_emb.ndim}" + #assert pos_emb.shape[0] == x.shape[0], f"pos_emb batch size mismatch: {pos_emb.shape[0]} != {x.shape[0]}" + + x=rearrange(x, "b n 1 c -> (b n) 1 c") + x=self.to_cond_embed(x) + x=rearrange(x, "(b n) 1 c -> b n 1 c", b=B, n=N) + + pos_emb_track=torch.zeros((B, N, 1, pos_emb_crossattn_dim), device=x.device, dtype=x.dtype) + for i in range(B): + for j in range(N): + if use_taxonomy_in_pos_emb: + assert taxonomy is not None, "taxonomy must be provided if use_taxonomy_in_pos_emb is True" + if taxonomy[i][j]=="92": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([0], device=x.device), num_classes=pos_emb_crossattn_dim).to(x.dtype) + elif taxonomy[i][j]=="2": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([1], device=x.device), num_classes=pos_emb_crossattn_dim).to(x.dtype) + elif taxonomy[i][j]=="11": + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([2], device=x.device), num_classes=pos_emb_crossattn_dim).to(x.dtype) + else: + pos_emb_track[i, j, :, :]= F.one_hot(torch.tensor([j], device=x.device), num_classes=pos_emb_crossattn_dim).to(x.dtype) + + return torch.cat((x, pos_emb_track), dim=-1) + + + + self.pos_emb_crossattn_fn = concat_pos_emb_crossattn + + else: + raise ValueError(f"Unknown pos_emb_type: {pos_emb_crossattn_type}") + + + + global_dim = None + + if self.global_cond_type == "adaLN": + # The global conditioning is projected to the embed_dim already at this point + global_dim = embed_dim + + from networks.transformer import ContinuousTransformer + + self.transformer = ContinuousTransformer( + dim=embed_dim, + depth=depth, + num_heads= num_heads, + dim_in=dim_in + pos_emb_dim, + dim_out=io_channels , + cross_attend = cond_token_dim > 0, + cond_token_dim = cond_embed_dim + pos_emb_crossattn_dim, + global_cond_dim=global_dim, + **kwargs + ) + + + self.preprocess_conv = nn.Conv1d(dim_in, dim_in, 1, bias=False) + nn.init.zeros_(self.preprocess_conv.weight) + self.postprocess_conv = nn.Conv1d(io_channels, io_channels, 1, bias=False) + nn.init.zeros_(self.postprocess_conv.weight) + + + + + def _forward( + self, + x, + t, + mask=None, + cross_attn_cond=None, + cross_attn_cond_mask=None, + input_concat_cond=None, + global_embed=None, + prepend_cond=None, + prepend_cond_mask=None, + return_info=False, + **kwargs): + + t=t.squeeze(1) + + #if cross_attn_cond is not None: + # cross_attn_cond = self.to_cond_embed(cross_attn_cond) + + if global_embed is not None: + # Project the global conditioning to the embedding dimension + global_embed = self.to_global_embed(global_embed) + + prepend_inputs = None + prepend_mask = None + prepend_length = 0 + if prepend_cond is not None: + # Project the prepend conditioning to the embedding dimension + prepend_cond = self.to_prepend_embed(prepend_cond) + + prepend_inputs = prepend_cond + if prepend_cond_mask is not None: + prepend_mask = prepend_cond_mask + + + # Get the batch of timestep embeddings + timestep_embed = self.to_timestep_embed(self.timestep_features(t[:, None])) # (b, embed_dim) + + # Timestep embedding is considered a global embedding. Add to the global conditioning if it exists + + if self.timestep_cond_type == "global": + if global_embed is not None: + global_embed = global_embed + timestep_embed + else: + global_embed = timestep_embed + elif self.timestep_cond_type == "input_concat": + x = torch.cat([x, timestep_embed.unsqueeze(1).expand(-1, -1, x.shape[2])], dim=1) + + # Add the global_embed to the prepend inputs if there is no global conditioning support in the transformer + if self.global_cond_type == "prepend" and global_embed is not None: + if prepend_inputs is None: + # Prepend inputs are just the global embed, and the mask is all ones + prepend_inputs = global_embed.unsqueeze(1) + prepend_mask = torch.ones((x.shape[0], 1), device=x.device, dtype=torch.bool) + else: + # Prepend inputs are the prepend conditioning + the global embed + prepend_inputs = torch.cat([prepend_inputs, global_embed.unsqueeze(1)], dim=1) + prepend_mask = torch.cat([prepend_mask, torch.ones((x.shape[0], 1), device=x.device, dtype=torch.bool)], dim=1) + + prepend_length = prepend_inputs.shape[1] + + extra_args = {} + + if self.global_cond_type == "adaLN": + extra_args["global_cond"] = global_embed + + + output = self.transformer(x, prepend_embeds=prepend_inputs, context=cross_attn_cond, context_mask=cross_attn_cond_mask, mask=mask, prepend_mask=prepend_mask, return_info=return_info, **extra_args, **kwargs) + + if return_info: + output, info = output + + output=output[:,prepend_length:,:] + + + if return_info: + return output, info + + return output + + def forward( + self, + x, + t, + cross_attn_cond=None, + cross_attn_cond_mask=None, + input_concat_cond=None, + global_embed=None, + taxonomy=None, + mask=None, + return_info=False, + **kwargs): + + + model_dtype = next(self.parameters()).dtype + + x = x.to(model_dtype) + + t = t.to(model_dtype) + + if cross_attn_cond is not None: + cross_attn_cond = cross_attn_cond.to(model_dtype) + + if input_concat_cond is not None: + input_concat_cond = input_concat_cond.to(model_dtype) + # Interpolate input_concat_cond to the same length as x + assert input_concat_cond.ndim == 4, f"input_concat_cond must be 4D, got {input_concat_cond.ndim}" + assert input_concat_cond.shape[0] == x.shape[0] + assert input_concat_cond.shape[1] == x.shape[1] + assert input_concat_cond.shape[-1] == x.shape[-1] + assert input_concat_cond.shape[-2] == self.input_concat_dim, f"input_concat_cond shape mismatch: {input_concat_cond.shape[-2]} != {self.input_concat_dim}" + + x = torch.cat([x, input_concat_cond], dim=-2) + + if global_embed is not None: + global_embed = global_embed.to(model_dtype) + + if cross_attn_cond_mask is not None: + cross_attn_cond_mask = cross_attn_cond_mask.bool() + + #cross_attn_cond_mask = None # Temporarily disabling conditioning masks due to kernel issue for flash attention + + + orig_shape = x.shape + x= rearrange(x, "b n c t -> (b n) c t") + + + x = self.preprocess_conv(x) + x + + x=x.view(orig_shape) + + x= rearrange(x, "b n c t -> b n t c") + #shape of contecxt is already [B, N, T, C] so no need to rearrange + + orig_shape = x.shape + + x= self.pos_emb_fn(x, taxonomy=taxonomy) + cross_attn_cond= self.pos_emb_crossattn_fn(cross_attn_cond, taxonomy=taxonomy) + + + x=rearrange(x, "b n t c -> b (n t) c") + + cross_attn_cond_orig_shape = cross_attn_cond.shape + cross_attn_cond = rearrange(cross_attn_cond, "b n t c -> b (n t) c") + + # rehape to [B, N \times T, C] for the transformer + + + # mask has shape [B, N ], I need to expand it to [B, N, T] for the convolution + mask= mask.unsqueeze(-1).expand(orig_shape[0], orig_shape[1], orig_shape[2]) + mask= rearrange(mask, "b n t -> b (n t)") + + cross_attn_cond_mask = cross_attn_cond_mask.unsqueeze(-1).expand(cross_attn_cond_orig_shape[0], cross_attn_cond_orig_shape[1], cross_attn_cond_orig_shape[2]) + cross_attn_cond_mask = rearrange(cross_attn_cond_mask, "b n t -> b (n t)") + + + out= self._forward( + x, + t, + cross_attn_cond=cross_attn_cond, + cross_attn_cond_mask=cross_attn_cond_mask, + input_concat_cond=input_concat_cond, + global_embed=global_embed, + mask=mask, + return_info=return_info, + **kwargs + ) + + #print("out shape", out.shape) + + out = rearrange(out, "b t c -> b c t") + out= self.postprocess_conv(out) + out + out = rearrange(out, "b c t -> b t c") + + #print("out shape after postprocess", out.shape) + + #now we reshape... + out = rearrange(out, "b (n t) c -> b n t c", n=orig_shape[1], t=orig_shape[2]) + + out=rearrange(out, "b n t c -> b n c t") + + return out \ No newline at end of file diff --git a/networks/transformer.py b/networks/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..9cd6851f942cbae55b6176938319cc37790067ea --- /dev/null +++ b/networks/transformer.py @@ -0,0 +1,932 @@ +#adapted from https://github.com/Stability-AI/stable-audio-tools/blob/main/stable_audio_tools/models/transformer.py + +from functools import reduce, partial +from packaging import version + +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import torch +import torch.nn.functional as F +from torch import nn, einsum +from torch.amp import autocast +from torch.nn.utils.parametrizations import weight_norm +from typing import Callable, Literal + +#from .utils import compile + +import os +enable_torch_compile = os.environ.get("ENABLE_TORCH_COMPILE", "0") == "1" + +def compile(function, *args, **kwargs): + + if enable_torch_compile: + try: + return torch.compile(function, *args, **kwargs) + except RuntimeError: + return function + + return function + +try: + from flash_attn import flash_attn_func, flash_attn_kvpacked_func +except ImportError as e: + print(e) + print('flash_attn not installed, disabling Flash Attention') + flash_attn_kvpacked_func = None + flash_attn_func = None + +try: + import natten +except ImportError: + natten = None + +def checkpoint(function, *args, **kwargs): + kwargs.setdefault("use_reentrant", False) + return torch.utils.checkpoint.checkpoint(function, *args, **kwargs) + + +# Copied and modified from https://github.com/lucidrains/x-transformers/blob/main/x_transformers/attend.py under MIT License +# License can be found in LICENSES/LICENSE_XTRANSFORMERS.txt + +def create_causal_mask(i, j, device): + return torch.ones((i, j), device = device, dtype = torch.bool).triu(j - i + 1) + +def or_reduce(masks): + head, *body = masks + for rest in body: + head = head | rest + return head + +class FourierFeatures(nn.Module): + def __init__(self, in_features, out_features, std=1.): + super().__init__() + assert out_features % 2 == 0 + self.weight = nn.Parameter(torch.randn( + [out_features // 2, in_features]) * std) + + def forward(self, input): + f = 2 * math.pi * input @ self.weight.T + return torch.cat([f.cos(), f.sin()], dim=-1) + + +class AbsolutePositionalEmbedding(nn.Module): + def __init__(self, dim, max_seq_len, fixed = False, seed = 42): + super().__init__() + self.scale = dim ** -0.5 + self.max_seq_len = max_seq_len + self.fixed=fixed + if fixed: + torch.manual_seed(seed) + emb = torch.randn(max_seq_len, dim) + self.register_buffer('emb', emb) # replaces nn.Embedding + else: + self.emb = nn.Embedding(max_seq_len, dim) + + def forward(self, x, pos = None, seq_start_pos = None): + seq_len, device = x.shape[1], x.device + assert seq_len <= self.max_seq_len, f'you are passing in a sequence length of {seq_len} but your absolute positional embedding has a max sequence length of {self.max_seq_len}' + + if pos is None: + pos = torch.arange(seq_len, device = device) + + if seq_start_pos is not None: + pos = (pos - seq_start_pos[..., None]).clamp(min = 0) + + if not self.fixed: + pos_emb = self.emb(pos) + else: + pos_emb = self.emb[pos] + pos_emb = pos_emb * self.scale + return pos_emb + +class ScaledSinusoidalEmbedding(nn.Module): + def __init__(self, dim, theta = 10000): + super().__init__() + assert (dim % 2) == 0, 'dimension must be divisible by 2' + self.scale = nn.Parameter(torch.ones(1) * dim ** -0.5) + + half_dim = dim // 2 + freq_seq = torch.arange(half_dim).float() / half_dim + inv_freq = theta ** -freq_seq + self.register_buffer('inv_freq', inv_freq, persistent = False) + + def forward(self, x, pos = None, seq_start_pos = None): + seq_len, device = x.shape[1], x.device + + if pos is None: + pos = torch.arange(seq_len, device = device) + + if seq_start_pos is not None: + pos = pos - seq_start_pos[..., None] + + emb = einsum('i, j -> i j', pos, self.inv_freq) + emb = torch.cat((emb.sin(), emb.cos()), dim = -1) + return emb * self.scale + +class RotaryEmbedding(nn.Module): + def __init__( + self, + dim, + use_xpos = False, + scale_base = 512, + interpolation_factor = 1., + base = 10000, + base_rescale_factor = 1. + ): + super().__init__() + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + # https://www.reddit.com/r/LocalLLaMA/comments/14lz7j5/ntkaware_scaled_rope_allows_llama_models_to_have/ + base *= base_rescale_factor ** (dim / (dim - 2)) + + inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float() / dim)) + self.register_buffer('inv_freq', inv_freq) + + assert interpolation_factor >= 1. + self.interpolation_factor = interpolation_factor + + if not use_xpos: + self.register_buffer('scale', None) + return + + scale = (torch.arange(0, dim, 2) + 0.4 * dim) / (1.4 * dim) + + self.scale_base = scale_base + self.register_buffer('scale', scale) + + def forward_from_seq_len(self, seq_len): + device = self.inv_freq.device + + t = torch.arange(seq_len, device = device) + return self.forward(t) + + @autocast("cuda", enabled = False) + def forward(self, t): + device = self.inv_freq.device + + t = t.to(torch.float32) + + t = t / self.interpolation_factor + + freqs = torch.einsum('i , j -> i j', t, self.inv_freq) + freqs = torch.cat((freqs, freqs), dim = -1) + + if self.scale is None: + return freqs, 1. + + power = (torch.arange(seq_len, device = device) - (seq_len // 2)) / self.scale_base + scale = self.scale ** rearrange(power, 'n -> n 1') + scale = torch.cat((scale, scale), dim = -1) + + return freqs, scale + +def rotate_half(x): + x = rearrange(x, '... (j d) -> ... j d', j = 2) + x1, x2 = x.unbind(dim = -2) + return torch.cat((-x2, x1), dim = -1) + +@autocast("cuda", enabled = False) +def apply_rotary_pos_emb(t, freqs, scale = 1): + out_dtype = t.dtype + + # cast to float32 if necessary for numerical stability + dtype = reduce(torch.promote_types, (t.dtype, freqs.dtype, torch.float32)) + rot_dim, seq_len = freqs.shape[-1], t.shape[-2] + freqs, t = freqs.to(dtype), t.to(dtype) + freqs = freqs[-seq_len:, :] + + if t.ndim == 4 and freqs.ndim == 3: + freqs = rearrange(freqs, 'b n d -> b 1 n d') + + # partial rotary embeddings, Wang et al. GPT-J + t, t_unrotated = t[..., :rot_dim], t[..., rot_dim:] + t = (t * freqs.cos() * scale) + (rotate_half(t) * freqs.sin() * scale) + + t, t_unrotated = t.to(out_dtype), t_unrotated.to(out_dtype) + + return torch.cat((t, t_unrotated), dim = -1) + +# norms +class LayerNorm(nn.Module): + def __init__(self, dim, bias=False, fix_scale=False, force_fp32=False, eps=1e-5): + """ + bias-less layernorm has been shown to be more stable. most newer models have moved towards rmsnorm, also bias-less + """ + super().__init__() + + if fix_scale: + self.register_buffer("gamma", torch.ones(dim)) + else: + self.gamma = nn.Parameter(torch.ones(dim)) + + if bias: + self.beta = nn.Parameter(torch.zeros(dim)) + else: + self.register_buffer("beta", torch.zeros(dim)) + + self.eps = eps + + self.force_fp32 = force_fp32 + + def forward(self, x): + if not self.force_fp32: + return F.layer_norm(x, x.shape[-1:], weight=self.gamma, bias=self.beta, eps=self.eps) + else: + output = F.layer_norm(x.float(), x.shape[-1:], weight=self.gamma.float(), bias=self.beta.float(), eps=self.eps) + return output.to(x.dtype) + +class LayerScale(nn.Module): + def __init__(self, dim, init_val = 1e-2): + super().__init__() + self.scale = nn.Parameter(torch.full([dim], init_val)) + def forward(self, x): + return x * self.scale + + +# feedforward + +class GLU(nn.Module): + def __init__( + self, + dim_in, + dim_out, + activation: Callable, + use_conv = False, + conv_kernel_size = 3, + ): + super().__init__() + self.act = activation + self.proj = nn.Linear(dim_in, dim_out * 2) if not use_conv else nn.Conv1d(dim_in, dim_out * 2, conv_kernel_size, padding = (conv_kernel_size // 2)) + self.use_conv = use_conv + + def forward(self, x): + if self.use_conv: + x = rearrange(x, 'b n d -> b d n') + x = self.proj(x) + x = rearrange(x, 'b d n -> b n d') + else: + x = self.proj(x) + + x, gate = x.chunk(2, dim = -1) + return x * self.act(gate) + +class FeedForward(nn.Module): + def __init__( + self, + dim, + dim_out = None, + mult = 4, + no_bias = False, + glu = True, + use_conv = False, + conv_kernel_size = 3, + zero_init_output = True, + ): + super().__init__() + inner_dim = int(dim * mult) + + # Default to SwiGLU + + activation = nn.SiLU() + + dim_out = dim if dim_out is None else dim_out + + if glu: + linear_in = GLU(dim, inner_dim, activation) + else: + linear_in = nn.Sequential( + Rearrange('b n d -> b d n') if use_conv else nn.Identity(), + nn.Linear(dim, inner_dim, bias = not no_bias) if not use_conv else nn.Conv1d(dim, inner_dim, conv_kernel_size, padding = (conv_kernel_size // 2), bias = not no_bias), + Rearrange('b n d -> b d n') if use_conv else nn.Identity(), + activation + ) + + linear_out = nn.Linear(inner_dim, dim_out, bias = not no_bias) if not use_conv else nn.Conv1d(inner_dim, dim_out, conv_kernel_size, padding = (conv_kernel_size // 2), bias = not no_bias) + + # init last linear layer to 0 + if zero_init_output: + nn.init.zeros_(linear_out.weight) + if not no_bias: + nn.init.zeros_(linear_out.bias) + + + self.ff = nn.Sequential( + linear_in, + Rearrange('b d n -> b n d') if use_conv else nn.Identity(), + linear_out, + Rearrange('b n d -> b d n') if use_conv else nn.Identity(), + ) + + #@compile + def forward(self, x): + return self.ff(x) + +class Attention(nn.Module): + def __init__( + self, + dim, + dim_heads = 64, + dim_context = None, + causal = False, + zero_init_output=True, + qk_norm: Literal['l2', 'ln', 'none'] = 'none', + natten_kernel_size = None, + sliding_window = [-1, -1], + feat_scale = False + ): + super().__init__() + self.dim = dim + self.dim_heads = dim_heads + self.causal = causal + + dim_kv = dim_context if dim_context is not None else dim + + self.num_heads = dim // dim_heads + self.kv_heads = dim_kv // dim_heads + #print("num_heads", self.num_heads, "kv_heads", self.kv_heads) + #print("dim", dim, "dim_heads", dim_heads, "dim_kv", dim_kv, dim/dim_heads, dim_kv/dim_heads) + assert dim % dim_heads == 0, f'dim {dim} must be divisible by dim_heads {dim_heads}' + assert dim_kv % dim_heads == 0, f'dim_kv {dim_kv} must be divisible by dim_heads {dim_heads}' + + + if dim_context is not None: + self.to_q = nn.Linear(dim, dim, bias=False) + self.to_kv = nn.Linear(dim_kv, dim_kv * 2, bias=False) + else: + self.to_qkv = nn.Linear(dim, dim * 3, bias=False) + + self.to_out = nn.Linear(dim, dim, bias=False) + + if zero_init_output: + nn.init.zeros_(self.to_out.weight) + + if qk_norm not in ['l2', 'ln', 'none']: + raise ValueError(f'qk_norm must be one of ["l2", "ln", "none"], got {qk_norm}') + + self.qk_norm = qk_norm + + if self.qk_norm == "ln": + self.q_norm = nn.LayerNorm(dim_heads, elementwise_affine=True, eps=1.0e-6) + self.k_norm = nn.LayerNorm(dim_heads, elementwise_affine=True, eps=1.0e-6) + + # Using 1d neighborhood attention + self.natten_kernel_size = natten_kernel_size + if natten_kernel_size is not None: + return + + self.use_pt_flash = torch.cuda.is_available() and version.parse(torch.__version__) >= version.parse('2.0.0') + + self.use_fa_flash = torch.cuda.is_available() and flash_attn_func is not None + + self.sdp_kwargs = dict( + enable_flash = True, + enable_math = True, + enable_mem_efficient = True + ) + + self.sliding_window = sliding_window + if not (sliding_window[0] == -1 and sliding_window[1] == -1) and not self.use_fa_flash: + print('Sliding window is being used, but Flash Attention is not. Please install Flash Attention to get correct results') + + self.feat_scale = feat_scale + + if self.feat_scale: + self.lambda_dc = nn.Parameter(torch.zeros(dim)) + self.lambda_hf = nn.Parameter(torch.zeros(dim)) + + def flash_attn( + self, + q, + k, + v, + mask = None, + causal = None + ): + batch, heads, q_len, _, k_len, device = *q.shape, k.shape[-2], q.device + kv_heads = k.shape[1] + # Recommended for multi-query single-key-value attention by Tri Dao + # kv shape torch.Size([1, 512, 64]) -> torch.Size([1, 8, 512, 64]) + + if heads != kv_heads: + # Repeat interleave kv_heads to match q_heads + heads_per_kv_head = heads // kv_heads + assert heads % kv_heads == 0, f'heads {heads} must be divisible by kv_heads {kv_heads}' + k, v = map(lambda t: t.repeat_interleave(heads_per_kv_head, dim = 1), (k, v)) + + if k.ndim == 3: + k = rearrange(k, 'b ... -> b 1 ...').expand_as(q) + + if v.ndim == 3: + v = rearrange(v, 'b ... -> b 1 ...').expand_as(q) + + causal = self.causal if causal is None else causal + + if q_len == 1 and causal: + causal = False + + if mask is not None: + assert mask.ndim == 4 + mask = mask.expand(batch, heads, q_len, k_len) + + # handle kv cache - this should be bypassable in updated flash attention 2 + + if k_len > q_len and causal: + causal_mask = self.create_causal_mask(q_len, k_len, device = device) + if mask is None: + mask = ~causal_mask + else: + mask = mask & ~causal_mask + causal = False + + # manually handle causal mask, if another mask was given + + row_is_entirely_masked = None + + if mask is not None and causal: + causal_mask = self.create_causal_mask(q_len, k_len, device = device) + mask = mask & ~causal_mask + + # protect against an entire row being masked out + + row_is_entirely_masked = ~mask.any(dim = -1) + mask[..., 0] = mask[..., 0] | row_is_entirely_masked + + causal = False + + #with torch.backends.cuda.sdp_kernel(**self.sdp_kwargs): + #print("q.shape", q.shape, "k.shape", k.shape, "v.shape", v.shape, "mask.shape", mask.shape if mask is not None else None, "causal", causal) + out = F.scaled_dot_product_attention( + q, k, v, + attn_mask = mask, + is_causal = causal + ) + + # for a row that is entirely masked out, should zero out the output of that row token + + if row_is_entirely_masked is not None: + out = out.masked_fill(row_is_entirely_masked[..., None], 0.) + + return out + + @compile + def apply_qk_layernorm(self, q, k): + q = self.q_norm(q) + k = self.k_norm(k) + return q, k + + #@compile + def forward( + self, + x, + context = None, + mask = None, + context_mask = None, + rotary_pos_emb = None, + causal = None + ): + h, kv_h, has_context = self.num_heads, self.kv_heads, context is not None + + #print("h", h, "kv_h", kv_h, "has_context", has_context) + #print("mask", mask, "context_mask", context_mask) + + kv_input = context if has_context else x + + if hasattr(self, 'to_q'): + # Use separate linear projections for q and k/v + q = self.to_q(x) + q = rearrange(q, 'b n (h d) -> b h n d', h = h) + + k, v = self.to_kv(kv_input).chunk(2, dim=-1) + + k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = kv_h), (k, v)) + else: + # Use fused linear projection + q, k, v = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), (q, k, v)) + + # Normalize q and k for cosine sim attention + if self.qk_norm == "l2": + q = F.normalize(q, dim=-1) + k = F.normalize(k, dim=-1) + elif self.qk_norm == "ln": + q, k = self.apply_qk_layernorm(q, k) + + if rotary_pos_emb is not None and not has_context: + freqs, _ = rotary_pos_emb + + q_dtype = q.dtype + k_dtype = k.dtype + + q = q.to(torch.float32) + k = k.to(torch.float32) + freqs = freqs.to(torch.float32) + + q = apply_rotary_pos_emb(q, freqs) + k = apply_rotary_pos_emb(k, freqs) + + q = q.to(q_dtype) + k = k.to(k_dtype) + + input_mask = context_mask + + if input_mask is None and not has_context: + input_mask = mask + + # determine masking + masks = [] + final_attn_mask = None # The mask that will be applied to the attention matrix, taking all masks into account + + if input_mask is not None: + input_mask = rearrange(input_mask, 'b j -> b 1 1 j') + masks.append(~input_mask) + + # Other masks will be added here later + + if len(masks) > 0: + final_attn_mask = ~or_reduce(masks) + + n, device = q.shape[-2], q.device + + causal = self.causal if causal is None else causal + + if n == 1 and causal: + causal = False + + if self.natten_kernel_size is not None: + if natten is None: + raise ImportError('natten not installed, please install natten to use neighborhood attention') + + dtype_in = q.dtype + q, k, v = map(lambda t: t.to(torch.float32), (q, k, v)) + + attn = natten.functional.na1d_qk(q, k, kernel_size = self.natten_kernel_size, dilation=1) + + if final_attn_mask is not None: + attn = attn.masked_fill(final_attn_mask, -torch.finfo(attn.dtype).max) + + attn = F.softmax(attn, dim=-1, dtype=torch.float32) + + out = natten.functional.na1d_av(attn, v, kernel_size = self.natten_kernel_size, dilation=1).to(dtype_in) + + # Prioritize Flash Attention 2 + elif self.use_fa_flash: + assert final_attn_mask is None, 'masking not yet supported for Flash Attention 2' + # Flash Attention 2 requires FP16 inputs + fa_dtype_in = q.dtype + + q, k, v = map(lambda t: rearrange(t, 'b h n d -> b n h d'), (q, k, v)) + + if fa_dtype_in != torch.float16 and fa_dtype_in != torch.bfloat16: + q, k, v = map(lambda t: t.to(torch.float16), (q, k, v)) + + out = flash_attn_func(q, k, v, causal = causal, window_size=self.sliding_window) + + out = rearrange(out.to(fa_dtype_in), 'b n h d -> b h n d') + + # Fall back to PyTorch implementation + elif self.use_pt_flash: + out = self.flash_attn(q, k, v, causal = causal, mask = final_attn_mask) + + else: + # Fall back to custom implementation + + if h != kv_h: + # Repeat interleave kv_heads to match q_heads + heads_per_kv_head = h // kv_h + k, v = map(lambda t: t.repeat_interleave(heads_per_kv_head, dim = 1), (k, v)) + + scale = 1. / (q.shape[-1] ** 0.5) + + kv_einsum_eq = 'b j d' if k.ndim == 3 else 'b h j d' + + dots = einsum(f'b h i d, {kv_einsum_eq} -> b h i j', q, k) * scale + + i, j, dtype = *dots.shape[-2:], dots.dtype + + mask_value = -torch.finfo(dots.dtype).max + + if final_attn_mask is not None: + dots = dots.masked_fill(~final_attn_mask, mask_value) + + if causal: + causal_mask = self.create_causal_mask(i, j, device = device) + dots = dots.masked_fill(causal_mask, mask_value) + + attn = F.softmax(dots, dim=-1, dtype=torch.float32) + attn = attn.type(dtype) + + out = einsum(f'b h i j, {kv_einsum_eq} -> b h i d', attn, v) + + # merge heads + out = rearrange(out, ' b h n d -> b n (h d)') + + # Communicate between heads + + # with autocast(enabled = False): + # out_dtype = out.dtype + # out = out.to(torch.float32) + # out = self.to_out(out).to(out_dtype) + out = self.to_out(out) + + if self.feat_scale: + out_dc = out.mean(dim=-2, keepdim=True) + out_hf = out - out_dc + + # Selectively modulate DC and high frequency components + out = out + self.lambda_dc * out_dc + self.lambda_hf * out_hf + + if mask is not None: + mask = rearrange(mask, 'b n -> b n 1') + out = out.masked_fill(~mask, 0.) + + return out + +class ConformerModule(nn.Module): + def __init__( + self, + dim, + norm_kwargs = {}, + ): + + super().__init__() + + self.dim = dim + + self.in_norm = LayerNorm(dim, **norm_kwargs) + self.pointwise_conv = nn.Conv1d(dim, dim, kernel_size=1, bias=False) + self.glu = GLU(dim, dim, nn.SiLU()) + self.depthwise_conv = nn.Conv1d(dim, dim, kernel_size=17, groups=dim, padding=8, bias=False) + self.mid_norm = LayerNorm(dim, **norm_kwargs) # This is a batch norm in the original but I don't like batch norm + self.swish = nn.SiLU() + self.pointwise_conv_2 = nn.Conv1d(dim, dim, kernel_size=1, bias=False) + + #@compile + def forward(self, x): + x = self.in_norm(x) + x = rearrange(x, 'b n d -> b d n') + x = self.pointwise_conv(x) + x = rearrange(x, 'b d n -> b n d') + x = self.glu(x) + x = rearrange(x, 'b n d -> b d n') + x = self.depthwise_conv(x) + x = rearrange(x, 'b d n -> b n d') + x = self.mid_norm(x) + x = self.swish(x) + x = rearrange(x, 'b n d -> b d n') + x = self.pointwise_conv_2(x) + x = rearrange(x, 'b d n -> b n d') + + return x + +class TransformerBlock(nn.Module): + def __init__( + self, + dim, + dim_heads = 64, + cross_attend = False, + dim_context = None, + global_cond_dim = None, + causal = False, + zero_init_branch_outputs = True, + conformer = False, + layer_ix = -1, + remove_norms = False, + add_rope = False, + layer_scale = False, + attn_kwargs = {}, + ff_kwargs = {}, + norm_kwargs = {} + ): + + super().__init__() + self.dim = dim + self.dim_heads = dim_heads + self.cross_attend = cross_attend + self.dim_context = dim_context + self.causal = causal + + if layer_scale and zero_init_branch_outputs: + print('zero_init_branch_outputs is redundant with layer_scale, setting zero_init_branch_outputs to False') + zero_init_branch_outputs = False + + self.pre_norm = LayerNorm(dim,**norm_kwargs) if not remove_norms else nn.Identity() + + self.add_rope = add_rope + + self.self_attn = Attention( + dim, + dim_heads = dim_heads, + causal = causal, + zero_init_output=zero_init_branch_outputs, + **attn_kwargs + ) + + self.self_attn_scale = LayerScale(dim) if layer_scale else nn.Identity() + + self.cross_attend = cross_attend + if cross_attend: + self.cross_attend_norm = LayerNorm(dim, **norm_kwargs) if not remove_norms else nn.Identity() + self.cross_attn = Attention( + dim, + dim_heads = dim_heads, + dim_context=dim_context, + causal = causal, + zero_init_output=zero_init_branch_outputs, + **attn_kwargs + ) + self.cross_attn_scale = LayerScale(dim) if layer_scale else nn.Identity() + + self.ff_norm = LayerNorm(dim, **norm_kwargs) if not remove_norms else nn.Identity() + self.ff = FeedForward(dim, zero_init_output=zero_init_branch_outputs, **ff_kwargs) + self.ff_scale = LayerScale(dim) if layer_scale else nn.Identity() + + self.layer_ix = layer_ix + + self.conformer = None + if conformer: + self.conformer = ConformerModule(dim, norm_kwargs=norm_kwargs) + self.conformer_scale = LayerScale(dim) if layer_scale else nn.Identity() + + self.global_cond_dim = global_cond_dim + + if global_cond_dim is not None: + self.to_scale_shift_gate = nn.Parameter(torch.randn(6*dim)/dim**0.5) + + self.rope = RotaryEmbedding(max(dim_heads // 2, 32)) if add_rope else None + + @compile + def forward( + self, + x, + context = None, + global_cond=None, + mask = None, + context_mask = None, + rotary_pos_emb = None + ): + if rotary_pos_emb is None and self.add_rope: + rotary_pos_emb = self.rope.forward_from_seq_len(x.shape[-2]) + + if self.global_cond_dim is not None and self.global_cond_dim > 0 and global_cond is not None: + + scale_self, shift_self, gate_self, scale_ff, shift_ff, gate_ff = (self.to_scale_shift_gate + global_cond).unsqueeze(1).chunk(6, dim=-1) + + # self-attention with adaLN + residual = x + x = self.pre_norm(x) + x = x * (1 + scale_self) + shift_self + x = self.self_attn(x, mask = mask, rotary_pos_emb = rotary_pos_emb) + x = x * torch.sigmoid(1 - gate_self) + x = self.self_attn_scale(x) + x = x + residual + + #print("self.cross_attend", self.cross_attend) + #print("context", context.shape) + if context is not None and self.cross_attend: + #print("mask", mask) + #print("context_mask", context_mask) + + x = x + self.cross_attn_scale(self.cross_attn(self.cross_attend_norm(x), context = context, context_mask = context_mask)) + + if self.conformer is not None: + x = x + self.conformer_scale(self.conformer(x)) + + # feedforward with adaLN + residual = x + x = self.ff_norm(x) + x = x * (1 + scale_ff) + shift_ff + x = self.ff(x) + x = x * torch.sigmoid(1 - gate_ff) + x = self.ff_scale(x) + x = x + residual + + else: + x = x + self.self_attn_scale(self.self_attn(self.pre_norm(x), mask = mask, rotary_pos_emb = rotary_pos_emb)) + + if context is not None and self.cross_attend: + #print(x.shape, context.shape, context_mask.shape if context_mask is not None else None) + #print(self.dim, self.dim_context, self.dim_heads) + x = x + self.cross_attn_scale(self.cross_attn(self.cross_attend_norm(x), context = context, context_mask = context_mask)) + + if self.conformer is not None: + x = x + self.conformer_scale(self.conformer(x)) + + x = x + self.ff_scale(self.ff(self.ff_norm(x))) + return x + +class ContinuousTransformer(nn.Module): + def __init__( + self, + dim, + depth, + *, + dim_in = None, + dim_out = None, + num_heads = 64, + cross_attend=False, + cond_token_dim=None, + final_cross_attn_ix=-1, + global_cond_dim=None, + causal=False, + zero_init_branch_outputs=True, + conformer=False, + **kwargs + ): + + super().__init__() + + self.dim = dim + self.depth = depth + self.causal = causal + self.layers = nn.ModuleList([]) + + self.project_in = nn.Linear(dim_in, dim, bias=False) if dim_in is not None else nn.Identity() + self.project_out = nn.Linear(dim, dim_out, bias=False) if dim_out is not None else nn.Identity() + + + + + self.global_cond_embedder = None + if global_cond_dim is not None: + self.global_cond_embedder = nn.Sequential( + nn.Linear(global_cond_dim, dim), + nn.SiLU(), + nn.Linear(dim, dim * 6) + ) + + self.final_cross_attn_ix = final_cross_attn_ix + + for i in range(depth): + should_cross_attend = cross_attend and (self.final_cross_attn_ix == -1 or i <= (self.final_cross_attn_ix)) + self.layers.append( + TransformerBlock( + dim, + dim_heads = (dim) // num_heads, + cross_attend = should_cross_attend, + dim_context = cond_token_dim, + global_cond_dim = global_cond_dim, + causal = causal, + zero_init_branch_outputs = zero_init_branch_outputs, + conformer=conformer, + layer_ix=i, + **kwargs + ) + ) + + + def forward( + self, + x, + mask = None, + prepend_embeds = None, + prepend_mask = None, + global_cond = None, + return_info = False, + context = None, + context_mask = None, + **kwargs + ): + batch, seq, device = *x.shape[:2], x.device + + model_dtype = next(self.parameters()).dtype + x = x.to(model_dtype) + + info = { + "hidden_states": [], + } + + + x = self.project_in(x) + + if prepend_embeds is not None: + prepend_length, prepend_dim = prepend_embeds.shape[1:] + + assert prepend_dim == x.shape[-1], 'prepend dimension must match sequence dimension' + + x = torch.cat((prepend_embeds, x), dim = -2) + + if prepend_mask is not None or mask is not None: + mask = mask if mask is not None else torch.ones((batch, seq), device = device, dtype = torch.bool) + prepend_mask = prepend_mask if prepend_mask is not None else torch.ones((batch, prepend_length), device = device, dtype = torch.bool) + + mask = torch.cat((prepend_mask, mask), dim = -1) + + + if global_cond is not None and self.global_cond_embedder is not None: + global_cond = self.global_cond_embedder(global_cond) + + # Iterate over the transformer layers + for layer in self.layers: + #x = layer(x, rotary_pos_emb = rotary_pos_emb, global_cond=global_cond, **kwargs) + x = checkpoint(layer, x, rotary_pos_emb = None, global_cond=global_cond, context=context, mask=mask, context_mask=context_mask, **kwargs) + + if return_info: + info["hidden_states"].append(x) + + x = self.project_out(x) + + + if return_info: + return x, info + + return x \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e9fe2057ee42759ef794bd2754aa453d814d372f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +git+https://github.com/TEAMuP-dev/pyharp.git@v0.3.0 +torch +torchaudio +omegaconf>=2.3.0 +hydra-core>=1.3.0 +numpy +scipy +soundfile +pyloudnorm +einops +packaging +librosa +transformers +wget +huggingface-hub diff --git a/utils/MSS_loss.py b/utils/MSS_loss.py new file mode 100755 index 0000000000000000000000000000000000000000..924492945a37ebd280fb818bc3c1090d8fa4c9dc --- /dev/null +++ b/utils/MSS_loss.py @@ -0,0 +1,260 @@ +""" + Implementation of objective functions used in the task 'ITO-Master' + https://github.com/SonyResearch/ITO-Master/blob/master/ito_master/modules/loss.py +""" + +import numpy as np +import math +import torch +import torch.nn.functional as F +import torch.nn as nn +import auraloss +import torchaudio +import warnings + + +import os +import sys + +currentdir = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.dirname(currentdir)) + +class FrontEnd(nn.Module): + def __init__(self, channel='stereo', \ + n_fft=2048, \ + n_mels=128, \ + sample_rate=44100, \ + hop_length=None, \ + win_length=None, \ + window="hann", \ + eps=1e-7, \ + device=torch.device("cpu")): + super(FrontEnd, self).__init__() + self.channel = channel + self.n_fft = n_fft + self.n_mels = n_mels + self.sample_rate = sample_rate + self.hop_length = n_fft//4 if hop_length==None else hop_length + self.win_length = n_fft if win_length==None else win_length + self.eps = eps + if window=="hann": + self.window = torch.hann_window(window_length=self.win_length, periodic=True).to(device) + elif window=="hamming": + self.window = torch.hamming_window(window_length=self.win_length, periodic=True).to(device) + self.melscale_transform = torchaudio.transforms.MelScale(n_mels=self.n_mels, \ + sample_rate=self.sample_rate, \ + n_stft=self.n_fft//2+1).to(device) + + + def forward(self, input, mode): + # front-end function which channel-wise combines all demanded features + # input shape : batch x channel x raw waveform + # output shape : batch x channel x frequency x time + phase_output = None + + front_output_list = [] + for cur_mode in mode: + # Real & Imaginary + if cur_mode=="cplx": + if self.channel=="mono": + output = torch.stft(input, n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window=self.window) + elif self.channel=="stereo": + output_l = torch.stft(input[:,0], n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window=self.window) + output_r = torch.stft(input[:,1], n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window=self.window) + output = torch.cat((output_l, output_r), axis=-1) + if input.shape[-1] % round(self.n_fft/4) == 0: + output = output[:, :, :-1] + if self.n_fft % 2 == 0: + output = output[:, :-1] + front_output_list.append(output.permute(0, 3, 1, 2)) + # Magnitude & Phase or Mel + elif "mag" in cur_mode or "mel" in cur_mode: + if self.channel=="mono": + cur_cplx = torch.stft(input, n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window=self.window, return_complex=True) + output = self.mag(cur_cplx).unsqueeze(-1)[..., 0:1] + if "mag_phase" in cur_mode: + phase = self.phase(cur_cplx) + if "mel" in cur_mode: + output = self.melscale_transform(output.squeeze(-1)).unsqueeze(-1) + elif self.channel=="stereo": + cplx_l = torch.stft(input[:,0], n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window=self.window, return_complex=True) + cplx_r = torch.stft(input[:,1], n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window=self.window, return_complex=True) + mag_l = self.mag(cplx_l).unsqueeze(-1) + mag_r = self.mag(cplx_r).unsqueeze(-1) + output = torch.cat((mag_l, mag_r), axis=-1) + if "mag_phase" in cur_mode: + phase_l = self.phase(cplx_l).unsqueeze(-1) + phase_r = self.phase(cplx_r).unsqueeze(-1) + output = torch.cat((mag_l, phase_l, mag_r, phase_r), axis=-1) + if "mel" in cur_mode: + output = torch.cat((self.melscale_transform(mag_l.squeeze(-1)).unsqueeze(-1), self.melscale_transform(mag_r.squeeze(-1)).unsqueeze(-1)), axis=-1) + + if "log" in cur_mode: + output = torch.log(output+self.eps) + + if input.shape[-1] % round(self.n_fft/4) == 0: + output = output[:, :, :-1] + if cur_mode!="mel" and self.n_fft % 2 == 0: # discard highest frequency + output = output[:, 1:] + front_output_list.append(output.permute(0, 3, 1, 2)) + + # combine all demanded features + if not front_output_list: + raise NameError("NameError at FrontEnd: check using features for front-end") + elif len(mode)!=1: + for i, cur_output in enumerate(front_output_list): + if i==0: + front_output = cur_output + else: + front_output = torch.cat((front_output, cur_output), axis=1) + else: + front_output = front_output_list[0] + + return front_output + + + def mag(self, cplx_input, eps=1e-07): + # mag_summed = cplx_input.pow(2.).sum(-1) + eps + mag_summed = cplx_input.real.pow(2.) + cplx_input.imag.pow(2.) + eps + return mag_summed.pow(0.5) + + + def phase(self, cplx_input, ): + return torch.atan2(cplx_input.imag, cplx_input.real) + # return torch.angle(cplx_input) + + + +# Multi-Scale Spectral Loss proposed at the paper "DDSP: DIFFERENTIABLE DIGITAL SIGNAL PROCESSING" (https://arxiv.org/abs/2001.04643) +# we extend this loss by applying it to mid/side channels +class MultiScale_Spectral_Loss_MidSide_DDSP(nn.Module): + def __init__(self, mode='midside', \ + reduce=True, \ + n_filters=None, \ + windows_size=None, \ + hops_size=None, \ + window="hann", \ + eps=1e-7, \ + device=torch.device("cpu")): + super(MultiScale_Spectral_Loss_MidSide_DDSP, self).__init__() + self.mode = mode + self.eps = eps + self.mid_weight = 0.5 # value in the range of 0.0 ~ 1.0 + self.logmag_weight = 0.1 + + if n_filters is None: + n_filters = [4096, 2048, 1024, 512] + if windows_size is None: + windows_size = [4096, 2048, 1024, 512] + if hops_size is None: + hops_size = [1024, 512, 256, 128] + + self.multiscales = [] + for i in range(len(windows_size)): + cur_scale = {'window_size' : float(windows_size[i])} + if self.mode=='midside': + cur_scale['front_end'] = FrontEnd(channel='mono', \ + n_fft=n_filters[i], \ + hop_length=hops_size[i], \ + win_length=windows_size[i], \ + window=window, \ + device=device) + elif self.mode=='ori': + cur_scale['front_end'] = FrontEnd(channel='stereo', \ + n_fft=n_filters[i], \ + hop_length=hops_size[i], \ + win_length=windows_size[i], \ + window=window, \ + device=device) + self.multiscales.append(cur_scale) + + self.reduce=reduce + self.objective_l1 = nn.L1Loss(reduce=reduce) + self.objective_l2 = nn.MSELoss(reduce=reduce) + + + def forward(self, est_targets, targets): + if self.mode=='midside': + return self.forward_midside(est_targets, targets) + elif self.mode=='ori': + return self.forward_ori(est_targets, targets) + + + def forward_ori(self, est_targets, targets): + if self.reduce: + total_mag_loss = 0.0 + total_logmag_loss = 0.0 + else: + total_mag_loss=torch.zeros(est_targets.shape[0], 1).to(est_targets.device) + total_logmag_loss=torch.zeros(est_targets.shape[0], 1).to(est_targets.device) + for cur_scale in self.multiscales: + est_mag = cur_scale['front_end'](est_targets, mode=["mag"]) + tgt_mag = cur_scale['front_end'](targets, mode=["mag"]) + + mag_loss = self.magnitude_loss(est_mag, tgt_mag) + logmag_loss = self.log_magnitude_loss(est_mag, tgt_mag) + if self.reduce: + total_mag_loss += mag_loss + total_logmag_loss += logmag_loss + else: + total_logmag_loss += logmag_loss.mean((1, 2, 3)).unsqueeze(-1) + total_mag_loss += mag_loss.mean((1, 2, 3)).unsqueeze(-1) + # return total_loss + return (1-self.logmag_weight)*total_mag_loss + \ + (self.logmag_weight)*total_logmag_loss + + + def forward_midside(self, est_targets, targets): + est_mid, est_side = self.to_mid_side(est_targets) + tgt_mid, tgt_side = self.to_mid_side(targets) + if self.reduce: + total_mag_loss = 0.0 + total_logmag_loss = 0.0 + else: + total_logmag_loss=torch.zeros(est_targets.shape[0], 1).to(est_targets.device) + total_mag_loss=torch.zeros(est_targets.shape[0], 1).to(est_targets.device) + + for cur_scale in self.multiscales: + est_mid_mag = cur_scale['front_end'](est_mid, mode=["mag"]) + est_side_mag = cur_scale['front_end'](est_side, mode=["mag"]) + tgt_mid_mag = cur_scale['front_end'](tgt_mid, mode=["mag"]) + tgt_side_mag = cur_scale['front_end'](tgt_side, mode=["mag"]) + + mag_loss = self.mid_weight*self.magnitude_loss(est_mid_mag, tgt_mid_mag) + \ + (1-self.mid_weight)*self.magnitude_loss(est_side_mag, tgt_side_mag) + logmag_loss = self.mid_weight*self.log_magnitude_loss(est_mid_mag, tgt_mid_mag) + \ + (1-self.mid_weight)*self.log_magnitude_loss(est_side_mag, tgt_side_mag) + + #take mean over all dimensions except batch + if self.reduce: + total_mag_loss += mag_loss + total_logmag_loss += logmag_loss + else: + total_mag_loss += mag_loss.mean((1, 2, 3)).unsqueeze(-1) + #mean over dims 1, 2, 3 + total_logmag_loss += logmag_loss.mean((1, 2, 3)).unsqueeze(-1) + # return total_loss + return (1-self.logmag_weight)*total_mag_loss + \ + (self.logmag_weight)*total_logmag_loss + + + def to_mid_side(self, stereo_in): + mid = stereo_in[:,0] + stereo_in[:,1] + side = stereo_in[:,0] - stereo_in[:,1] + return mid, side + + + def magnitude_loss(self, est_mag_spec, tgt_mag_spec): + if self.reduce: + return torch.norm(self.objective_l1(est_mag_spec, tgt_mag_spec)) + else: + return self.objective_l1(est_mag_spec, tgt_mag_spec) + + + def log_magnitude_loss(self, est_mag_spec, tgt_mag_spec): + est_log_mag_spec = torch.log10(est_mag_spec+self.eps) + tgt_log_mag_spec = torch.log10(tgt_mag_spec+self.eps) + return self.objective_l2(est_log_mag_spec, tgt_log_mag_spec) + + + diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/collators.py b/utils/collators.py new file mode 100644 index 0000000000000000000000000000000000000000..0b147ddf11fe31717261cf066b75e23f48b58e38 --- /dev/null +++ b/utils/collators.py @@ -0,0 +1,262 @@ + + +import torch +import torch.nn.functional as F +import torchaudio + +def collate_multitrack_sim(batch, max_tracks=None): + + x= [ data_i[0] for data_i in batch ] # x is a list of tensors, each tensor is a track + clusters=[ torch.tensor(data_i[1]) for data_i in batch ] # cluster is a list of tensors, each tensor is a cluster + taxonomies=[ data_i[2] for data_i in batch ] # taxonomy is a + + if max_tracks is None: + max_tracks = max(track.shape[0] for track in x) # Find the maximum number of tracks in the batch + else: + if max_tracks > max(track.shape[0] for track in x): + print(f"Warning: max_tracks is set to {max_tracks}, but the maximum number of tracks in the batch is {max(track.shape[0] for track in x)}. I dont know what will happen, consider increasing max_tracks," ) + + # Pad each examples with zeros to the maximum length (dimension 0) + + padded_x = [] + padded_taxonomies = [] + masks = torch.zeros((len(x), max_tracks), dtype=torch.bool) # Create a mask tensor + + for i in range(len(x)): + current_x = x[i] + current_taxonomies = taxonomies[i] + + # Get current number of tracks + current_tracks = current_x.shape[0] + + masks[i, :current_tracks] = 1 # Set mask for current tracks + + if current_tracks < max_tracks: + # Pad dimension N (first dimension) + # For a tensor of shape [N, C, L], we need to pad the first dimension + # F.pad expects padding as (pad_left, pad_right, pad_top, pad_bottom, pad_front, pad_back) + # To pad the first dimension, we need to specify padding for the third-from-last dimension + pad_size = (0, 0, # last dim (L): no padding + 0, 0, # second-to-last dim (C): no padding + 0, max_tracks - current_tracks) # third-to-last dim (N): pad at the end + + padded_x.append(F.pad(current_x, pad_size)) + + # Pad taxonomies + padded_taxonomies.append(current_taxonomies + [None] * (max_tracks - len(current_taxonomies))) + elif current_tracks > max_tracks : + raise ValueError(f"Number of tracks {current_tracks} exceeds maximum allowed {max_tracks}. that is impossible") + else: + padded_x.append(current_x) + padded_taxonomies.append(current_taxonomies[:max_tracks]) + + + + x_stacked = torch.stack(padded_x, dim=0) # Shape: [B, max_tracks, C, L] + + clusters_stacked = torch.stack(clusters, dim=0) # Shape: [B, max_tracks] + + return { + 'x': x_stacked, + 'clusters': clusters_stacked, + 'taxonomies': padded_taxonomies, + "masks": masks + } + +def collate_multitrack(batch, max_tracks=None, sample_rate=None, segment_length=None, device=None): + + x= [ data_i[0] for data_i in batch ] # x is a list of tensors, each tensor is a track + + paths=[ data_i[1] for data_i in batch ] # paths is a list of paths to the audio files, each path is a string + fs= [ data_i[2] for data_i in batch ] # fs is a list of sample rates, each sample rate is an integer + + if max_tracks is None: + max_tracks = max(track.shape[0] for track in x) # Find the maximum number of tracks in the batch + else: + if max_tracks < max(track.shape[0] for track in x): + print(f"Warning: max_tracks is set to {max_tracks}, but the maximum number of tracks in the batch is {max(track.shape[0] for track in x)}. I will crop the last tracks. I hope the order is random..." ) + + for i in range(len(x)): + if x[i].shape[0] > max_tracks: + print("cropping x[i] to max_tracks") + x[i] = x[i][:max_tracks] + + is_x_none= any(x_i is None for x_i in x) + + assert not (is_x_none ), "Either x or y should be None, but not both. This is a bug in the collator" + + #now resample audio to 44100 Hz and cut to segment length + for i in range(len(x)): + x[i] = x[i].to(device) # Move to device + + if fs[i] != sample_rate: + if fs[i] == 48000 and sample_rate == 44100: + #print(f"Resampling audio from {fs[i]} Hz to {sample_rate} Hz") + x[i]=torchaudio.functional.resample(x[i], orig_freq=160, new_freq=147) + else: + #print(f"Resampling audio from {fs[i]} Hz to {sample_rate} Hz") + x[i]=torchaudio.functional.resample(x[i], fs[i], sample_rate) + + assert segment_length is not None, "segment_length should be set to the length of the audio segment in samples" + if x[i].shape[-1] > segment_length: + x[i] = x[i][..., :segment_length] + elif x[i].shape[-1] < segment_length: + raise ValueError(f"Audio length {x[i].shape[-1]} is less than segment length {segment_length}. Please check your data.") + + # Pad each examples with zeros to the maximum length (dimension 0) + padded_x = [] + masks = torch.zeros((len(x), max_tracks), dtype=torch.bool) # Create a mask tensor + + + for i in range(len(x)): + if not is_x_none: + current_x = x[i] + + # Get current number of tracks + current_tracks = current_x.shape[0] + + masks[i, :current_tracks] = 1 # Set mask for current tracks + + if current_tracks < max_tracks: + # Pad dimension N (first dimension) + # For a tensor of shape [N, C, L], we need to pad the first dimension + # F.pad expects padding as (pad_left, pad_right, pad_top, pad_bottom, pad_front, pad_back) + # To pad the first dimension, we need to specify padding for the third-from-last dimension + pad_size = (0, 0, # last dim (L): no padding + 0, 0, # second-to-last dim (C): no padding + 0, max_tracks - current_tracks) # third-to-last dim (N): pad at the end + + + if not is_x_none: + padded_x.append(F.pad(current_x, pad_size)) + + elif current_tracks > max_tracks : + raise ValueError(f"Number of tracks {current_tracks} exceeds maximum allowed {max_tracks}. that is impossible") + else: + if not is_x_none: + padded_x.append(current_x) + + + if not is_x_none: + x_stacked = torch.stack(padded_x, dim=0) # Shape: [B, max_tracks, C, L] + + return { + 'y': x_stacked.to(device), # Shape: [B, max_tracks, C, L] + "masks": masks.to(device), # Shape: [B, max_tracks] + "paths": paths, + "fs": fs + } + +def collate_multitrack_paired(batch, max_tracks=None, sample_rate=None, segment_length=None, device=None): + + x= [ data_i[0] for data_i in batch ] # x is a list of tensors, each tensor is a track + y= [ data_i[1] for data_i in batch ] # x is a list of tensors, each tensor is a track + + taxonomies=[ data_i[2] for data_i in batch ] # taxonomy is a + + paths=[ data_i[3] for data_i in batch ] # paths is a list of paths to the audio files, each path is a string + + if max_tracks is None: + max_tracks = max(track.shape[0] for track in x) # Find the maximum number of tracks in the batch + else: + if max_tracks < max(track.shape[0] for track in x): + print(f"Warning: max_tracks is set to {max_tracks}, but the maximum number of tracks in the batch is {max(track.shape[0] for track in x)}. I will crop the last tracks. I hope the order is random..." ) + + for i in range(len(x)): + if x[i].shape[0] > max_tracks: + print("cropping x[i] to max_tracks") + x[i] = x[i][:max_tracks] + if y[i].shape[0] > max_tracks: + print("cropping y[i] to max_tracks") + y[i] = y[i][:max_tracks] + if len(taxonomies[i]) > max_tracks: + print("cropping taxonomies[i] to max_tracks") + taxonomies[i] = taxonomies[i][:max_tracks] + + + + is_x_none= any(x_i is None for x_i in x) + is_y_none= any(y_i is None for y_i in y) + + assert not (is_x_none and is_y_none), "Either x or y should be None, but not both. This is a bug in the collator" + + # Pad each examples with zeros to the maximum length (dimension 0) + padded_x = [] + padded_y = [] + padded_taxonomies = [] + masks = torch.zeros((len(x), max_tracks), dtype=torch.bool) # Create a mask tensor + + for i in range(len(x)): + + if not is_x_none: + current_x = x[i] + if not is_y_none: + current_y = y[i] + current_taxonomies = taxonomies[i] + + # Get current number of tracks + if is_x_none: + current_tracks = current_y.shape[0] + else: + current_tracks = current_x.shape[0] + if not is_y_none: + assert current_tracks == current_y.shape[0], f"Number of tracks in x ({current_tracks}) does not match number of tracks in y ({current_y.shape[0]})" + + masks[i, :current_tracks] = 1 # Set mask for current tracks + + if current_tracks < max_tracks: + # Pad dimension N (first dimension) + # For a tensor of shape [N, C, L], we need to pad the first dimension + # F.pad expects padding as (pad_left, pad_right, pad_top, pad_bottom, pad_front, pad_back) + # To pad the first dimension, we need to specify padding for the third-from-last dimension + pad_size = (0, 0, # last dim (L): no padding + 0, 0, # second-to-last dim (C): no padding + 0, max_tracks - current_tracks) # third-to-last dim (N): pad at the end + + + if not is_x_none: + padded_x.append(F.pad(current_x, pad_size)) + if not is_y_none: + padded_y.append(F.pad(current_y, pad_size)) + + # Pad taxonomies + padded_taxonomies.append(current_taxonomies + [None] * (max_tracks - len(current_taxonomies))) + elif current_tracks > max_tracks : + raise ValueError(f"Number of tracks {current_tracks} exceeds maximum allowed {max_tracks}. that is impossible") + else: + if not is_x_none: + padded_x.append(current_x) + if not is_y_none: + padded_y.append(current_y) + + padded_taxonomies.append(current_taxonomies[:max_tracks]) + + + if not is_x_none: + x_stacked = torch.stack(padded_x, dim=0) # Shape: [B, max_tracks, C, L] + + if not is_y_none: + y_stacked = torch.stack(padded_y, dim=0) # Shape: [B, max_tracks, C, L] + + if is_x_none: + return { + 'y': y_stacked, + 'taxonomies': padded_taxonomies, + "masks": masks, + "paths": paths + } + if is_y_none: + return { + 'x': x_stacked, + 'taxonomies': padded_taxonomies, + "masks": masks, + "paths": paths + } + + return { + 'x': x_stacked, + 'y': y_stacked, + 'taxonomies': padded_taxonomies, + "masks": masks, + "paths": paths + } \ No newline at end of file diff --git a/utils/common_audioeffects.py b/utils/common_audioeffects.py new file mode 100644 index 0000000000000000000000000000000000000000..cff7eb0c164324af90d7c5980ce7086d354ffad4 --- /dev/null +++ b/utils/common_audioeffects.py @@ -0,0 +1,1729 @@ +""" +Audio effects for data augmentation. + +Several audio effects can be combined into an augmentation chain. + +Important note: We assume that the parallelization during training is done using + multi-processing and not multi-threading. Hence, we do not need the + `@sox.sox_context()` decorators as discussed in this + [thread](https://github.com/pseeth/soxbindings/issues/4). + +Section 2, TL21 +AI Speech and Sound Group, SL1 +""" + +from itertools import permutations +import os +import io +import functools +import lameenc +import logging +import numpy as np +import pymixconsole as pymc +from pymixconsole.parameter import Parameter +from pymixconsole.parameter_list import ParameterList +from pymixconsole.processor import Processor +from random import shuffle +from scipy.signal import oaconvolve +import soundfile as sf +#import soxbindings as sox +from typing import List, Optional, Tuple, Union, Dict +from numba import jit + +#from common_dataprocessing import sample_data + +# prevent pysox from logging warnings regarding non-opimal timestretch factors +logging.getLogger('sox').setLevel(logging.ERROR) + +# set maximum peak value if we pass a signal through SOX +MAX_SOX_PROCESSING_PEAK = 0.707 + + +def convert_audio2data(x): + """ + Convert audio data from the format it was stored in to float32. + + Args: + x (Numpy array): input with `x.dtype` either `np.int16`, `np.int32`, `np.float32` or `np.float64`. + + Returns: + Numpy array: output with values in [-1., 1.) where `dtype` is `np.float32`. + """ + if x.dtype in [np.float32, np.float64]: + return x.astype(dtype=np.float32) + else: + return (x.astype(dtype=np.float64) / (1. + np.iinfo(x.dtype).max)).astype(np.float32) + + + +def sample_data(data: Tuple[int, Union[np.ndarray, functools.partial]], + start: int = 0, length: Optional[int] = None) -> np.ndarray: + """ + Load one stem specified by `data`. + + Returns the audio beginning from `start` either up to the end (if `length` is None) + or until the provided `length`. For the case that `start + length > n_samples`, we do a wrap-around and + load the remaining samples from the beginning of `data`. + + Args: + data: Data with shape (n_samples, data). + start: Start index. + length: Length of sample. If `length` is not None, `length` samples are returned (possibly with a wrap-around). + Otherwise, everything until the end of `data` is returned. + + Returns: + samples: data with shape `n_samples x n_channels` + """ + n_samples, audio = data + + # determine whether we have to load the audio or whether it was already loaded + is_loaded = True if type(audio) is np.ndarray else False + + # if `length` is not None, then only select subset + do_wrap_around = False + if length is not None: + if start + length > n_samples: + # we need to wrap around and concatenate `start:` and `:stop` + do_wrap_around = True + stop = length - (n_samples - start) + else: + # no wrap around as it is inside the array/file boundaries + stop = start + length + else: + stop = None + + if is_loaded: + if not do_wrap_around: + samples = convert_audio2data(audio[start:stop]) + else: + samples = np.vstack((convert_audio2data(audio[start:]), + convert_audio2data(audio[:stop]))) + else: + if not do_wrap_around: + samples = convert_audio2data(audio(start=start, stop=stop)[0]) + else: + samples = np.vstack((convert_audio2data(audio(start=start)[0]), + convert_audio2data(audio(stop=stop)[0]))) + + return samples + + +# Monkey-Patch `Processor` for convenience +# (a) Allow `None` as blocksize if processor can work on variable-length audio +def new_init(self, name, parameters, block_size, sample_rate=None, normalize=None, dtype='float32'): + """ + Initialize processor. + + Args: + self: Reference to object + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + block_size (int): Size of blocks for blockwise processing. + Can also be `None` if full audio can be processed at once. + sample_rate (int): Sample rate of input audio. Use `None` if effect is independent of this value. + normalize (str): Defines, whether the processed signal is normalized. + Possible values are `'rms'`, `'max'` and `None`. + dtype (str): data type of samples + + Raises: + ValueError: If `normalize` is not equal to `'rms'`, `'max'` or `False`. + """ + self.name = name + self.parameters = parameters + self.block_size = block_size + self.sample_rate = sample_rate + self.dtype = dtype + if normalize not in [None, 'rms', 'max']: + raise ValueError(f'Unknown value {normalize} for `normalize`. Must be either `rms`, `max` or `None`') + self.normalize = normalize + + +# (b) make code simpler +def new_update(self, parameter_name): + """ + Update processor after randomization of parameters. + + Args: + self: Reference to object. + parameter_name (str): Parameter whose value has changed. + """ + pass + + +# (c) representation for nice print +def new_repr(self): + """ + Create human-readable representation. + + Args: + self: Reference to object. + + Returns: + string representation of object. + """ + return f'Processor(name={self.name!r}, parameters={self.parameters!r}' + + +Processor.__init__ = new_init +Processor.__repr__ = new_repr +Processor.update = new_update + + +class AugmentationChain: + """Basic audio Fx chain which is used for data augmentation.""" + + def __init__(self, + fxs: Optional[List[Tuple[Union[Processor, 'AugmentationChain'], float]]] = [], + shuffle: Optional[bool] = False, + apply_to: Optional[str] = 'both'): + """ + Create augmentation chain from the dictionary `fxs`. + + Args: + fxs (list of tuples): Each tuple has three elements: + First tuple element is an instance of `pymc.processor` or `AugmentationChain` that + we want to use for data augmentation. + Second element gives probability that effect should be applied. + shuffle (bool): If `True` then order of Fx are changed whenever chain is applied. + apply_to (str): Apply the chain to both input and target or one of them only. + Possible values are `'both'`, `'input'` and `target`. + + Raises: + ValueError: If `apply_to` is not equal to `both`, `input` or `target`. + """ + self.fxs = fxs + self.shuffle = shuffle + if apply_to not in ['both', 'input', 'target']: + raise ValueError(f'Unknown value {apply_to} for `apply_to`. Must be `both`, `input` or `target`') + else: + self.apply_to = apply_to + + def apply_processor(self, x, processor: Processor): + """ + Pass audio in `x` through `processor` and output the respective processed audio. + + Args: + x (Numpy array): Input audio of shape `n_samples` x `n_channels`. + processor (Processor): Audio effect that we want to apply. + + Returns: + Numpy array: Processed audio of shape `n_samples` x `n_channels` (same size as `x') + """ + n_samples_input = x.shape[0] + + if processor.block_size is None: + y = processor.process(x) + else: + # make sure that n_samples is a multiple of `processor.block_size` + if x.shape[0] % processor.block_size != 0: + n_pad = processor.block_size - x.shape[0] % processor.block_size + x = np.pad(x, ((0, n_pad), (0, 0)), mode='reflective') + + y = np.zeros_like(x) + for idx in range(0, x.shape[0], processor.block_size): + y[idx:idx+processor.block_size, :] = processor.process(x[idx:idx+processor.block_size, :]) + + if processor.normalize is not None: + if processor.normalize == 'rms': # normalize output energy such that it is the same as the input energy + scale = np.sqrt(np.mean(np.square(x)) / np.maximum(1e-7, np.mean(np.square(y)))) + elif processor.normalize == 'max': # normalize output signal by its max. amplitude + scale = (1 + 1e-7)/(np.max(np.abs(y)) + 1e-7) + y *= scale + + # return audio of same length as x + return y[:n_samples_input, :] + + def __call__(self, input_x, target_x): + """ + Apply augmentation chain to audio in `input_x` and `target_x`. + + Args: + input_x (Numpy array): Audio samples of shape `n_samples` x `n_channels`. + target_x (Numpy array): Audio samples of shape `n_samples` x `n_channels`. + + Returns: + input_y (Numpy array): Processed audio of same shape as `input_x` where effects have been applied. + target_y (Numpy array): Processed audio of same shape as `target_x` where effects have been applied. + """ + # randomly shuffle effect order if `self.shuffle` is True + if self.shuffle: + shuffle(self.fxs) + + input_y = input_x + target_y = target_x + + # check whether we only need to process once later + if self.apply_to == 'both' and np.allclose(input_y, target_y): + is_input_equal_target = True + else: + is_input_equal_target = False + + # apply effects with probabilities given in `self.fxs` + for fx, p in self.fxs: + if np.random.rand() < p: + if isinstance(fx, Processor): + # randomize all effect parameters (also calls `update()` for each processor) + fx.randomize() + # apply processor dependent on `apply_to` + if self.apply_to == 'both': + input_y = self.apply_processor(input_y, fx) + if not is_input_equal_target: + target_y = self.apply_processor(target_y, fx) + elif self.apply_to == 'input': + input_y = self.apply_processor(input_y, fx) + elif self.apply_to == 'target': + target_y = self.apply_processor(target_y, fx) + else: + # apply effect chain + if is_input_equal_target: + target_y = input_y + input_y, target_y = fx(input_y, target_y) + # check whether input and target are still the same + if not fx.apply_to == 'both': + is_input_equal_target = False + + if is_input_equal_target: + target_y = input_y + + return input_y, target_y + + def __repr__(self): + """ + Human-readable representation. + + Returns: + string representation of object. + """ + return f'AugmentationChain(fxs={self.fxs!r}, shuffle={self.shuffle!r})' + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% DISTORTION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +def hard_clip(x, threshold_dB, drive): + """ + Hard clip distortion. + + Args: + x: input audio + threshold_dB: threshold + drive: drive + + Returns: + (Numpy array): distorted audio + """ + drive_linear = np.power(10., drive / 20.).astype(np.float32) + threshold_linear = 10. ** (threshold_dB / 20.) + return np.clip(x * drive_linear, -threshold_linear, threshold_linear) + + +def overdrive(x, drive, colour, sample_rate): + """ + Overdrive distortion. + + Args: + x: input audio + drive: Controls the amount of distortion (dB). + colour: Controls the amount of even harmonic content in the output(dB) + sample_rate: sampling rate + + Returns: + (Numpy array): distorted audio + """ + scale = np.max(np.abs(x)) + if scale > MAX_SOX_PROCESSING_PEAK: + clips = True + x = x * (MAX_SOX_PROCESSING_PEAK / scale) + else: + clips = False + + tfm = sox.Transformer() + tfm.overdrive(gain_db=drive, colour=colour) + y = tfm.build_array(input_array=x, sample_rate_in=sample_rate).astype(np.float32) + + if clips: + y *= scale / MAX_SOX_PROCESSING_PEAK # rescale output to original scale + return y + + +def hyperbolic_tangent(x, drive): + """ + Hyperbolic Tanh distortion. + + Args: + x: input audio + drive: drive + + Returns: + (Numpy array): distorted audio + """ + drive_linear = np.power(10., drive / 20.).astype(np.float32) + return np.tanh(2. * x * drive_linear) + + +def soft_sine(x, drive): + """ + Soft sine distortion. + + Args: + x: input audio + drive: drive + + Returns: + (Numpy array): distorted audio + """ + drive_linear = np.power(10., drive / 20.).astype(np.float32) + y = np.clip(x * drive_linear, -np.pi/4.0, np.pi/4.0) + return np.sin(2. * y) + + +def bit_crusher(x, bits): + """ + Bit crusher distortion. + + Args: + x: input audio + bits: bits + + Returns: + (Numpy array): distorted audio + """ + return np.rint(x * (2 ** bits)) / (2 ** bits) + + +class Distortion(Processor): + """ + Distortion processor. + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + mode (str): Currently supports the following five modes: hard_clip, waveshaper, soft_sine, tanh, bit_crusher. + Each mode has different parameters such as threshold, factor, or bits. + threshold (float): threshold + drive (float): drive + factor (float): factor + limit_range (float): limit range + bits (int): bits + """ + + def __init__(self, sample_rates, name='Distortion', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): sample rates of audio. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name, None, block_size=None, **kwargs) + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('mode', 'hard_clip', 'string', + options=['hard_clip', + 'overdrive', + 'soft_sine', + 'tanh', + 'bit_crusher'])) + self.parameters.add(Parameter('threshold', 0.0, 'float', + units='dB', maximum=0.0, minimum=-20.0)) + self.parameters.add(Parameter('drive', 0.0, 'float', + units='dB', maximum=20.0, minimum=0.0)) + self.parameters.add(Parameter('colour', 20.0, 'float', + maximum=100.0, minimum=0.0)) + self.parameters.add(Parameter('bits', 12, 'int', + maximum=12, minimum=8)) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): distorted audio of size `n_samples x n_channels`. + """ + if self.parameters.mode.value == 'hard_clip': + y = hard_clip(x, self.parameters.threshold.value, self.parameters.drive.value) + elif self.parameters.mode.value == 'overdrive': + y = overdrive(x, self.parameters.drive.value, + self.parameters.colour.value, self.parameters.sample_rate.value) + elif self.parameters.mode.value == 'soft_sine': + y = soft_sine(x, self.parameters.drive.value) + elif self.parameters.mode.value == 'tanh': + y = hyperbolic_tangent(x, self.parameters.drive.value) + elif self.parameters.mode.value == 'bit_crusher': + y = bit_crusher(x, self.parameters.bits.value) + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% EQUALISER %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class Equaliser(Processor): + """ + Five band parametric equaliser (two shelves and three central bands). + + All gains are set in dB values and range from `MIN_GAIN` dB to `MAX_GAIN` dB. + This processor is implemented as cascade of five biquad IIR filters + that are implemented using the infamous cookbook formulae from RBJ. + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + low_shelf_gain (float), low_shelf_freq (float) + first_band_gain (float), first_band_freq (float), first_band_q (float) + second_band_gain (float), second_band_freq (float), second_band_q (float) + third_band_gain (float), third_band_freq (float), third_band_q (float) + + original from https://github.com/csteinmetz1/pymixconsole/blob/master/pymixconsole/processors/equaliser.py + """ + + def __init__(self, n_channels, sample_rates, gain_range=(-15.0, 15.0), q_range=(0.1, 2.0), hard_clip=False, + name='Equaliser', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + n_channels (int): Number of audio channels. + sample_rates (list of ints): Sample rates of audio. + gain_range (tuple of floats): minimum and maximum gain that can be used. + q_range (tuple of floats): minimum and maximum q value. + hard_clip (bool): Whether we clip to [-1, 1.] after processing. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name, parameters=parameters, block_size=None, **kwargs) + + self.n_channels = n_channels + + MIN_GAIN, MAX_GAIN = gain_range + MIN_Q, MAX_Q = q_range + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + # low shelf parameters ------- + self.parameters.add(Parameter('low_shelf_gain', 0.0, 'float', minimum=MIN_GAIN, maximum=MAX_GAIN)) + self.parameters.add(Parameter('low_shelf_freq', 80.0, 'float', minimum=30.0, maximum=200.0)) + # first band parameters ------ + self.parameters.add(Parameter('first_band_gain', 0.0, 'float', minimum=MIN_GAIN, maximum=MAX_GAIN)) + self.parameters.add(Parameter('first_band_freq', 400.0, 'float', minimum=200.0, maximum=1000.0)) + self.parameters.add(Parameter('first_band_q', 0.7, 'float', minimum=MIN_Q, maximum=MAX_Q)) + # second band parameters ----- + self.parameters.add(Parameter('second_band_gain', 0.0, 'float', minimum=MIN_GAIN, maximum=MAX_GAIN)) + self.parameters.add(Parameter('second_band_freq', 2000.0, 'float', minimum=1000.0, maximum=3000.0)) + self.parameters.add(Parameter('second_band_q', 0.7, 'float', minimum=MIN_Q, maximum=MAX_Q)) + # third band parameters ------ + self.parameters.add(Parameter('third_band_gain', 0.0, 'float', minimum=MIN_GAIN, maximum=MAX_GAIN)) + self.parameters.add(Parameter('third_band_freq', 4000.0, 'float', minimum=3000.0, maximum=8000.0)) + self.parameters.add(Parameter('third_band_q', 0.7, 'float', minimum=MIN_Q, maximum=MAX_Q)) + # high shelf parameters ------ + self.parameters.add(Parameter('high_shelf_gain', 0.0, 'float', minimum=MIN_GAIN, maximum=MAX_GAIN)) + self.parameters.add(Parameter('high_shelf_freq', 8000.0, 'float', minimum=5000.0, maximum=10000.0)) + else: + self.parameters = parameters + + self.bands = ['low_shelf', 'first_band', 'second_band', 'third_band', 'high_shelf'] + self.filters = self.setup_filters() + self.hard_clip = hard_clip + + def setup_filters(self): + """ + Create IIR filters. + + Returns: + IIR filters + """ + filters = {} + + for band in self.bands: + + G = getattr(self.parameters, band + '_gain').value + fc = getattr(self.parameters, band + '_freq').value + rate = self.parameters.sample_rate.value + + if band in ['low_shelf', 'high_shelf']: + Q = 0.707 + filter_type = band + else: + Q = getattr(self.parameters, band + '_q').value + filter_type = 'peaking' + + filters[band] = pymc.components.iirfilter.IIRfilter(G, Q, fc, rate, filter_type, n_channels=self.n_channels) + + return filters + + def update_filter(self, band): + """ + Update filters. + + Args: + band (str): Band that should be updated. + """ + self.filters[band].G = getattr(self.parameters, band + '_gain').value + self.filters[band].fc = getattr(self.parameters, band + '_freq').value + self.filters[band].rate = self.parameters.sample_rate.value + + if band in ['first_band', 'second_band', 'third_band']: + self.filters[band].Q = getattr(self.parameters, band + '_q').value + + def update(self, parameter_name=None): + """ + Update processor after randomization of parameters. + + Args: + parameter_name (str): Parameter whose value has changed. + """ + if parameter_name is not None: + bands = ['_'.join(parameter_name.split('_')[:2])] + else: + bands = self.bands + + for band in bands: + self.update_filter(band) + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): equalized audio of size `n_samples x n_channels`. + """ + for _band, iirfilter in self.filters.items(): + iirfilter.reset_state() + x = iirfilter.apply_filter(x) + + if self.hard_clip: + x = np.clip(x, -1.0, 1.0) + + # make sure that we have float32 as IIR filtering returns float64 + x = x.astype(np.float32) + + # make sure that we have two dimensions (if `n_channels == 1`) + if x.ndim == 1: + x = x[:, np.newaxis] + + return x + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% COMPRESSOR %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +@jit(nopython=True) +def compressor_process(x, threshold, attack_time, release_time, ratio, makeup_gain, sample_rate): + """ + Apply compressor. + + Args: + x (Numpy array): audio data. + threshold: threshold in dB. + attack_time: attack_time in ms. + release_time: release_time in ms. + ratio: ratio. + makeup_gain: makeup_gain. + sample_rate: sample rate. + + Returns: + compressed audio. + """ + M = x.shape[0] + x_g = np.zeros(M) + x_l = np.zeros(M) + y_g = np.zeros(M) + y_l = np.zeros(M) + c = np.zeros(M) + yL_prev = 0. + + alpha_attack = np.exp(-1/(0.001 * sample_rate * attack_time)) + alpha_release = np.exp(-1/(0.001 * sample_rate * release_time)) + + for i in np.arange(M): + if np.abs(x[i]) < 0.000001: + x_g[i] = -120.0 + else: + x_g[i] = 20 * np.log10(np.abs(x[i])) + + if x_g[i] >= threshold: + y_g[i] = threshold + (x_g[i] - threshold) / ratio + else: + y_g[i] = x_g[i] + + x_l[i] = x_g[i] - y_g[i] + + if x_l[i] > yL_prev: + y_l[i] = alpha_attack * yL_prev + (1 - alpha_attack) * x_l[i] + else: + y_l[i] = alpha_release * yL_prev + (1 - alpha_release) * x_l[i] + + c[i] = np.power(10.0, (makeup_gain - y_l[i]) / 20.0) + yL_prev = y_l[i] + + y = x * c + + return y + + +class Compressor(Processor): + """ + Single band stereo dynamic range compressor. + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + threshold (float) + attack_time (float) + release_time (float) + ratio (float) + makeup_gain (float) + """ + + def __init__(self, sample_rates, name='Compressor', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('threshold', 0.0, 'float', units='dB', minimum=-40.0, maximum=0.0)) + self.parameters.add(Parameter('attack_time', 2.0, 'float', units='ms', minimum=0.03, maximum=30.0)) + self.parameters.add(Parameter('release_time', 50.0, 'float', units='ms', minimum=50.0, maximum=100.0)) + self.parameters.add(Parameter('ratio', 2.0, 'float', minimum=2.0, maximum=10.0)) + self.parameters.add(Parameter('makeup_gain', 0.0, 'float', units='dB', minimum=-3.0, maximum=6.0)) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): compressed audio of size `n_samples x n_channels`. + """ + if not self.parameters.threshold.value == 0.0: + y = np.zeros_like(x) + + for ch in range(x.shape[1]): + y[:, ch] = compressor_process(x[:, ch], + self.parameters.threshold.value, + self.parameters.attack_time.value, + self.parameters.release_time.value, + self.parameters.ratio.value, + self.parameters.makeup_gain.value, + self.parameters.sample_rate.value) + else: + y = x + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%% CONVOLUTIONAL REVERB %%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class ConvolutionalReverb(Processor): + """ + Convolutional Reverb. + + Important: Due to convolving the audio sequence with some impulse response, we should ignore the + first/last samples of the augmented audio sequence using `config['AUGMENTER_PADDING']`. + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + wet_dry (float): Wet/dry ratio. + decay (float): Applies a fade out to the impulse response. + pre_delay (float): Value in ms. Shifts the IR in time and allows. + A positive value produces a traditional delay between the dry signal and the wet. + A negative delay is, in reality, zero delay, but effectively trims off the start of IR, + so the reverb response begins at a point further in. + """ + + def __init__(self, impulse_responses, sample_rates, name='ConvolutionalReverb', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + impulse_responses (list): List with impulse responses created by `common_dataprocessing.create_dataset` + sample_rates (list of ints): Sample rates that we should assume (used for fade-out computation) + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + + Raises: + ValueError: if no impulse responses are provided. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if impulse_responses is None: + raise ValueError('List of impulse responses must be provided for ConvolutionalReverb processor.') + self.impulse_responses = impulse_responses + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('index', 0, 'int', minimum=0, maximum=len(impulse_responses))) + self.parameters.add(Parameter('wet_dry', 1.0, 'float', minimum=0.1, maximum=1.0)) + self.parameters.add(Parameter('decay', 1.0, 'float', minimum=0.1, maximum=1.0)) + self.parameters.add(Parameter('pre_delay', 0, 'int', units='ms', minimum=-100, maximum=100)) + else: + self.parameters = parameters + + def update(self, parameter_name=None): + """ + Update processor after randomization of parameters. + + Args: + parameter_name (str): Parameter whose value has changed. + """ + # copy IR from current index (to avoid modifying it in-place) + self.h = np.copy(sample_data(self.impulse_responses.loc[self.parameters.index.value]['impulse_response'])) + + # fade out the impulse based on the decay setting (starting from peak value) - constant 20ms fade-out + if self.parameters.decay.value < 1.: + idx_peak = np.argmax(np.max(np.abs(self.h), axis=1), axis=0) + fstart = np.minimum(self.h.shape[0], + idx_peak + int(self.parameters.decay.value * (self.h.shape[0] - idx_peak))) + fstop = np.minimum(self.h.shape[0], fstart + int(0.020*self.parameters.sample_rate.value)) + flen = fstop - fstart + + fade = np.arange(1, flen+1, dtype=self.dtype)/flen + fade = np.power(0.1, fade * 5) + self.h[fstart:fstop, :] *= fade[:, np.newaxis] + self.h = self.h[:fstop] + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): reverbed audio of size `n_samples x n_channels`. + """ + # reshape IR to the correct size + n_channels = x.shape[1] + if self.h.shape[1] == 1 and n_channels > 1: + self.h = np.hstack([self.h] * n_channels) # repeat mono IR for multi-channel input + if self.h.shape[1] > 1 and n_channels == 1: + self.h = self.h[:, np.random.randint(self.h.shape[1]), np.newaxis] # randomly choose one IR channel + + if self.parameters.wet_dry.value == 0.0: + return x + else: + # perform convolution to get wet signal + y = oaconvolve(x, self.h, mode='full', axes=0) + + # cut out wet signal (compensating for the delay that the IR is introducing + predelay) + idx = np.argmax(np.max(np.abs(self.h), axis=1), axis=0) + idx += int(0.001 * self.parameters.pre_delay.value * self.parameters.sample_rate.value) + + idx = np.clip(idx, 0, self.h.shape[0]-1) + + y = y[idx:idx+x.shape[0], :] + + # return weighted sum of dry and wet signal + return (1.0 - self.parameters.wet_dry.value) * x + self.parameters.wet_dry.value * y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%% HAAS EFFECT %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +def haas_process(x, delay, feedback, wet_channel): + """ + Add Haas effect to audio. + + Args: + x (Numpy array): input audio. + delay: Delay that we apply to one of the channels (in samples). + feedback: Feedback value. + wet_channel: Which channel we process (`left` or `right`). + + Returns: + (Numpy array): Audio with Haas effect. + """ + y = np.copy(x) + if wet_channel == 'left': + y[:, 0] += feedback * np.roll(x[:, 0], delay) + elif wet_channel == 'right': + y[:, 1] += feedback * np.roll(x[:, 1], delay) + + return y + + +class Haas(Processor): + """ + Haas Effect Processor. + + Randomly selects one channel and applies a short delay to it. + + Important: This audio effect uses `np.roll` to perform the shift of one channel. Hence, you should use + `config['AUGMENTER_PADDING']` to ignore the first samples of the augmented audio sequence. + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + delay (float) + feedback (float) + wet_channel (string) + """ + + def __init__(self, sample_rates, delay_range=(-0.040, 0.040), name='Haas', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + delay_range (tuple of floats): minimum/maximum delay in milliseconds for Haas effect. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('delay', delay_range[1], 'float', units='ms', + minimum=delay_range[0], maximum=delay_range[1])) + self.parameters.add(Parameter('feedback', 0.35, 'float', minimum=0.33, maximum=0.66)) + self.parameters.add(Parameter('wet_channel', 'left', 'string', options=['left', 'right'])) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): audio with Haas effect of size `n_samples x n_channels`. + """ + assert x.shape[1] == 1 or x.shape[1] == 2, 'Haas effect only works with monaural or stereo audio.' + + if x.shape[1] < 2: + x = np.repeat(x, 2, axis=1) + + y = haas_process(x, int(self.parameters.delay.value * self.parameters.sample_rate.value), + self.parameters.feedback.value, self.parameters.wet_channel.value) + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% PANNER %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class Panner(Processor): + """ + Simple stereo panner (adjusting amplitude and delay). + + If input is mono, output is stereo. + Original edited from https://github.com/csteinmetz1/pymixconsole/blob/master/pymixconsole/processors/panner.py + + Important: This audio effect uses `np.roll` to perform the shift of one channel. Hence, you should use + `config['AUGMENTER_PADDING']` to ignore the first samples of the augmented audio sequence. + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + pan (float): Panning angle. Can take values in [0, 1] where `0` corresponds to fully panned to left + and `1` corresponds to fully panned to the right. + pan_law (str): Pan law to be used for amplitude panning. Can be '-4.5dB', 'linear' or 'constant_power'. + pan_mode (str): Scheme that is used for panning. Can be 'amplitude', 'delay' or 'both'. + pan_maxdelay (float): Maximum delay if we pan a source fully to the left/right. + For example `2. / 343.` is the maximum delay if the microphones are 2 meters apart. + """ + + def __init__(self, sample_rates, maxdelay_range=(0, 2. / 343.0), name='Panner', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + maxdelay_range (tuple of floats): minimum/maximum delay for panning effect. `2. / 343.` corresponds to + the maximum delay if we have a microphone array where the microphones are 2 meters apart. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + # default processor class constructor + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('pan', 0.5, 'float', minimum=0.1, maximum=0.9)) + self.parameters.add(Parameter('pan_law', '-4.5dB', 'string', + options=['-4.5dB', 'linear', 'constant_power'])) + self.parameters.add(Parameter('pan_mode', 'amplitude', 'string', + options=['amplitude', 'delay', 'both'])) + self.parameters.add(Parameter('pan_maxdelay', (maxdelay_range[0] + maxdelay_range[1]) / 2., 'float', + units='ms', minimum=maxdelay_range[0], maximum=maxdelay_range[1])) + else: + self.parameters = parameters + + # setup the coefficents based on default params + self.update() + + def _calculate_pan_coefficents(self): + """ + Calculate panning coefficients from the chosen pan law. + + Based on the set pan law determine the gain value + to apply for the left and right channel to achieve panning effect. + This operates on the assumption that the input channel is mono. + The output data will be stereo at the moment, but could be expanded + to a higher channel count format. + The panning value is in the range [0, 1], where + 0 means the signal is panned completely to the left, and + 1 means the signal is apanned copletely to the right. + + Raises: + ValueError: `self.parameters.pan_law` is not supported. + """ + self.gains = np.zeros(2, dtype=self.dtype) + + # first scale the linear [0, 1] to [0, pi/2] + theta = self.parameters.pan.value * (np.pi/2) + + if self.parameters.pan_law.value == 'linear': + self.gains[0] = ((np.pi/2) - theta) * (2/np.pi) + self.gains[1] = theta * (2/np.pi) + elif self.parameters.pan_law.value == 'constant_power': + self.gains[0] = np.cos(theta) + self.gains[1] = np.sin(theta) + elif self.parameters.pan_law.value == '-4.5dB': + self.gains[0] = np.sqrt(((np.pi/2) - theta) * (2/np.pi) * np.cos(theta)) + self.gains[1] = np.sqrt(theta * (2/np.pi) * np.sin(theta)) + else: + raise ValueError(f'Invalid pan_law {self.parameters.pan_law.value}.') + + def _calculate_pan_delay(self): + """Calculate delay for the chosen pan angle.""" + self.shifts = np.zeros(2, dtype=np.int32) + + # compute overall shift that we need between the two channels + pan_maxdelay_samples = self.parameters.pan_maxdelay.value * self.parameters.sample_rate.value + shift = 2 * int(pan_maxdelay_samples * np.abs(self.parameters.pan.value - 0.5)) + + if self.parameters.pan.value < 0.5: + # panning to the left + self.shifts[0] = -shift // 2 + self.shifts[1] = shift - shift // 2 + else: + self.shifts[0] = shift - shift // 2 + self.shifts[1] = -shift // 2 + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): panned audio of size `n_samples x n_channels`. + """ + assert x.shape[1] == 1 or x.shape[1] == 2, 'Panner only works with monaural or stereo audio.' + + # convert to stereo if signal is monaural + if x.shape[1] < 2: + x = np.repeat(x, 2, axis=1) + + y = np.copy(x) + if self.parameters.pan_mode.value in ['delay', 'both']: + y[:, 0] = np.roll(y[:, 0], self.shifts[0]) + y[:, 1] = np.roll(y[:, 1], self.shifts[1]) + + if self.parameters.pan_mode.value in ['amplitude', 'both']: + y *= self.gains + + return y + + def update(self, parameter_name=None): + """ + Update processor after randomization of parameters. + + Args: + parameter_name (str): Parameter whose value has changed. + """ + self._calculate_pan_coefficents() + self._calculate_pan_delay() + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% GAIN %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class Gain(Processor): + """ + Gain Processor. + + Applies gain in dB and optionally inverts polarity. + + Processor parameters: + gain (float): Gain that should be applied (dB scale). + invert (bool): If True, then we also invert the waveform (all channels jointly). + """ + + def __init__(self, name='Gain', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('gain', 1.0, 'float', units='dB', minimum=-6.0, maximum=6.0)) + self.parameters.add(Parameter('invert', False, 'bool')) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): gain-augmented audio of size `n_samples x n_channels`. + """ + gain = 10 ** (self.parameters.gain.value / 20.) + if self.parameters.invert.value: + gain = -gain + return gain * x + + +# %%%%%%%%%%%%%%%%%%%%%%% SIMPLE CHANNEL SWAP %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class SwapChannels(Processor): + """ + Swap channels in multi-channel audio. + + Processor parameters: + index (int) Selects the permutation that we are using. + Please note that "no permutation" is one of the permutations in `self.permutations` at index `0`. + Hence, this effect should be applied with probability "1.". + """ + + def __init__(self, n_channels, name='SwapChannels', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + n_channels (int): Number of channels in audio that we want to process. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + self.permutations = tuple(permutations(range(n_channels), n_channels)) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('index', 0, 'int', minimum=0, maximum=len(self.permutations))) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): channel-swapped audio of size `n_samples x n_channels`. + """ + return x[:, self.permutations[self.parameters.index.value]] + + +# %%%%%%%%%%%%%%%%%%%%%%% Monauralize %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class Monauralize(Processor): + """ + Monauralizes audio (i.e., removes spatial information). + + Process parameters: + seed_channel (int): channel that we use for overwriting the others. + `-1` refers to using the mean over all channels. + """ + + def __init__(self, n_channels, name='Monauralize', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + n_channels (int): Number of channels in audio that we want to process. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('seed_channel', 0, 'int', minimum=-1, maximum=n_channels)) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): monauralized audio of size `n_samples x n_channels`. + """ + tile_reps = (1, x.shape[1]) + if self.parameters.seed_channel.value == -1: + # use average as seed + return np.tile(np.mean(x, axis=1, keepdims=True), tile_reps) + else: + # use one channel as seed + return np.tile(x[:, [self.parameters.seed_channel.value]], tile_reps) + + +# %%%%%%%%%%%%%%%%%%%%%%% ChunkShuffle %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class ChunkShuffle(Processor): + """ + Split audio into chunks and randomly re-arrange it. + + Process parameters: + n_chunks (int): number of chunks into which we split + """ + + def __init__(self, name='ChunkShuffle', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('n_chunks', 2, 'int', minimum=2, maximum=6)) + else: + self.parameters = parameters + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): chunk-shuffled audio of size `n_samples x n_channels`. + """ + # determine random chunk boundaries + boundaries = np.sort(np.random.randint(x.shape[0], size=self.parameters.n_chunks.value-1)) + + # create chunks + chunks = np.split(x, boundaries) + + # shuffle them + np.random.shuffle(chunks) + + # return chunks in new random order + return np.concatenate(chunks) + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% PITCH SHIFT %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class PitchShift(Processor): + """ + Simple pitch shifter using SoX and soxbindings (https://github.com/pseeth/soxbindings). + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + steps (float): Pitch shift as positive/negative semitones + quick (bool): If True, this effect will run faster but with lower sound quality. + """ + + def __init__(self, sample_rates, fix_length=True, name='PitchShift', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + fix_length (bool): If True, then output has same length as input. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('steps', 0.0, 'float', minimum=-6., maximum=6.)) + self.parameters.add(Parameter('quick', False, 'bool')) + else: + self.parameters = parameters + + self.fix_length = fix_length + self.clips = False + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): pitch-shifted audio of size `n_samples x n_channels`. + """ + if self.parameters.steps.value == 0.0: + y = x + else: + scale = np.max(np.abs(x)) + if scale > MAX_SOX_PROCESSING_PEAK: + clips = True + x = x * (MAX_SOX_PROCESSING_PEAK / scale) + else: + clips = False + + tfm = sox.Transformer() + tfm.pitch(self.parameters.steps.value, quick=bool(self.parameters.quick.value)) + y = tfm.build_array(input_array=x, sample_rate_in=self.parameters.sample_rate.value).astype(np.float32) + + if clips: + y *= scale / MAX_SOX_PROCESSING_PEAK # rescale output to original scale + + if self.fix_length: + n_samples_input = x.shape[0] + n_samples_output = y.shape[0] + if n_samples_input < n_samples_output: + idx1 = (n_samples_output - n_samples_input) // 2 + idx2 = idx1 + n_samples_input + y = y[idx1:idx2] + elif n_samples_input > n_samples_output: + n_pad = n_samples_input - n_samples_output + y = np.pad(y, ((n_pad//2, n_pad - n_pad//2), (0, 0))) + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% TIME STRETCH %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class TimeStretch(Processor): + """ + Simple time stretcher using SoX and soxbindings (https://github.com/pseeth/soxbindings). + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + factor (float): Time stretch factor. + quick (bool): If True, this effect will run faster but with lower sound quality. + stretch_type (str): Algorithm used for stretching (`tempo` or `stretch`). + audio_type (str): Sets which time segments are most optmial when finding + the best overlapping points for time stretching. + """ + + def __init__(self, sample_rates, fix_length=True, name='TimeStretch', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + fix_length (bool): If True, then output has same length as input. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('factor', 1.0, 'float', minimum=1/1.33, maximum=1.33)) + self.parameters.add(Parameter('quick', False, 'bool')) + self.parameters.add(Parameter('stretch_type', 'tempo', 'string', options=['tempo', 'stretch'])) + self.parameters.add(Parameter('audio_type', 'l', 'string', options=['m', 's', 'l'])) + else: + self.parameters = parameters + + self.fix_length = fix_length + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): time-stretched audio of size `n_samples x n_channels`. + """ + if self.parameters.factor.value == 1.0: + y = x + else: + scale = np.max(np.abs(x)) + if scale > MAX_SOX_PROCESSING_PEAK: + clips = True + x = x * (MAX_SOX_PROCESSING_PEAK / scale) + else: + clips = False + + tfm = sox.Transformer() + if self.parameters.stretch_type.value == 'stretch': + tfm.stretch(self.parameters.factor.value) + elif self.parameters.stretch_type.value == 'tempo': + tfm.tempo(1/self.parameters.factor.value, + audio_type=self.parameters.audio_type.value, + quick=bool(self.parameters.quick.value)) + y = tfm.build_array(input_array=x, sample_rate_in=self.parameters.sample_rate.value).astype(np.float32) + + if clips: + y *= scale / MAX_SOX_PROCESSING_PEAK # rescale output to original scale + + if self.fix_length: + n_samples_input = x.shape[0] + n_samples_output = y.shape[0] + if n_samples_input < n_samples_output: + idx1 = (n_samples_output - n_samples_input) // 2 + idx2 = idx1 + n_samples_input + y = y[idx1:idx2] + elif n_samples_input > n_samples_output: + n_pad = n_samples_input - n_samples_output + y = np.pad(y, ((n_pad//2, n_pad - n_pad//2), (0, 0))) + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% PLAYBACK SPEED %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class PlaybackSpeed(Processor): + """ + Simple playback speed effect using SoX and soxbindings (https://github.com/pseeth/soxbindings). + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + factor (float): Playback speed factor. + """ + + def __init__(self, sample_rates, fix_length=True, name='PlaybackSpeed', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + fix_length (bool): If True, then output has same length as input. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('factor', 1.0, 'float', minimum=1./1.33, maximum=1.33)) + else: + self.parameters = parameters + + self.fix_length = fix_length + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): resampled audio of size `n_samples x n_channels`. + """ + if self.parameters.factor.value == 1.0: + y = x + else: + scale = np.max(np.abs(x)) + if scale > MAX_SOX_PROCESSING_PEAK: + clips = True + x = x * (MAX_SOX_PROCESSING_PEAK / scale) + else: + clips = False + + tfm = sox.Transformer() + tfm.speed(self.parameters.factor.value) + y = tfm.build_array(input_array=x, sample_rate_in=self.parameters.sample_rate.value).astype(np.float32) + + if clips: + y *= scale / MAX_SOX_PROCESSING_PEAK # rescale output to original scale + + if self.fix_length: + n_samples_input = x.shape[0] + n_samples_output = y.shape[0] + if n_samples_input < n_samples_output: + idx1 = (n_samples_output - n_samples_input) // 2 + idx2 = idx1 + n_samples_input + y = y[idx1:idx2] + elif n_samples_input > n_samples_output: + n_pad = n_samples_input - n_samples_output + y = np.pad(y, ((n_pad//2, n_pad - n_pad//2), (0, 0))) + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% BEND %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class Bend(Processor): + """ + Simple bend effect using SoX and soxbindings (https://github.com/pseeth/soxbindings). + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + n_bends (int): Number of segments or intervals to pitch shift + """ + + def __init__(self, sample_rates, pitch_range=(-600, 600), fix_length=True, name='Bend', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + pitch_range (tuple of ints): min and max pitch bending ranges in cents + fix_length (bool): If True, then output has same length as input. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('n_bends', 2, 'int', minimum=2, maximum=10)) + else: + self.parameters = parameters + self.pitch_range_min, self.pitch_range_max = pitch_range + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): pitch-bended audio of size `n_samples x n_channels`. + """ + n_bends = self.parameters.n_bends.value + max_length = x.shape[0] / self.parameters.sample_rate.value + + # Generates random non-overlapping segments + delta = 1. / self.parameters.sample_rate.value + boundaries = np.sort(delta + np.random.rand(n_bends-1) * (max_length - delta)) + + start, end = np.zeros(n_bends), np.zeros(n_bends) + start[0] = delta + for i, b in enumerate(boundaries): + end[i] = b + start[i+1] = b + end[-1] = max_length + + # randomly sample pitch-shifts in cents + cents = np.random.randint(self.pitch_range_min, self.pitch_range_max+1, n_bends) + + # remove segment if cent value is zero or start == end (as SoX does not allow such values) + idx_keep = np.logical_and(cents != 0, start != end) + n_bends, start, end, cents = sum(idx_keep), start[idx_keep], end[idx_keep], cents[idx_keep] + + scale = np.max(np.abs(x)) + if scale > MAX_SOX_PROCESSING_PEAK: + clips = True + x = x * (MAX_SOX_PROCESSING_PEAK / scale) + else: + clips = False + + tfm = sox.Transformer() + tfm.bend(n_bends=int(n_bends), start_times=list(start), end_times=list(end), cents=list(cents)) + y = tfm.build_array(input_array=x, sample_rate_in=self.parameters.sample_rate.value).astype(np.float32) + + if clips: + y *= scale / MAX_SOX_PROCESSING_PEAK # rescale output to original scale + + return y + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% MASTERING %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class Mastering(Processor): + """ + Mastering Processor. + + Models different mastering effects, i.e., ways how we deal with sample values outside [-1, 1]. + + Processor parameters: + method (str): Method that should be applied. Can take the following values: + `none`: Do nothing. E.g., useful if training a network for VST plugin where DAW can also handle peaks + larger/smaller than `+1`/`-1`. + `scale`: Scale to max-abs peak to avoid clipping. + `hardclip`: Apply hardclipping to [-1, 1]. + `softclip`: Apply softclipping to [-1, 1] using tanh. + """ + + def __init__(self, name='Mastering', parameters=None, maximum_amplitude=1.0, **kwargs): + """ + Initialize processor. + + Args: + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + maximum_amplitude (float): Maximum amplitude after applying the mastering processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + self.parameters = ParameterList() + self.parameters.add(Parameter('method', 'none', 'string', + options=['none', 'scale', 'hardclip', 'softclip'])) + else: + self.parameters = parameters + + self.maximum_amplitude = maximum_amplitude + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): mastered audio of size `n_samples x n_channels`. + """ + y = np.copy(x) + if self.parameters.method.value == 'none': + return y + elif self.parameters.method.value == 'scale': + maxabs = np.maximum(self.maximum_amplitude, np.max(np.abs(y))) + return y / (maxabs / self.maximum_amplitude) + elif self.parameters.method.value == 'hardclip': + return np.clip(y, -self.maximum_amplitude, self.maximum_amplitude) + elif self.parameters.method.value == 'softclip': + return self.maximum_amplitude * np.tanh(y) + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% MP3 COMPRESSION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +class MP3Compression(Processor): + """ + Models the effect of an mp3 compression using LAME. + + LAME adds some delay at the beginning - we therefore truncate the first samples + + Processor parameters: + sample_rate (int): Current (assumed) sample rate of the audio. + quality (int): in the range from 2 ... 7 (2 = highest, 7 = fastest) + bitrate (int): supported bitrates are + [8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 192, 224, 256, 320] + """ + + def __init__(self, sample_rates, fix_length=True, name='MP3Compression', parameters=None, **kwargs): + """ + Initialize processor. + + Args: + sample_rates (list of ints): Sample rates of input audio. + fix_length (bool): If True, then output has same length as input. + name (str): Name of processor. + parameters (parameter_list): Parameters for this processor. + **kwargs (dict): Keyword arguments that are passed to the Processor class. + """ + super().__init__(name=name, parameters=parameters, block_size=None, **kwargs) + + if not parameters: + supported_bitrates = [8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 192, 224, 256, 320] + self.parameters = ParameterList() + self.parameters.add(Parameter('sample_rate', 44100, 'string', options=sample_rates)) + self.parameters.add(Parameter('quality', 2, 'string', options=[2, 3, 4, 5, 6, 7])) + self.parameters.add(Parameter('bitrate', 96, 'string', options=supported_bitrates)) + else: + self.parameters = parameters + + self.fix_length = fix_length + + def process(self, x): + """ + Process audio. + + Args: + x (Numpy array): input audio of size `n_samples x n_channels`. + + Returns: + (Numpy array): MP3 compressed audio of size `n_samples x n_channels`. + """ + # scale if max-abs peak is larger than `MAX_SOX_PROCESSING_PEAK` + scale = np.max(np.abs(x)) + if scale > MAX_SOX_PROCESSING_PEAK: + clips = True + x = x * (MAX_SOX_PROCESSING_PEAK / scale) + else: + clips = False + + # convert to int16 + x_int = (x * np.iinfo(np.int16).max).astype(np.int16) + + encoder = lameenc.Encoder() + encoder.set_bit_rate(self.parameters.bitrate.value) + encoder.set_in_sample_rate(self.parameters.sample_rate.value) + encoder.set_channels(x.shape[1]) + encoder.set_quality(self.parameters.quality.value) + encoder.silence() + + mp3_data = encoder.encode(x_int.tobytes()) + mp3_data += encoder.flush() + + memory_file = io.BytesIO(initial_bytes=mp3_data) + y, fs = sf.read(memory_file, always_2d=True, dtype=np.float32) + + # resample to original sampling rate if LAME changed it + if fs != self.parameters.sample_rate.value: + tfm = sox.Transformer() + tfm.speed(fs / self.parameters.sample_rate.value) + y = tfm.build_array(input_array=y, sample_rate_in=fs).astype(np.float32) + + if clips: + y *= scale / MAX_SOX_PROCESSING_PEAK # rescale output to original scale + + if self.fix_length: + # LAME adds some samples at the beginning - remove them here + n_samples_input = x.shape[0] + n_samples_output = y.shape[0] + if n_samples_input < n_samples_output: + y = y[-n_samples_input:, :] + + return y + + +def __main__(): + """ + Main function for testing the audio effects. + + examples: + + config['AUGMENTER_CHAIN'] = AugmentationChain([(ConvolutionalReverb(impulse_responses=impulse_responses, + sample_rates=config['ACCEPTED_SAMPLING_RATES']), 0.5), + (Gain(), 0.5), + (Haas(sample_rates=config['ACCEPTED_SAMPLING_RATES']), 0.5), + (Panner(sample_rates=config['ACCEPTED_SAMPLING_RATES']), 0.5), + (SwapChannels(n_channels=config['N_CHANNELS']), 1.), + (Monauralize(n_channels=config['N_CHANNELS']), 0.5)], + shuffle=True) + + config['AUGMENTER_CHAIN'] = AugmentationChain([(Gain(), 1.), + (SwapChannels(n_channels=config['N_CHANNELS']), 1.), + (Monauralize(n_channels=config['N_CHANNELS']), 0.5)], shuffle=True) + """ + pass + diff --git a/utils/data_utils.py b/utils/data_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..894bdcc3a3cfa13c48f8fe13107dce95fcda8380 --- /dev/null +++ b/utils/data_utils.py @@ -0,0 +1,190 @@ +import soundfile as sf +import torchaudio + +def read_wav_segment(file_path, start=None, end=None, dtype="float32"): + """ + Reads a specific segment from a .wav file efficiently. + + Args: + file_path (str): Path to the .wav file. + start (int): Start frame index. + end (int): End frame index. + + Returns: + numpy.ndarray: Audio data for the specified segment. + int: Sample rate of the audio file. + """ + # Open the .wav file + if start is None or end is None: + data, samplerate = sf.read(file_path, dtype=dtype) + else: + with sf.SoundFile(file_path) as audio_file: + # Read only the required frames + audio_file.seek(start) + data = audio_file.read(frames=end-start, dtype=dtype) + samplerate = audio_file.samplerate + + return data, samplerate + +def get_audio_length(file_path): + """ + Retrieves the length of an audio file in seconds and frames. + + Args: + file_path (str): Path to the audio file. + + Returns: + float: Length of the audio file in seconds. + int: Total number of frames in the audio file. + int: Sample rate of the audio file. + """ + with sf.SoundFile(file_path) as audio_file: + total_frames = len(audio_file) # Total number of frames + samplerate = audio_file.samplerate # Sample rate + duration = total_frames / samplerate # Duration in seconds + + return duration, total_frames, samplerate + +def taxonomy2track(input_class, num_instr=8): + + assert num_instr==8, "num_instr should be 8 for this function, the rest is not implemented yet" + + if input_class is None: + return 'unknown' + if num_instr == 8: + mapping = {0000: 'other', 1100: 'drums', 1200: 'drums', 1300: 'other', 2000: 'bass', 3000: 'guitar', 4100: 'piano', 4200: 'piano', 4300: 'piano', 4400: 'other', 4500: 'other', 4600: 'other', 4700: 'other', 4900: 'other', 5000: 'brass', 6100: 'strings', 6210: 'brass', 6220: 'brass', 8100: 'guitar', 8200: 'brass', 9000: 'vocals'} + else: + raise NotImplementedError() + + code_length = len(str(input_class)) + if code_length < 4: + #pad zeros to the right to make it 4 digits + input_class = int(str(input_class) + "0" * (4 - code_length)) + + class_str = str(input_class) + for i in range(len(class_str), 0, -1): + general_class = int(class_str[:i] + "0" * (len(class_str) - i)) + if general_class in mapping: + return mapping[general_class] + + try: + raise ValueError(f"No mapping found for input class {input_class} with num_instr {num_instr}") + except ValueError as e: + print(f"Error: {e}") + return "other" # Return a default value if no mapping is found + +import torch +def efficient_roll(x, shift, dims=-1): + """ + Efficiently roll tensor elements along a dimension without creating a full copy. + + Args: + x: Input tensor + shift: Number of places to roll (negative for left roll) + dim: Dimension along which to roll + + Returns: + Rolled tensor view where possible, minimal copy where necessary + """ + if shift == 0: + return x + + # Get the size of the dimension + dim_size = x.size(dims) + + # Handle shift larger than dimension size + shift = shift % dim_size + if shift < 0: + shift += dim_size + + # Create indices for the roll + indices = torch.cat([torch.arange(dim_size-shift, dim_size), + torch.arange(0, dim_size-shift)]) + + # Use index_select for the roll + return torch.index_select(x, dims, indices) + +#import loudness +import numpy as np + +def apply_loud_normalization(x, lufs=-23, sample_rate=44100,device=None): + """ + x shaPe: (batch_size, channels, time) + """ + + in_shape= x.shape + if x.ndim != 3: + x=x.view(-1, in_shape[-2], in_shape[-1]) # Ensure x is 3D + + B, C, T = x.shape + + if device is None: + device = x.device + + x_out = torch.zeros_like(x) + #for b in range(B): + # x_i=x[b].cpu().numpy().T + # lufs_in=loudness.integrated_loudness(x_i, sample_rate) + + # delta_loudness= lufs - lufs_in + # gain=np.power(10, delta_loudness / 20) # Convert dB to linear gain + + # x_out[b] = torch.tensor(x_i.T * gain, device=device) + + x=x.view(B* C,1, T) # Ensure x is 3D + + loudness=torchaudio.functional.loudness(x+1e-5, sample_rate=sample_rate) + delta_loudness = lufs - loudness + gain= torch.pow(10, delta_loudness / 20) # Convert dB to linear gain + if gain.isnan().any(): + print("NaN detected in gain, setting to -30 dB") + gain = torch.nan_to_num(gain, nan=-30.0) + + x_out = x * gain.view(B * C, 1, 1) # Apply gain to each channel + + + x_out = x_out.view(in_shape) + + return x_out + + + +from utils.feature_extractors.dsp_features import compute_log_rms_gated_max, compute_crest_factor, compute_stereo_width, compute_stereo_imbalance, compute_log_spread + +def apply_RMS_normalization(x, RMS_norm=-25, device=None, use_gate=False): + if device is None: + device = x.device + + RMS= torch.tensor(RMS_norm, device=device).view(1, 1, 1).repeat(x.shape[0],1,1) # Use fixed RMS for evaluation + + x_RMS_ref=20*torch.log10(torch.sqrt(torch.mean(x**2, dim=(-1), keepdim=True).mean(dim=-2, keepdim=True))) + if use_gate: + x_RMS = compute_log_rms_gated_max(x).unsqueeze(-1) + else: + x_RMS=20*torch.log10(torch.sqrt(torch.mean(x**2, dim=(-1), keepdim=True).mean(dim=-2, keepdim=True))) + + + gain= RMS - x_RMS + gain_linear = 10 ** (gain / 20 + 1e-6) # Convert dB gain to linear scale, adding a small value to avoid division by zero + x=x* gain_linear + + return x + + +import pyloudnorm as pyln + +def loudness_normalize(audio, target_loudness=-23.0, sample_rate=44100): + """ + Normalize the loudness of the audio to a target level. + """ + + pylnmeter = pyln.Meter(sample_rate) # Create a meter for 44100 Hz sampling rate + + audio= np.array(audio, dtype=np.float32).T + loudness = pylnmeter.integrated_loudness(audio) + + + # loudness normalize audio to -12 dB LUFS + loudness_normalized_audio = pyln.normalize.loudness(audio, loudness, -14.0) + + return torch.tensor(loudness_normalized_audio.T, dtype=torch.float32) diff --git a/utils/dnnlib/__init__.py b/utils/dnnlib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c4640c2e64304cf9b498218b5880101a0752045e --- /dev/null +++ b/utils/dnnlib/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# This work is licensed under a Creative Commons +# Attribution-NonCommercial-ShareAlike 4.0 International License. +# You should have received a copy of the license along with this +# work. If not, see http://creativecommons.org/licenses/by-nc-sa/4.0/ + +from .util import EasyDict, make_cache_dir_path, call_func_by_name diff --git a/utils/dnnlib/util.py b/utils/dnnlib/util.py new file mode 100644 index 0000000000000000000000000000000000000000..b7f9c01be98eaed593198e957992ce83284c80cc --- /dev/null +++ b/utils/dnnlib/util.py @@ -0,0 +1,491 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# This work is licensed under a Creative Commons +# Attribution-NonCommercial-ShareAlike 4.0 International License. +# You should have received a copy of the license along with this +# work. If not, see http://creativecommons.org/licenses/by-nc-sa/4.0/ + +"""Miscellaneous utility classes and functions.""" + +import ctypes +import fnmatch +import importlib +import inspect +import numpy as np +import os +import shutil +import sys +import types +import io +import pickle +import re +import requests +import html +import hashlib +import glob +import tempfile +import urllib +import urllib.request +import uuid + +from distutils.util import strtobool +from typing import Any, List, Tuple, Union, Optional + + +# Util classes +# ------------------------------------------------------------------------------------------ + + +class EasyDict(dict): + """Convenience class that behaves like a dict but allows access with the attribute syntax.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name: str, value: Any) -> None: + self[name] = value + + def __delattr__(self, name: str) -> None: + del self[name] + + +class Logger(object): + """Redirect stderr to stdout, optionally print stdout to a file, and optionally force flushing on both stdout and the file.""" + + def __init__(self, file_name: Optional[str] = None, file_mode: str = "w", should_flush: bool = True): + self.file = None + + if file_name is not None: + self.file = open(file_name, file_mode) + + self.should_flush = should_flush + self.stdout = sys.stdout + self.stderr = sys.stderr + + sys.stdout = self + sys.stderr = self + + def __enter__(self) -> "Logger": + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.close() + + def write(self, text: Union[str, bytes]) -> None: + """Write text to stdout (and a file) and optionally flush.""" + if isinstance(text, bytes): + text = text.decode() + if len(text) == 0: # workaround for a bug in VSCode debugger: sys.stdout.write(''); sys.stdout.flush() => crash + return + + if self.file is not None: + self.file.write(text) + + self.stdout.write(text) + + if self.should_flush: + self.flush() + + def flush(self) -> None: + """Flush written text to both stdout and a file, if open.""" + if self.file is not None: + self.file.flush() + + self.stdout.flush() + + def close(self) -> None: + """Flush, close possible files, and remove stdout/stderr mirroring.""" + self.flush() + + # if using multiple loggers, prevent closing in wrong order + if sys.stdout is self: + sys.stdout = self.stdout + if sys.stderr is self: + sys.stderr = self.stderr + + if self.file is not None: + self.file.close() + self.file = None + + +# Cache directories +# ------------------------------------------------------------------------------------------ + +_dnnlib_cache_dir = None + +def set_cache_dir(path: str) -> None: + global _dnnlib_cache_dir + _dnnlib_cache_dir = path + +def make_cache_dir_path(*paths: str) -> str: + if _dnnlib_cache_dir is not None: + return os.path.join(_dnnlib_cache_dir, *paths) + if 'DNNLIB_CACHE_DIR' in os.environ: + return os.path.join(os.environ['DNNLIB_CACHE_DIR'], *paths) + if 'HOME' in os.environ: + return os.path.join(os.environ['HOME'], '.cache', 'dnnlib', *paths) + if 'USERPROFILE' in os.environ: + return os.path.join(os.environ['USERPROFILE'], '.cache', 'dnnlib', *paths) + return os.path.join(tempfile.gettempdir(), '.cache', 'dnnlib', *paths) + +# Small util functions +# ------------------------------------------------------------------------------------------ + + +def format_time(seconds: Union[int, float]) -> str: + """Convert the seconds to human readable string with days, hours, minutes and seconds.""" + s = int(np.rint(seconds)) + + if s < 60: + return "{0}s".format(s) + elif s < 60 * 60: + return "{0}m {1:02}s".format(s // 60, s % 60) + elif s < 24 * 60 * 60: + return "{0}h {1:02}m {2:02}s".format(s // (60 * 60), (s // 60) % 60, s % 60) + else: + return "{0}d {1:02}h {2:02}m".format(s // (24 * 60 * 60), (s // (60 * 60)) % 24, (s // 60) % 60) + + +def format_time_brief(seconds: Union[int, float]) -> str: + """Convert the seconds to human readable string with days, hours, minutes and seconds.""" + s = int(np.rint(seconds)) + + if s < 60: + return "{0}s".format(s) + elif s < 60 * 60: + return "{0}m {1:02}s".format(s // 60, s % 60) + elif s < 24 * 60 * 60: + return "{0}h {1:02}m".format(s // (60 * 60), (s // 60) % 60) + else: + return "{0}d {1:02}h".format(s // (24 * 60 * 60), (s // (60 * 60)) % 24) + + +def ask_yes_no(question: str) -> bool: + """Ask the user the question until the user inputs a valid answer.""" + while True: + try: + print("{0} [y/n]".format(question)) + return strtobool(input().lower()) + except ValueError: + pass + + +def tuple_product(t: Tuple) -> Any: + """Calculate the product of the tuple elements.""" + result = 1 + + for v in t: + result *= v + + return result + + +_str_to_ctype = { + "uint8": ctypes.c_ubyte, + "uint16": ctypes.c_uint16, + "uint32": ctypes.c_uint32, + "uint64": ctypes.c_uint64, + "int8": ctypes.c_byte, + "int16": ctypes.c_int16, + "int32": ctypes.c_int32, + "int64": ctypes.c_int64, + "float32": ctypes.c_float, + "float64": ctypes.c_double +} + + +def get_dtype_and_ctype(type_obj: Any) -> Tuple[np.dtype, Any]: + """Given a type name string (or an object having a __name__ attribute), return matching Numpy and ctypes types that have the same size in bytes.""" + type_str = None + + if isinstance(type_obj, str): + type_str = type_obj + elif hasattr(type_obj, "__name__"): + type_str = type_obj.__name__ + elif hasattr(type_obj, "name"): + type_str = type_obj.name + else: + raise RuntimeError("Cannot infer type name from input") + + assert type_str in _str_to_ctype.keys() + + my_dtype = np.dtype(type_str) + my_ctype = _str_to_ctype[type_str] + + assert my_dtype.itemsize == ctypes.sizeof(my_ctype) + + return my_dtype, my_ctype + + +def is_pickleable(obj: Any) -> bool: + try: + with io.BytesIO() as stream: + pickle.dump(obj, stream) + return True + except: + return False + + +# Functionality to import modules/objects by name, and call functions by name +# ------------------------------------------------------------------------------------------ + +def get_module_from_obj_name(obj_name: str) -> Tuple[types.ModuleType, str]: + """Searches for the underlying module behind the name to some python object. + Returns the module and the object name (original name with module part removed).""" + + # allow convenience shorthands, substitute them by full names + obj_name = re.sub("^np.", "numpy.", obj_name) + obj_name = re.sub("^tf.", "tensorflow.", obj_name) + + # list alternatives for (module_name, local_obj_name) + parts = obj_name.split(".") + name_pairs = [(".".join(parts[:i]), ".".join(parts[i:])) for i in range(len(parts), 0, -1)] + + # try each alternative in turn + for module_name, local_obj_name in name_pairs: + try: + module = importlib.import_module(module_name) # may raise ImportError + get_obj_from_module(module, local_obj_name) # may raise AttributeError + return module, local_obj_name + except: + pass + + # maybe some of the modules themselves contain errors? + for module_name, _local_obj_name in name_pairs: + try: + importlib.import_module(module_name) # may raise ImportError + except ImportError: + if not str(sys.exc_info()[1]).startswith("No module named '" + module_name + "'"): + raise + + # maybe the requested attribute is missing? + for module_name, local_obj_name in name_pairs: + try: + module = importlib.import_module(module_name) # may raise ImportError + get_obj_from_module(module, local_obj_name) # may raise AttributeError + except ImportError: + pass + + # we are out of luck, but we have no idea why + raise ImportError(obj_name) + + +def get_obj_from_module(module: types.ModuleType, obj_name: str) -> Any: + """Traverses the object name and returns the last (rightmost) python object.""" + if obj_name == '': + return module + obj = module + for part in obj_name.split("."): + obj = getattr(obj, part) + return obj + + +def get_obj_by_name(name: str) -> Any: + """Finds the python object with the given name.""" + module, obj_name = get_module_from_obj_name(name) + return get_obj_from_module(module, obj_name) + + +def call_func_by_name(*args, func_name: str = None, **kwargs) -> Any: + """Finds the python object with the given name and calls it as a function.""" + assert func_name is not None + func_obj = get_obj_by_name(func_name) + assert callable(func_obj) + return func_obj(*args, **kwargs) + + +def construct_class_by_name(*args, class_name: str = None, **kwargs) -> Any: + """Finds the python class with the given name and constructs it with the given arguments.""" + return call_func_by_name(*args, func_name=class_name, **kwargs) + + +def get_module_dir_by_obj_name(obj_name: str) -> str: + """Get the directory path of the module containing the given object name.""" + module, _ = get_module_from_obj_name(obj_name) + return os.path.dirname(inspect.getfile(module)) + + +def is_top_level_function(obj: Any) -> bool: + """Determine whether the given object is a top-level function, i.e., defined at module scope using 'def'.""" + return callable(obj) and obj.__name__ in sys.modules[obj.__module__].__dict__ + + +def get_top_level_function_name(obj: Any) -> str: + """Return the fully-qualified name of a top-level function.""" + assert is_top_level_function(obj) + module = obj.__module__ + if module == '__main__': + module = os.path.splitext(os.path.basename(sys.modules[module].__file__))[0] + return module + "." + obj.__name__ + + +# File system helpers +# ------------------------------------------------------------------------------------------ + +def list_dir_recursively_with_ignore(dir_path: str, ignores: List[str] = None, add_base_to_relative: bool = False) -> List[Tuple[str, str]]: + """List all files recursively in a given directory while ignoring given file and directory names. + Returns list of tuples containing both absolute and relative paths.""" + assert os.path.isdir(dir_path) + base_name = os.path.basename(os.path.normpath(dir_path)) + + if ignores is None: + ignores = [] + + result = [] + + for root, dirs, files in os.walk(dir_path, topdown=True): + for ignore_ in ignores: + dirs_to_remove = [d for d in dirs if fnmatch.fnmatch(d, ignore_)] + + # dirs need to be edited in-place + for d in dirs_to_remove: + dirs.remove(d) + + files = [f for f in files if not fnmatch.fnmatch(f, ignore_)] + + absolute_paths = [os.path.join(root, f) for f in files] + relative_paths = [os.path.relpath(p, dir_path) for p in absolute_paths] + + if add_base_to_relative: + relative_paths = [os.path.join(base_name, p) for p in relative_paths] + + assert len(absolute_paths) == len(relative_paths) + result += zip(absolute_paths, relative_paths) + + return result + + +def copy_files_and_create_dirs(files: List[Tuple[str, str]]) -> None: + """Takes in a list of tuples of (src, dst) paths and copies files. + Will create all necessary directories.""" + for file in files: + target_dir_name = os.path.dirname(file[1]) + + # will create all intermediate-level directories + if not os.path.exists(target_dir_name): + os.makedirs(target_dir_name) + + shutil.copyfile(file[0], file[1]) + + +# URL helpers +# ------------------------------------------------------------------------------------------ + +def is_url(obj: Any, allow_file_urls: bool = False) -> bool: + """Determine whether the given object is a valid URL string.""" + if not isinstance(obj, str) or not "://" in obj: + return False + if allow_file_urls and obj.startswith('file://'): + return True + try: + res = requests.compat.urlparse(obj) + if not res.scheme or not res.netloc or not "." in res.netloc: + return False + res = requests.compat.urlparse(requests.compat.urljoin(obj, "/")) + if not res.scheme or not res.netloc or not "." in res.netloc: + return False + except: + return False + return True + + +def open_url(url: str, cache_dir: str = None, num_attempts: int = 10, verbose: bool = True, return_filename: bool = False, cache: bool = True) -> Any: + """Download the given URL and return a binary-mode file object to access the data.""" + assert num_attempts >= 1 + assert not (return_filename and (not cache)) + + # Doesn't look like an URL scheme so interpret it as a local filename. + if not re.match('^[a-z]+://', url): + return url if return_filename else open(url, "rb") + + # Handle file URLs. This code handles unusual file:// patterns that + # arise on Windows: + # + # file:///c:/foo.txt + # + # which would translate to a local '/c:/foo.txt' filename that's + # invalid. Drop the forward slash for such pathnames. + # + # If you touch this code path, you should test it on both Linux and + # Windows. + # + # Some internet resources suggest using urllib.request.url2pathname() but + # but that converts forward slashes to backslashes and this causes + # its own set of problems. + if url.startswith('file://'): + filename = urllib.parse.urlparse(url).path + if re.match(r'^/[a-zA-Z]:', filename): + filename = filename[1:] + return filename if return_filename else open(filename, "rb") + + assert is_url(url) + + # Lookup from cache. + if cache_dir is None: + cache_dir = make_cache_dir_path('downloads') + + url_md5 = hashlib.md5(url.encode("utf-8")).hexdigest() + if cache: + cache_files = glob.glob(os.path.join(cache_dir, url_md5 + "_*")) + if len(cache_files) == 1: + filename = cache_files[0] + return filename if return_filename else open(filename, "rb") + + # Download. + url_name = None + url_data = None + with requests.Session() as session: + if verbose: + print("Downloading %s ..." % url, end="", flush=True) + for attempts_left in reversed(range(num_attempts)): + try: + with session.get(url) as res: + res.raise_for_status() + if len(res.content) == 0: + raise IOError("No data received") + + if len(res.content) < 8192: + content_str = res.content.decode("utf-8") + if "download_warning" in res.headers.get("Set-Cookie", ""): + links = [html.unescape(link) for link in content_str.split('"') if "export=download" in link] + if len(links) == 1: + url = requests.compat.urljoin(url, links[0]) + raise IOError("Google Drive virus checker nag") + if "Google Drive - Quota exceeded" in content_str: + raise IOError("Google Drive download quota exceeded -- please try again later") + + match = re.search(r'filename="([^"]*)"', res.headers.get("Content-Disposition", "")) + url_name = match[1] if match else url + url_data = res.content + if verbose: + print(" done") + break + except KeyboardInterrupt: + raise + except: + if not attempts_left: + if verbose: + print(" failed") + raise + if verbose: + print(".", end="", flush=True) + + # Save to cache. + if cache: + safe_name = re.sub(r"[^0-9a-zA-Z-._]", "_", url_name) + safe_name = safe_name[:min(len(safe_name), 128)] + cache_file = os.path.join(cache_dir, url_md5 + "_" + safe_name) + temp_file = os.path.join(cache_dir, "tmp_" + uuid.uuid4().hex + "_" + url_md5 + "_" + safe_name) + os.makedirs(cache_dir, exist_ok=True) + with open(temp_file, "wb") as f: + f.write(url_data) + os.replace(temp_file, cache_file) # atomic + if return_filename: + return cache_file + + # Return data as file object. + assert not return_filename + return io.BytesIO(url_data) diff --git a/utils/evaluation/KAD_evaluate.py b/utils/evaluation/KAD_evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..8fa13b17886bb67720ecd0e4dcd06f43ef7978c7 --- /dev/null +++ b/utils/evaluation/KAD_evaluate.py @@ -0,0 +1,307 @@ + +import pandas as pd + +import torch +import os +import omegaconf +import numpy as np +import glob +import pyloudnorm as pyln +import sys +#move to the parent directory to import datasets +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from datasets.eval_benchmark import load_audio +from utils.evaluation.dist_metrics import KADFeatures + +# see device 1 +os.environ["CUDA_VISIBLE_DEVICES"] = "0" + +path_test_set="/scratch/elec/t412-asp/automix/MDX_TM_benchmark" + +set="MDX_TM" + +features=[ + "AFxRep", + "FxEncoder", + "FxEncoder++", + "CLAP", +] + + +KAD_metrics={} + +KAD_args = { + "do_PCA_figure": False, + "do_TSNE_figure": False, + "kernel": "gaussian", #kernel to use for the KAD metric +} + +KAD_args = omegaconf.OmegaConf.create(KAD_args) + +for feature in features: + if feature=="AFxRep": + AFxRep_args = { + "distance_type": "cosine", # not used + "ckpt_path": "/scratch/work/molinee2/projects/project_mfm_eloi/src/tmp/afx-rep.ckpt" + } + AFxRep_args = omegaconf.OmegaConf.create(AFxRep_args) + KAD_metrics[feature] = KADFeatures(type="AFxRep", sample_rate=44100, AFxRep_args=AFxRep_args, KAD_args=KAD_args) + elif feature=="FxEncoder": + fx_encoder_args = { + "distance_type": "cosine", # not used + "ckpt_path": "/scratch/work/molinee2/projects/project_mfm_eloi/src/utils/feature_extractors/ckpt/fxenc_default.pt" + } + fx_encoder_args = omegaconf.OmegaConf.create(fx_encoder_args) + KAD_metrics[feature] = KADFeatures(type="fx_encoder", sample_rate=44100, fx_encoder_args=fx_encoder_args, KAD_args=KAD_args) + elif feature=="FxEncoder++": + fx_encoder_plusplus_args = { + "distance_type": "cosine", # not used + "ckpt_path": "/scratch/work/molinee2/projects/project_mfm_eloi/src_clean/checkpoints/fxenc_plusplus_default.pt" + } + fx_encoder_plusplus_args = omegaconf.OmegaConf.create(fx_encoder_plusplus_args) + KAD_metrics[feature] = KADFeatures(type="fx_encoder_++", sample_rate=44100, fx_encoder_plusplus_args=fx_encoder_plusplus_args, KAD_args=KAD_args) + + elif feature=="CLAP": + clap_args = { + "ckpt_path": "/scratch/work/molinee2/projects/project_mfm_eloi/src_clean/checkpoints/music_audioset_epoch_15_esc_90.14.patched.pt", + "distance_type": "cosine", + "normalize": True, # if True, the features will be normalized + "use_adaptor": False, # if True, the features will be adapted to the CLAP space + "adaptor_checkpoint":None, + "adaptor_type":None, + "add_noise": False, # if True, the features will be augmented with orthogonal noise + "noise_sigma": 0 # sigma of the orthogonal noise to + } + + clap_args = omegaconf.OmegaConf.create(clap_args) + KAD_metrics[feature] = KADFeatures(type="CLAP", sample_rate=44100, CLAP_args=clap_args, KAD_args=KAD_args) + + elif feature=="bark": + bark_args = { + "distance_type": "cosine", # not used + "normalize": True, # if True, the features will be normalized + } + bark_args = omegaconf.OmegaConf.create(bark_args) + KAD_metrics[feature] = KADFeatures(type="bark", sample_rate=44100, bark_args=bark_args, KAD_args=KAD_args, normalize=True) + + else: + raise ValueError(f"Unknown feature: {feature}") + + +def get_ref_file_path(dir): + ref_mixture = os.path.join(dir,"mix", "mix.wav") + return ref_mixture + +pylnmeter = pyln.Meter(44100) # Create a meter for 44100 Hz sampling rate + +def loudness_normalize(audio, target_loudness=-23.0): + """ + Normalize the loudness of the audio to a target level. + """ + audio= np.array(audio, dtype=np.float32).T + loudness = pylnmeter.integrated_loudness(audio) + + # loudness normalize audio to -12 dB LUFS + loudness_normalized_audio = pyln.normalize.loudness(audio, loudness, -14.0) + + return torch.tensor(loudness_normalized_audio.T, dtype=torch.float32) + +song_dirs= sorted(glob.glob(os.path.join(path_test_set, "*"))) + +reference_dict = {} + +for song_dir in song_dirs: + + song_name = os.path.basename(song_dir) + segment_subdirs = sorted(glob.glob(os.path.join(song_dir, "*"))) + + song_id=os.path.basename(song_dir) + + for segment_dir in segment_subdirs: + + segment= os.path.basename(segment_dir) + id= f"{song_id}_{segment}" + + ref_file_path= get_ref_file_path(segment_dir) + mix_ref, fs=load_audio(str(ref_file_path), stereo=True) + assert fs==44100, "Expected sampling rate of 44100 Hz" + mix_ref = loudness_normalize(mix_ref) + reference_dict[id] = mix_ref + + +# create dataframe colums are features, rows are methods +dataframe= pd.DataFrame(columns=["method"] +list(KAD_metrics.keys())) + +filename_results="results_eval/results_KAD"+f"_{set}_test.csv" + +#methods=[ "fxnorm_automix_S_Lb", "fxnorm_automix_L_Lb",] +#methods=[ "proposed_random_churn","S3_random_churn", "S4_random_churn", "only_rms_S4_random_churn", "only_rms_random_churn" ] +#methods=[ "proposed_public", "only_rms_public" ]j +#methods=[ "S1internal_S2public"] +#methods=[ "S1internal_S2publicv3", "S1publicv3_S2publicv3", "S1publicv3_S2internal"] +#methods=[ "S1public_S2publicv3"] +#methods=[ "S1public_S2publicv3", "S1public_S2publicv3_oracle", "S1public_S2publicv3_rms"] +#methods=["mst_oracle", "equal_loudness", "diff_baseline", "proposed_oracle", "proposed_random", "only_rms_random" , "proposed_centroid_close", "proposed_centroid_far", "only_rms_centroid_close", "only_rms_centroid_far"] +#methods=["S4_random"] +#methods=["onlyrms_3108"] +#methods=["publicv3_2"] +#methods=["mst_oracle", "equal_loudness", "diff_baseline", "proposed_oracle", "proposed_random_3108_v2", "prop_random_indep_3108", "S4_random", "S3_random", "WUN_4instr", "publicv3", "DMC_14instr", "fxnorm_automix_S_Lb", "fxnorm_automix_L_Lb"] +methods=[ "tencydb_test", "publicv3_test"] +#methods=["prop_random_indep", "only_rms_indep"] +#methods=["WUN_4instr"] +#methods=["diff_baseline"] +#methods=["internal_2708", "equal_loudness", "internal_2708_oracle"] +#methods=["WUN_4instr", "WUN_14instr", "DMC_14instr"] + + +for method in methods: + + if method=="mst_oracle": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"mst_oracle", "mixture_output.wav") + return pred_mixture + elif method=="mst_oracle_multi": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"mst_oracle_multi", "mixture_output.wav") + return pred_mixture + elif method=="equal_loudness": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"anchor_equal_loudness", "mix.wav") + return pred_mixture + elif method=="diff_baseline": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"diff_baseline", "pred_mixture.wav") + return pred_mixture + elif method=="fxnorm_automix_S_Lb": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"fxnorm_automix_S_Lb_v2", "mixture_output.wav") + return pred_mixture + elif method=="fxnorm_automix_L_Lb": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"fxnorm_automix_L_Lb_v2", "mixture_output.wav") + return pred_mixture + elif method=="proposed_random": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S9v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30", "random.wav") + return pred_mixture + elif method=="proposed_random_3108_v2": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S9v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30_3108_v2", "random.wav") + return pred_mixture + elif method=="onlyrms_3108": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S9v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30_3108_v2", "only_rms_random.wav") + return pred_mixture + elif method=="proposed_random_3108_v2_oracle": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S9v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30_3108_v2_oracle", "random.wav") + return pred_mixture + elif method=="only_rms_random": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S9v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30", "only_rms_random.wav") + return pred_mixture + elif method=="S3_random": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S3v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30", "random.wav") + return pred_mixture + elif method=="S4_3108": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S4v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30_3108_v2", "random.wav") + return pred_mixture + elif method=="publicv3": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_publicv3_publicv3_MDX_TM_benchmark_cfg_1_T30_publicv3", "random.wav") + return pred_mixture + elif method=="publicv3_test": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"public_public_MDX_TM_benchmark_public_test_Sep29", "random.wav") + return pred_mixture + elif method=="tencydb_test": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"internal_TencyDB_internal_TencyMastering_MDX_TM_benchmark_internal_test_Sep29", "random.wav") + return pred_mixture + elif method=="prop_random_indep_3108": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"stylediffpipeline_S9v6_MF3wetv6_MDX_TM_benchmark_cfg_1_T30_3108_v2_independent", "random.wav") + return pred_mixture + elif method=="WUN_4instr": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"WUN_4instr", "pred_mixture.wav") + return pred_mixture + elif method=="DMC_14instr": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"DMC_14instr", "pred_mixture.wav") + return pred_mixture + elif method=="only_rms_public": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"ours_public", "mixture_processed_onlyrms.wav") + return pred_mixture + elif method=="proposed_oracle": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"oracle_proposed", "mix.wav") + return pred_mixture + elif method=="publicv3_oracle": + def get_pred_file_path(dir): + pred_mixture= os.path.join(dir,"oracle_proposed_public", "mix.wav") + return pred_mixture + else: + raise ValueError(f"Unknown method: {method}") + + + method_dict = {} + + for song_dir in song_dirs: + + song_name = os.path.basename(song_dir) + segment_subdirs = sorted(glob.glob(os.path.join(song_dir, "*"))) + + song_id=os.path.basename(song_dir) + + for segment_dir in segment_subdirs: + + segment= os.path.basename(segment_dir) + id= f"{song_id}_{segment}" + + pred_file_path= get_pred_file_path(segment_dir) + pred_mixture, fs=load_audio(str(pred_file_path), stereo=True) + assert fs==44100, "Expected sampling rate of 44100 Hz" + pred_mixture = loudness_normalize(pred_mixture) + assert not np.isnan(pred_mixture).any(), f"NaN values found in predicted mixture for {id}" + method_dict[id] = pred_mixture + + + #create new empty row with key as "method" + dataframe.loc[method] = [None] * (len(KAD_metrics)+1) + + for feature, metric in KAD_metrics.items(): + + KAD_distance, dict_output=metric.compute(reference_dict, method_dict, None) + + print(f"KAD distance for method {method} and feature {feature}: {KAD_distance}") + + #write the KAD distance to the dataframe + dataframe.at[method, "method"] = method + dataframe.at[method, feature] = KAD_distance + + for output_key, output_value in dict_output.items(): + if "figure" in output_key: + output_value.savefig(f"results_eval/{output_key}_{set}_{method}_{feature}.png") + print(f"Saved figure: {output_key}_{set}_{method}_{feature}.png") + + + + dataframe.to_csv(filename_results, index=False) + print(f"Results saved to {filename_results}") + + #except Exception as e: + + # print(f"Error processing method {method}: {e}") + + # dataframe.to_csv(filename_results, index=False) + # print(f"Probably incompleted Results saved to {filename_results}") + # continue + + + + diff --git a/utils/evaluation/dist_metrics.py b/utils/evaluation/dist_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..f5f057c5fee7ef6361b94486c7e1a350d28498b9 --- /dev/null +++ b/utils/evaluation/dist_metrics.py @@ -0,0 +1,971 @@ + +from numpy.lib.scimath import sqrt as scisqrt +from scipy import linalg +import numpy as np +import torchaudio +import os +from importlib import import_module +import torch + + +from utils.feature_extractors.load_features import load_AFxRep, load_fx_encoder, load_fx_encoder_plusplus, load_CLAP + +from utils.log import make_PCA_figure + +SCALE_FACTOR = 100 + +def calc_kernel_audio_distance( + x: torch.Tensor, + y: torch.Tensor, + device: str, + bandwidth=None, + kernel='gaussian', + precision=torch.float32, + eps=1e-8 +) -> torch.Tensor: + """ + Compute the Kernel Audio Distance (KAD) between two samples using PyTorch. + + Args: + x: The first set of embeddings of shape (m, embedding_dim). + y: The second set of embeddings of shape (n, embedding_dim). + cache_dirs: Directories to cache kernel statistics. + bandwidth: The bandwidth value for the Gaussian RBF kernel. + kernel: Kernel function to use ('gaussian', 'iq', 'imq'). + precision: Type setting for matrix calculation precision. + eps: Small value to prevent division by zero. + + Returns: + The KAD between x and y embedding sets. + """ + # Ensure x and y are of the correct precision + x = x.to(dtype=precision, device=device) + y = y.to(dtype=precision, device=device) + + assert bandwidth is not None, "Bandwidth must be provided for KAD calculation" + + m, n = x.shape[0], y.shape[0] + + # Define kernel functions + gamma = 1 / (2 * bandwidth**2 + eps) + if kernel == 'gaussian': # Gaussian Kernel + kernel = lambda a: torch.exp(-gamma * a) + elif kernel == 'iq': # Inverse Quadratic Kernel + kernel = lambda a: 1 / (1 + gamma * a) + elif kernel == 'imq': # Inverse Multiquadric Kernel + kernel = lambda a: 1 / torch.sqrt(1 + gamma * a) + else: + raise ValueError("Invalid kernel type. Valid kernels: 'gaussian', 'iq', 'imq'") + + # Load x kernel statistics + xx = x @ x.T + x_sqnorms = torch.diagonal(xx) + d2_xx = x_sqnorms.unsqueeze(1) + x_sqnorms.unsqueeze(0) - 2 * xx # shape (m, m) + + k_xx = kernel(d2_xx) + k_xx = k_xx - torch.diag(torch.diagonal(k_xx)) + k_xx_mean = k_xx.sum() / (m * (m - 1)) + + # Load y kernel statistics + yy = y @ y.T + y_sqnorms = torch.diagonal(yy) + d2_yy = y_sqnorms.unsqueeze(1) + y_sqnorms.unsqueeze(0) - 2 * yy # shape (n, n) + + k_yy = kernel(d2_yy) + k_yy = k_yy - torch.diag(torch.diagonal(k_yy)) + k_yy_mean = k_yy.sum() / (n * (n - 1)) + + # Compute kernel statistics for xy + xy = x @ y.T + d2_xy = x_sqnorms.unsqueeze(1) + y_sqnorms.unsqueeze(0) - 2 * xy # shape (m, n) + k_xy = kernel(d2_xy) + k_xy_mean = k_xy.mean() + + # Compute MMD + result = k_xx_mean + k_yy_mean - 2 * k_xy_mean + + return result * SCALE_FACTOR + + +def median_pairwise_distance(x, subsample=None): + """ + Compute the median pairwise distance of an embedding set. + + Args: + x: torch.Tensor of shape (n_samples, embedding_dim) + subsample: int, number of random pairs to consider (optional) + + Returns: + The median pairwise distance between points in x. + """ + x = torch.tensor(x, dtype=torch.float32) + n_samples = x.shape[0] + + if subsample is not None and subsample < n_samples * (n_samples - 1) / 2: + # Randomly select pairs of indices + idx1 = torch.randint(0, n_samples, (subsample,)) + idx2 = torch.randint(0, n_samples, (subsample,)) + + # Ensure idx1 != idx2 + mask = idx1 == idx2 + idx2[mask] = (idx2[mask] + 1) % n_samples + + # Compute distances for selected pairs + distances = torch.sqrt(torch.sum((x[idx1] - x[idx2])**2, dim=1)) + else: + # Compute all pairwise distances + distances = torch.pdist(x) + + return torch.median(distances).item() + +class DistMetric: + """ + Base class for pairwise metrics. + + This class should be subclassed to implement specific pairwise metrics. + """ + def __init__(self,type, sample_rate,*args, **kwargs): + """ + Initialize the PairwiseMetric instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type=type + self.sample_rate=sample_rate + + if self.type == "fx_encoder": + self.model_args= kwargs.get("fx_encoder_args", None) + + assert self.model_args is not None, "model_args must be provided for fx_encoder type" + + self.distance_type=self.model_args.distance_type + + self.feat_extractor = load_fx_encoder(self.model_args, self.device) + + #self.feat_extractor = load_effects_encoder(ckpt_path=ckpt_path).to(self.device) + + elif self.type == "fx_encoder_++": + self.model_args= kwargs.get("fx_encoder_plusplus_args", None) + + assert self.model_args is not None, "model_args must be provided for fx_encoder_plusplus type" + + print(self.model_args) + + self.distance_type=self.model_args.distance_type + + self.feat_extractor = load_fx_encoder_plusplus(self.model_args, self.device) + + #self.feat_extractor = load_effects_encoder(ckpt_path=ckpt_path).to(self.device) + + elif self.type== "AFxRep-mid" or self.type== "AFxRep-side" or self.type== "AFxRep": + + self.model_args= kwargs.get("AFxRep_args", None) + + assert self.model_args is not None, "model_args must be provided for AFxRep type" + + self.distance_type=self.model_args.distance_type + + feat_extractor = load_AFxRep(self.model_args, self.device, sample_rate=self.sample_rate) + + if self.type == "AFxRep-mid": + def feat_extractor_mid(x): + + features= feat_extractor(x) + + #print("features shape:", features.shape) + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_mid + + self.feat_extractor = feat_extractor_mid + + elif self.type == "AFxRep-side": + def feat_extractor_side(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_side + + self.feat_extractor = feat_extractor_side + else: + self.feat_extractor = feat_extractor + + elif self.type == "CLAP": + self.model_args= kwargs.get("CLAP_args", None) + assert self.model_args is not None, "model_args must be provided for CLAP type" + + CLAP_fn = load_CLAP(self.model_args, self.device) + self.feat_extractor=lambda x: CLAP_fn(x, None) + + else: + raise ValueError(f"Unknown type: {self.type}. Supported types: fx_encoder, fx_encoder_plusplus, AFxRep-mid, AFxRep-side, AFxRep") + + + def compute(self, *args, **kwargs): + raise NotImplementedError("Subclasses should implement this method.") + + def extract_features(self, y, y_hat, x=None): + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + feat_y_hat= self.feat_extractor(y_hat) + + if x is not None: + x=torch.tensor(x).permute(1,0).unsqueeze(0).to(self.device) + with torch.no_grad(): + feat_x= self.feat_extractor(x) + else: + feat_x=None + + + return feat_y, feat_y_hat, feat_x + + + def do_TSNE_figure(self, dict_features_y, dict_features_y_hat, dict_features_x=None, fit_mode="target", dict_cluster=None): + """ + Perform PCA on the features and create a figure. + + Args: + dict_features_y (dict): Dictionary containing features for the first set. + dict_features_y_hat (dict): Dictionary containing features for the second set. + + Returns: + fig: The created figure. + """ + import matplotlib.pyplot as plt + from sklearn.decomposition import PCA + from sklearn.manifold import TSNE + + y_values = list(dict_features_y.values()) + y_values = torch.cat(y_values, dim=0) + + y_hat_values = list(dict_features_y_hat.values()) + y_hat_values = torch.cat(y_hat_values, dim=0) + + + if dict_cluster is not None: + clusters= list(dict_cluster.values()) + clusters = [c.unsqueeze(0) if c.dim() == 0 else c for c in clusters] + clusters = torch.cat(clusters, dim=0) + + #check different clusters (0,1,2,3...) + assert len(torch.unique(clusters)) == 2, "Only two clusters are supported for PCA visualization" + C0= clusters == 0 + C1= clusters == 1 + + #manage Nans in y_values + y_values=torch.nan_to_num(y_values, nan=0) + y_hat_values=torch.nan_to_num(y_hat_values, nan=0) + + #if self.pca is None: + self.tsne =TSNE(n_components=2, perplexity=30) + + combined_data = torch.cat([y_values, y_hat_values], dim=0).cpu().numpy() + combined_result = self.tsne.fit_transform(combined_data) + + # Split the results back + n_y = y_values.shape[0] + tsne_result = combined_result[:n_y] + tsne_result_hat = combined_result[n_y:] + + + if dict_cluster is not None: + data_dict = { + "y_C0": tsne_result[C0], + "y_hat_C0": tsne_result_hat[C0], + "y_C1": tsne_result[C1], + "y_hat_C1": tsne_result_hat[C1] + } + else: + data_dict = { + "y": tsne_result, + "y_hat": tsne_result_hat + } + + + + fig= make_PCA_figure(data_dict, title=self.type + " TSNE") + + + return fig + + + def do_PCA_figure(self, dict_features_y, dict_features_y_hat, dict_features_x=None, fit_mode="target", dict_cluster=None): + """ + Perform PCA on the features and create a figure. + + Args: + dict_features_y (dict): Dictionary containing features for the first set. + dict_features_y_hat (dict): Dictionary containing features for the second set. + + Returns: + fig: The created figure. + """ + import matplotlib.pyplot as plt + from sklearn.decomposition import PCA + + y_values = list(dict_features_y.values()) + y_values = torch.cat(y_values, dim=0) + + y_hat_values = list(dict_features_y_hat.values()) + y_hat_values = torch.cat(y_hat_values, dim=0) + + + if dict_cluster is not None: + clusters= list(dict_cluster.values()) + clusters = [c.unsqueeze(0) if c.dim() == 0 else c for c in clusters] + clusters = torch.cat(clusters, dim=0) + + #check different clusters (0,1,2,3...) + assert len(torch.unique(clusters)) == 2, "Only two clusters are supported for PCA visualization" + C0= clusters == 0 + C1= clusters == 1 + + #manage Nans in y_values + y_values=torch.nan_to_num(y_values, nan=0) + y_hat_values=torch.nan_to_num(y_hat_values, nan=0) + + if self.pca is None: + self.pca = PCA(n_components=2) + if fit_mode == "target": + pca_result = self.pca.fit_transform(y_values.cpu().numpy()) + elif fit_mode == "all": + self.pca = self.pca.fit(torch.cat([y_values,y_hat_values], dim=0).cpu().numpy()) + pca_result= self.pca.transform(y_values.cpu().numpy()) + else: + pca_result = self.pca.transform(y_values.cpu().numpy()) + + + #project y_hat values into the same PCA space + pca_result_hat = self.pca.transform(y_hat_values.cpu().numpy()) + + if dict_cluster is not None: + data_dict = { + "y_C0": pca_result[C0], + "y_hat_C0": pca_result_hat[C0], + "y_C1": pca_result[C1], + "y_hat_C1": pca_result_hat[C1] + } + else: + data_dict = { + "y": pca_result, + "y_hat": pca_result_hat + } + + if dict_features_x is not None: + x_values = list(dict_features_x.values()) + x_values = torch.cat(x_values, dim=0) + + pca_result_x = self.pca.transform(x_values.cpu().numpy()) + + data_dict["x"] = pca_result_x + + + + fig= make_PCA_figure(data_dict, title=self.type + " PCA") + + + return fig + + + + +class KADFeatures(DistMetric): + """ + Class for computing the pairwise spectral metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise spectral metric. + """ + def __init__(self, + type=None, + sample_rate=44100, + KAD_args=None, + classwise=False, + *args, **kwargs): + """ + Initialize the PairwiseSpectral instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type = type + + self.classwise = classwise + + self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + + + self.KAD_args = KAD_args + + assert self.KAD_args is not None, "FAD_args must be provided" + + if self.KAD_args.do_PCA_figure: + self.pca = None + + self.bandwidth = None + + super().__init__(self.type, sample_rate, *args, **kwargs) + + def calculate_KAD_distance_classwise(self, dict_features_y, dict_features_y_hat, dict_cluster): + + if dict_cluster is not None: + clusters= list(dict_cluster.values()) + clusters = [c.unsqueeze(0) if c.dim() == 0 else c for c in clusters] + clusters = torch.cat(clusters, dim=0) + + #check different clusters (0,1,2,3...) + assert len(torch.unique(clusters)) == 2, "Only two clusters are supported for PCA visualization" + C0= clusters == 0 + C1= clusters == 1 + + y= torch.cat(list(dict_features_y.values()), dim=0) + y_hat= torch.cat(list(dict_features_y_hat.values()), dim=0) + + y_C0= y[C0] + y_hat_C0= y_hat[C0] + + with torch.no_grad(): + KAD_C0=calc_kernel_audio_distance(y_C0, y_hat_C0, device=self.device, bandwidth=self.bandwidth, kernel=self.KAD_args.kernel, precision=torch.float32) + + y_C1= y[C1] + y_hat_C1= y_hat[C1] + with torch.no_grad(): + KAD_C1=calc_kernel_audio_distance(y_C1, y_hat_C1, device=self.device, bandwidth=self.bandwidth, kernel=self.KAD_args.kernel, precision=torch.float32) + + KAD= (KAD_C0 + KAD_C1) / 2 + + return KAD.cpu().item() + def calculate_KAD_distance(self, dict_features_y, dict_features_y_hat): + y= torch.cat(list(dict_features_y.values()), dim=0) + y_hat= torch.cat(list(dict_features_y_hat.values()), dim=0) + with torch.no_grad(): + KAD=calc_kernel_audio_distance(y, y_hat, device=self.device, bandwidth=self.bandwidth, kernel=self.KAD_args.kernel, precision=torch.float32) + return KAD.cpu().item() + + def calculate_bandwidth(self, dict_features_y): + print("Calculating bandwidth...") + y= torch.cat(list(dict_features_y.values()), dim=0) + print("y shape:", y.shape) + self.bandwidth = median_pairwise_distance(y) + print("Bandwidth:", self.bandwidth) + + + + def compute(self, dict_y, dict_y_hat, dict_x, dict_p_hat=None, dict_cluster=None, normalize=False, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + print("Computing KAD distance...") + + + if self.classwise: + assert dict_cluster is not None, "dict_cluster must be provided if classwise is True" + + dict_features_y={} + dict_features_y_hat={} + #dict_features_x={} + + if dict_y_hat is None: + print("computing KAD with style embeddings") + assert dict_p_hat is not None, "dict_p_hat must be provided if dict_y_hat is None" + + for key in dict_y.keys(): + y= dict_y[key] + #x= dict_x[key] + + embed= dict_p_hat[key] + embed=torch.tensor(embed).to(self.device).unsqueeze(0) + + embed_mid, embed_side = torch.chunk(embed, 2, dim=-1) + + if self.type== "AFxRep-mid": + p_hat= embed_mid + elif self.type== "AFxRep-side": + p_hat= embed_side + elif self.type== "AFxRep": + p_hat= embed + + #if x.shape[-2] == 1: + # x = x.repeat( 2, 1) + + #assert x.shape == y.shape, f"Shape mismatch for key {key}: {x.shape} vs {y.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + #x=x.T + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + #x=torch.tensor(x).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + #feat_x= self.feat_extractor(x) + + assert p_hat.shape == feat_y.shape, f"Shape mismatch for key {key}: {p_hat.shape} vs {feat_y.shape}" + #assert p_hat.shape == feat_x.shape, f"Shape mismatch for key {key}: {p_hat.shape} vs {feat_x.shape}" + + dict_features_y[key] = feat_y + #dict_features_x[key] = feat_x + dict_features_y_hat[key] = p_hat + + else: + for key in dict_y.keys(): + y= dict_y[key] + #x= dict_x[key] + y_hat= dict_y_hat[key] + + #if x.shape[-2] == 1: + # x = x.repeat( 2, 1) + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + #assert x.shape == y.shape, f"Shape mismatch for key {key}: {x.shape} vs {y.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + y_hat=y_hat.T + #x=x.T + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + #x=torch.tensor(x).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + #try: + feat_y= self.feat_extractor(y) + feat_y_hat= self.feat_extractor(y_hat) + #except Exception as e: + #print(f"Error extracting features for key {key}: {e}") + # continue + #feat_x= self.feat_extractor(x) + + dict_features_y[key] = feat_y + dict_features_y_hat[key] = feat_y_hat + #dict_features_x[key] = feat_x + + if normalize: + #compute normalization statistics wrt to reference features + y_values = list(dict_features_y.values()) + y_values = torch.cat(y_values, dim=0) + mean= y_values.mean(dim=0, keepdim=True) + std= y_values.std(dim=0, keepdim=True) + + #normalize features + for key in dict_features_y.keys(): + dict_features_y[key] = (dict_features_y[key] - mean) / (std + 1e-8) + dict_features_y_hat[key] = (dict_features_y_hat[key] - mean) / (std + 1e-8) + + + dict_output = {} + if self.KAD_args.do_PCA_figure: + fig=self.do_PCA_figure(dict_features_y, dict_features_y_hat, fit_mode=self.KAD_args.PCA_fit_mode, dict_cluster=dict_cluster) + key= self.type+ "_PCA_figure" + dict_output = {key: fig} + + if self.KAD_args.do_TSNE_figure: + fig=self.do_TSNE_figure(dict_features_y, dict_features_y_hat, fit_mode="all", dict_cluster=dict_cluster) + key= self.type+ "_TSNE_figure" + dict_output[key] = fig + + + # Compute mean features + + + + # Compute the distance between the mean features + #FAD_distance = self.calculate_FAD_distance(y_mean, y_cov, y_hat_mean, y_hat_cov) + if self.bandwidth is None: + self.calculate_bandwidth(dict_features_y) + + if self.classwise: + #calculate KAD distance for each class + KAD_distance = self.calculate_KAD_distance_classwise(dict_features_y, dict_features_y_hat, dict_cluster) + else: + + KAD_distance = self.calculate_KAD_distance(dict_features_y, dict_features_y_hat) + + + return KAD_distance, dict_output + +class FADFeatures(DistMetric): + """ + Class for computing the pairwise spectral metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise spectral metric. + """ + def __init__(self, + type=None, + sample_rate=44100, + FAD_args=None, + *args, **kwargs): + """ + Initialize the PairwiseSpectral instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type = type + + self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + + + self.FAD_args = FAD_args + + assert self.FAD_args is not None, "FAD_args must be provided" + + if self.FAD_args.do_PCA_figure: + self.pca = None + + super().__init__(self.type, sample_rate, *args, **kwargs) + + def calculate_FAD_distance(self, mu1, sigma1, mu2, sigma2, eps=1e-6): + + """Numpy implementation of the Frechet Distance. + The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1) + and X_2 ~ N(mu_2, C_2) is + d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)). + + Stable version by Dougal J. Sutherland. + + Params: + -- mu1 : Numpy array containing the activations of a layer of the + inception net (like returned by the function 'get_predictions') + for generated samples. + -- mu2 : The sample mean over activations, precalculated on an + representative data set. + -- sigma1: The covariance matrix over activations for generated samples. + -- sigma2: The covariance matrix over activations, precalculated on an + representative data set. + + Returns: + -- : The Frechet Distance. + """ + mu1=mu1.detach().cpu().numpy() + mu2=mu2.detach().cpu().numpy() + + sigma1=sigma1.detach().cpu().numpy() + sigma2=sigma2.detach().cpu().numpy() + + + mu1 = np.atleast_1d(mu1) + mu2 = np.atleast_1d(mu2) + + sigma1 = np.atleast_2d(sigma1) + sigma2 = np.atleast_2d(sigma2) + + assert ( + mu1.shape == mu2.shape + ), "Training and test mean vectors have different lengths" + assert ( + sigma1.shape == sigma2.shape + ), "Training and test covariances have different dimensions" + + diff = mu1 - mu2 + + # Product might be almost singular + covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False) + if not np.isfinite(covmean).all(): + msg = ( + "fid calculation produces singular product; " + "adding %s to diagonal of cov estimates" + ) % eps + offset = np.eye(sigma1.shape[0]) * eps + covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset)) + + # Numerical error might give slight imaginary component + if np.iscomplexobj(covmean): + if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3): + m = np.max(np.abs(covmean.imag)) + raise ValueError("Imaginary component {}".format(m)) + covmean = covmean.real + + tr_covmean = np.trace(covmean) + + return diff.dot(diff) + np.trace(sigma1) + np.trace(sigma2) - 2 * tr_covmean + + + def calculate_emb_statistics(self, features_dicto): + """ + Calculate the mean and standard deviation of the features. + + Args: + features_dicto (dict): Dictionary containing features for each key. + + Returns: + mean_features (torch.Tensor): Mean of the features. + std_features (torch.Tensor): Standard deviation of the features. + """ + + all_features = torch.cat(list(features_dicto.values()), dim=0) + + + #mean + #mean_features = all_features.mean(dim=0) + + #cov + #cov_features = torch.cov(all_features.T) + mean_features = np.mean(all_features.cpu().numpy(), axis=0) + + cov_features= np.cov(all_features.cpu().numpy(), rowvar=False) + + + + return torch.tensor(mean_features), torch.tensor(cov_features) + + + def do_PCA_figure(self, dict_features_y, dict_features_y_hat, dict_features_x=None): + """ + Perform PCA on the features and create a figure. + + Args: + dict_features_y (dict): Dictionary containing features for the first set. + dict_features_y_hat (dict): Dictionary containing features for the second set. + + Returns: + fig: The created figure. + """ + import matplotlib.pyplot as plt + from sklearn.decomposition import PCA + + y_values = list(dict_features_y.values()) + y_values = torch.cat(y_values, dim=0) + + + + if self.pca is None: + self.pca = PCA(n_components=2) + pca_result = self.pca.fit_transform(y_values.cpu().numpy()) + else: + pca_result = self.pca.transform(y_values.cpu().numpy()) + + + y_hat_values = list(dict_features_y_hat.values()) + y_hat_values = torch.cat(y_hat_values, dim=0) + + #project y_hat values into the same PCA space + pca_result_hat = self.pca.transform(y_hat_values.cpu().numpy()) + + data_dict = { + "y": pca_result, + "y_hat": pca_result_hat + } + + if dict_features_x is not None: + x_values = list(dict_features_x.values()) + x_values = torch.cat(x_values, dim=0) + + pca_result_x = self.pca.transform(x_values.cpu().numpy()) + + data_dict["x"] = pca_result_x + + + + fig= make_PCA_figure(data_dict, title=self.type + " PCA") + + + return fig + + + + + + def compute(self, dict_y, dict_y_hat, dict_x, dict_p_hat=None, dict_cluster=None, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + print("Computing FAD distance...") + + + + dict_features_y={} + dict_features_y_hat={} + #dict_features_x={} + + if dict_y_hat is None: + print("computing FAD with style embeddings") + assert dict_p_hat is not None, "dict_p_hat must be provided if dict_y_hat is None" + + for key in dict_y.keys(): + y= dict_y[key] + #x= dict_x[key] + + embed= dict_p_hat[key] + embed=torch.tensor(embed).to(self.device).unsqueeze(0) + + print("embed shape:", embed.shape) + embed_mid, embed_side = torch.chunk(embed, 2, dim=-1) + + if self.type== "AFxRep-mid": + p_hat= embed_mid + elif self.type== "AFxRep-side": + p_hat= embed_side + elif self.type== "AFxRep": + p_hat= embed + + #if x.shape[-2] == 1: + # x = x.repeat( 2, 1) + + #assert x.shape == y.shape, f"Shape mismatch for key {key}: {x.shape} vs {y.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + #x=x.T + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + #x=torch.tensor(x).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + #feat_x= self.feat_extractor(x) + + assert p_hat.shape == feat_y.shape, f"Shape mismatch for key {key}: {p_hat.shape} vs {feat_y.shape}" + #assert p_hat.shape == feat_x.shape, f"Shape mismatch for key {key}: {p_hat.shape} vs {feat_x.shape}" + + dict_features_y[key] = feat_y + #dict_features_x[key] = feat_x + dict_features_y_hat[key] = p_hat + + else: + for key in dict_y.keys(): + y= dict_y[key] + #x= dict_x[key] + y_hat= dict_y_hat[key] + + #if x.shape[-2] == 1: + # x = x.repeat( 2, 1) + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + #assert x.shape == y.shape, f"Shape mismatch for key {key}: {x.shape} vs {y.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + y_hat=y_hat.T + #x=x.T + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + #x=torch.tensor(x).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + feat_y_hat= self.feat_extractor(y_hat) + #feat_x= self.feat_extractor(x) + + dict_features_y[key] = feat_y + dict_features_y_hat[key] = feat_y_hat + #dict_features_x[key] = feat_x + + dict_output = {} + if self.FAD_args.do_PCA_figure: + fig=self.do_PCA_figure(dict_features_y, dict_features_y_hat, fit_mode="target") + key= self.type+ "_PCA_figure" + dict_output[key] = fig + + if self.FAD_args.do_TSNE_figure: + fig=self.do_TSNE_figure(dict_features_y, dict_features_y_hat, fit_mode="target") + key= self.type+ "_TSNE_figure" + dict_output[key] = fig + + # Compute mean features + + y_mean, y_cov = self.calculate_emb_statistics(dict_features_y) + + y_hat_mean, y_hat_cov = self.calculate_emb_statistics(dict_features_y_hat) + + + # Compute the distance between the mean features + #FAD_distance = self.calculate_FAD_distance(y_mean, y_cov, y_hat_mean, y_hat_cov) + FAD_distance = self.calculate_FAD_distance(y_mean, y_cov, y_hat_mean, y_hat_cov) + + + return FAD_distance, dict_output + + +def metric_factory(metric_name, sample_rate, *args, **kwargs): + """ + Factory function to create a metric function based on the metric name. + + Args: + metric_name (str): The name of the metric to create. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + An instance of a class that implements the metric function. + """ + if metric_name == "fad-fx_encoder": + return FADFeatures(*args, **kwargs, type="fx_encoder", sample_rate=sample_rate ) + elif metric_name == "fad-AFxRep": + return FADFeatures(*args, **kwargs, type="AFxRep", sample_rate=sample_rate) + elif metric_name == "fad-AFxRep-mid": + return FADFeatures(*args, **kwargs, type="AFxRep-side", sample_rate=sample_rate) + elif metric_name == "fad-AFxRep-side": + return FADFeatures(*args, **kwargs, type="AFxRep-mid", sample_rate=sample_rate) + elif metric_name == "kad-fx_encoder": + return KADFeatures(*args, **kwargs, type="fx_encoder", sample_rate=sample_rate) + elif metric_name == "kad-AFxRep": + return KADFeatures(*args, **kwargs, type="AFxRep", sample_rate=sample_rate) + elif metric_name == "kad-AFxRep-mid": + return KADFeatures(*args, **kwargs, type="AFxRep-side", sample_rate=sample_rate) + elif metric_name == "kad-AFxRep-side": + return KADFeatures(*args, **kwargs, type="AFxRep-mid", sample_rate=sample_rate) + elif metric_name == "kad-class-fx_encoder": + return KADFeatures(*args, **kwargs, type="fx_encoder", sample_rate=sample_rate, classwise=True) + elif metric_name == "kad-class-AFxRep": + return KADFeatures(*args, **kwargs, type="AFxRep", sample_rate=sample_rate, classwise=True) + elif metric_name == "kad-class-AFxRep-mid": + return KADFeatures(*args, **kwargs, type="AFxRep-side", sample_rate=sample_rate, classwise=True) + elif metric_name == "kad-class-AFxRep-side": + return KADFeatures(*args, **kwargs, type="AFxRep-mid", sample_rate=sample_rate, classwise=True) + else: + raise ValueError(f"Unknown metric: {metric_name}") + +# Example usage: +#metric_instance = metric_factory("pairwise-spectral") +#``` diff --git a/utils/evaluation/dist_metrics_multitrack.py b/utils/evaluation/dist_metrics_multitrack.py new file mode 100644 index 0000000000000000000000000000000000000000..4423b3fdaa2cc15c023ca795e81e3b31ec824ff6 --- /dev/null +++ b/utils/evaluation/dist_metrics_multitrack.py @@ -0,0 +1,619 @@ + +import math +from numpy.lib.scimath import sqrt as scisqrt +from scipy import linalg +import numpy as np +import torchaudio +import os +from importlib import import_module +import torch + + +from utils.feature_extractors.load_features import load_AFxRep, load_fx_encoder, load_fx_encoder_plusplus, load_MERT, load_CLAP, load_fx_encoder_plusplus_2048 + +from utils.log import make_PCA_figure + +SCALE_FACTOR = 100 + +def calc_kernel_audio_distance( + x: torch.Tensor, + y: torch.Tensor, + device: str, + bandwidth=None, + kernel='gaussian', + precision=torch.float32, + eps=1e-8 +) -> torch.Tensor: + """ + Compute the Kernel Audio Distance (KAD) between two samples using PyTorch. + + Args: + x: The first set of embeddings of shape (m, embedding_dim). + y: The second set of embeddings of shape (n, embedding_dim). + cache_dirs: Directories to cache kernel statistics. + bandwidth: The bandwidth value for the Gaussian RBF kernel. + kernel: Kernel function to use ('gaussian', 'iq', 'imq'). + precision: Type setting for matrix calculation precision. + eps: Small value to prevent division by zero. + + Returns: + The KAD between x and y embedding sets. + """ + # Ensure x and y are of the correct precision + x = x.to(dtype=precision, device=device) + y = y.to(dtype=precision, device=device) + + assert bandwidth is not None, "Bandwidth must be provided for KAD calculation" + + m, n = x.shape[0], y.shape[0] + + # Define kernel functions + gamma = 1 / (2 * bandwidth**2 + eps) + if kernel == 'gaussian': # Gaussian Kernel + kernel = lambda a: torch.exp(-gamma * a) + elif kernel == 'iq': # Inverse Quadratic Kernel + kernel = lambda a: 1 / (1 + gamma * a) + elif kernel == 'imq': # Inverse Multiquadric Kernel + kernel = lambda a: 1 / torch.sqrt(1 + gamma * a) + else: + raise ValueError("Invalid kernel type. Valid kernels: 'gaussian', 'iq', 'imq'") + + # Load x kernel statistics + xx = x @ x.T + x_sqnorms = torch.diagonal(xx) + d2_xx = x_sqnorms.unsqueeze(1) + x_sqnorms.unsqueeze(0) - 2 * xx # shape (m, m) + + k_xx = kernel(d2_xx) + k_xx = k_xx - torch.diag(torch.diagonal(k_xx)) + k_xx_mean = k_xx.sum() / (m * (m - 1)) + + # Load y kernel statistics + yy = y @ y.T + y_sqnorms = torch.diagonal(yy) + d2_yy = y_sqnorms.unsqueeze(1) + y_sqnorms.unsqueeze(0) - 2 * yy # shape (n, n) + + k_yy = kernel(d2_yy) + k_yy = k_yy - torch.diag(torch.diagonal(k_yy)) + k_yy_mean = k_yy.sum() / (n * (n - 1)) + + # Compute kernel statistics for xy + xy = x @ y.T + d2_xy = x_sqnorms.unsqueeze(1) + y_sqnorms.unsqueeze(0) - 2 * xy # shape (m, n) + k_xy = kernel(d2_xy) + k_xy_mean = k_xy.mean() + + # Compute MMD + result = k_xx_mean + k_yy_mean - 2 * k_xy_mean + + return result * SCALE_FACTOR + + +def median_pairwise_distance(x, subsample=None): + """ + Compute the median pairwise distance of an embedding set. + + Args: + x: torch.Tensor of shape (n_samples, embedding_dim) + subsample: int, number of random pairs to consider (optional) + + Returns: + The median pairwise distance between points in x. + """ + x = torch.tensor(x, dtype=torch.float32) + n_samples = x.shape[0] + + if subsample is not None and subsample < n_samples * (n_samples - 1) / 2: + # Randomly select pairs of indices + idx1 = torch.randint(0, n_samples, (subsample,)) + idx2 = torch.randint(0, n_samples, (subsample,)) + + # Ensure idx1 != idx2 + mask = idx1 == idx2 + idx2[mask] = (idx2[mask] + 1) % n_samples + + # Compute distances for selected pairs + distances = torch.sqrt(torch.sum((x[idx1] - x[idx2])**2, dim=1)) + else: + # Compute all pairwise distances + distances = torch.pdist(x) + + return torch.median(distances).item() + +class DistMetric: + """ + Base class for pairwise metrics. + + This class should be subclassed to implement specific pairwise metrics. + """ + def __init__(self,type, sample_rate,*args, **kwargs): + """ + Initialize the PairwiseMetric instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type=type + self.sample_rate=sample_rate + + self.taxonomy_ref= kwargs.get("taxonomy_ref", None) + + if self.type == "DynamicFeatures": + from utils.feature_extractors.AF_features_embedding import AF_fourier_embedding + AFembedding= AF_fourier_embedding(device=self.device) + + def feat_extracfor_fn(x): + z, _ = AFembedding.encode(x) + return z + + self.feat_extractor = feat_extracfor_fn + elif self.type == "FxEncoder++": + self.model_args= kwargs.get("fx_encoder_plusplus_args", None) + assert self.model_args is not None, "model_args must be provided for fxencAFv2-fxenc++ type" + + self.distance_type=self.model_args.distance_type + fxencoder = load_fx_encoder_plusplus_2048(self.model_args, self.device) + + def feat_extractor_fn(x): + z= fxencoder(x) + z=torch.nn.functional.normalize(z, dim=-1, p=2) # normalize to unit variance + return z + + self.feat_extractor = feat_extractor_fn + + else: + raise ValueError(f"Unknown type: {self.type}. Supported types: fx_encoder, fx_encoder_plusplus, AFxRep-mid, AFxRep-side, AFxRep") + + + def compute(self, *args, **kwargs): + raise NotImplementedError("Subclasses should implement this method.") + + def extract_features(self, y, y_hat, x=None): + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + feat_y_hat= self.feat_extractor(y_hat) + + if x is not None: + x=torch.tensor(x).permute(1,0).unsqueeze(0).to(self.device) + with torch.no_grad(): + feat_x= self.feat_extractor(x) + else: + feat_x=None + + + return feat_y, feat_y_hat, feat_x + + + def do_TSNE_figure(self, dict_features_y, dict_features_y_hat, dict_taxonomy=None): + """ + Perform PCA on the features and create a figure. + + Args: + dict_features_y (dict): Dictionary containing features for the first set. + dict_features_y_hat (dict): Dictionary containing features for the second set. + + Returns: + fig: The created figure. + """ + import matplotlib.pyplot as plt + from sklearn.manifold import TSNE + + + y_values = {} + y_hat_values = {} + for key, track_name in self.taxonomy_ref.items(): + + + y_values[track_name] = [] + for k in range(len(dict_features_y.keys())): + index = [i for i, tax_id in enumerate(dict_taxonomy[k]) if tax_id == key] + print("taxonomy",k, dict_taxonomy[k]) + if index: # If any matches were found + # Select the corresponding rows from dict_features_y[k] + selected_features = dict_features_y[k][index] + y_values[track_name].append(selected_features) + + + if not y_values[track_name]: + print(f"Warning: No features found for track {track_name} with key {key}. Skipping this track.") + #remove the track from the dictionaries + y_values.pop(track_name, None) + continue + + y_values[track_name]=torch.cat(y_values[track_name], dim=0) + y_values[track_name]=torch.nan_to_num(y_values[track_name], nan=0) + + #print(clusters[track_name]) + + y_hat_values[track_name] = [] + + for k in range(len(dict_features_y_hat.keys())): + index = [i for i, tax_id in enumerate(dict_taxonomy[k]) if tax_id == key] + if index: # If any matches were found + # Select the corresponding rows from dict_features_y[k] + selected_features = dict_features_y_hat[k][index] + y_hat_values[track_name].append(selected_features) + #else: + # y_hat_values[track_name].append(None) # Append zeros if no match found + + + y_hat_values[track_name]=torch.cat(y_hat_values[track_name], dim=0) + y_hat_values[track_name]=torch.nan_to_num(y_hat_values[track_name], nan=0) + + #print(clusters_hat[track_name]) + + if self.pca is None: + self.pca= {} + + + tsne_result = {} + tsne_result_hat = {} + data_dict = {} + figs= {} + + for k in y_values.keys(): + print("Processing track:", k) + self.tsne =TSNE(n_components=2, perplexity=30) + combined_data = torch.cat([y_values[k], y_hat_values[k]], dim=0).cpu().numpy() + combined_result = self.tsne.fit_transform(combined_data) + + n_y = y_values[k].shape[0] + tsne_result[k] = combined_result[:n_y] + tsne_result_hat[k] = combined_result[n_y:] + + + data_dict[k] = { + "y": tsne_result[k], + "y_hat": tsne_result_hat[k] } + + figs[k]= make_PCA_figure(data_dict[k], title=self.type + " TSNE; track: "+k) + + + return figs + + def do_PCA_figure(self, dict_features_y, dict_features_y_hat, fit_mode="target", dict_cluster=None, dict_taxonomy=None): + """ + Perform PCA on the features and create a figure. + + Args: + dict_features_y (dict): Dictionary containing features for the first set. + dict_features_y_hat (dict): Dictionary containing features for the second set. + + Returns: + fig: The created figure. + """ + import matplotlib.pyplot as plt + from sklearn.decomposition import PCA + + + y_values = {} + y_hat_values = {} + if dict_cluster is not None: + clusters={} + clusters_hat= {} + + for key, track_name in self.taxonomy_ref.items(): + + + y_values[track_name] = [] + if dict_cluster is not None: + clusters[track_name] = [] + for k in range(len(dict_features_y.keys())): + print("taxonomy",k, dict_taxonomy[k]) + index = [i for i, tax_id in enumerate(dict_taxonomy[k]) if tax_id == key] + if index: # If any matches were found + # Select the corresponding rows from dict_features_y[k] + selected_features = dict_features_y[k][index] + y_values[track_name].append(selected_features) + + if dict_cluster is not None: + clusters[track_name].append(dict_cluster[k].unsqueeze(0)) + + #check if y_values[track_name] is empty + if not y_values[track_name]: + print(f"Warning: No features found for track {track_name} with key {key}. Skipping this track.") + #remove the track from the dictionaries + y_values.pop(track_name, None) + continue + + y_values[track_name]=torch.cat(y_values[track_name], dim=0) + y_values[track_name]=torch.nan_to_num(y_values[track_name], nan=0) + + if dict_cluster is not None: + clusters[track_name]=torch.cat(clusters[track_name], dim=0) + + y_hat_values[track_name] = [] + if dict_cluster is not None: + clusters_hat[track_name] = [] + + for k in range(len(dict_features_y_hat.keys())): + index = [i for i, tax_id in enumerate(dict_taxonomy[k]) if tax_id == key] + if index: # If any matches were found + # Select the corresponding rows from dict_features_y[k] + selected_features = dict_features_y_hat[k][index] + y_hat_values[track_name].append(selected_features) + if dict_cluster is not None: + clusters_hat[track_name].append(dict_cluster[k].unsqueeze(0)) + + y_hat_values[track_name]=torch.cat(y_hat_values[track_name], dim=0) + y_hat_values[track_name]=torch.nan_to_num(y_hat_values[track_name], nan=0) + + if dict_cluster is not None: + clusters_hat[track_name]=torch.cat(clusters_hat[track_name], dim=0) + + + if self.pca is None: + self.pca= {} + + pca_result = {} + pca_result_hat = {} + data_dict = {} + figs= {} + + for k in y_values.keys(): + print("Processing track:", k) + #check if pca already exists for this key + if k not in self.pca.keys(): + if fit_mode == "target": + print("Fitting PCA for target values only track:", k) + self.pca[k] = PCA(n_components=2).fit(y_values[k].cpu().numpy()) + elif fit_mode == "all": + print("Fitting PCA for all values track:", k) + self.pca[k] = PCA(n_components=2).fit(torch.cat([y_values[k], y_hat_values[k]], dim=0).cpu().numpy()) + else: + raise ValueError(f"Unknown fit_mode: {fit_mode}. Supported modes: target, all") + + pca_result[k]=self.pca[k].transform(y_values[k].cpu().numpy()) + pca_result_hat[k] = self.pca[k].transform(y_hat_values[k].cpu().numpy()) + + + if dict_cluster is not None: + data_dict[k] = { + "y_C0": pca_result[k][clusters[k] == 0], + "y_hat_C0": pca_result_hat[k][clusters_hat[k] == 0], + "y_C1": pca_result[k][clusters[k] == 1], + "y_hat_C1": pca_result_hat[k][clusters_hat[k] == 1] + } + else: + data_dict[k] = { + "y": pca_result[k], + "y_hat": pca_result_hat[k] + } + + + figs[k]= make_PCA_figure(data_dict[k], title=self.type + " PCA; track: "+k) + + + return figs + + +class KADFeatures(DistMetric): + """ + Class for computing the pairwise spectral metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise spectral metric. + """ + def __init__(self, + type=None, + sample_rate=44100, + KAD_args=None, + *args, **kwargs): + """ + Initialize the PairwiseSpectral instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type = type + + self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + + self.KAD_args = KAD_args + + assert self.KAD_args is not None, "FAD_args must be provided" + + if self.KAD_args.do_PCA_figure: + self.pca = None + + self.bandwidth = None + + super().__init__(self.type, sample_rate, *args, **kwargs) + + def calculate_KAD_distance(self, dict_features_y, dict_features_y_hat, dict_taxonomy=None): + + y_values = {} + y_hat_values = {} + for key, track_name in self.taxonomy_ref.items(): + y_values[track_name] = [] + for k in range(len(dict_features_y.keys())): + index = [i for i, tax_id in enumerate(dict_taxonomy[k]) if tax_id == key] + if index: # If any matches were found + # Select the corresponding rows from dict_features_y[k] + selected_features = dict_features_y[k][index] + y_values[track_name].append(selected_features) + #else: + # y_values[track_name].append(None) # Append zeros if no match found + if not y_values[track_name]: + print(f"Warning: No features found for track {track_name} with key {key}. Skipping this track.") + #remove the track from the dictionaries + y_values.pop(track_name, None) + continue + + y_values[track_name]=torch.cat(y_values[track_name], dim=0) + y_values[track_name]=torch.nan_to_num(y_values[track_name], nan=0) + + y_hat_values[track_name] = [] + for k in range(len(dict_features_y_hat.keys())): + index = [i for i, tax_id in enumerate(dict_taxonomy[k]) if tax_id == key] + if index: # If any matches were found + # Select the corresponding rows from dict_features_y[k] + selected_features = dict_features_y_hat[k][index] + y_hat_values[track_name].append(selected_features) + #else: + # y_hat_values[track_name].append(None) # Append zeros if no match found + + y_hat_values[track_name]=torch.cat(y_hat_values[track_name], dim=0) + y_hat_values[track_name]=torch.nan_to_num(y_hat_values[track_name], nan=0) + + + KAD={} + weight={} + total_examples= 0 + for key in y_values.keys(): + y= y_values[key] + y_hat= y_hat_values[key] + with torch.no_grad(): + KAD[key]=calc_kernel_audio_distance(y, y_hat, device=self.device, bandwidth=self.bandwidth, kernel=self.KAD_args.kernel, precision=torch.float32) + + weight[key]= y.shape[0] + total_examples += weight[key] + + KAD= sum([KAD[k] * weight[k] for k in KAD.keys()]) / total_examples + + return KAD.cpu().item() + + def calculate_bandwidth(self, dict_features_y): + + print("Calculating bandwidth...") + y= torch.cat(list(dict_features_y.values()), dim=0) + print("y shape:", y.shape) + self.bandwidth = median_pairwise_distance(y) + print("Bandwidth:", self.bandwidth) + + + + def compute(self, dict_y, dict_y_hat, dict_taxonomy=None, dict_p_hat=None, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + print("Computing KAD distance...") + + dict_features_y={} + dict_features_y_hat={} + + if dict_y_hat is None and dict_p_hat is not None: + + for key in dict_y.keys(): + y= dict_y[key] + + embed= dict_p_hat[key] + embed=torch.tensor(embed).to(self.device) + + embed=embed*math.sqrt(embed.shape[-1]) # Scale the embedding + + embed_fxenc=embed[...,:2048]/ math.sqrt(2048) # Scale the first 128 dimensions + embed_AF=embed[...,2048:]/ math.sqrt(64) # Scale the last 128 dimensions + + + if self.type == "FxEncoder++": + p_hat= embed_fxenc + + elif self.type == "DynamicFeatures": + p_hat= embed_AF + + n, c, d=y.shape + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=torch.tensor(y).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + + assert p_hat.shape == feat_y.shape, f"Shape mismatch for key {key}: {p_hat.shape} vs {feat_y.shape}" + + dict_features_y[key] = feat_y + dict_features_y_hat[key] = p_hat + + else: + for key in dict_y.keys(): + y= dict_y[key] + y_hat= dict_y_hat[key] + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + + n, c, d=y.shape + if dict_taxonomy is not None: + taxonomy= dict_taxonomy[key] + assert len(taxonomy) == n, f"Taxonomy length mismatch for key {key}: {len(taxonomy)} vs {n}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=torch.tensor(y).to(self.device) + y_hat=torch.tensor(y_hat).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + feat_y_hat= self.feat_extractor(y_hat) + + dict_features_y[key] = feat_y + dict_features_y_hat[key] = feat_y_hat + + dict_output = {} + if self.KAD_args.do_PCA_figure: + dict_figs=self.do_PCA_figure(dict_features_y, dict_features_y_hat, dict_taxonomy=dict_taxonomy, fit_mode=self.KAD_args.PCA_fit_mode) + #returns a dictionary with figures for each taxonomy key + #modify the key to include the type + + for k, v in dict_figs.items(): + #modify the key to include the type + key= self.type+ "_PCA_figure_" + k + dict_output[key] = v + + if self.KAD_args.do_TSNE_figure: + dict_figs=self.do_TSNE_figure(dict_features_y, dict_features_y_hat, dict_taxonomy=dict_taxonomy) + for k, fig in dict_figs.items(): + key= self.type+ "_TSNE_figure"+k + dict_output[key] = fig + + + + # Compute the distance between the mean features + #FAD_distance = self.calculate_FAD_distance(y_mean, y_cov, y_hat_mean, y_hat_cov) + if self.bandwidth is None: + self.calculate_bandwidth(dict_features_y) + + + KAD_distance = self.calculate_KAD_distance(dict_features_y, dict_features_y_hat, dict_taxonomy) + + + return KAD_distance, dict_output + + + +def metric_factory(metric_name, sample_rate, *args, **kwargs): + """ + Factory function to create a metric function based on the metric name. + + Args: + metric_name (str): The name of the metric to create. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + An instance of a class that implements the metric function. + """ + if metric_name == "kad-FxEncoder++": + return KADFeatures(*args, **kwargs, type="FxEncoder++", sample_rate=sample_rate) + elif metric_name == "kad-DynamicFeatures": + return KADFeatures(*args, **kwargs, type="DynamicFeatures", sample_rate=sample_rate) + else: + raise ValueError(f"Unknown metric: {metric_name}") + +# Example usage: +#metric_instance = metric_factory("pairwise-spectral") +#``` diff --git a/utils/evaluation/pairwise_metrics.py b/utils/evaluation/pairwise_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..ccdd3f330e3bb16e3a6bdc5a1e550fd8731351dc --- /dev/null +++ b/utils/evaluation/pairwise_metrics.py @@ -0,0 +1,958 @@ + +import math +import torchaudio +import os +from importlib import import_module +import yaml +import torch +from utils.feature_extractors.load_features import load_AFxRep, load_fx_encoder, load_fx_encoder_plusplus, load_fx_encoder_plusplus_2048 +from utils.MSS_loss import MultiScale_Spectral_Loss_MidSide_DDSP + +class PairwiseMetric: + """ + Base class for pairwise metrics. + + This class should be subclassed to implement specific pairwise metrics. + """ + def __init__(self, *args, **kwargs): + """ + Initialize the PairwiseMetric instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass + + def compute(self, *args, **kwargs): + raise NotImplementedError("Subclasses should implement this method.") + +class PairwiseFeatures(PairwiseMetric): + """ + Class for computing the pairwise spectral metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise spectral metric. + """ + def __init__(self, + type=None, + sample_rate=44100, + *args, **kwargs): + """ + Initialize the PairwiseSpectral instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type = type + self.sample_rate = sample_rate + + self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + + if self.type == "fx_encoder": + self.model_args= kwargs.get("fx_encoder_args", None) + + assert self.model_args is not None, "model_args must be provided for fx_encoder type" + + self.distance_type=self.model_args.distance_type + + self.feat_extractor = load_fx_encoder(self.model_args, self.device) + + + #self.feat_extractor = load_effects_encoder(ckpt_path=ckpt_path).to(self.device) + elif self.type == "fxenc++": + self.model_args= kwargs.get("fx_encoder_plusplus_args", None) + + assert self.model_args is not None, "model_args must be provided for fx_encoder type" + + self.distance_type=self.model_args.distance_type + + self.feat_extractor = load_fx_encoder_plusplus(self.model_args, self.device) + + + #self.feat_extractor = load_effects_encoder(ckpt_path=ckpt_path).to(self.device) + + elif self.type == "logrms": + self.model_args= kwargs.get("logrms_args", None) + + assert self.model_args is not None, "model_args must be provided for logrms type" + self.distance_type=self.model_args.distance_type + + from utils.feature_extractors.dsp_features import compute_log_rms + + self.feat_extractor = lambda x: compute_log_rms(x) + + elif self.type == "crestfactor": + self.model_args= kwargs.get("crestfactor_args", None) + assert self.model_args is not None, "model_args must be provided for crestfactor type" + + self.distance_type=self.model_args.distance_type + + from utils.feature_extractors.dsp_features import compute_crest_factor + + self.feat_extractor = lambda x: compute_crest_factor(x) + elif self.type == "logspread": + self.model_args= kwargs.get("logspread_args", None) + assert self.model_args is not None, "model_args must be provided for logspread type" + + self.distance_type=self.model_args.distance_type + + from utils.feature_extractors.dsp_features import compute_log_spread + + self.feat_extractor = lambda x: compute_log_spread(x).view(-1, 1) + elif self.type == "stereowidth": + self.model_args= kwargs.get("stereowidth_args", None) + assert self.model_args is not None, "model_args must be provided for stereowidth type" + + self.distance_type=self.model_args.distance_type + + from utils.feature_extractors.dsp_features import compute_stereo_width + self.feat_extractor = lambda x: compute_stereo_width(x).view(-1, 1) + + elif self.type == "stereoimbalance": + self.model_args= kwargs.get("stereoimbalance_args", None) + assert self.model_args is not None, "model_args must be provided for stereoimbalance type" + self.distance_type=self.model_args.distance_type + from utils.feature_extractors.dsp_features import compute_stereo_imbalance + self.feat_extractor = lambda x: compute_stereo_imbalance(x).view(-1, 1) + + elif self.type== "AFxRep-mid" or self.type== "AFxRep-side" or self.type== "AFxRep": + + self.model_args= kwargs.get("AFxRep_args", None) + + assert self.model_args is not None, "model_args must be provided for AFxRep type" + + self.distance_type=self.model_args.distance_type + + feat_extractor = load_AFxRep(self.model_args, self.device) + + if self.type == "AFxRep-mid": + def feat_extractor_mid(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_mid + + self.feat_extractor = feat_extractor_mid + + elif self.type == "AFxRep-side": + def feat_extractor_side(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_side + + self.feat_extractor = feat_extractor_side + else: + self.feat_extractor = feat_extractor + + + super().__init__(*args, **kwargs) + + def compute_feature_distance(self, y, y_hat, sample_rate, type): + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + feat_y_hat= self.feat_extractor(y_hat) + + if self.distance_type == "cosine": + cos_dist= 1- torch.cosine_similarity(feat_y_hat, feat_y, dim=1) + + + return {"distance": cos_dist.mean().item()} + elif self.distance_type == "l1": + l1_dist = torch.abs(feat_y_hat - feat_y).mean(dim=1) + + return {"distance": l1_dist.mean().item()} + + + def compute(self, dict_y, dict_y_hat, dict_x, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + + + dict_features={} + + for key in dict_y.keys(): + y= dict_y[key] + y_hat= dict_y_hat[key] + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + y_hat=y_hat.T + + + if self.type == "spectral": + from evaluation.automix_evaluation import compute_spectral_features + dict_features_out= compute_spectral_features(y_hat, y ,self.sample_rate) + dict_features[key] = dict_features_out['mean_mape_spectral'] + elif self.type=="panning": + from evaluation.automix_evaluation import compute_panning_features + dict_features_out = compute_panning_features(y_hat, y, self.sample_rate) + dict_features[key] = dict_features_out['mean_mape_panning'] + elif self.type=="loudness": + from evaluation.automix_evaluation import compute_loudness_features + dict_features_out = compute_loudness_features(y_hat, y, self.sample_rate) + dict_features[key] = dict_features_out['mean_mape_loudness'] + elif self.type=="dynamic": + from evaluation.automix_evaluation import compute_dynamic_features + dict_features_out = compute_dynamic_features(y_hat, y, self.sample_rate) + dict_features[key] = dict_features_out['mean_mape_dynamic'] + elif self.type=="fx_encoder": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep-mid": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep-side": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="fxenc++": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="logrms": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="crestfactor": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="logspread": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="stereowidth": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="stereoimbalance": + dict_features_out = self.compute_feature_distance(y, y_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + + + # Compute the mean of the features across all keys + + mean_features = sum(dict_features.values()) / len(dict_features) + + return mean_features, {} + +class PairwiseStyleMultitrackFeatures(PairwiseMetric): + """ + Class for computing the pairwise spectral metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise spectral metric. + """ + def __init__(self, + type=None, + sample_rate=44100, + *args, **kwargs): + """ + Initialize the PairwiseSpectral instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type = type + self.sample_rate = sample_rate + + self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + + if self.type == "fx_encoder": + raise NotImplementedError("Style features for fx_encoder not implemented yet") + self.model_args= kwargs.get("fx_encoder_args", None) + + assert self.model_args is not None, "model_args must be provided for fx_encoder type" + + self.distance_type=self.model_args.distance_type + + self.feat_extractor = load_fx_encoder(self.model_args, self.device) + + + #self.feat_extractor = load_effects_encoder(ckpt_path=ckpt_path).to(self.device) + + elif self.type== "AFxRep-mid" or self.type== "AFxRep-side" or self.type== "AFxRep": + + self.model_args= kwargs.get("AFxRep_args", None) + + assert self.model_args is not None, "model_args must be provided for AFxRep type" + + self.distance_type=self.model_args.distance_type + + feat_extractor = load_AFxRep(self.model_args, self.device) + + if self.type == "AFxRep-mid": + def feat_extractor_mid(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_mid + + self.feat_extractor = feat_extractor_mid + + elif self.type == "AFxRep-side": + def feat_extractor_side(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_side + + self.feat_extractor = feat_extractor_side + else: + self.feat_extractor = feat_extractor + + elif self.type == "fxenc2048AFv3-fxenc++": + + self.model_args= kwargs.get("fx_encoder_plusplus_args", None) + assert self.model_args is not None, "model_args must be provided for fxencAFv2-fxenc++ type" + + self.distance_type= self.model_args.distance_type + + fxencoder = load_fx_encoder_plusplus_2048(self.model_args, self.device) + + def feat_extractor_fn(x): + z= fxencoder(x) + z=torch.nn.functional.normalize(z, dim=-1, p=2) # normalize to unit variance + return z + + self.feat_extractor = feat_extractor_fn + + elif self.type == "fxenc2048AFv3-AF": + + from utils.AF_features_embedding_v2 import AF_fourier_embedding + AFembedding= AF_fourier_embedding(device=self.device) + + self.distance_type="cosine" + + def feat_extracfor_fn(x): + z, _ = AFembedding.encode(x) + return z + + self.feat_extractor = feat_extracfor_fn + + + super().__init__(*args, **kwargs) + + def compute_feature_distance(self, y, p_hat, sample_rate, type): + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + #y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + #feat_y_hat= self.feat_extractor(y_hat) + + assert p_hat.shape == feat_y.shape, f"Shape mismatch: p_hat {p_hat.shape} vs feat_y {feat_y.shape}" + + if self.distance_type == "cosine": + cos_dist= 1- torch.cosine_similarity(p_hat, feat_y, dim=1) + + return {"distance": cos_dist.mean().item()} + + + + def compute(self, dict_y, dict_y_hat, dict_x, dict_p_hat=None, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + dict_features={} + + index=0 + + for key in dict_y.keys(): + y= dict_y[key] + + embed= dict_p_hat[key] + embed=torch.tensor(embed).to(self.device).unsqueeze(0) + + print(f"Processing key: {key}, embed shape: {embed.shape}") + if "AFxRep" in self.type: + embed_mid, embed_side = torch.chunk(embed, 2, dim=-1) + + if self.type== "AFxRep-mid": + p_hat= embed_mid + elif self.type== "AFxRep-side": + p_hat= embed_side + elif self.type== "AFxRep": + p_hat= embed + elif "fxenc2048AFv3" in self.type: + + embed=embed*math.sqrt(embed.shape[-1]) # Scale the embedding + embed_fxenc=embed[...,:2048]/ math.sqrt(2048) # Scale the first 128 dimensions + embed_AF=embed[...,2048:]/ math.sqrt(64) # Scale the last 128 dimensions + + if self.type == "fxenc2048AFv3-fxenc++": + p_hat= embed_fxenc + elif self.type == "fxenc2048AFv3-AF": + p_hat= embed_AF + + + n, c, d=y.shape + + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + for i in range(n): + y_i=y[i].T + + if self.type=="fx_encoder": + raise NotImplementedError("Style features for fx_encoder not implemented yet") + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep": + dict_features_out = self.compute_feature_distance(y_i, p_hat[i], self.sample_rate, type=self.type) + dict_features[index] = dict_features_out["distance"] + elif self.type=="AFxRep-mid": + dict_features_out = self.compute_feature_distance(y_i, p_hat[i], self.sample_rate, type=self.type) + dict_features[index] = dict_features_out["distance"] + elif self.type=="AFxRep-side": + dict_features_out = self.compute_feature_distance(y_i, p_hat[i], self.sample_rate, type=self.type) + dict_features[index] = dict_features_out["distance"] + elif self.type=="fxenc2048AFv3-fxenc++": + dict_features_out = self.compute_feature_distance(y_i, p_hat[i], self.sample_rate, type=self.type) + dict_features[index] = dict_features_out["distance"] + elif self.type=="fxenc2048AFv3-AF": + dict_features_out = self.compute_feature_distance(y_i, p_hat, self.sample_rate, type=self.type) + dict_features[index] = dict_features_out["distance"] + + + mean_features = sum(dict_features.values()) / len(dict_features) + + return mean_features, {} +class PairwiseStyleFeatures(PairwiseMetric): + """ + Class for computing the pairwise spectral metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise spectral metric. + """ + def __init__(self, + type=None, + sample_rate=44100, + *args, **kwargs): + """ + Initialize the PairwiseSpectral instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + self.type = type + self.sample_rate = sample_rate + + self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + + if self.type == "fx_encoder": + raise NotImplementedError("Style features for fx_encoder not implemented yet") + self.model_args= kwargs.get("fx_encoder_args", None) + + assert self.model_args is not None, "model_args must be provided for fx_encoder type" + + self.distance_type=self.model_args.distance_type + + self.feat_extractor = load_fx_encoder(self.model_args, self.device) + + + #self.feat_extractor = load_effects_encoder(ckpt_path=ckpt_path).to(self.device) + + elif self.type== "AFxRep-mid" or self.type== "AFxRep-side" or self.type== "AFxRep": + + self.model_args= kwargs.get("AFxRep_args", None) + + assert self.model_args is not None, "model_args must be provided for AFxRep type" + + self.distance_type=self.model_args.distance_type + + feat_extractor = load_AFxRep(self.model_args, self.device) + + if self.type == "AFxRep-mid": + def feat_extractor_mid(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_mid + + self.feat_extractor = feat_extractor_mid + + elif self.type == "AFxRep-side": + def feat_extractor_side(x): + + features= feat_extractor(x) + + #divide by 2 to get mid and side features + + feat_mid, feat_side = features.chunk(2, dim=-1) + + return feat_side + + self.feat_extractor = feat_extractor_side + else: + self.feat_extractor = feat_extractor + + elif self.type == "fxenc2048AFv3-fxenc++": + + self.model_args= kwargs.get("fx_encoder_plusplus_args", None) + assert self.model_args is not None, "model_args must be provided for fxencAFv2-fxenc++ type" + + self.distance_type= self.model_args.distance_type + + fxencoder = load_fx_encoder_plusplus_2048(self.model_args, self.device) + + def feat_extractor_fn(x): + z= fxencoder(x) + z=torch.nn.functional.normalize(z, dim=-1, p=2) # normalize to unit variance + return z + + self.feat_extractor = feat_extractor_fn + + elif self.type == "fxenc2048AFv3-AF": + + from utils.AF_features_embedding_v2 import AF_fourier_embedding + AFembedding= AF_fourier_embedding(device=self.device) + + self.distance_type="cosine" + + def feat_extracfor_fn(x): + z, _ = AFembedding.encode(x) + return z + + self.feat_extractor = feat_extracfor_fn + + + super().__init__(*args, **kwargs) + + def compute_feature_distance(self, y, p_hat, sample_rate, type): + + y=torch.tensor(y).permute(1,0).unsqueeze(0).to(self.device) + #y_hat=torch.tensor(y_hat).permute(1,0).unsqueeze(0).to(self.device) + + with torch.no_grad(): + feat_y= self.feat_extractor(y) + #feat_y_hat= self.feat_extractor(y_hat) + + assert p_hat.shape == feat_y.shape, f"Shape mismatch: p_hat {p_hat.shape} vs feat_y {feat_y.shape}" + + if self.distance_type == "cosine": + cos_dist= 1- torch.cosine_similarity(p_hat, feat_y, dim=1) + + return {"distance": cos_dist.mean().item()} + + + + def compute(self, dict_y, dict_y_hat, dict_x, dict_p_hat=None, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + dict_features={} + + for key in dict_y.keys(): + y= dict_y[key] + + embed= dict_p_hat[key] + embed=torch.tensor(embed).to(self.device).unsqueeze(0) + + print(f"Processing key: {key}, embed shape: {embed.shape}") + if "AFxRep" in self.type: + embed_mid, embed_side = torch.chunk(embed, 2, dim=-1) + + if self.type== "AFxRep-mid": + p_hat= embed_mid + elif self.type== "AFxRep-side": + p_hat= embed_side + elif self.type== "AFxRep": + p_hat= embed + elif "fxenc2048AFv3" in self.type: + + embed=embed*math.sqrt(embed.shape[-1]) # Scale the embedding + embed_fxenc=embed[...,:2048]/ math.sqrt(2048) # Scale the first 128 dimensions + embed_AF=embed[...,2048:]/ math.sqrt(64) # Scale the last 128 dimensions + + if self.type == "fxenc2048AFv3-fxenc++": + p_hat= embed_fxenc + elif self.type == "fxenc2048AFv3-AF": + p_hat= embed_AF + + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + #y_hat=y_hat.T + + if self.type=="fx_encoder": + raise NotImplementedError("Style features for fx_encoder not implemented yet") + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep": + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep-mid": + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="AFxRep-side": + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="fxenc2048AFv3-fxenc++": + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + elif self.type=="fxenc2048AFv3-AF": + dict_features_out = self.compute_feature_distance(y, p_hat, self.sample_rate, type=self.type) + dict_features[key] = dict_features_out["distance"] + + + + # Compute the mean of the features across all keys + + mean_features = sum(dict_features.values()) / len(dict_features) + + return mean_features, {} + +class PairwiseIMMSS(PairwiseMetric): + """ + Class for computing the pairwise LDR metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise LDR metric. + """ + def __init__(self, mode=None, *args, **kwargs): + """ + Initialize the PairwiseLDR instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + super().__init__(*args, **kwargs) + + assert mode is not None, "Mode must be specified for PairwiseLDR" + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if mode == "mss-lr": + multi_scale_spectral_ori = MultiScale_Spectral_Loss_MidSide_DDSP(mode='ori', eps=1e-6, device=device) + self.metric= lambda y_hat, y: multi_scale_spectral_ori(y_hat, y) + elif mode == "mss-ms": + multi_scale_spectral_midside = MultiScale_Spectral_Loss_MidSide_DDSP(mode='midside', eps=1e-6, device=device) + self.metric= lambda y_hat, y: multi_scale_spectral_midside(y_hat, y) + + def compute(self, dict_y, dict_y_hat, dict_x, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + dict_metrics={} + + for key in dict_y.keys(): + y= dict_y[key] + y_hat= dict_y_hat[key] + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=torch.from_numpy(y).cuda().unsqueeze(0) + y_hat=torch.from_numpy(y_hat).cuda().unsqueeze(0) + + metric=self.metric(y_hat, y) + + dict_metrics[key] = metric.item() + + mean_features = sum(dict_metrics.values()) / len(dict_metrics) + + return mean_features, {} +class PairwiseLDR(PairwiseMetric): + """ + Class for computing the pairwise LDR metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise LDR metric. + """ + def __init__(self, mode=None, *args, **kwargs): + """ + Initialize the PairwiseLDR instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + super().__init__(*args, **kwargs) + + assert mode is not None, "Mode must be specified for PairwiseLDR" + + if mode == "mldr-lr": + from evaluation.ldr import MLDRLoss + raw_metric= MLDRLoss( + sr=44100, + s_taus=[50, 100], + l_taus=[1000, 2000], + ).cuda() + self.metric=lambda y_hat, y: 0.5*raw_metric(y_hat, y) + elif mode == "mldr-ms": + from evaluation.ldr import MLDRLoss + raw_metric= MLDRLoss( + sr=44100, + s_taus=[50, 100], + l_taus=[1000, 2000], + mid_side=True + ).cuda() + self.metric=lambda y_hat, y: 0.25*raw_metric(y_hat, y) + + def compute(self, dict_y, dict_y_hat, dict_x, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + dict_metrics={} + + for key in dict_y.keys(): + y= dict_y[key] + y_hat= dict_y_hat[key] + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + y=y.T + y_hat=y_hat.T + + y=torch.from_numpy(y).cuda().unsqueeze(0) + y_hat=torch.from_numpy(y_hat).cuda().unsqueeze(0) + + metric=self.metric(y_hat, y) + + dict_metrics[key] = metric.item() + + mean_features = sum(dict_metrics.values()) / len(dict_metrics) + + return mean_features, {} + + +class PairwiseAuraloss(PairwiseMetric): + """ + Class for computing the pairwise MSS metric. + + This class inherits from PairwiseMetric and implements the compute method + to calculate the pairwise MSS metric. + """ + def __init__(self, mode=None, *args, **kwargs): + """ + Initialize the PairwiseMSS instance. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + super().__init__(*args, **kwargs) + + assert mode is not None, "Mode must be specified for PairwiseMSS" + + if mode == "mss-lr": + from auraloss.freq import MultiResolutionSTFTLoss + raw_metric=MultiResolutionSTFTLoss( + [128, 512, 2048], + [32, 128, 512], + [128, 512, 2048], + sample_rate=44100, + perceptual_weighting=True, + ).cuda() + self.metric=lambda y_hat, y: raw_metric(y_hat, y) + elif mode == "mss-ms": + from auraloss.freq import SumAndDifferenceSTFTLoss + raw_metric=SumAndDifferenceSTFTLoss( + [128, 512, 2048], + [32, 128, 512], + [128, 512, 2048], + sample_rate=44100, + perceptual_weighting=True, + ).cuda() + self.metric=lambda y_hat, y: 0.5*raw_metric(y_hat, y) + + def compute(self, dict_y, dict_y_hat, dict_x, *args, **kwargs): + """ + Compute the pairwise spectral metric. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The computed pairwise spectral metric. + """ + + dict_metrics={} + + for key in dict_y.keys(): + y= dict_y[key] + y_hat= dict_y_hat[key] + + assert y.shape == y_hat.shape, f"Shape mismatch for key {key}: {y.shape} vs {y_hat.shape}" + + c, d=y.shape + #assert b==1, f"Expected batch size of 1, got {b} for key {key}" + + assert c==2, f"Expected 2 channels, got {c} for key {key}" + + #y=y.T + #y_hat=y_hat.T + + y=torch.from_numpy(y).cuda().unsqueeze(0) + y_hat=torch.from_numpy(y_hat).cuda().unsqueeze(0) + + metric=self.metric(y_hat, y) + + dict_metrics[key] = metric.item() + + mean_features = sum(dict_metrics.values()) / len(dict_metrics) + + return mean_features, {} + +def metric_factory(metric_name, sample_rate, *args, **kwargs): + """ + Factory function to create a metric function based on the metric name. + + Args: + metric_name (str): The name of the metric to create. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + An instance of a class that implements the metric function. + """ + if metric_name == "pairwise-spectral": + return PairwiseFeatures(*args, **kwargs, type="spectral", sample_rate=sample_rate) + elif metric_name == "pairwise-panning": + return PairwiseFeatures(*args, **kwargs, type="panning", sample_rate=sample_rate) + elif metric_name == "pairwise-loudness": + return PairwiseFeatures(*args, **kwargs, type="loudness", sample_rate=sample_rate) + elif metric_name == "pairwise-dynamic": + return PairwiseFeatures(*args, **kwargs, type="dynamic", sample_rate=sample_rate) + elif metric_name == "pairwise-mss-lr": + return PairwiseAuraloss(mode="mss-lr",*args, **kwargs) + elif metric_name == "pairwise-mss-ms": + return PairwiseAuraloss(mode="mss-ms",*args, **kwargs) + elif metric_name == "pairwise-IM-mss-lr": + return PairwiseIMMSS(mode="mss-lr",*args, **kwargs) + elif metric_name == "pairwise-IM-mss-ms": + return PairwiseIMMSS(mode="mss-ms",*args, **kwargs) + elif metric_name == "pairwise-mldr-lr": + return PairwiseLDR(mode="mldr-lr",*args, **kwargs) + elif metric_name == "pairwise-mldr-ms": + return PairwiseLDR(mode="mldr-ms",*args, **kwargs) + elif metric_name == "pairwise-fx_encoder": + return PairwiseFeatures(*args, **kwargs, type="fx_encoder", sample_rate=sample_rate, model_args=kwargs.get('fx_encoder_args', None)) + elif metric_name == "pairwise-AFxRep": + return PairwiseFeatures(*args, **kwargs, type="AFxRep", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-AFxRep-mid": + return PairwiseFeatures(*args, **kwargs, type="AFxRep-mid", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-AFxRep-side": + return PairwiseFeatures(*args, **kwargs, type="AFxRep-side", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-AFxRep": + return PairwiseStyleFeatures(*args, **kwargs, type="AFxRep", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-AFxRep-mid": + return PairwiseStyleFeatures(*args, **kwargs, type="AFxRep-mid", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-AFxRep-side": + return PairwiseStyleFeatures(*args, **kwargs, type="AFxRep-side", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-fxenc2048AFv3-fxenc++": + return PairwiseStyleFeatures(*args, **kwargs, type="fxenc2048AFv3-fxenc++", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-fxenc2048AFv3-AF": + return PairwiseStyleFeatures(*args, **kwargs, type="fxenc2048AFv3-AF", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-multitrack-fxenc2048AFv3-fxenc++": + return PairwiseStyleMultitrackFeatures(*args, **kwargs, type="fxenc2048AFv3-fxenc++", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-style-multitrack-fxenc2048AFv3-AF": + return PairwiseStyleMultitrackFeatures(*args, **kwargs, type="fxenc2048AFv3-AF", sample_rate=sample_rate, model_args=kwargs.get('AFxRep_args', None)) + elif metric_name == "pairwise-fxenc++": + return PairwiseFeatures(*args, **kwargs, type="fxenc++", sample_rate=sample_rate, model_args=kwargs.get('fx_encoder_plusplus_args', None)) + elif metric_name == "pairwise-logrms": + return PairwiseFeatures(*args, **kwargs, type="logrms", sample_rate=sample_rate, model_args=kwargs.get('logrms_args', None)) + elif metric_name == "pairwise-crestfactor": + return PairwiseFeatures(*args, **kwargs, type="crestfactor", sample_rate=sample_rate, model_args=kwargs.get('crestfactor_args', None)) + elif metric_name == "pairwise-logspread": + return PairwiseFeatures(*args, **kwargs, type="logspread", sample_rate=sample_rate, model_args=kwargs.get('logspread_args', None)) + elif metric_name == "pairwise-stereowidth": + return PairwiseFeatures(*args, **kwargs, type="stereowidth", sample_rate=sample_rate, model_args=kwargs.get('stereowidth_args', None)) + elif metric_name == "pairwise-stereoimbalance": + return PairwiseFeatures(*args, **kwargs, type="stereoimbalance", sample_rate=sample_rate, model_args=kwargs.get('stereoimbalance_args', None)) + else: + raise ValueError(f"Unknown metric: {metric_name}") + +# Example usage: +#metric_instance = metric_factory("pairwise-spectral") +#``` diff --git a/utils/feature_extractors/AF_features_embedding.py b/utils/feature_extractors/AF_features_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..34fa0e82e1eb34819dc032b0e594d2df500044da --- /dev/null +++ b/utils/feature_extractors/AF_features_embedding.py @@ -0,0 +1,166 @@ +import math +from utils.feature_extractors.dsp_features import compute_log_rms_gated_max, compute_crest_factor, compute_stereo_width, compute_stereo_imbalance, compute_log_spread +import torch + +class AF_fourier_embedding: + def __init__(self, + input_dim=8, + output_dim=64, + sigma=0.2, + log_rms_shift=-26.5, #calculated as the mean from the dataset + log_rms_scale=7.0, #calculated as the std from the dataset + crest_shift=16.7, #calculated as the mean from the dataset + crest_scale=6.3, + log_spread_shift=-20.0, #calculated as the mean from the dataset + log_spread_scale=20.0, #calculated as the std from the dataset + stereo_width_shift=0.28, + stereo_width_scale=0.39, + stereo_imbalance_shift=0.0, + stereo_imbalance_scale=0.35, + device="cpu" + ): + """ + Deterministic Fourier feature transformer using fixed cosine-based projection + """ + + self.device = device + # Ensure output_dim is even and >= 2 * input_dim + self.output_dim = max(input_dim * 2, output_dim) + if self.output_dim % 2 != 0: + self.output_dim += 1 + + self.input_dim = input_dim + self.sigma = sigma + + # Create deterministic projection matrix + self.projection = self._create_deterministic_projection(input_dim, self.output_dim // 2, sigma) + self.projection = self.projection.to(self.device) + + # Normalization factor + self.scale_factor = math.sqrt(2.0 / self.output_dim) + + self.log_rms_shift = log_rms_shift + self.log_rms_scale = log_rms_scale + self.crest_shift = crest_shift + self.crest_scale = crest_scale + self.log_spread_shift = log_spread_shift + self.log_spread_scale = log_spread_scale + self.stereo_width_shift = stereo_width_shift + self.stereo_width_scale = stereo_width_scale + self.stereo_imbalance_shift = stereo_imbalance_shift + self.stereo_imbalance_scale = stereo_imbalance_scale + + def _create_deterministic_projection(self, input_dim, proj_dim, sigma): + """ + Create a deterministic projection matrix using a cosine basis + """ + # Cosine-based matrix (like DCT type-II) + projection = torch.zeros(input_dim, proj_dim) + for i in range(input_dim): + for j in range(proj_dim): + projection[i, j] = math.cos(math.pi * (i + 0.5) * (j + 1) / proj_dim) + + return projection * sigma + + def encode(self, x): + + log_rms=compute_log_rms_gated_max(x) + crest_factor= compute_crest_factor(x) + log_spread= compute_log_spread(x) + stereo_width= compute_stereo_width(x) + stereo_imbalance= compute_stereo_imbalance(x) + + + log_rms_std, crest_factor_std, log_spread_std, stereo_width_std, stereo_imbalance_std = self.standardize_features( + log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance + ) + + embedding= self.transform( + log_rms_std, crest_factor_std, log_spread_std, stereo_width_std, stereo_imbalance_std + ) + + + return embedding, (log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance) + + def decode(self, fourier_features): + """ + Invert Fourier features back to original feature space + (approximate due to phase-only reconstruction) + """ + reconstructed = self.inverse_transform(fourier_features) + + # Reshape back to original feature dimensions + log_rms= reconstructed[:,0:2] + crest_factor = reconstructed[:,2:4] + log_spread= reconstructed[:,4:6] + stereo_width = reconstructed[:,6:7] + stereo_imbalance = reconstructed[:,7:8] + + log_rms, crest_factor, log_spread,stereo_width, stereo_imbalance = self.destandardize_features( + log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance + ) + + return log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance + + def standardize_features(self, log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance): + """ + Standardize features using pre-computed mean and std + """ + log_rms = (log_rms - self.log_rms_shift) / self.log_rms_scale + crest_factor = (crest_factor - self.crest_shift) / self.crest_scale + log_spread = (log_spread - self.log_spread_shift) / self.log_spread_scale + stereo_width = (stereo_width - self.stereo_width_shift) / self.stereo_width_scale + stereo_imbalance = (stereo_imbalance - self.stereo_imbalance_shift) / self.stereo_imbalance_scale + + return log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance + + def destandardize_features(self, log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance): + """ + Reverse standardization to get back to original feature space + """ + log_rms = log_rms * self.log_rms_scale + self.log_rms_shift + crest_factor = crest_factor * self.crest_scale + self.crest_shift + log_spread = log_spread * self.log_spread_scale + self.log_spread_shift + stereo_width = stereo_width * self.stereo_width_scale + self.stereo_width_shift + stereo_imbalance = stereo_imbalance * self.stereo_imbalance_scale + self.stereo_imbalance_shift + + return log_rms, crest_factor, log_spread, stereo_width, stereo_imbalance + + def transform(self, log_rms, crest_factor,log_spread, stereo_width, stereo_imbalance): + """ + Transform features using the stored projection matrix + """ + + flat_features=torch.cat([log_rms, crest_factor, log_spread, stereo_width.unsqueeze(-1), stereo_imbalance.unsqueeze(-1)], dim=-1) + + # Project and transform + projected = flat_features @ self.projection + cos_features = torch.cos(projected) + sin_features = torch.sin(projected) + + # Concatenate and normalize + return torch.cat([cos_features, sin_features], dim=-1) * self.scale_factor + + def inverse_transform(self, fourier_features): + """ + Invert Fourier features back to original feature space + (approximate due to phase-only reconstruction) + """ + # Split into cosine and sine components + feature_dim = fourier_features.shape[-1] // 2 + cos_features = fourier_features[:, :feature_dim] + sin_features = fourier_features[:, feature_dim:] + + # Denormalize + cos_features = cos_features / self.scale_factor + sin_features = sin_features / self.scale_factor + + # Compute phase angles + phases = torch.atan2(sin_features, cos_features) + + # Use pseudo-inverse for approximate inversion + projection_pinv = torch.pinverse(self.projection) + reconstructed = phases @ projection_pinv + + return reconstructed + diff --git a/utils/feature_extractors/__init__.py b/utils/feature_extractors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/feature_extractors/audio_features.py b/utils/feature_extractors/audio_features.py new file mode 100755 index 0000000000000000000000000000000000000000..c379861562a3c85ac9db8231a811410a14087496 --- /dev/null +++ b/utils/feature_extractors/audio_features.py @@ -0,0 +1,76 @@ +import numpy as np +import librosa + +def compute_spectrogram(file_path, set_limit=False, limit_seconds=10): + """Compute the spectrogram of an audio file.""" + y, sr = librosa.load(file_path, sr=None) + if set_limit: + y = y[:int(sr*limit_seconds)] + S = librosa.stft(y) + S_db = librosa.amplitude_to_db(np.abs(S), ref=np.max) + return S_db, sr, y + +def smooth_curve(data, window_size=30): + """Smooth a curve using a moving average.""" + return np.convolve(data, np.ones(window_size)/window_size, mode='same') + +def compute_crest_factor(y, sr, frame_length=2048, hop_length=512): + # compute crest factor by windowing + crest_factor_total = [] + for i in range(0, len(y), hop_length): + frame = y[i:i+frame_length] + peak = np.max(np.abs(frame)) + rms = np.sqrt(np.mean(frame**2)) + cur_crest_factor = peak / (rms + 1e-6) + crest_factor_total.append(cur_crest_factor) + crest_factor = np.asarray(crest_factor_total) + return crest_factor + +def compute_audio_features(y, sr, smooth_window_size=1, average_time=True): + # features to compute: RMS energy, crest factor, dynamic spread, spectral centroid, spectral contrast, spectral flatness, spectral balance, and spectral bandwidth + y_mono = y.mean(axis=0) + spectral_centroid = smooth_curve(librosa.feature.spectral_centroid(y=y_mono, sr=sr)[0], window_size=smooth_window_size) + spectral_contrast = smooth_curve(np.mean(librosa.feature.spectral_contrast(y=y_mono, sr=sr), axis=0), window_size=smooth_window_size) + spectral_flatness = smooth_curve(librosa.feature.spectral_flatness(y=y_mono)[0], window_size=smooth_window_size) + rms_energy = smooth_curve(librosa.feature.rms(y=y_mono)[0], window_size=smooth_window_size) + # Crest Factor: Peak-to-RMS ratio + crest_factor = smooth_curve(compute_crest_factor(y_mono, sr, frame_length=2048, hop_length=512), window_size=smooth_window_size) + # Dynamic Spread + dynamic_spread = np.std(rms_energy) + # Spectral Balance (Low, Mid, High Frequency Energy Ratios) + spec = np.abs(librosa.stft(y_mono))**2 + freqs = librosa.fft_frequencies(sr=sr) + low_energy = np.sum(spec[freqs < 200]) + mid_energy = np.sum(spec[(freqs >= 200) & (freqs < 2000)]) + high_energy = np.sum(spec[freqs >= 2000]) + total_energy = low_energy + mid_energy + high_energy + spectral_balance = np.array([low_energy / total_energy, mid_energy / total_energy, high_energy / total_energy]) + spectral_bandwidth = smooth_curve(librosa.feature.spectral_bandwidth(y=y_mono, sr=sr)[0], window_size=smooth_window_size) + # Stereo Width + Mid-Side Ratio + y_mid = (y[:len(y)//2] + y[len(y)//2:]) / 2 + y_side = (y[:len(y)//2] - y[len(y)//2:]) / 2 + stereo_width = np.mean(np.abs(y_side)) / (np.mean(np.abs(y_mid)) + 1e-6) + mid_side_ratio = np.mean(np.abs(y_mid)) / (np.mean(np.abs(y_side)) + 1e-6) + # average across time + if average_time: + spectral_centroid = np.mean(spectral_centroid) + spectral_flatness = np.mean(spectral_flatness) + spectral_contrast = np.mean(spectral_contrast) + spectral_flatness = np.mean(spectral_flatness) + rms_energy = np.mean(rms_energy) + crest_factor = np.mean(crest_factor) + spectral_bandwidth = np.mean(spectral_bandwidth) + + # return as a dictionary + return {'spectral_centroid': spectral_centroid, + 'spectral_contrast': spectral_contrast, + 'spectral_flatness': spectral_flatness, + 'rms_energy': rms_energy, + 'crest_factor': crest_factor, + 'dynamic_spread': dynamic_spread, + 'spectral_balance': spectral_balance, + 'spectral_bandwidth': spectral_bandwidth, + 'stereo_width': stereo_width, + 'mid_side_ratio': mid_side_ratio} + + diff --git a/utils/feature_extractors/dsp_features.py b/utils/feature_extractors/dsp_features.py new file mode 100644 index 0000000000000000000000000000000000000000..9b222f1a0b4af06e8db83710af79d77e95e44750 --- /dev/null +++ b/utils/feature_extractors/dsp_features.py @@ -0,0 +1,141 @@ +import torch + +def compute_rms(x: torch.Tensor, **kwargs): + """Compute root mean square energy. + + Args: + x: (bs, 1, seq_len) + + Returns: + rms: (bs, ) + """ + rms = torch.sqrt(torch.mean(x**2, dim=-1).clamp(min=1e-8)) + return rms + +def compute_log_rms_gated_max(x: torch.Tensor, sample_rate=44100, **kwargs): + """Compute gated log RMS energy. + + Frames the signal in 400 ms windows with 75% overlap, computes RMS, + discards frames with RMS < -60 dBFS, and averages the log-RMS. + + If all frames in a given (batch, channel) are below -60 dBFS, + returns -60 for that entry. + + Args: + x: Tensor of shape (bs, c, seq_len) + + Returns: + log_rms: Tensor of shape (bs, c) + """ + seg_size = int(sample_rate * 0.4) + hop_size = int(sample_rate * 0.1) + + # (bs, c, num_frames, seg_size) + B, C, L = x.size() + + assert C==1 or C==2 + + x_frames = x.unfold(2, seg_size, hop_size) # (bs, c, num_frames, seg_size) + + # RMS over last dimension (seg_size) + rms = torch.sqrt((x_frames ** 2).mean(dim=-1)) # (bs, c, num_frames) + + # dB conversion + rms_db = 20 * torch.log10(rms.clamp(min=1e-8)) # (bs, c, num_frames) + #print("rms db shape", rms_db.shape) + + #take the maximum RMS across all frames + rms_max = rms_db.max(dim=2)[0] # (bs, c) + #print(f"RMS max shape: {rms_max.shape}") + + return rms_max + + +def compute_log_rms(x: torch.Tensor, **kwargs): + """Compute root mean square energy. + + Args: + x: (bs, 1, seq_len) + + Returns: + rms: (bs, ) + """ + rms=compute_rms(x) + return 20 * torch.log10(rms.clamp(min=1e-8)) + +def compute_crest_factor(x: torch.Tensor, **kwargs): + """Compute crest factor as ratio of peak to rms energy in dB. + + Args: + x: (bs, 2, seq_len) + + """ + num = torch.max(torch.abs(x), dim=-1)[0] + den = compute_rms(x).clamp(min=1e-8) + cf = 20 * torch.log10((num / den).clamp(min=1e-8)) + return cf + +def compute_log_spread(x: torch.Tensor, **kwargs): + """Compute log spread as the mean difference between log magnitude of samples and log RMS. + + Args: + x: (bs, 1, seq_len) + + Returns: + log_spread: (bs, ) + """ + # Compute log RMS + log_rms = compute_log_rms(x).unsqueeze(-1) # (bs, 1, 1) + + # Compute log magnitude of each sample + log_magnitude = 20 * torch.log10(torch.abs(x).clamp(min=1e-8)) # (bs, 1, seq_len) + + # Compute the difference and take the mean + log_spread = torch.mean(log_magnitude - log_rms, dim=-1).squeeze(1) # (bs, ) + + return log_spread + + +def compute_stereo_width(x: torch.Tensor, **kwargs): + """Compute stereo width as ratio of energy in sum and difference signals. + + Args: + x: (bs, 2, seq_len) + + """ + bs, chs, seq_len = x.size() + + assert chs == 2, "Input must be stereo" + + # compute sum and diff of stereo channels + x_sum = x[:, 0, :] + x[:, 1, :] + x_diff = x[:, 0, :] - x[:, 1, :] + + # compute power of sum and diff + sum_energy = torch.mean(x_sum**2, dim=-1) + diff_energy = torch.mean(x_diff**2, dim=-1) + + # compute stereo width as ratio + stereo_width = diff_energy / sum_energy.clamp(min=1e-8) + + return stereo_width + + +def compute_stereo_imbalance(x: torch.Tensor, **kwargs): + """Compute stereo imbalance as ratio of energy in left and right channels. + + Args: + x: (bs, 2, seq_len) + + Returns: + stereo_imbalance: (bs, ) + + """ + left_energy = torch.mean(x[:, 0, :] ** 2, dim=-1) + right_energy = torch.mean(x[:, 1, :] ** 2, dim=-1) + + stereo_imbalance = (right_energy - left_energy) / ( + right_energy + left_energy + ).clamp(min=1e-8) + + return stereo_imbalance diff --git a/utils/feature_extractors/fx_encoder_plus_plus.py b/utils/feature_extractors/fx_encoder_plus_plus.py new file mode 100644 index 0000000000000000000000000000000000000000..56ef9f6b1fbe1e3ec835ecda9342e7d574abeb76 --- /dev/null +++ b/utils/feature_extractors/fx_encoder_plus_plus.py @@ -0,0 +1,82 @@ +import torch +from utils.fxencoder_plusplus import FxEncoderPlusPlus +from utils.fxencoder_plusplus.model import get_model_path + + +def load_model(model_name="default", model_path=None, device="cuda", auto_download=True, cache_dir=None): + """ + Load FxEncoderPlusPlus model. + + Args: + model_name: Name of pretrained model ('default', 'musdb', 'medleydb') + model_path: Custom checkpoint path. If provided, ignores model_name + device: Device to load model on ('cuda' or 'cpu') + auto_download: Automatically download if model not found + cache_dir: Custom cache directory for downloaded models + + Returns: + Loaded FxEncoderPlusPlus model + + Examples: + # Load default base model + model = load_model() + + # Load musdb model + model = load_model(model_name="musdb") + + # Load medleydb model + model = load_model(model_name="medleydb") + + # Load custom checkpoint + model = load_model(model_path="/path/to/custom.pt") + + # List available models + list_available_models() + """ + # Handle device + if device == "cuda" and not torch.cuda.is_available(): + print("CUDA not available, using CPU") + device = "cpu" + + + # Determine model path + if model_path is None: + if auto_download: + model_path = get_model_path(model_name, cache_dir=cache_dir) + else: + raise ValueError("model_path is None and auto_download is False") + + # Create model instance with specified device + model = FxEncoderPlusPlus( + embed_dim=2048, + audio_clap_module=False, + text_clap_module=False, + extractor_module=False, + device=device + ) + + # Load checkpoint + checkpoint = torch.load(model_path, map_location=device, weights_only=False) + + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if next(iter(sd.items()))[0].startswith("module"): + sd = {k[len("module."):]: v for k, v in sd.items()} + model.load_state_dict(sd, strict=False ) + print(f"Loaded checkpoint from epoch {start_epoch}") + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + print("Loaded model checkpoint") + + model.to(device) + model.eval() + + # Freeze parameters for inference + for param in model.parameters(): + param.requires_grad = False + + print(f"Model loaded successfully on {device}") + return model diff --git a/utils/feature_extractors/load_features.py b/utils/feature_extractors/load_features.py new file mode 100644 index 0000000000000000000000000000000000000000..47ab6f1818ea7d066fe91e9969e66710f523f251 --- /dev/null +++ b/utils/feature_extractors/load_features.py @@ -0,0 +1,260 @@ +import os +import math +import sys +import torchaudio +from importlib import import_module +import torch +import yaml +import torch.nn.functional as F + +def load_fx_encoder_plusplus(model_args, device, *args, **kwargs): + from utils.feature_extractors.fx_encoder_plus_plus import load_model + + assert model_args is not None, "model_args must be provided for fx_encoder type" + + ckpt_path=model_args.ckpt_path + + model=load_model( + model_path=ckpt_path, + device=device, + ) + + def effects_encoder_fn(x): + assert x.ndim == 3, f"Input tensor x must be 2D, got {x.ndim}D" + assert x.shape[1] == 2, f"Input tensor x must have 2 channels, got {x.shape[1]} channels" + + emb=model.get_fx_embedding(x) + #l2 normalize the embeddings + emb = torch.nn.functional.normalize(emb, p=2, dim=-1) + + return emb + + + return lambda x: effects_encoder_fn(x) + +def load_fx_encoder_plusplus_2048(model_args, device, *args, **kwargs): + from utils.feature_extractors.fx_encoder_plus_plus import load_model + + assert model_args is not None, "model_args must be provided for fx_encoder type" + + ckpt_path=model_args.ckpt_path + + model=load_model( + model_path=ckpt_path, + device=device, + ) + + def effects_encoder_fn(x): + assert x.ndim == 3, f"Input tensor x must be 2D, got {x.ndim}D" + assert x.shape[1] == 2, f"Input tensor x must have 2 channels, got {x.shape[1]} channels" + + emb=model.fx_encoder(x) + emb=emb["embedding"] # Extract the embedding from the dictionary + + return emb + + + return lambda x: effects_encoder_fn(x) + +def add_isotropic_noise(z, sigma=0.1): + """ + z: [..., D] normalized embeddings (e.g., from CLAP or a regressor) + sigma: scale of noise to inject + Returns: z with orthogonal Gaussian noise added + """ + n=torch.randn_like(z) # isotropic noise + z_noisy = F.normalize(z + sigma * n, dim=-1) + return z_noisy + + + + +def load_CLAP(model_args, device, *args, **kwargs): + + #original_path = sys.path.copy() + from utils.laion_clap.hook import CLAP_Module + model= CLAP_Module(enable_fusion=False, amodel= 'HTSAT-base') + #sys.path = original_path + + + print("checkpoint",model_args.ckpt_path) + #print current sys.path + print("sys.path", sys.path) + model.load_ckpt(model_args.ckpt_path) + model.to(device) + + normalize = model_args.normalize + + if model_args.use_adaptor: + if model_args.adaptor_type == "MLP_CLAP_regressor": + from networks.MLP_CLAP_regressor import MLP_CLAP_regressor + adaptor=MLP_CLAP_regressor() + ckpt=torch.load(model_args.adaptor_checkpoint, map_location=device, weights_only=False) + adaptor.load_state_dict(ckpt["network"], strict=True) + adaptor.to(device) + + + + + def clap_fn(x, type=None): + B, C, T = x.shape + if C > 1: + x= x.mean(dim=1, keepdim=True) # Convert to mono if stereo + + with torch.no_grad(): + x=torchaudio.functional.resample(x, orig_freq=44100, new_freq=48000) + x= x.squeeze(1) # Remove channel dimension for CLAP + emb=model.get_audio_embedding_from_data(x,use_tensor=True) + + if type is not None: + if type == "wet": + #print("wet mode") + if model_args.use_adaptor: + emb= adaptor(emb) # Apply the adaptor if specified + + if model_args.add_noise: + emb= torch.nn.functional.normalize(emb, p=2, dim=-1) # Normalize before adding noise + emb = add_isotropic_noise(emb, sigma=model_args.noise_sigma) + + # Normalize the embeddings + if normalize: + emb = torch.nn.functional.normalize(emb, p=2, dim=-1) + + return emb + + return lambda x, type: clap_fn(x, type=type) + + + +def load_fx_encoder(model_args, device, *args, **kwargs): + """ + Load the FX Encoder model. + + Args: + model_args: Arguments for the FX Encoder model. + device: Device to load the model on (CPU or GPU). + + Returns: + a function that extracts features from audio. + """ + assert model_args is not None, "model_args must be provided for fx_encoder type" + + ckpt_path=model_args.ckpt_path + + #from utils.feature_extractors.fx_encoder import load_effects_encoder + from utils.feature_extractors.networks import Effects_Encoder + + def reload_weights(model, ckpt_path, device): + checkpoint = torch.load(ckpt_path, map_location=device) + + from collections import OrderedDict + new_state_dict = OrderedDict() + for k, v in checkpoint["model"].items(): + name = k[7:] # remove `module.` + new_state_dict[name] = v + model.load_state_dict(new_state_dict, strict=False) + + + try: + with open(os.path.join('.','utils','feature_extractors', 'networks', 'configs.yaml'), 'r') as f: + configs = yaml.full_load(f) + except: + with open(model_args.config_file, 'r') as f: + configs = yaml.full_load(f) + + cfg_enc = configs['Effects_Encoder']['default'] + + effects_encoder = Effects_Encoder(cfg_enc) + reload_weights(effects_encoder, ckpt_path, device) + effects_encoder.to(device) + effects_encoder.eval() + + def effects_encoder_fn(x): + emb=effects_encoder(x) + #l2 normalize the embeddings + emb = torch.nn.functional.normalize(emb, p=2, dim=-1) + return emb + + + return lambda x, *args: effects_encoder_fn(x) + +def load_AFxRep(model_args, device, sample_rate=44100, peak_scaling=True, *args, **kwargs): + + assert model_args is not None, "model_args must be provided for AFxRep type" + + ckpt_path=model_args.ckpt_path + + config_path = os.path.join(os.path.dirname(ckpt_path), "config.yaml") + + with open(config_path) as f: + config = yaml.safe_load(f) + + encoder_configs = config["model"]["init_args"]["encoder"] + + module_path, class_name = encoder_configs["class_path"].rsplit(".", 1) + module_path = module_path.replace("lcap", "utils.st_ito") + + module = import_module(module_path) + + model = getattr(module, class_name)(**encoder_configs["init_args"]) + + checkpoint = torch.load(ckpt_path, map_location="cpu") + + # load state dicts + state_dict = {} + for k, v in checkpoint["state_dict"].items(): + if k.startswith("encoder"): + state_dict[k.replace("encoder.", "", 1)] = v + + model.load_state_dict(state_dict) + + model.eval() + + model.to(device) + + def wrapper_fn(x, sample_rate): + + x=x.to(device) + + #x=torch.transpose(x,-1,-2) + + if sample_rate != 48000: + x=torchaudio.functional.resample(x, sample_rate, 48000) + + bs= x.shape[0] + #peak normalization. I do it because this is what ST-ITO get_param_embeds does. Not sure if it is good that this representation is invariant to gain + if peak_scaling: + x_max=[] + for batch_idx in range(bs): + #x[batch_idx, ...] /= x[batch_idx, ...].abs().max().clamp(1e-8) + x_max.append( x[batch_idx, ...].abs().max().clamp(1e-8) ) + + if x.ndim == 3: + x_max=torch.stack(x_max, dim=0).view(bs, 1, 1) + elif x.ndim == 2: + x_max=torch.stack(x_max, dim=0).view(bs, 1) + + x=x/ x_max + + mid_embeddings, side_embeddings = model(x) + + # check for nan + if torch.isnan(mid_embeddings).any(): + print("Warning: NaNs found in mid_embeddings") + mid_embeddings = torch.nan_to_num(mid_embeddings) + elif torch.isnan(side_embeddings).any(): + print("Warning: NaNs found in side_embeddings") + side_embeddings = torch.nan_to_num(side_embeddings) + + mid_embeddings = torch.nn.functional.normalize(mid_embeddings, p=2, dim=-1) + side_embeddings = torch.nn.functional.normalize(side_embeddings, p=2, dim=-1) + + embeddings_all= torch.cat([mid_embeddings, side_embeddings], dim=-1) + + return embeddings_all + + feat_extractor = lambda x, *args: wrapper_fn(x, sample_rate=sample_rate) + + return feat_extractor + + diff --git a/utils/feature_extractors/networks/__init__.py b/utils/feature_extractors/networks/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..4fe695aec7c837c75a665bae0091975a9131056a --- /dev/null +++ b/utils/feature_extractors/networks/__init__.py @@ -0,0 +1,2 @@ +from .architectures import * +from .network_utils import * \ No newline at end of file diff --git a/utils/feature_extractors/networks/architectures.py b/utils/feature_extractors/networks/architectures.py new file mode 100755 index 0000000000000000000000000000000000000000..19bb9aaba277551ab4e46455441643e27998482a --- /dev/null +++ b/utils/feature_extractors/networks/architectures.py @@ -0,0 +1,163 @@ +""" + Implementation of neural networks used in the task 'Music Mastering Style Transfer' + - 'Effects Encoder' + - 'Mastering Style Transfer' + - 'Differentiable Mastering Style Transfer' +""" +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.init as init +#import dasp_pytorch + +import os +import sys +import time + +# compute receptive field +def compute_receptive_field(kernels, strides, dilations): + rf = 0 + for i in range(len(kernels)): + rf += rf * strides[i] + (kernels[i]-strides[i]) * dilations[i] + return rf + + +# Encoder of music effects for contrastive learning of music effects +class Effects_Encoder(nn.Module): + def __init__(self, config): + super(Effects_Encoder, self).__init__() + # input is stereo channeled audio + config["channels"].insert(0, 2) + + # encoder layers + encoder = [] + for i in range(len(config["kernels"])): + if config["conv_block"]=='res': + encoder.append(Res_ConvBlock(dimension=1, \ + in_channels=config["channels"][i], \ + out_channels=config["channels"][i+1], \ + kernel_size=config["kernels"][i], \ + stride=config["strides"][i], \ + padding="SAME", \ + dilation=config["dilation"][i], \ + norm=config["norm"], \ + activation=config["activation"], \ + last_activation=config["activation"])) + self.encoder = nn.Sequential(*encoder) + + # pooling method + self.glob_pool = nn.AdaptiveAvgPool1d(1) + + + # network forward operation + def forward(self, input): + enc_output = self.encoder(input) + glob_pooled = self.glob_pool(enc_output).squeeze(-1) + + # outputs c feature + return glob_pooled + + +# Residual Block + # the input is added after the first convolutional layer, retaining its original channel size + # therefore, the second convolutional layer's output channel may differ +class Res_ConvBlock(nn.Module): + def __init__(self, dimension, \ + in_channels, out_channels, \ + kernel_size, \ + stride=1, padding="SAME", \ + dilation=1, \ + bias=True, \ + norm="batch", \ + activation="relu", last_activation="relu", \ + mode="conv"): + super(Res_ConvBlock, self).__init__() + + if dimension==1: + self.conv1 = Conv1d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation) + self.conv2 = Conv1d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode) + + def forward(self, input): + c1_out = self.conv1(input) + input + c2_out = self.conv2(c1_out) + return c2_out + + +# 1-dimensional convolutional layer +# in the order of conv -> norm -> activation +class Conv1d_layer(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, \ + stride=1, \ + padding="SAME", dilation=1, bias=True, \ + norm="batch", activation="relu", \ + mode="conv"): + super(Conv1d_layer, self).__init__() + + self.conv1d = nn.Sequential() + + ''' padding ''' + if mode=="deconv": + padding = int(dilation * (kernel_size-1) / 2) + out_padding = 0 if stride==1 else 1 + elif mode=="conv" or "alias_free" in mode: + if padding == "SAME": + pad = int((kernel_size-1) * dilation) + l_pad = int(pad//2) + r_pad = pad - l_pad + padding_area = (l_pad, r_pad) + elif padding == "VALID": + padding_area = (0, 0) + else: + pass + + ''' convolutional layer ''' + if mode=="deconv": + self.conv1d.add_module("deconv1d", nn.ConvTranspose1d(in_channels, out_channels, kernel_size, \ + stride=stride, padding=padding, output_padding=out_padding, \ + dilation=dilation, \ + bias=bias)) + elif mode=="conv": + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module(f"{mode}1d", nn.Conv1d(in_channels, out_channels, kernel_size, \ + stride=stride, padding=0, \ + dilation=dilation, \ + bias=bias)) + elif "alias_free" in mode: + if "up" in mode: + up_factor = stride * 2 + down_factor = 2 + elif "down" in mode: + up_factor = 2 + down_factor = stride * 2 + else: + raise ValueError("choose alias-free method : 'up' or 'down'") + # procedure : conv -> upsample -> lrelu -> low-pass filter -> downsample + # the torchaudio.transforms.Resample's default resampling_method is 'sinc_interpolation' which performs low-pass filter during the process + # details at https://pytorch.org/audio/stable/transforms.html + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module(f"{mode}1d", nn.Conv1d(in_channels, out_channels, kernel_size, \ + stride=1, padding=0, \ + dilation=dilation, \ + bias=bias)) + self.conv1d.add_module(f"{mode}upsample", torchaudio.transforms.Resample(orig_freq=1, new_freq=up_factor)) + self.conv1d.add_module(f"{mode}lrelu", nn.LeakyReLU()) + self.conv1d.add_module(f"{mode}downsample", torchaudio.transforms.Resample(orig_freq=down_factor, new_freq=1)) + + ''' normalization ''' + if norm=="batch": + self.conv1d.add_module("batch_norm", nn.BatchNorm1d(out_channels)) + # self.conv1d.add_module("batch_norm", nn.SyncBatchNorm(out_channels)) + + ''' activation ''' + if 'alias_free' not in mode: + if activation=="relu": + self.conv1d.add_module("relu", nn.ReLU()) + elif activation=="lrelu": + self.conv1d.add_module("lrelu", nn.LeakyReLU()) + + + def forward(self, input): + # input shape should be : batch x channel x height x width + output = self.conv1d(input) + return output + diff --git a/utils/feature_extractors/networks/configs.yaml b/utils/feature_extractors/networks/configs.yaml new file mode 100755 index 0000000000000000000000000000000000000000..a7174ab5e1582d280607dfb8dfb540a52879f6a7 --- /dev/null +++ b/utils/feature_extractors/networks/configs.yaml @@ -0,0 +1,14 @@ +# model architecture configurations + +# Music Effects Encoder +Effects_Encoder: + + default: + channels: [16, 32, 64, 128, 256, 256, 512, 512, 1024, 1024, 2048, 2048] + kernels: [25, 25, 15, 15, 10, 10, 10, 10, 5, 5, 5, 5] + strides: [4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1] + dilation: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + bias: True + norm: 'batch' + conv_block: 'res' + activation: "relu" diff --git a/utils/feature_extractors/networks/network_utils.py b/utils/feature_extractors/networks/network_utils.py new file mode 100755 index 0000000000000000000000000000000000000000..0b819fd9c1b083aacabfcce973433c89c84bbee7 --- /dev/null +++ b/utils/feature_extractors/networks/network_utils.py @@ -0,0 +1,254 @@ +""" + Utility File + containing functions for neural networks +""" +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.init as init +import torch +import torchaudio + + + +# 2-dimensional convolutional layer +# in the order of conv -> norm -> activation +class Conv2d_layer(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, \ + stride=1, \ + padding="SAME", dilation=(1,1), bias=True, \ + norm="batch", activation="relu", \ + mode="conv"): + super(Conv2d_layer, self).__init__() + + self.conv2d = nn.Sequential() + + if isinstance(kernel_size, int): + kernel_size = [kernel_size, kernel_size] + if isinstance(stride, int): + stride = [stride, stride] + if isinstance(dilation, int): + dilation = [dilation, dilation] + + ''' padding ''' + if mode=="deconv": + padding = tuple(int((current_kernel - 1)/2) for current_kernel in kernel_size) + out_padding = tuple(0 if current_stride == 1 else 1 for current_stride in stride) + elif mode=="conv": + if padding == "SAME": + f_pad = int((kernel_size[0]-1) * dilation[0]) + t_pad = int((kernel_size[1]-1) * dilation[1]) + t_l_pad = int(t_pad//2) + t_r_pad = t_pad - t_l_pad + f_l_pad = int(f_pad//2) + f_r_pad = f_pad - f_l_pad + padding_area = (t_l_pad, t_r_pad, f_l_pad, f_r_pad) + elif padding == "VALID": + padding = 0 + else: + pass + + ''' convolutional layer ''' + if mode=="deconv": + self.conv2d.add_module("deconv2d", nn.ConvTranspose2d(in_channels, out_channels, \ + (kernel_size[0], kernel_size[1]), \ + stride=stride, \ + padding=padding, output_padding=out_padding, \ + dilation=dilation, \ + bias=bias)) + elif mode=="conv": + self.conv2d.add_module(f"{mode}2d_pad", nn.ReflectionPad2d(padding_area)) + self.conv2d.add_module(f"{mode}2d", nn.Conv2d(in_channels, out_channels, \ + (kernel_size[0], kernel_size[1]), \ + stride=stride, \ + padding=0, \ + dilation=dilation, \ + bias=bias)) + + ''' normalization ''' + if norm=="batch": + self.conv2d.add_module("batch_norm", nn.BatchNorm2d(out_channels)) + + ''' activation ''' + if activation=="relu": + self.conv2d.add_module("relu", nn.ReLU()) + elif activation=="lrelu": + self.conv2d.add_module("lrelu", nn.LeakyReLU()) + + + def forward(self, input): + # input shape should be : batch x channel x height x width + output = self.conv2d(input) + return output + + + +# 1-dimensional convolutional layer +# in the order of conv -> norm -> activation +class Conv1d_layer(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, \ + stride=1, \ + padding="SAME", dilation=1, bias=True, \ + norm="batch", activation="relu", \ + mode="conv"): + super(Conv1d_layer, self).__init__() + + self.conv1d = nn.Sequential() + + ''' padding ''' + if mode=="deconv": + padding = int(dilation * (kernel_size-1) / 2) + out_padding = 0 if stride==1 else 1 + elif mode=="conv" or "alias_free" in mode: + if padding == "SAME": + pad = int((kernel_size-1) * dilation) + l_pad = int(pad//2) + r_pad = pad - l_pad + padding_area = (l_pad, r_pad) + elif padding == "VALID": + padding_area = (0, 0) + else: + pass + + ''' convolutional layer ''' + if mode=="deconv": + self.conv1d.add_module("deconv1d", nn.ConvTranspose1d(in_channels, out_channels, kernel_size, \ + stride=stride, padding=padding, output_padding=out_padding, \ + dilation=dilation, \ + bias=bias)) + elif mode=="conv": + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module(f"{mode}1d", nn.Conv1d(in_channels, out_channels, kernel_size, \ + stride=stride, padding=0, \ + dilation=dilation, \ + bias=bias)) + elif "alias_free" in mode: + if "up" in mode: + up_factor = stride * 2 + down_factor = 2 + elif "down" in mode: + up_factor = 2 + down_factor = stride * 2 + else: + raise ValueError("choose alias-free method : 'up' or 'down'") + # procedure : conv -> upsample -> lrelu -> low-pass filter -> downsample + # the torchaudio.transforms.Resample's default resampling_method is 'sinc_interpolation' which performs low-pass filter during the process + # details at https://pytorch.org/audio/stable/transforms.html + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module(f"{mode}1d", nn.Conv1d(in_channels, out_channels, kernel_size, \ + stride=1, padding=0, \ + dilation=dilation, \ + bias=bias)) + self.conv1d.add_module(f"{mode}upsample", torchaudio.transforms.Resample(orig_freq=1, new_freq=up_factor)) + self.conv1d.add_module(f"{mode}lrelu", nn.LeakyReLU()) + self.conv1d.add_module(f"{mode}downsample", torchaudio.transforms.Resample(orig_freq=down_factor, new_freq=1)) + + ''' normalization ''' + if norm=="batch": + self.conv1d.add_module("batch_norm", nn.BatchNorm1d(out_channels)) + # self.conv1d.add_module("batch_norm", nn.SyncBatchNorm(out_channels)) + + ''' activation ''' + if 'alias_free' not in mode: + if activation=="relu": + self.conv1d.add_module("relu", nn.ReLU()) + elif activation=="lrelu": + self.conv1d.add_module("lrelu", nn.LeakyReLU()) + + + def forward(self, input): + # input shape should be : batch x channel x height x width + output = self.conv1d(input) + return output + + + +# Residual Block + # the input is added after the first convolutional layer, retaining its original channel size + # therefore, the second convolutional layer's output channel may differ +class Res_ConvBlock(nn.Module): + def __init__(self, dimension, \ + in_channels, out_channels, \ + kernel_size, \ + stride=1, padding="SAME", \ + dilation=1, \ + bias=True, \ + norm="batch", \ + activation="relu", last_activation="relu", \ + mode="conv"): + super(Res_ConvBlock, self).__init__() + + if dimension==1: + self.conv1 = Conv1d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation) + self.conv2 = Conv1d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode) + elif dimension==2: + self.conv1 = Conv2d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation) + self.conv2 = Conv2d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode) + + + def forward(self, input): + c1_out = self.conv1(input) + input + c2_out = self.conv2(c1_out) + return c2_out + + + +# Convoluaionl Block + # consists of multiple (number of layer_num) convolutional layers + # only the final convoluational layer outputs the desired 'out_channels' +class ConvBlock(nn.Module): + def __init__(self, dimension, layer_num, \ + in_channels, out_channels, \ + kernel_size, \ + stride=1, padding="SAME", \ + dilation=1, \ + bias=True, \ + norm="batch", \ + activation="relu", last_activation="relu", \ + mode="conv"): + super(ConvBlock, self).__init__() + + conv_block = [] + if dimension==1: + for i in range(layer_num-1): + conv_block.append(Conv1d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation)) + conv_block.append(Conv1d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode)) + elif dimension==2: + for i in range(layer_num-1): + conv_block.append(Conv2d_layer(in_channels, in_channels, kernel_size, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=activation)) + conv_block.append(Conv2d_layer(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias, norm=norm, activation=last_activation, mode=mode)) + self.conv_block = nn.Sequential(*conv_block) + + + def forward(self, input): + return self.conv_block(input) + + +# Feature-wise Linear Modulation +class FiLM(nn.Module): + def __init__(self, condition_len=2048, feature_len=1024): + super(FiLM, self).__init__() + self.film_fc = nn.Linear(condition_len, feature_len*2) + self.feat_len = feature_len + + + def forward(self, feature, condition, sefa=None): + # SeFA + if sefa: + weight = self.film_fc.weight.T + weight = weight / torch.linalg.norm((weight+1e-07), dim=0, keepdims=True) + eigen_values, eigen_vectors = torch.eig(torch.matmul(weight, weight.T), eigenvectors=True) + + ####### custom parameters ####### + chosen_eig_idx = sefa[0] + alpha = eigen_values[chosen_eig_idx][0] * sefa[1] + ################################# + + An = eigen_vectors[chosen_eig_idx].repeat(condition.shape[0], 1) + alpha_An = alpha * An + + condition += alpha_An + + film_factor = self.film_fc(condition).unsqueeze(-1) + r, b = torch.split(film_factor, self.feat_len, dim=1) + return r*feature + b + diff --git a/utils/feature_extractors/networks/pytorch_utils.py b/utils/feature_extractors/networks/pytorch_utils.py new file mode 100755 index 0000000000000000000000000000000000000000..de05d330a89255e5b20245e6764df6f6c597fa2b --- /dev/null +++ b/utils/feature_extractors/networks/pytorch_utils.py @@ -0,0 +1,256 @@ +""" +From: https://github.com/qiuqiangkong/audioset_tagging_cnn/blob/master/pytorch/pytorch_utils.py + +Copyright (c) 2018-2020 Qiuqiang Kong +""" +import numpy as np +import time +import torch +import torch.nn as nn + + +def move_data_to_device(x, device): + if 'float' in str(x.dtype): + x = torch.Tensor(x) + elif 'int' in str(x.dtype): + x = torch.LongTensor(x) + else: + return x + + return x.to(device) + + +def do_mixup(x, mixup_lambda): + """Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes + (1, 3, 5, ...). + + Args: + x: (batch_size * 2, ...) + mixup_lambda: (batch_size * 2,) + + Returns: + out: (batch_size, ...) + """ + out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \ + x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1) + return out + + +def append_to_dict(dict, key, value): + if key in dict.keys(): + dict[key].append(value) + else: + dict[key] = [value] + + +def forward(model, generator, return_input=False, + return_target=False): + """Forward data to a model. + + Args: + model: object + generator: object + return_input: bool + return_target: bool + + Returns: + audio_name: (audios_num,) + clipwise_output: (audios_num, classes_num) + (ifexist) segmentwise_output: (audios_num, segments_num, classes_num) + (ifexist) framewise_output: (audios_num, frames_num, classes_num) + (optional) return_input: (audios_num, segment_samples) + (optional) return_target: (audios_num, classes_num) + """ + output_dict = {} + device = next(model.parameters()).device + time1 = time.time() + + # Forward data to a model in mini-batches + for n, batch_data_dict in enumerate(generator): + print(n) + batch_waveform = move_data_to_device(batch_data_dict['waveform'], device) + + with torch.no_grad(): + model.eval() + batch_output = model(batch_waveform) + + append_to_dict(output_dict, 'audio_name', batch_data_dict['audio_name']) + + append_to_dict(output_dict, 'clipwise_output', + batch_output['clipwise_output'].data.cpu().numpy()) + + if 'segmentwise_output' in batch_output.keys(): + append_to_dict(output_dict, 'segmentwise_output', + batch_output['segmentwise_output'].data.cpu().numpy()) + + if 'framewise_output' in batch_output.keys(): + append_to_dict(output_dict, 'framewise_output', + batch_output['framewise_output'].data.cpu().numpy()) + + if return_input: + append_to_dict(output_dict, 'waveform', batch_data_dict['waveform']) + + if return_target: + if 'target' in batch_data_dict.keys(): + append_to_dict(output_dict, 'target', batch_data_dict['target']) + + if n % 10 == 0: + print(' --- Inference time: {:.3f} s / 10 iterations ---'.format( + time.time() - time1)) + time1 = time.time() + + for key in output_dict.keys(): + output_dict[key] = np.concatenate(output_dict[key], axis=0) + + return output_dict + + +def interpolate(x, ratio): + """Interpolate data in time domain. This is used to compensate the + resolution reduction in downsampling of a CNN. + + Args: + x: (batch_size, time_steps, classes_num) + ratio: int, ratio to interpolate + + Returns: + upsampled: (batch_size, time_steps * ratio, classes_num) + """ + (batch_size, time_steps, classes_num) = x.shape + upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1) + upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num) + return upsampled + + +def pad_framewise_output(framewise_output, frames_num): + """Pad framewise_output to the same length as input frames. The pad value + is the same as the value of the last frame. + + Args: + framewise_output: (batch_size, frames_num, classes_num) + frames_num: int, number of frames to pad + + Outputs: + output: (batch_size, frames_num, classes_num) + """ + pad = framewise_output[:, -1 :, :].repeat(1, frames_num - framewise_output.shape[1], 1) + """tensor for padding""" + + output = torch.cat((framewise_output, pad), dim=1) + """(batch_size, frames_num, classes_num)""" + + return output + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +def count_flops(model, audio_length): + """Count flops. Code modified from others' implementation. + """ + multiply_adds = True + list_conv2d=[] + def conv2d_hook(self, input, output): + batch_size, input_channels, input_height, input_width = input[0].size() + output_channels, output_height, output_width = output[0].size() + + kernel_ops = self.kernel_size[0] * self.kernel_size[1] * (self.in_channels / self.groups) * (2 if multiply_adds else 1) + bias_ops = 1 if self.bias is not None else 0 + + params = output_channels * (kernel_ops + bias_ops) + flops = batch_size * params * output_height * output_width + + list_conv2d.append(flops) + + list_conv1d=[] + def conv1d_hook(self, input, output): + batch_size, input_channels, input_length = input[0].size() + output_channels, output_length = output[0].size() + + kernel_ops = self.kernel_size[0] * (self.in_channels / self.groups) * (2 if multiply_adds else 1) + bias_ops = 1 if self.bias is not None else 0 + + params = output_channels * (kernel_ops + bias_ops) + flops = batch_size * params * output_length + + list_conv1d.append(flops) + + list_linear=[] + def linear_hook(self, input, output): + batch_size = input[0].size(0) if input[0].dim() == 2 else 1 + + weight_ops = self.weight.nelement() * (2 if multiply_adds else 1) + bias_ops = self.bias.nelement() + + flops = batch_size * (weight_ops + bias_ops) + list_linear.append(flops) + + list_bn=[] + def bn_hook(self, input, output): + list_bn.append(input[0].nelement() * 2) + + list_relu=[] + def relu_hook(self, input, output): + list_relu.append(input[0].nelement() * 2) + + list_pooling2d=[] + def pooling2d_hook(self, input, output): + batch_size, input_channels, input_height, input_width = input[0].size() + output_channels, output_height, output_width = output[0].size() + + kernel_ops = self.kernel_size * self.kernel_size + bias_ops = 0 + params = output_channels * (kernel_ops + bias_ops) + flops = batch_size * params * output_height * output_width + + list_pooling2d.append(flops) + + list_pooling1d=[] + def pooling1d_hook(self, input, output): + batch_size, input_channels, input_length = input[0].size() + output_channels, output_length = output[0].size() + + kernel_ops = self.kernel_size[0] + bias_ops = 0 + + params = output_channels * (kernel_ops + bias_ops) + flops = batch_size * params * output_length + + list_pooling2d.append(flops) + + def foo(net): + childrens = list(net.children()) + if not childrens: + if isinstance(net, nn.Conv2d): + net.register_forward_hook(conv2d_hook) + elif isinstance(net, nn.Conv1d): + net.register_forward_hook(conv1d_hook) + elif isinstance(net, nn.Linear): + net.register_forward_hook(linear_hook) + elif isinstance(net, nn.BatchNorm2d) or isinstance(net, nn.BatchNorm1d): + net.register_forward_hook(bn_hook) + elif isinstance(net, nn.ReLU): + net.register_forward_hook(relu_hook) + elif isinstance(net, nn.AvgPool2d) or isinstance(net, nn.MaxPool2d): + net.register_forward_hook(pooling2d_hook) + elif isinstance(net, nn.AvgPool1d) or isinstance(net, nn.MaxPool1d): + net.register_forward_hook(pooling1d_hook) + else: + print('Warning: flop of module {} is not counted!'.format(net)) + return + for c in childrens: + foo(c) + + # Register hook + foo(model) + + device = device = next(model.parameters()).device + input = torch.rand(1, audio_length).to(device) + + out = model(input) + + total_flops = sum(list_conv2d) + sum(list_conv1d) + sum(list_linear) + \ + sum(list_bn) + sum(list_relu) + sum(list_pooling2d) + sum(list_pooling1d) + + return total_flops \ No newline at end of file diff --git a/utils/fx_normalization/__init__.py b/utils/fx_normalization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/fx_normalization/features.npy b/utils/fx_normalization/features.npy new file mode 100644 index 0000000000000000000000000000000000000000..b59c9065d675a53a384737831b3911efd5eb2794 --- /dev/null +++ b/utils/fx_normalization/features.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1271839aa88a321deb76fb3e4f3059eeee26cf64c4071cd116e1e4a69d896d74 +size 1574572 diff --git a/utils/fx_normalization/fxnorm.py b/utils/fx_normalization/fxnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..9c02887b99181be502ea71a70dc0c56d80ec0586 --- /dev/null +++ b/utils/fx_normalization/fxnorm.py @@ -0,0 +1,154 @@ +import torch +import numpy as np + +from utils.training_utils import Gauss_smooth_vectorized, prepare_smooth_filter + +def T602logmag(t60, sample_rate=44100, hop_length=512): + return 6.908 / (t60 * (sample_rate / hop_length)) # Convert T60 to delta log magnitude + +from utils.data_utils import apply_RMS_normalization + + +class FxNormAug: + + def __init__(self, + sample_rate=44100, # Sample rate of the audio + device="cuda" if torch.cuda.is_available() else "cpu", + mode="train", # Mode can be "train" or "eval" + seed=42, + features_path="features_tency1_4instr_v4.npy", # Path to the features file + ): + + torch.random.manual_seed(seed) + + #the path is in the same directory as this file + self.features_path = features_path + + self.sample_rate = sample_rate + self.device = device + self.train_setup() + + self.EQ_normalize_setup() # Initialize EQ normalization function + + + def train_setup(self): + #hardcoded setuo used for training + + self.RMS_norm=-25 # Target RMS level in dB, used for normalization + + def EQ_normalize_setup(self ): + + features_mean = np.load(self.features_path, allow_pickle='TRUE')[()] + + target_cuves_original= { + "vocals": torch.tensor(features_mean["eq"]["vocals"]).to(torch.float32).to(self.device), + "drums": torch.tensor(features_mean["eq"]["drums"]).to(torch.float32).to(self.device), + "bass": torch.tensor(features_mean["eq"]["bass"]).to(torch.float32).to(self.device), + "other": torch.tensor(features_mean["eq"]["other"]).to(torch.float32).to(self.device), + } + + + nfft=4096 # FFT size hardcoded + nfft_orig = 65536 # FFT size for the smoothing filter + + win_length=2048 # Window length hardcoded + hop_length=1024 # Hop length hardcoded + + window = torch.sqrt(torch.hann_window(win_length, device=self.device)) + window_energy = window.pow(2).sum().sqrt() # Energy of the window + + freqs = torch.fft.rfftfreq(nfft, d=1.0).to(self.device) + freqs_Hz=torch.fft.rfftfreq(nfft, d=1.0).to(self.device) * self.sample_rate + + smooth_filter = prepare_smooth_filter(freqs_Hz, Noct=3).to(self.device) # Prepare the smoothing filter + + freqs_Hz_orig=torch.fft.rfftfreq(nfft_orig, d=1.0).to(self.device) * self.sample_rate + smooth_filter_orig = prepare_smooth_filter(freqs_Hz_orig, Noct=3).to(self.device) # Prepare the smoothing filter + + def downsample_curve(x): + return torch.nn.functional.interpolate( + x.unsqueeze(0).unsqueeze(0), + size=(nfft // 2 + 1,), + mode='linear', + align_corners=False + ).squeeze(0).squeeze(0) + + target_curves = { + "vocals": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["vocals"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + "drums": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["drums"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + "bass": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["bass"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + "other": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["other"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + } + + + def EQ_normalize_fn(x): + + shape= x.shape + + target_curves_tensor=torch.zeros((shape[0], nfft // 2 + 1), device=self.device, dtype=torch.float32) + + for i in range(shape[0]): + track_class = "other" + + assert track_class in target_curves, f"track_class {track_class} not found in target_curves" + target_curves_tensor[i] = target_curves[track_class] + + x=x.view(-1, shape[-1]) + #ensure x.shape[-1] is divisible by hop_length + if x.shape[-1] % hop_length != 0: + # Pad the input to make it divisible by hop_length + pad_length = hop_length - (x.shape[-1] % hop_length) + x = torch.nn.functional.pad(x, (0, pad_length), mode='constant', value=0) + X=torch.stft(x, n_fft=nfft, hop_length=hop_length, win_length=win_length, window=window, return_complex=True)/ window_energy + X_pow=X.abs().pow(2) + X_mean= torch.sqrt(X_pow.mean(dim=-1, keepdim=False)) # Mean power spectrum + + + + ratio= target_curves_tensor / (X_mean + 1e-6) + + ratio = torch.clamp(ratio, max=10.0**(20.0/20.0)) + + + ratio_smooth = Gauss_smooth_vectorized(ratio, freqs_Hz, Noct=3, smooth_filter=smooth_filter) + + + + X= X * ratio_smooth.unsqueeze(-1) + + X_unnormalized=X* window_energy + + x_reconstructed = torch.istft(X_unnormalized, + n_fft=nfft, + hop_length=hop_length, + win_length=win_length, + window=window, + return_complex=False) # Set to True if you want complex outpu + + #remove the padding if it was added + if x_reconstructed.shape[-1] > shape[-1]: + x_reconstructed = x_reconstructed[..., :shape[-1]] + x_reconstructed = x_reconstructed.view(shape) + return x_reconstructed + + self.EQ_normalize = EQ_normalize_fn + + + def __call__(self, x, use_gate=False, RMS=None): + + B, C, T = x.shape + if C > 1: + x = x.mean(dim=1, keepdim=True) + + + x= apply_RMS_normalization(x, self.RMS_norm , use_gate=use_gate) # Apply RMS normalization to the input + + x=self.EQ_normalize(x) + + + + x= apply_RMS_normalization(x, self.RMS_norm, use_gate=use_gate) + assert not torch.isnan(x).any(), "NaN detected in x after EQ normalization" + + + return x diff --git a/utils/fx_normalization/fxnorm_v2_public.py b/utils/fx_normalization/fxnorm_v2_public.py new file mode 100644 index 0000000000000000000000000000000000000000..f0d7decc84b45150a64d32d458b5d695d2fb1ec1 --- /dev/null +++ b/utils/fx_normalization/fxnorm_v2_public.py @@ -0,0 +1,142 @@ +import torch +import numpy as np + +from utils.training_utils import Gauss_smooth_vectorized, prepare_smooth_filter + +def T602logmag(t60, sample_rate=44100, hop_length=512): + return 6.908 / (t60 * (sample_rate / hop_length)) # Convert T60 to delta log magnitude + +from utils.data_utils import apply_RMS_normalization + + +class FxNormAug: + + def __init__(self, + sample_rate=44100, # Sample rate of the audio + device="cuda" if torch.cuda.is_available() else "cpu", + mode="train", # Mode can be "train" or "eval" + seed=42, + features_path="features_tency1_4instr_v4.npy", # Path to the features file + ): + + torch.random.manual_seed(seed) + + #the path is in the same directory as this file + self.features_path = features_path + + self.sample_rate = sample_rate + self.device = device + + self.EQ_normalize_setup() # Initialize EQ normalization function + + + + def EQ_normalize_setup(self ): + + features_mean = np.load(self.features_path, allow_pickle='TRUE')[()] + + target_cuves_original= { + "vocals": torch.tensor(features_mean["eq"]["vocals"]).to(torch.float32).to(self.device), + "drums": torch.tensor(features_mean["eq"]["drums"]).to(torch.float32).to(self.device), + "bass": torch.tensor(features_mean["eq"]["bass"]).to(torch.float32).to(self.device), + "other": torch.tensor(features_mean["eq"]["other"]).to(torch.float32).to(self.device), + } + + + nfft=4096 # FFT size hardcoded + nfft_orig = 65536 # FFT size for the smoothing filter + + win_length=2048 # Window length hardcoded + hop_length=1024 # Hop length hardcoded + + window = torch.sqrt(torch.hann_window(win_length, device=self.device)) + window_energy = window.pow(2).sum().sqrt() # Energy of the window + + freqs = torch.fft.rfftfreq(nfft, d=1.0).to(self.device) + freqs_Hz=torch.fft.rfftfreq(nfft, d=1.0).to(self.device) * self.sample_rate + + smooth_filter = prepare_smooth_filter(freqs_Hz, Noct=3).to(self.device) # Prepare the smoothing filter + + freqs_Hz_orig=torch.fft.rfftfreq(nfft_orig, d=1.0).to(self.device) * self.sample_rate + smooth_filter_orig = prepare_smooth_filter(freqs_Hz_orig, Noct=3).to(self.device) # Prepare the smoothing filter + + def downsample_curve(x): + return torch.nn.functional.interpolate( + x.unsqueeze(0).unsqueeze(0), + size=(nfft // 2 + 1,), + mode='linear', + align_corners=False + ).squeeze(0).squeeze(0) + + target_curves = { + "vocals": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["vocals"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + "drums": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["drums"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + "bass": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["bass"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + "other": downsample_curve(Gauss_smooth_vectorized(target_cuves_original["other"], freqs_Hz_orig, Noct=3, smooth_filter=smooth_filter_orig)), + } + + + def EQ_normalize_fn(x): + + shape= x.shape + + target_curves_tensor=torch.zeros((shape[0], nfft // 2 + 1), device=self.device, dtype=torch.float32) + + for i in range(shape[0]): + track_class = "other" + + assert track_class in target_curves, f"track_class {track_class} not found in target_curves" + target_curves_tensor[i] = target_curves[track_class] + + x=x.view(-1, shape[-1]) + #ensure x.shape[-1] is divisible by hop_length + if x.shape[-1] % hop_length != 0: + # Pad the input to make it divisible by hop_length + pad_length = hop_length - (x.shape[-1] % hop_length) + x = torch.nn.functional.pad(x, (0, pad_length), mode='constant', value=0) + X=torch.stft(x, n_fft=nfft, hop_length=hop_length, win_length=win_length, window=window, return_complex=True)/ window_energy + X_pow=X.abs().pow(2) + X_mean= torch.sqrt(X_pow.mean(dim=-1, keepdim=False)) # Mean power spectrum + + ratio= target_curves_tensor / (X_mean + 1e-6) + + ratio = torch.clamp(ratio, max=10.0**(40.0/20.0)) + + + ratio_smooth = Gauss_smooth_vectorized(ratio, freqs_Hz, Noct=3, smooth_filter=smooth_filter) + + + X= X * ratio_smooth.unsqueeze(-1) + + X_unnormalized=X* window_energy + + x_reconstructed = torch.istft(X_unnormalized, + n_fft=nfft, + hop_length=hop_length, + win_length=win_length, + window=window, + return_complex=False) # Set to True if you want complex outpu + + #remove the padding if it was added + if x_reconstructed.shape[-1] > shape[-1]: + x_reconstructed = x_reconstructed[..., :shape[-1]] + x_reconstructed = x_reconstructed.view(shape) + return x_reconstructed + + self.EQ_normalize = EQ_normalize_fn + + + def __call__(self, x, use_gate=False, RMS=-25): + + B, C, T = x.shape + if C > 1: + x = x.mean(dim=1, keepdim=True) + + x=x/x.max() + + x=self.EQ_normalize(x) + + x= apply_RMS_normalization(x, RMS, use_gate=use_gate) + assert not torch.isnan(x).any(), "NaN detected in x after EQ normalization" + + return x diff --git a/utils/fxencoder_plusplus/__init__.py b/utils/fxencoder_plusplus/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..23474e5d2346c56802428ed054bc3be533889a0d --- /dev/null +++ b/utils/fxencoder_plusplus/__init__.py @@ -0,0 +1,12 @@ +from .model import FxEncoderPlusPlus, load_model, load_default_model + + +__version__ = "0.1.0" + +# This defines what gets imported when someone does "from fxencoder_plusplus import *" +__all__ = [ + 'FxEncoderPlusPlus', + 'load_model', + 'load_default_model' +] + diff --git a/utils/fxencoder_plusplus/model.py b/utils/fxencoder_plusplus/model.py new file mode 100644 index 0000000000000000000000000000000000000000..e23cb32a9db85089d45dcbe5e5b8c75cc0570319 --- /dev/null +++ b/utils/fxencoder_plusplus/model.py @@ -0,0 +1,676 @@ +import torch +#import laion_clap +import logging +import sys +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchlibrosa.stft import Spectrogram, LogmelFilterBank +from collections import OrderedDict +from dataclasses import dataclass +import numpy as np +import os +from pathlib import Path +from huggingface_hub import hf_hub_download + +def init_layer(layer): + """Initialize a Linear or Convolutional layer. """ + nn.init.xavier_uniform_(layer.weight) + + if hasattr(layer, 'bias'): + if layer.bias is not None: + layer.bias.data.fill_(0.) + +def init_bn(bn): + """Initialize a Batchnorm layer. """ + bn.bias.data.fill_(0.) + bn.weight.data.fill_(1.) + +class ConvBlock(nn.Module): + def __init__(self, in_channels, out_channels): + + super(ConvBlock, self).__init__() + + self.conv1 = nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=(3, 3), stride=(1, 1), + padding=(1, 1), bias=False) + + self.conv2 = nn.Conv2d(in_channels=out_channels, + out_channels=out_channels, + kernel_size=(3, 3), stride=(1, 1), + padding=(1, 1), bias=False) + + self.bn1 = nn.BatchNorm2d(out_channels) + self.bn2 = nn.BatchNorm2d(out_channels) + + self.init_weight() + + def init_weight(self): + init_layer(self.conv1) + init_layer(self.conv2) + init_bn(self.bn1) + init_bn(self.bn2) + + + def forward(self, input, pool_size=(2, 2), pool_type='avg'): + + x = input + x = F.relu_(self.bn1(self.conv1(x))) + x = F.relu_(self.bn2(self.conv2(x))) + if pool_type == 'max': + x = F.max_pool2d(x, kernel_size=pool_size) + elif pool_type == 'avg': + x = F.avg_pool2d(x, kernel_size=pool_size) + elif pool_type == 'avg+max': + x1 = F.avg_pool2d(x, kernel_size=pool_size) + x2 = F.max_pool2d(x, kernel_size=pool_size) + x = x1 + x2 + else: + raise Exception('Incorrect argument!') + + return x + +class CLAP_AUDIO_ENCODER(torch.nn.Module): + def __init__(self, pretrained: bool = True, frozen: bool = False) -> None: + super().__init__() + self.pretrained = pretrained + self.frozen = frozen + + # load the model + self.encoder = laion_clap.CLAP_Module(enable_fusion=False, amodel= 'HTSAT-tiny', tmodel='roberta') + if self.pretrained: + self.encoder.load_ckpt() # download the default pretrained checkpoint. + + self.embed_dim = 512 + + def forward(self, x: torch.Tensor): + if self.frozen: + with torch.no_grad(): + embed = self.encoder.get_audio_embedding_from_data( + x=x, use_tensor=True + ) + else: + embed = self.encoder.get_audio_embedding_from_data( + x=x, use_tensor=True + ) + + return embed + +class CLAP_TEXT_ENCODER(torch.nn.Module): + def __init__(self, pretrained: bool = True, frozen: bool = False) -> None: + super().__init__() + self.pretrained = pretrained + self.frozen = frozen + + # load the model + self.encoder = laion_clap.CLAP_Module(enable_fusion=False, amodel= 'HTSAT-tiny', tmodel='roberta') + if self.pretrained: + self.encoder.load_ckpt() # download the default pretrained checkpoint. + + self.embed_dim = 512 + + def forward(self, x): + if self.frozen: + with torch.no_grad(): + embed = self.encoder.get_text_embedding( + x=x, use_tensor=True + ) + else: + embed = self.encoder.get_text_embedding( + x=x, use_tensor=True + ) + + return embed + +# > ================ Proposed =================== < +class MixtureFxEncoder(nn.Module): + def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, enable_fusion=False, fusion_type='None'): + super().__init__() + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + + window = "hann" + center = True + pad_mode = "reflect" + ref = 1.0 + amin = 1e-10 + top_db = None + self.input_norm = "minmax" + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram( + n_fft=window_size, + hop_length=hop_size, + win_length=window_size, + window=window, + center=center, + pad_mode=pad_mode, + freeze_parameters=True, + ) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank( + sr=sample_rate, + n_fft=window_size, + n_mels=mel_bins, + fmin=fmin, + fmax=fmax, + ref=ref, + amin=amin, + top_db=top_db, + freeze_parameters=True, + ) + + self.bn0 = nn.BatchNorm2d(64) + + self.conv_block1 = ConvBlock(in_channels=2, out_channels=64) + self.conv_block2 = ConvBlock(in_channels=64, out_channels=128) + self.conv_block3 = ConvBlock(in_channels=128, out_channels=256) + self.conv_block4 = ConvBlock(in_channels=256, out_channels=512) + self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024) + self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048) + + self.fc_1 = nn.Linear(2048, 2048, bias=True) + + self.init_weight() + + def init_weight(self): + init_bn(self.bn0) + init_layer(self.fc_1) + + def forward(self, x): + """ + Input: (batch_size, 2, data_length) + """ + batch_size, chs, seq_len = x.size() + + # move to batch dim + x = x.view(batch_size * chs, seq_len) + + # extract logmel features + x = self.spectrogram_extractor(x) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + + if self.input_norm == "batchnorm": + # this normalizes over mel bins which is problematic for equalization + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + elif self.input_norm == "minmax": + x = x.clamp(-80, 40.0) # clamp the logmels between -80 and 40 + x = (x + 80) / 120 # normalize the logmels between 0 and 1 + x = (x * 2) - 1 # normalize the logmels between -1 and 1 + elif self.input_norm == "none": + pass + else: + raise ValueError(f"Invalid input_norm: {self.input_norm}") + + x = x.view(batch_size, chs, x.size(-2), x.size(-1)) + + x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = torch.mean(x, dim=3) + + + (x1, _) = torch.max(x, dim=2) + x2 = torch.mean(x, dim=2) + x = x1 + x2 + x = F.relu_(self.fc_1(x)) + embedding = x + + output_dict = { + 'embedding': embedding, + } + + return output_dict + +def create_MixtureFxEncoder(): + model = MixtureFxEncoder( + sample_rate = 44100, #audio_cfg.sample_rate, + window_size = 2048, #audio_cfg.window_size, + hop_size = 512, #audio_cfg.hop_size, + mel_bins = 64, #audio_cfg.mel_bins, + fmin = 50, #audio_cfg.fmin, + fmax = 18000, #audio_cfg.fmax, + ) + return model + +class MLPLayers(nn.Module): + def __init__(self, units=[512, 512, 512], nonlin=nn.ReLU(), dropout=0.1): + super(MLPLayers, self).__init__() + self.nonlin = nonlin + self.dropout = dropout + + sequence = [] + for u0, u1 in zip(units[:-1], units[1:]): + sequence.append(nn.Linear(u0, u1)) + sequence.append(self.nonlin) + sequence.append(nn.Dropout(self.dropout)) + sequence = sequence[:-2] + + self.sequential = nn.Sequential(*sequence) + + def forward(self, X): + X = self.sequential(X) + return X + +class BernoulliDynamicDropout(nn.Module): + def __init__(self): + super().__init__() + self.p_min = 0.75 + self.p_max = 0.95 + + def get_random_dropout_rate(self): + return torch.empty(1).uniform_(self.p_min, self.p_max).item() + + def forward(self, x): + if self.training: + p = self.get_random_dropout_rate() + mask = torch.bernoulli(torch.full_like(x, 1-p)) + return x * mask / (1 - p) + return x + +class AudioExtracter(nn.Module): + def __init__(self, fx_embedding_dim=128, clap_embedding_dim=512): + super().__init__() + + # Simple fusion network + self.fusion = nn.Sequential( + nn.Linear(fx_embedding_dim+clap_embedding_dim, 128), + nn.LeakyReLU(0.1), + nn.Linear(128, 128), + nn.LeakyReLU(0.1), + nn.Linear(128, 128), + ) + + def forward(self, mixture_emb, query_emb): + # Concatenate and project + x = torch.cat([mixture_emb, query_emb], dim=-1) # [B, 2D] + stem_emb = self.fusion(x) # [B, D] + + return stem_emb + +@dataclass +class AudioCfg: + model_type: str = "PANN" + model_name: str = "Cnn14" + sample_rate: int = 44100 + # Param + audio_length: int = 1024 + window_size: int = 1024 + hop_size: int = 1024 + fmin: int = 50 + fmax: int = 14000 + mel_bins: int = 64 + clip_samples: int = 441000 + class_num: int = 527 + condition_dim: int = 512 + +class FxEncoderPlusPlus(nn.Module): + def __init__( + self, + embed_dim: int = 2048, + mixture_cfg: AudioCfg = None, + enable_fusion: bool = False, + fusion_type: str = 'None', + joint_embed_shape: int = 128, + mlp_act: str = 'relu', + audio_clap_module: bool = True, + text_clap_module: bool = False, + extractor_module: bool = True, + device: str = "cpu", + ): + super().__init__() + + self.mixture_cfg = mixture_cfg + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + self.joint_embed_shape = joint_embed_shape + self.mlp_act = mlp_act + self.device = device + + if mlp_act == 'relu': + mlp_act_layer = nn.ReLU() + elif mlp_act == 'gelu': + mlp_act_layer = nn.GELU() + else: + raise NotImplementedError + + # > ========================= FX Encoder ========================= < + self.fx_encoder = create_MixtureFxEncoder() + self.fx_encoder_transform = MLPLayers(units=[self.joint_embed_shape, self.joint_embed_shape, self.joint_embed_shape], dropout=0.1) + + self.fx_encoder_projection = nn.Sequential( + nn.Linear(embed_dim, self.joint_embed_shape), + mlp_act_layer, + nn.Linear(self.joint_embed_shape, self.joint_embed_shape) + ) + + self.logit_scale_m = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + self.logit_scale_t = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + if audio_clap_module: + # Freeze all layers + # print("Loading CLAP Audio Model") + self.audio_clap_model = CLAP_AUDIO_ENCODER(pretrained=True, frozen=True) + self.audio_clap_model.to(device) + for param in self.audio_clap_model.parameters(): + param.requires_grad = False + self.clap_dropout = BernoulliDynamicDropout() + if text_clap_module: + # Freeze all layers + # print("Loading CLAP Text Model") + self.text_clap_model = CLAP_TEXT_ENCODER(pretrained=True, frozen=True) + self.text_clap_model.to(device) + for param in self.text_clap_model.parameters(): + param.requires_grad = False + + if extractor_module: + # extractor + self.extractor = AudioExtracter() + + self.use_audio_clap_module = audio_clap_module + self.use_text_clap_module = text_clap_module + self.use_extractor_module = extractor_module + + def get_fx_embedding(self, x): + fx_emb = self.fx_encoder(x) + fx_emb = self.fx_encoder_projection(fx_emb["embedding"]) + fx_emb = F.normalize(fx_emb, dim=-1) + return fx_emb + + def get_fx_embedding_by_audio_query(self, x, audio_query): + # mixture fx embedding + fx_mixture_emb = self.fx_encoder(x) + fx_mixture_emb = self.fx_encoder_projection(fx_mixture_emb["embedding"]) + fx_mixture_emb = F.normalize(fx_mixture_emb, dim=-1) + + # stem fx embedding + query_content_embeded = self.audio_clap_model(torch.mean(audio_query, dim=1)) + fx_stem_emb = self.extractor(fx_mixture_emb, query_content_embeded) + fx_stem_emb = F.normalize(fx_stem_emb, dim=-1) + return fx_mixture_emb, fx_stem_emb + + def get_fx_embedding_by_text_query(self, x, text_query): + # mixture fx embedding + fx_mixture_emb = self.fx_encoder(x) + fx_mixture_emb = self.fx_encoder_projection(fx_mixture_emb["embedding"]) + fx_mixture_emb = F.normalize(fx_mixture_emb, dim=-1) + + # stem fx embedding + query_embeded = self.text_clap_model(text_query) + fx_stem_emb = self.extractor(fx_mixture_emb, query_embeded) + fx_stem_emb = F.normalize(fx_stem_emb, dim=-1) + return fx_mixture_emb, fx_stem_emb + + def forward( + self, + mixture_a, + mixture_b, + stem_a, + query_stem, + device = None + ): + + if device is None: + if mixture_a is not None: + device = mixture_a.device + elif mixture_b is not None: + device = mixture_b.device + if mixture_a is None and mixture_b is None: + # a hack to get the logit scale + return self.logit_scale_m.exp(), self.logit_scale_t.exp() + + # ======== Global ======== + mixture_a_features = self.fx_encoder_projection( + self.fx_encoder(mixture_a)["embedding"] + ) + mixture_a_features = F.normalize(mixture_a_features, dim=-1) + + mixture_b_features = self.fx_encoder_projection( + self.fx_encoder(mixture_b)["embedding"] + ) + mixture_b_features = F.normalize(mixture_b_features, dim=-1) + + mixture_a_features_mlp = self.fx_encoder_transform(mixture_a_features) + mixture_b_features_mlp = self.fx_encoder_transform(mixture_b_features) + + # ======= Local ======== + stem_a_features = self.fx_encoder_projection( + self.fx_encoder(stem_a)["embedding"] + ) + stem_a_features = F.normalize(stem_a_features, dim=-1) + + if self.use_audio_clap_module and self.use_extractor_module: + query_stem_content_embeded = self.clap_dropout( + self.audio_clap_model( + torch.mean(query_stem, dim=1) + ) + ) + extracted_stem_a_features = self.extractor(mixture_a_features, query_stem_content_embeded) + extracted_stem_a_features = F.normalize(extracted_stem_a_features, dim=-1) + elif self.use_text_clap_module and self.use_extractor_module: + query_stem_content_embeded = self.text_clap_model(query_stem) + extracted_stem_a_features = self.extractor(mixture_a_features, query_stem_content_embeded) + extracted_stem_a_features = F.normalize(extracted_stem_a_features, dim=-1) + + return ( + mixture_a_features, # global + mixture_b_features, # global + stem_a_features, # local + extracted_stem_a_features, # local + mixture_a_features_mlp, + mixture_b_features_mlp, + self.logit_scale_m.exp(), + self.logit_scale_t.exp(), + ) + + def get_logit_scale(self): + return self.logit_scale_m.exp(), self.logit_scale_t.exp() + +# def load_model(model_path, device): + +# model = FxEncoderPlusPlus( +# embed_dim = 2048, +# audio_clap_module = True, +# extractor_module = True +# ) + +# # load model +# checkpoint = torch.load(model_path, map_location=device, weights_only=False) +# if "epoch" in checkpoint: +# # resuming a train checkpoint w/ epoch and optimizer state +# start_epoch = checkpoint["epoch"] +# sd = checkpoint["state_dict"] +# if next(iter(sd.items()))[0].startswith( +# "module" +# ): +# sd = {k[len("module."):]: v for k, v in sd.items()} +# model.load_state_dict(sd) +# logging.info( +# f"=> resuming checkpoint '{model_path}' (epoch {start_epoch})" +# ) +# else: +# # loading a bare (model only) checkpoint for fine-tune or evaluation +# model.load_state_dict(checkpoint) +# start_epoch = 0 + +# model.to(device) +# model.eval() +# for param in model.parameters(): +# param.requires_grad = False +# return model + +# Define available models +MODEL_REGISTRY = { + "default": { + "repo_id": "yytung/fxencoder-plusplus", + "filename": "fxenc_plusplus_default.pt", + "description": "Default model", + }, + # "musdb": { + # "repo_id": "yytung/fxencoder-plusplus", + # "filename": "fxenc_plusplus_musdb.pt", + # "description": "Fx-Encoder++ trained on musdb", + # }, + # "medleydb": { + # "repo_id": "yytung/fxencoder-plusplus", + # "filename": "fxenc_plusplus_medleydb.pt", + # "description": "Fx-Encoder++ trained on medleydb", + # }, +} + +def get_model_path(model_name="default", cache_dir=None, force_download=False): + """ + Download or retrieve the path to a pretrained model. + + Args: + model_name: Name of the model variant ('default', 'musdb', 'medleydb') + cache_dir: Custom cache directory. If None, uses ~/.cache/fxencoder_plusplus + force_download: Force re-download even if file exists + + Returns: + Path to the model file + """ + if model_name not in MODEL_REGISTRY: + available = ", ".join(MODEL_REGISTRY.keys()) + raise ValueError(f"Unknown model: {model_name}. Available models: {available}") + + if cache_dir is None: + cache_dir = Path.home() / ".cache" / "fxencoder_plusplus" + else: + cache_dir = Path(cache_dir) + + cache_dir.mkdir(parents=True, exist_ok=True) + + model_info = MODEL_REGISTRY[model_name] + model_path = cache_dir / model_info["filename"] + + # Check if already downloaded + if model_path.exists() and not force_download: + print(f"Using cached model: {model_path}") + return str(model_path) + + print(f"Description: {model_info['description']}") + + # Download from Hugging Face + downloaded_path = hf_hub_download( + repo_id=model_info["repo_id"], + filename=model_info["filename"], + cache_dir=str(cache_dir), + force_download=force_download + ) + + print(f"Model downloaded successfully to: {downloaded_path}") + return downloaded_path + +def list_available_models(): + """List all available pretrained models.""" + print("Available FxEncoder++ models:") + print("-" * 50) + for name, info in MODEL_REGISTRY.items(): + print(f" {name}:") + print(f" - Description: {info['description']}") + print("-" * 50) + +def load_model(model_name="default", model_path=None, device="cuda", auto_download=True, cache_dir=None): + """ + Load FxEncoderPlusPlus model. + + Args: + model_name: Name of pretrained model ('default', 'musdb', 'medleydb') + model_path: Custom checkpoint path. If provided, ignores model_name + device: Device to load model on ('cuda' or 'cpu') + auto_download: Automatically download if model not found + cache_dir: Custom cache directory for downloaded models + + Returns: + Loaded FxEncoderPlusPlus model + + Examples: + # Load default base model + model = load_model() + + # Load musdb model + model = load_model(model_name="musdb") + + # Load medleydb model + model = load_model(model_name="medleydb") + + # Load custom checkpoint + model = load_model(model_path="/path/to/custom.pt") + + # List available models + list_available_models() + """ + # Handle device + if device == "cuda" and not torch.cuda.is_available(): + print("CUDA not available, using CPU") + device = "cpu" + + + # Determine model path + if model_path is None: + if auto_download: + model_path = get_model_path(model_name, cache_dir=cache_dir) + else: + raise ValueError("model_path is None and auto_download is False") + + # Create model instance with specified device + model = FxEncoderPlusPlus( + embed_dim=2048, + audio_clap_module=True, + text_clap_module=True, + extractor_module=True, + device=device + ) + + # Load checkpoint + checkpoint = torch.load(model_path, map_location=device, weights_only=False) + + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if next(iter(sd.items()))[0].startswith("module"): + sd = {k[len("module."):]: v for k, v in sd.items()} + model.load_state_dict(sd) + print(f"Loaded checkpoint from epoch {start_epoch}") + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + print("Loaded model checkpoint") + + model.to(device) + model.eval() + + # Freeze parameters for inference + for param in model.parameters(): + param.requires_grad = False + + print(f"Model loaded successfully on {device}") + return model + +# Convenience functions for specific models +def load_default_model(device="cuda", **kwargs): + """Load the default FxEncoder++ model.""" + return load_model(model_name="default", device=device, **kwargs) + +# def load_musdb_model(device="cuda", **kwargs): +# """Load the musdb FxEncoder++ model.""" +# return load_model(model_name="musdb", device=device, **kwargs) + +# def load_medleydb_model(device="cuda", **kwargs): +# """Load the medleydb FxEncoder++ model.""" +# return load_model(model_name="medleydb", device=device, **kwargs) + diff --git a/utils/laion_clap/__init__.py b/utils/laion_clap/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/laion_clap/clap_module/__init__.py b/utils/laion_clap/clap_module/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b585be6540fe21eef8bc6594375baee5017877ef --- /dev/null +++ b/utils/laion_clap/clap_module/__init__.py @@ -0,0 +1,8 @@ +from .factory import list_models, create_model, create_model_and_transforms, add_model_config +from .loss import ClipLoss, gather_features, LPLoss, lp_gather_features, LPMetrics +from .model import CLAP, CLAPTextCfg, CLAPVisionCfg, CLAPAudioCfp, convert_weights_to_fp16, trace_model +from .openai import load_openai_model, list_openai_models +from .pretrained import list_pretrained, list_pretrained_tag_models, list_pretrained_model_tags,\ + get_pretrained_url, download_pretrained +from .tokenizer import SimpleTokenizer, tokenize +from .transform import image_transform \ No newline at end of file diff --git a/utils/laion_clap/clap_module/bert.py b/utils/laion_clap/clap_module/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..005e72dec67e4b1c05063dbd4d024166344fd2c4 --- /dev/null +++ b/utils/laion_clap/clap_module/bert.py @@ -0,0 +1,32 @@ +from transformers import BertTokenizer, BertModel +tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') +model = BertModel.from_pretrained("bert-base-uncased") +text = "Replace me by any text you'd like." + +def bert_embeddings(text): + # text = "Replace me by any text you'd like." + encoded_input = tokenizer(text, return_tensors='pt') + output = model(**encoded_input) + return output + +from transformers import RobertaTokenizer, RobertaModel + +tokenizer = RobertaTokenizer.from_pretrained('roberta-base') +model = RobertaModel.from_pretrained('roberta-base') +text = "Replace me by any text you'd like." +def Roberta_embeddings(text): + # text = "Replace me by any text you'd like." + encoded_input = tokenizer(text, return_tensors='pt') + output = model(**encoded_input) + return output + +from transformers import BartTokenizer, BartModel + +tokenizer = BartTokenizer.from_pretrained('facebook/bart-base') +model = BartModel.from_pretrained('facebook/bart-base') +text = "Replace me by any text you'd like." +def bart_embeddings(text): + # text = "Replace me by any text you'd like." + encoded_input = tokenizer(text, return_tensors='pt') + output = model(**encoded_input) + return output \ No newline at end of file diff --git a/utils/laion_clap/clap_module/bpe_simple_vocab_16e6.txt.gz b/utils/laion_clap/clap_module/bpe_simple_vocab_16e6.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..36a15856e00a06a9fbed8cdd34d2393fea4a3113 --- /dev/null +++ b/utils/laion_clap/clap_module/bpe_simple_vocab_16e6.txt.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a +size 1356917 diff --git a/utils/laion_clap/clap_module/factory.py b/utils/laion_clap/clap_module/factory.py new file mode 100644 index 0000000000000000000000000000000000000000..9ea26287e07f354b44565b9a6d70edc65b16889a --- /dev/null +++ b/utils/laion_clap/clap_module/factory.py @@ -0,0 +1,257 @@ +import json +import logging +import os +import pathlib +import re +from copy import deepcopy +from pathlib import Path + +import torch + +from .model import CLAP, convert_weights_to_fp16 +from .openai import load_openai_model +from .pretrained import get_pretrained_url, download_pretrained +from .transform import image_transform + +_MODEL_CONFIG_PATHS = [Path(__file__).parent / f"model_configs/"] +_MODEL_CONFIGS = {} # directory (model_name: config) of model architecture configs + + +def _natural_key(string_): + return [int(s) if s.isdigit() else s for s in re.split(r"(\d+)", string_.lower())] + + +def _rescan_model_configs(): + global _MODEL_CONFIGS + + config_ext = (".json",) + config_files = [] + for config_path in _MODEL_CONFIG_PATHS: + if config_path.is_file() and config_path.suffix in config_ext: + config_files.append(config_path) + elif config_path.is_dir(): + for ext in config_ext: + config_files.extend(config_path.glob(f"*{ext}")) + + for cf in config_files: + with open(cf, "r") as f: + model_cfg = json.load(f) + if all(a in model_cfg for a in ("embed_dim", "audio_cfg", "text_cfg")): + _MODEL_CONFIGS[cf.stem] = model_cfg + + _MODEL_CONFIGS = { + k: v + for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0])) + } + + +_rescan_model_configs() # initial populate of model config registry + + +def load_state_dict(checkpoint_path: str, map_location="cpu", skip_params=True): + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + if skip_params: + if next(iter(state_dict.items()))[0].startswith("module"): + state_dict = {k[7:]: v for k, v in state_dict.items()} + # for k in state_dict: + # if k.startswith('transformer'): + # v = state_dict.pop(k) + # state_dict['text_branch.' + k[12:]] = v + return state_dict + + +def create_model( + amodel_name: str, + tmodel_name: str, + pretrained: str = "", + precision: str = "fp32", + device: torch.device = torch.device("cpu"), + jit: bool = False, + force_quick_gelu: bool = False, + openai_model_cache_dir: str = os.path.expanduser("~/.cache/clip"), + skip_params=True, + pretrained_audio: str = "", + pretrained_text: str = "", + enable_fusion: bool = False, + fusion_type: str = 'None' + # pretrained_image: bool = False, +): + amodel_name = amodel_name.replace( + "/", "-" + ) # for callers using old naming with / in ViT names + pretrained_orig = pretrained + pretrained = pretrained.lower() + if pretrained == "openai": + if amodel_name in _MODEL_CONFIGS: + logging.info(f"Loading {amodel_name} model config.") + model_cfg = deepcopy(_MODEL_CONFIGS[amodel_name]) + else: + logging.error( + f"Model config for {amodel_name} not found; available models {list_models()}." + ) + raise RuntimeError(f"Model config for {amodel_name} not found.") + + logging.info(f"Loading pretrained ViT-B-16 text encoder from OpenAI.") + # Hard Code in model name + model_cfg["text_cfg"]["model_type"] = tmodel_name + model = load_openai_model( + "ViT-B-16", + model_cfg, + device=device, + jit=jit, + cache_dir=openai_model_cache_dir, + enable_fusion=enable_fusion, + fusion_type=fusion_type + ) + # See https://discuss.pytorch.org/t/valueerror-attemting-to-unscale-fp16-gradients/81372 + if precision == "amp" or precision == "fp32": + model = model.float() + else: + if amodel_name in _MODEL_CONFIGS: + logging.info(f"Loading {amodel_name} model config.") + model_cfg = deepcopy(_MODEL_CONFIGS[amodel_name]) + else: + logging.error( + f"Model config for {amodel_name} not found; available models {list_models()}." + ) + raise RuntimeError(f"Model config for {amodel_name} not found.") + + if force_quick_gelu: + # override for use of QuickGELU on non-OpenAI transformer models + model_cfg["quick_gelu"] = True + + # if pretrained_image: + # if 'timm_amodel_name' in model_cfg.get('vision_cfg', {}): + # # pretrained weight loading for timm models set via vision_cfg + # model_cfg['vision_cfg']['timm_model_pretrained'] = True + # else: + # assert False, 'pretrained image towers currently only supported for timm models' + model_cfg["text_cfg"]["model_type"] = tmodel_name + model_cfg["enable_fusion"] = enable_fusion + model_cfg["fusion_type"] = fusion_type + model = CLAP(**model_cfg) + + if pretrained: + checkpoint_path = "" + url = get_pretrained_url(amodel_name, pretrained) + if url: + checkpoint_path = download_pretrained(url, root=openai_model_cache_dir) + elif os.path.exists(pretrained_orig): + checkpoint_path = pretrained_orig + if checkpoint_path: + logging.info(f"Loading pretrained {amodel_name}-{tmodel_name} weights ({pretrained}).") + ckpt = load_state_dict(checkpoint_path, skip_params=True) + model.load_state_dict(ckpt) + param_names = [n for n, p in model.named_parameters()] + for n in param_names: + print(n, "\t", "Loaded" if n in ckpt else "Unloaded") + else: + logging.warning( + f"Pretrained weights ({pretrained}) not found for model {amodel_name}." + ) + raise RuntimeError( + f"Pretrained weights ({pretrained}) not found for model {amodel_name}." + ) + + if pretrained_audio: + if amodel_name.startswith('PANN'): + if 'Cnn14_mAP' in pretrained_audio: # official checkpoint + audio_ckpt = torch.load(pretrained_audio, map_location='cpu') + audio_ckpt = audio_ckpt['model'] + keys = list(audio_ckpt.keys()) + for key in keys: + if 'spectrogram_extractor' not in key and 'logmel_extractor' not in key: + v = audio_ckpt.pop(key) + audio_ckpt['audio_branch.' + key] = v + elif os.path.basename(pretrained_audio).startswith('PANN'): # checkpoint trained via HTSAT codebase + audio_ckpt = torch.load(pretrained_audio, map_location='cpu') + audio_ckpt = audio_ckpt['state_dict'] + keys = list(audio_ckpt.keys()) + for key in keys: + if key.startswith('sed_model'): + v = audio_ckpt.pop(key) + audio_ckpt['audio_branch.' + key[10:]] = v + elif os.path.basename(pretrained_audio).startswith('finetuned'): # checkpoint trained via linear probe codebase + audio_ckpt = torch.load(pretrained_audio, map_location='cpu') + else: + raise ValueError('Unknown audio checkpoint') + elif amodel_name.startswith('HTSAT'): + if 'HTSAT_AudioSet_Saved' in pretrained_audio: # official checkpoint + audio_ckpt = torch.load(pretrained_audio, map_location='cpu') + audio_ckpt = audio_ckpt['state_dict'] + keys = list(audio_ckpt.keys()) + for key in keys: + if key.startswith('sed_model') and ('spectrogram_extractor' not in key + and 'logmel_extractor' not in key): + v = audio_ckpt.pop(key) + audio_ckpt['audio_branch.' + key[10:]] = v + elif os.path.basename(pretrained_audio).startswith('HTSAT'): # checkpoint trained via HTSAT codebase + audio_ckpt = torch.load(pretrained_audio, map_location='cpu') + audio_ckpt = audio_ckpt['state_dict'] + keys = list(audio_ckpt.keys()) + for key in keys: + if key.startswith('sed_model'): + v = audio_ckpt.pop(key) + audio_ckpt['audio_branch.' + key[10:]] = v + elif os.path.basename(pretrained_audio).startswith('finetuned'): # checkpoint trained via linear probe codebase + audio_ckpt = torch.load(pretrained_audio, map_location='cpu') + else: + raise ValueError('Unknown audio checkpoint') + else: + raise f'this audio encoder pretrained checkpoint is not support' + + model.load_state_dict(audio_ckpt, strict=False) + logging.info(f"Loading pretrained {amodel_name} weights ({pretrained_audio}).") + param_names = [n for n, p in model.named_parameters()] + for n in param_names: + print(n, "\t", "Loaded" if n in audio_ckpt else "Unloaded") + + model.to(device=device) + if precision == "fp16": + assert device.type != "cpu" + convert_weights_to_fp16(model) + + if jit: + model = torch.jit.script(model) + + return model, model_cfg + + +def create_model_and_transforms( + model_name: str, + pretrained: str = "", + precision: str = "fp32", + device: torch.device = torch.device("cpu"), + jit: bool = False, + force_quick_gelu: bool = False, + # pretrained_image: bool = False, +): + model = create_model( + model_name, + pretrained, + precision, + device, + jit, + force_quick_gelu=force_quick_gelu, + # pretrained_image=pretrained_image + ) + preprocess_train = image_transform(model.visual.image_size, is_train=True) + preprocess_val = image_transform(model.visual.image_size, is_train=False) + return model, preprocess_train, preprocess_val + + +def list_models(): + """enumerate available model architectures based on config files""" + return list(_MODEL_CONFIGS.keys()) + + +def add_model_config(path): + """add model config path or file and update registry""" + if not isinstance(path, Path): + path = Path(path) + _MODEL_CONFIG_PATHS.append(path) + _rescan_model_configs() diff --git a/utils/laion_clap/clap_module/feature_fusion.py b/utils/laion_clap/clap_module/feature_fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..c2419516b76931f0aa801d78e1b5f04a92a909e6 --- /dev/null +++ b/utils/laion_clap/clap_module/feature_fusion.py @@ -0,0 +1,193 @@ +''' +Feature Fusion for Varible-Length Data Processing +AFF/iAFF is referred and modified from https://github.com/YimianDai/open-aff/blob/master/aff_pytorch/aff_net/fusion.py +According to the paper: Yimian Dai et al, Attentional Feature Fusion, IEEE Winter Conference on Applications of Computer Vision, WACV 2021 +''' + +import torch +import torch.nn as nn + + +class DAF(nn.Module): + ''' + 直接相加 DirectAddFuse + ''' + + def __init__(self): + super(DAF, self).__init__() + + def forward(self, x, residual): + return x + residual + + +class iAFF(nn.Module): + ''' + 多特征融合 iAFF + ''' + + def __init__(self, channels=64, r=4, type='2D'): + super(iAFF, self).__init__() + inter_channels = int(channels // r) + + if type == '1D': + # 本地注意力 + self.local_att = nn.Sequential( + nn.Conv1d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv1d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(channels), + ) + + # 全局注意力 + self.global_att = nn.Sequential( + nn.AdaptiveAvgPool1d(1), + nn.Conv1d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv1d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(channels), + ) + + # 第二次本地注意力 + self.local_att2 = nn.Sequential( + nn.Conv1d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv1d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(channels), + ) + # 第二次全局注意力 + self.global_att2 = nn.Sequential( + nn.AdaptiveAvgPool1d(1), + nn.Conv1d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv1d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(channels), + ) + elif type == '2D': + # 本地注意力 + self.local_att = nn.Sequential( + nn.Conv2d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + + # 全局注意力 + self.global_att = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + + # 第二次本地注意力 + self.local_att2 = nn.Sequential( + nn.Conv2d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + # 第二次全局注意力 + self.global_att2 = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + else: + raise f'the type is not supported' + + self.sigmoid = nn.Sigmoid() + + def forward(self, x, residual): + flag = False + xa = x + residual + if xa.size(0) == 1: + xa = torch.cat([xa,xa],dim=0) + flag = True + xl = self.local_att(xa) + xg = self.global_att(xa) + xlg = xl + xg + wei = self.sigmoid(xlg) + xi = x * wei + residual * (1 - wei) + + xl2 = self.local_att2(xi) + xg2 = self.global_att(xi) + xlg2 = xl2 + xg2 + wei2 = self.sigmoid(xlg2) + xo = x * wei2 + residual * (1 - wei2) + if flag: + xo = xo[0].unsqueeze(0) + return xo + + +class AFF(nn.Module): + ''' + 多特征融合 AFF + ''' + + def __init__(self, channels=64, r=4, type='2D'): + super(AFF, self).__init__() + inter_channels = int(channels // r) + + if type == '1D': + self.local_att = nn.Sequential( + nn.Conv1d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv1d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(channels), + ) + self.global_att = nn.Sequential( + nn.AdaptiveAvgPool1d(1), + nn.Conv1d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv1d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm1d(channels), + ) + elif type == '2D': + self.local_att = nn.Sequential( + nn.Conv2d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + self.global_att = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(channels, inter_channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(inter_channels), + nn.ReLU(inplace=True), + nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + else: + raise f'the type is not supported.' + + self.sigmoid = nn.Sigmoid() + + def forward(self, x, residual): + flag = False + xa = x + residual + if xa.size(0) == 1: + xa = torch.cat([xa,xa],dim=0) + flag = True + xl = self.local_att(xa) + xg = self.global_att(xa) + xlg = xl + xg + wei = self.sigmoid(xlg) + xo = 2 * x * wei + 2 * residual * (1 - wei) + if flag: + xo = xo[0].unsqueeze(0) + return xo + diff --git a/utils/laion_clap/clap_module/htsat.py b/utils/laion_clap/clap_module/htsat.py new file mode 100644 index 0000000000000000000000000000000000000000..bb8e7cf5f2307c57e094a122121f3ca7f527436a --- /dev/null +++ b/utils/laion_clap/clap_module/htsat.py @@ -0,0 +1,1031 @@ +# Ke Chen +# knutchen@ucsd.edu +# HTS-AT: A HIERARCHICAL TOKEN-SEMANTIC AUDIO TRANSFORMER FOR SOUND CLASSIFICATION AND DETECTION +# Some layers designed on the model +# below codes are based and referred from https://github.com/microsoft/Swin-Transformer +# Swin Transformer for Computer Vision: https://arxiv.org/pdf/2103.14030.pdf + +import torch +import torch.nn as nn +import torch.nn.functional as F +from itertools import repeat +import collections.abc +import math +import warnings + +from torch.nn.init import _calculate_fan_in_and_fan_out +import torch.utils.checkpoint as checkpoint + +import random + +from torchlibrosa.stft import Spectrogram, LogmelFilterBank +from torchlibrosa.augmentation import SpecAugmentation + +from itertools import repeat +from .utils import do_mixup, interpolate + +from .feature_fusion import iAFF, AFF, DAF + +# from PyTorch internals +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + return parse + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = _ntuple + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True, patch_stride = 16, + enable_fusion=False, fusion_type='None'): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patch_stride = to_2tuple(patch_stride) + self.img_size = img_size + self.patch_size = patch_size + self.patch_stride = patch_stride + self.grid_size = (img_size[0] // patch_stride[0], img_size[1] // patch_stride[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + + padding = ((patch_size[0] - patch_stride[0]) // 2, (patch_size[1] - patch_stride[1]) // 2) + + if (self.enable_fusion) and (self.fusion_type == 'channel_map'): + self.proj = nn.Conv2d(in_chans*4, embed_dim, kernel_size=patch_size, stride=patch_stride, padding=padding) + else: + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_stride, padding=padding) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + if (self.enable_fusion) and (self.fusion_type in ['daf_2d','aff_2d','iaff_2d']): + self.mel_conv2d = nn.Conv2d(in_chans, embed_dim, kernel_size=(patch_size[0], patch_size[1]*3), stride=(patch_stride[0], patch_stride[1] * 3), padding=padding) + if self.fusion_type == 'daf_2d': + self.fusion_model = DAF() + elif self.fusion_type == 'aff_2d': + self.fusion_model = AFF(channels=embed_dim, type='2D') + elif self.fusion_type == 'iaff_2d': + self.fusion_model = iAFF(channels=embed_dim, type='2D') + def forward(self, x, longer_idx = None): + if (self.enable_fusion) and (self.fusion_type in ['daf_2d','aff_2d','iaff_2d']): + global_x = x[:,0:1,:,:] + + + # global processing + B, C, H, W = global_x.shape + assert H == self.img_size[0] and W == self.img_size[1], \ + f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + global_x = self.proj(global_x) + TW = global_x.size(-1) + if len(longer_idx) > 0: + # local processing + local_x = x[longer_idx,1:,:,:].contiguous() + B, C, H, W = local_x.shape + local_x = local_x.view(B*C,1,H,W) + local_x = self.mel_conv2d(local_x) + local_x = local_x.view(B,C,local_x.size(1),local_x.size(2),local_x.size(3)) + local_x = local_x.permute((0,2,3,1,4)).contiguous().flatten(3) + TB,TC,TH,_ = local_x.size() + if local_x.size(-1) < TW: + local_x = torch.cat([local_x, torch.zeros((TB,TC,TH,TW-local_x.size(-1)), device=global_x.device)], dim=-1) + else: + local_x = local_x[:,:,:,:TW] + + global_x[longer_idx] = self.fusion_model(global_x[longer_idx],local_x) + x = global_x + else: + B, C, H, W = x.shape + assert H == self.img_size[0] and W == self.img_size[1], \ + f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x) + + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks + """ + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +def variance_scaling_(tensor, scale=1.0, mode='fan_in', distribution='normal'): + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + if mode == 'fan_in': + denom = fan_in + elif mode == 'fan_out': + denom = fan_out + elif mode == 'fan_avg': + denom = (fan_in + fan_out) / 2 + + variance = scale / denom + + if distribution == "truncated_normal": + # constant is stddev of standard normal truncated to (-2, 2) + trunc_normal_(tensor, std=math.sqrt(variance) / .87962566103423978) + elif distribution == "normal": + tensor.normal_(std=math.sqrt(variance)) + elif distribution == "uniform": + bound = math.sqrt(3 * variance) + tensor.uniform_(-bound, bound) + else: + raise ValueError(f"invalid distribution {distribution}") + + +def lecun_normal_(tensor): + variance_scaling_(tensor, mode='fan_in', distribution='truncated_normal') + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + r""" Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x, attn + + def extra_repr(self): + return f'dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}' + + +# We use the model based on Swintransformer Block, therefore we can use the swin-transformer pretrained model +class SwinTransformerBlock(nn.Module): + r""" Swin Transformer Block. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm, norm_before_mlp='ln'): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.norm_before_mlp = norm_before_mlp + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, window_size=to_2tuple(self.window_size), num_heads=num_heads, + qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + if self.norm_before_mlp == 'ln': + self.norm2 = nn.LayerNorm(dim) + elif self.norm_before_mlp == 'bn': + self.norm2 = lambda x: nn.BatchNorm1d(dim)(x.transpose(1, 2)).transpose(1, 2) + else: + raise NotImplementedError + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + if self.shift_size > 0: + # calculate attention mask for SW-MSA + H, W = self.input_resolution + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0)) + else: + attn_mask = None + + self.register_buffer("attn_mask", attn_mask) + + def forward(self, x): + # pdb.set_trace() + H, W = self.input_resolution + # print("H: ", H) + # print("W: ", W) + # pdb.set_trace() + B, L, C = x.shape + # assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows, attn = self.attn(x_windows, mask=self.attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x, attn + + def extra_repr(self): + return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \ + f"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}" + + + +class PatchMerging(nn.Module): + r""" Patch Merging Layer. + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." + + x = x.view(B, H, W, C) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + def extra_repr(self): + return f"input_resolution={self.input_resolution}, dim={self.dim}" + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, dim, input_resolution, depth, num_heads, window_size, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False, + norm_before_mlp='ln'): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock(dim=dim, input_resolution=input_resolution, + num_heads=num_heads, window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop, attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, norm_before_mlp=norm_before_mlp) + for i in range(depth)]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x): + attns = [] + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x, attn = blk(x) + if not self.training: + attns.append(attn.unsqueeze(0)) + if self.downsample is not None: + x = self.downsample(x) + if not self.training: + attn = torch.cat(attns, dim = 0) + attn = torch.mean(attn, dim = 0) + return x, attn + + def extra_repr(self): + return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}" + + +# The Core of HTSAT +class HTSAT_Swin_Transformer(nn.Module): + r"""HTSAT based on the Swin Transformer + Args: + spec_size (int | tuple(int)): Input Spectrogram size. Default 256 + patch_size (int | tuple(int)): Patch size. Default: 4 + path_stride (iot | tuple(int)): Patch Stride for Frequency and Time Axis. Default: 4 + in_chans (int): Number of input image channels. Default: 1 (mono) + num_classes (int): Number of classes for classification head. Default: 527 + embed_dim (int): Patch embedding dimension. Default: 96 + depths (tuple(int)): Depth of each HTSAT-Swin Transformer layer. + num_heads (tuple(int)): Number of attention heads in different layers. + window_size (int): Window size. Default: 8 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. Default: None + drop_rate (float): Dropout rate. Default: 0 + attn_drop_rate (float): Attention dropout rate. Default: 0 + drop_path_rate (float): Stochastic depth rate. Default: 0.1 + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False + patch_norm (bool): If True, add normalization after patch embedding. Default: True + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False + config (module): The configuration Module from config.py + """ + + def __init__(self, spec_size=256, patch_size=4, patch_stride=(4,4), + in_chans=1, num_classes=527, + embed_dim=96, depths=[2, 2, 6, 2], num_heads=[4, 8, 16, 32], + window_size=8, mlp_ratio=4., qkv_bias=True, qk_scale=None, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, + norm_layer=nn.LayerNorm, + ape=False, patch_norm=True, + use_checkpoint=False, norm_before_mlp='ln', config = None, + enable_fusion = False, fusion_type = 'None', **kwargs): + super(HTSAT_Swin_Transformer, self).__init__() + + self.config = config + self.spec_size = spec_size + self.patch_stride = patch_stride + self.patch_size = patch_size + self.window_size = window_size + self.embed_dim = embed_dim + self.depths = depths + self.ape = ape + self.in_chans = in_chans + self.num_classes = num_classes + self.num_heads = num_heads + self.num_layers = len(self.depths) + self.num_features = int(self.embed_dim * 2 ** (self.num_layers - 1)) + + self.drop_rate = drop_rate + self.attn_drop_rate = attn_drop_rate + self.drop_path_rate = drop_path_rate + + self.qkv_bias = qkv_bias + self.qk_scale = None + + self.patch_norm = patch_norm + self.norm_layer = norm_layer if self.patch_norm else None + self.norm_before_mlp = norm_before_mlp + self.mlp_ratio = mlp_ratio + + self.use_checkpoint = use_checkpoint + + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + + # process mel-spec ; used only once + self.freq_ratio = self.spec_size // self.config.mel_bins + window = 'hann' + center = True + pad_mode = 'reflect' + ref = 1.0 + amin = 1e-10 + top_db = None + self.interpolate_ratio = 32 # Downsampled ratio + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram(n_fft=config.window_size, hop_length=config.hop_size, + win_length=config.window_size, window=window, center=center, pad_mode=pad_mode, + freeze_parameters=True) + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank(sr=config.sample_rate, n_fft=config.window_size, + n_mels=config.mel_bins, fmin=config.fmin, fmax=config.fmax, ref=ref, amin=amin, top_db=top_db, + freeze_parameters=True) + # Spec augmenter + self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, + freq_drop_width=8, freq_stripes_num=2) # 2 2 + self.bn0 = nn.BatchNorm2d(self.config.mel_bins) + + + # split spctrogram into non-overlapping patches + self.patch_embed = PatchEmbed( + img_size=self.spec_size, patch_size=self.patch_size, in_chans=self.in_chans, + embed_dim=self.embed_dim, norm_layer=self.norm_layer, patch_stride = patch_stride, + enable_fusion=self.enable_fusion, fusion_type=self.fusion_type + ) + + num_patches = self.patch_embed.num_patches + patches_resolution = self.patch_embed.grid_size + self.patches_resolution = patches_resolution + + # absolute position embedding + if self.ape: + self.absolute_pos_embed = nn.Parameter(torch.zeros(1, num_patches, self.embed_dim)) + trunc_normal_(self.absolute_pos_embed, std=.02) + + self.pos_drop = nn.Dropout(p=self.drop_rate) + + # stochastic depth + dpr = [x.item() for x in torch.linspace(0, self.drop_path_rate, sum(self.depths))] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer(dim=int(self.embed_dim * 2 ** i_layer), + input_resolution=(patches_resolution[0] // (2 ** i_layer), + patches_resolution[1] // (2 ** i_layer)), + depth=self.depths[i_layer], + num_heads=self.num_heads[i_layer], + window_size=self.window_size, + mlp_ratio=self.mlp_ratio, + qkv_bias=self.qkv_bias, qk_scale=self.qk_scale, + drop=self.drop_rate, attn_drop=self.attn_drop_rate, + drop_path=dpr[sum(self.depths[:i_layer]):sum(self.depths[:i_layer + 1])], + norm_layer=self.norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + norm_before_mlp=self.norm_before_mlp) + self.layers.append(layer) + + self.norm = self.norm_layer(self.num_features) + self.avgpool = nn.AdaptiveAvgPool1d(1) + self.maxpool = nn.AdaptiveMaxPool1d(1) + + SF = self.spec_size // (2 ** (len(self.depths) - 1)) // self.patch_stride[0] // self.freq_ratio + self.tscam_conv = nn.Conv2d( + in_channels = self.num_features, + out_channels = self.num_classes, + kernel_size = (SF,3), + padding = (0,1) + ) + self.head = nn.Linear(num_classes, num_classes) + + if (self.enable_fusion) and (self.fusion_type in ['daf_1d','aff_1d','iaff_1d']): + self.mel_conv1d = nn.Sequential( + nn.Conv1d(64, 64, kernel_size=5, stride=3, padding=2), + nn.BatchNorm1d(64) + ) + if self.fusion_type == 'daf_1d': + self.fusion_model = DAF() + elif self.fusion_type == 'aff_1d': + self.fusion_model = AFF(channels=64, type='1D') + elif self.fusion_type == 'iaff_1d': + self.fusion_model = iAFF(channels=64, type='1D') + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'absolute_pos_embed'} + + @torch.jit.ignore + def no_weight_decay_keywords(self): + return {'relative_position_bias_table'} + + + def forward_features(self, x, longer_idx = None): + # A deprecated optimization for using a hierarchical output from different blocks + + frames_num = x.shape[2] + x = self.patch_embed(x, longer_idx = longer_idx) + if self.ape: + x = x + self.absolute_pos_embed + x = self.pos_drop(x) + for i, layer in enumerate(self.layers): + x, attn = layer(x) + # for x + x = self.norm(x) + B, N, C = x.shape + SF = frames_num // (2 ** (len(self.depths) - 1)) // self.patch_stride[0] + ST = frames_num // (2 ** (len(self.depths) - 1)) // self.patch_stride[1] + x = x.permute(0,2,1).contiguous().reshape(B, C, SF, ST) + B, C, F, T = x.shape + # group 2D CNN + c_freq_bin = F // self.freq_ratio + x = x.reshape(B, C, F // c_freq_bin, c_freq_bin, T) + x = x.permute(0,1,3,2,4).contiguous().reshape(B, C, c_freq_bin, -1) + # get latent_output + fine_grained_latent_output = torch.mean(x, dim = 2) + fine_grained_latent_output = interpolate(fine_grained_latent_output.permute(0,2,1).contiguous(), 8 * self.patch_stride[1]) + + latent_output = self.avgpool(torch.flatten(x,2)) + latent_output = torch.flatten(latent_output, 1) + + # display the attention map, if needed + + x = self.tscam_conv(x) + x = torch.flatten(x, 2) # B, C, T + + fpx = interpolate(torch.sigmoid(x).permute(0,2,1).contiguous(), 8 * self.patch_stride[1]) + + x = self.avgpool(x) + x = torch.flatten(x, 1) + + output_dict = { + 'framewise_output': fpx, # already sigmoided + 'clipwise_output': torch.sigmoid(x), + 'fine_grained_embedding': fine_grained_latent_output, + 'embedding': latent_output + } + + return output_dict + + def crop_wav(self, x, crop_size, spe_pos = None): + time_steps = x.shape[2] + tx = torch.zeros(x.shape[0], x.shape[1], crop_size, x.shape[3]).to(x.device) + for i in range(len(x)): + if spe_pos is None: + crop_pos = random.randint(0, time_steps - crop_size - 1) + else: + crop_pos = spe_pos + tx[i][0] = x[i, 0, crop_pos:crop_pos + crop_size,:] + return tx + + # Reshape the wavform to a img size, if you want to use the pretrained swin transformer model + def reshape_wav2img(self, x): + B, C, T, F = x.shape + target_T = int(self.spec_size * self.freq_ratio) + target_F = self.spec_size // self.freq_ratio + assert T <= target_T and F <= target_F, "the wav size should less than or equal to the swin input size" + # to avoid bicubic zero error + if T < target_T: + x = nn.functional.interpolate(x, (target_T, x.shape[3]), mode="bicubic", align_corners=True) + if F < target_F: + x = nn.functional.interpolate(x, (x.shape[2], target_F), mode="bicubic", align_corners=True) + x = x.permute(0,1,3,2).contiguous() + x = x.reshape(x.shape[0], x.shape[1], x.shape[2], self.freq_ratio, x.shape[3] // self.freq_ratio) + # print(x.shape) + x = x.permute(0,1,3,2,4).contiguous() + x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3], x.shape[4]) + return x + + # Repeat the wavform to a img size, if you want to use the pretrained swin transformer model + def repeat_wat2img(self, x, cur_pos): + B, C, T, F = x.shape + target_T = int(self.spec_size * self.freq_ratio) + target_F = self.spec_size // self.freq_ratio + assert T <= target_T and F <= target_F, "the wav size should less than or equal to the swin input size" + # to avoid bicubic zero error + if T < target_T: + x = nn.functional.interpolate(x, (target_T, x.shape[3]), mode="bicubic", align_corners=True) + if F < target_F: + x = nn.functional.interpolate(x, (x.shape[2], target_F), mode="bicubic", align_corners=True) + x = x.permute(0,1,3,2).contiguous() # B C F T + x = x[:,:,:,cur_pos:cur_pos + self.spec_size] + x = x.repeat(repeats = (1,1,4,1)) + return x + + def forward(self, x: torch.Tensor, mixup_lambda = None, infer_mode = False, device=None):# out_feat_keys: List[str] = None): + + if self.enable_fusion and x["longer"].sum() == 0: + # if no audio is longer than 10s, then randomly select one audio to be longer + if self.training: + x["longer"][torch.randint(0, x["longer"].shape[0], (1,))] = True + else: + x = x["mel_fusion"].to(device=device, non_blocking=True) + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + x = self.reshape_wav2img(x) + output_dict = self.forward_features(x, longer_idx=[]) + return output_dict + + if not self.enable_fusion: + x = x["waveform"].to(device=device, non_blocking=True) + x = self.spectrogram_extractor(x) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + if self.training: + x = self.spec_augmenter(x) + + if self.training and mixup_lambda is not None: + x = do_mixup(x, mixup_lambda) + + x = self.reshape_wav2img(x) + output_dict = self.forward_features(x) + else: + longer_list = x["longer"].to(device=device, non_blocking=True) + x = x["mel_fusion"].to(device=device, non_blocking=True) + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + longer_list_idx = torch.where(longer_list)[0] + if self.fusion_type in ['daf_1d','aff_1d','iaff_1d']: + new_x = x[:,0:1,:,:].clone().contiguous() + if len(longer_list_idx) > 0: + # local processing + fusion_x_local = x[longer_list_idx,1:,:,:].clone().contiguous() + FB,FC,FT,FF = fusion_x_local.size() + fusion_x_local = fusion_x_local.view(FB * FC, FT, FF) + fusion_x_local = torch.permute(fusion_x_local, (0,2,1)).contiguous() + fusion_x_local = self.mel_conv1d(fusion_x_local) + fusion_x_local = fusion_x_local.view(FB,FC,FF,fusion_x_local.size(-1)) + fusion_x_local = torch.permute(fusion_x_local, (0,2,1,3)).contiguous().flatten(2) + if fusion_x_local.size(-1) < FT: + fusion_x_local = torch.cat([fusion_x_local, torch.zeros((FB,FF,FT- fusion_x_local.size(-1)), device=device)], dim=-1) + else: + fusion_x_local = fusion_x_local[:,:,:FT] + # 1D fusion + new_x = new_x.squeeze(1).permute((0,2,1)).contiguous() + new_x[longer_list_idx] = self.fusion_model(new_x[longer_list_idx], fusion_x_local) + x = new_x.permute((0,2,1)).contiguous()[:,None,:,:] + else: + x = new_x + + elif self.fusion_type in ['daf_2d','aff_2d','iaff_2d','channel_map']: + x = x # no change + + if self.training: + x = self.spec_augmenter(x) + if self.training and mixup_lambda is not None: + x = do_mixup(x, mixup_lambda) + + x = self.reshape_wav2img(x) + output_dict = self.forward_features(x, longer_idx = longer_list_idx) + + # if infer_mode: + # # in infer mode. we need to handle different length audio input + # frame_num = x.shape[2] + # target_T = int(self.spec_size * self.freq_ratio) + # repeat_ratio = math.floor(target_T / frame_num) + # x = x.repeat(repeats=(1,1,repeat_ratio,1)) + # x = self.reshape_wav2img(x) + # output_dict = self.forward_features(x) + # else: + # if x.shape[2] > self.freq_ratio * self.spec_size: + # if self.training: + # x = self.crop_wav(x, crop_size=self.freq_ratio * self.spec_size) + # x = self.reshape_wav2img(x) + # output_dict = self.forward_features(x) + # else: + # # Change: Hard code here + # overlap_size = (x.shape[2] - 1) // 4 + # output_dicts = [] + # crop_size = (x.shape[2] - 1) // 2 + # for cur_pos in range(0, x.shape[2] - crop_size - 1, overlap_size): + # tx = self.crop_wav(x, crop_size = crop_size, spe_pos = cur_pos) + # tx = self.reshape_wav2img(tx) + # output_dicts.append(self.forward_features(tx)) + # clipwise_output = torch.zeros_like(output_dicts[0]["clipwise_output"]).float().to(x.device) + # framewise_output = torch.zeros_like(output_dicts[0]["framewise_output"]).float().to(x.device) + # for d in output_dicts: + # clipwise_output += d["clipwise_output"] + # framewise_output += d["framewise_output"] + # clipwise_output = clipwise_output / len(output_dicts) + # framewise_output = framewise_output / len(output_dicts) + # output_dict = { + # 'framewise_output': framewise_output, + # 'clipwise_output': clipwise_output + # } + # else: # this part is typically used, and most easy one + # x = self.reshape_wav2img(x) + # output_dict = self.forward_features(x) + # x = self.head(x) + + # We process the data in the dataloader part, in that here we only consider the input_T < fixed_T + + + + return output_dict + +def create_htsat_model(audio_cfg, enable_fusion=False, fusion_type='None'): + try: + + assert audio_cfg.model_name in ["tiny", "base", "large"], "model name for HTS-AT is wrong!" + if audio_cfg.model_name == "tiny": + model = HTSAT_Swin_Transformer( + spec_size=256, + patch_size=4, + patch_stride=(4,4), + num_classes=audio_cfg.class_num, + embed_dim=96, + depths=[2,2,6,2], + num_heads=[4,8,16,32], + window_size=8, + config = audio_cfg, + enable_fusion = enable_fusion, + fusion_type = fusion_type + ) + elif audio_cfg.model_name == "base": + model = HTSAT_Swin_Transformer( + spec_size=256, + patch_size=4, + patch_stride=(4,4), + num_classes=audio_cfg.class_num, + embed_dim=128, + depths=[2,2,12,2], + num_heads=[4,8,16,32], + window_size=8, + config = audio_cfg, + enable_fusion = enable_fusion, + fusion_type = fusion_type + ) + elif audio_cfg.model_name == "large": + model = HTSAT_Swin_Transformer( + spec_size=256, + patch_size=4, + patch_stride=(4,4), + num_classes=audio_cfg.class_num, + embed_dim=256, + depths=[2,2,12,2], + num_heads=[4,8,16,32], + window_size=8, + config = audio_cfg, + enable_fusion = enable_fusion, + fusion_type = fusion_type + ) + + return model + except: + raise RuntimeError(f'Import Model for {audio_cfg.model_name} not found, or the audio cfg parameters are not enough.') + \ No newline at end of file diff --git a/utils/laion_clap/clap_module/linear_probe.py b/utils/laion_clap/clap_module/linear_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..bb2841dd4e28201db8b5bd4a215e1b8b9a60d25a --- /dev/null +++ b/utils/laion_clap/clap_module/linear_probe.py @@ -0,0 +1,63 @@ +import numpy as np +import torch.nn.functional as F +from torch import nn +from .model import MLPLayers + + +class LinearProbe(nn.Module): + def __init__(self, model, mlp, freeze, in_ch, out_ch, act=None): + """ + Args: + model: nn.Module + mlp: bool, if True, then use the MLP layer as the linear probe module + freeze: bool, if Ture, then freeze all the CLAP model's layers when training the linear probe + in_ch: int, the output channel from CLAP model + out_ch: int, the output channel from linear probe (class_num) + act: torch.nn.functional, the activation function before the loss function + """ + super().__init__() + in_ch = 512 + self.clap_model = model + self.clap_model.text_branch = None # to save memory + self.freeze = freeze + if mlp: + self.lp_layer = MLPLayers(units=[in_ch, in_ch * 2, out_ch]) + else: + self.lp_layer = nn.Linear(in_ch, out_ch) + + if self.freeze: + for param in self.clap_model.parameters(): + param.requires_grad = False + + if act == 'None': + self.act = None + elif act == 'relu': + self.act = nn.ReLU() + elif act == 'elu': + self.act = nn.ELU() + elif act == 'prelu': + self.act = nn.PReLU(num_parameters=in_ch) + elif act == 'softmax': + self.act = nn.Softmax(dim=-1) + elif act == 'sigmoid': + self.act = nn.Sigmoid() + + def forward(self, x, mix_lambda=None, device=None): + """ + Args: + x: waveform, torch.tensor [batch, t_samples] / batch of mel_spec and longer list + mix_lambda: torch.tensor [batch], the mixup lambda + Returns: + class_prob: torch.tensor [batch, class_num] + + """ + # batchnorm cancel grandient + if self.freeze: + self.clap_model.eval() + + x = self.clap_model.audio_projection( + self.clap_model.audio_branch(x, mixup_lambda=mix_lambda, device=device)["embedding"]) + out = self.lp_layer(x) + if self.act is not None: + out = self.act(out) + return out diff --git a/utils/laion_clap/clap_module/loss.py b/utils/laion_clap/clap_module/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..53bbedd959813b072b146c16c14cd96df6cada14 --- /dev/null +++ b/utils/laion_clap/clap_module/loss.py @@ -0,0 +1,307 @@ +from multiprocessing.sharedctypes import Value +import torch +import torch.distributed.nn +from torch import distributed as dist, nn as nn +from torch.nn import functional as F +import numpy as np +from sklearn.metrics import average_precision_score, roc_auc_score, accuracy_score + +try: + import horovod.torch as hvd +except ImportError: + hvd = None + + +def gather_features( + audio_features, + text_features, + audio_features_mlp=None, + text_features_mlp=None, + local_loss=False, + gather_with_grad=False, + rank=0, + world_size=1, + use_horovod=False, + mlp_loss=False +): + if use_horovod: + assert hvd is not None, 'Please install horovod' + if gather_with_grad: + all_audio_features = hvd.allgather(audio_features) + all_text_features = hvd.allgather(text_features) + if mlp_loss: + all_audio_features_mlp = hvd.allgather(audio_features_mlp) + all_text_features_mlp = hvd.allgather(text_features_mlp) + else: + with torch.no_grad(): + all_audio_features = hvd.allgather(audio_features) + all_text_features = hvd.allgather(text_features) + if mlp_loss: + all_audio_features_mlp = hvd.allgather(audio_features_mlp) + all_text_features_mlp = hvd.allgather(text_features_mlp) + if not local_loss: + # ensure grads for local rank when all_* features don't have a gradient + gathered_audio_features = list(all_audio_features.chunk(world_size, dim=0)) + gathered_text_features = list(all_text_features.chunk(world_size, dim=0)) + gathered_audio_features[rank] = audio_features + gathered_text_features[rank] = text_features + all_audio_features = torch.cat(gathered_audio_features, dim=0) + all_text_features = torch.cat(gathered_text_features, dim=0) + if mlp_loss: + gathered_audio_features_mlp = list(all_audio_features_mlp.chunk(world_size, dim=0)) + gathered_text_features_mlp = list(all_text_features_mlp.chunk(world_size, dim=0)) + gathered_audio_features_mlp[rank] = audio_features_mlp + gathered_text_features_mlp[rank] = text_features_mlp + all_audio_features_mlp = torch.cat(gathered_audio_features_mlp, dim=0) + all_text_features_mlp = torch.cat(gathered_text_features_mlp, dim=0) + else: + # We gather tensors from all gpus + if gather_with_grad: + all_audio_features = torch.cat(torch.distributed.nn.all_gather(audio_features), dim=0) + all_text_features = torch.cat(torch.distributed.nn.all_gather(text_features), dim=0) + if mlp_loss: + all_audio_features_mlp = torch.cat(torch.distributed.nn.all_gather(audio_features_mlp), dim=0) + all_text_features_mlp = torch.cat(torch.distributed.nn.all_gather(text_features_mlp), dim=0) + else: + gathered_audio_features = [torch.zeros_like(audio_features) for _ in range(world_size)] + gathered_text_features = [torch.zeros_like(text_features) for _ in range(world_size)] + dist.all_gather(gathered_audio_features, audio_features) + dist.all_gather(gathered_text_features, text_features) + if mlp_loss: + gathered_audio_features_mlp = [torch.zeros_like(audio_features_mlp) for _ in range(world_size)] + gathered_text_features_mlp = [torch.zeros_like(text_features_mlp) for _ in range(world_size)] + dist.all_gather(gathered_audio_features_mlp, audio_features_mlp) + dist.all_gather(gathered_text_features_mlp, text_features_mlp) + if not local_loss: + # ensure grads for local rank when all_* features don't have a gradient + gathered_audio_features[rank] = audio_features + gathered_text_features[rank] = text_features + if mlp_loss: + gathered_audio_features_mlp[rank] = audio_features_mlp + gathered_text_features_mlp[rank] = text_features_mlp + + all_audio_features = torch.cat(gathered_audio_features, dim=0) + all_text_features = torch.cat(gathered_text_features, dim=0) + if mlp_loss: + all_audio_features_mlp = torch.cat(gathered_audio_features_mlp, dim=0) + all_text_features_mlp = torch.cat(gathered_text_features_mlp, dim=0) + if mlp_loss: + return all_audio_features, all_text_features, all_audio_features_mlp, all_text_features_mlp + else: + return all_audio_features, all_text_features + +class ClipLoss(nn.Module): + + def __init__( + self, + local_loss=False, + gather_with_grad=False, + cache_labels=False, + rank=0, + world_size=1, + use_horovod=False, + mlp_loss=False, + weight_loss_kappa=0, + ): + super().__init__() + self.local_loss = local_loss + self.gather_with_grad = gather_with_grad + self.cache_labels = cache_labels + self.rank = rank + self.world_size = world_size + self.use_horovod = use_horovod + self.mlp_loss = mlp_loss + self.weighted_loss = bool(weight_loss_kappa!=0) + self.weight_loss_kappa = weight_loss_kappa + # cache state + self.prev_num_logits = 0 + self.labels = {} + + def forward(self, audio_features, text_features, logit_scale_a, logit_scale_t=None, audio_features_mlp=None, text_features_mlp=None): + device = audio_features.device + if self.mlp_loss: + if self.world_size > 1: + all_audio_features, all_text_features, all_audio_features_mlp, all_text_features_mlp = gather_features( + audio_features=audio_features,text_features=text_features, + audio_features_mlp=audio_features_mlp,text_features_mlp=text_features_mlp, + local_loss=self.local_loss,gather_with_grad=self.gather_with_grad, + rank=self.rank,world_size=self.world_size,use_horovod=self.use_horovod, + mlp_loss=self.mlp_loss + ) + if self.local_loss: + a_logits_per_audio = logit_scale_a * audio_features @ all_text_features_mlp.T + a_logits_per_text = logit_scale_a * text_features_mlp @ all_audio_features.T + t_logits_per_audio = logit_scale_t * audio_features_mlp @ all_text_features.T + t_logits_per_text = logit_scale_t * text_features @ all_audio_features_mlp.T + else: + a_logits_per_audio = logit_scale_a * all_audio_features @ all_text_features_mlp.T + a_logits_per_text = a_logits_per_audio.T + t_logits_per_audio = logit_scale_t * all_audio_features_mlp @ all_text_features.T + t_logits_per_text = t_logits_per_audio.T + else: + a_logits_per_audio = logit_scale_a * audio_features @ text_features_mlp.T + a_logits_per_text = logit_scale_a * text_features_mlp @ audio_features.T + t_logits_per_audio = logit_scale_t * audio_features_mlp @ text_features.T + t_logits_per_text = logit_scale_t * text_features @ audio_features_mlp.T + + # calculated ground-truth and cache if enabled + num_logits = a_logits_per_audio.shape[0] + if self.prev_num_logits != num_logits or device not in self.labels: + labels = torch.arange(num_logits, device=device, dtype=torch.long) + if self.world_size > 1 and self.local_loss: + labels = labels + num_logits * self.rank + if self.cache_labels: + self.labels[device] = labels + self.prev_num_logits = num_logits + else: + labels = self.labels[device] + + if not self.weighted_loss: + total_loss = ( + F.cross_entropy(a_logits_per_audio, labels) + + F.cross_entropy(a_logits_per_text, labels) + + F.cross_entropy(t_logits_per_audio, labels) + + F.cross_entropy(t_logits_per_text, labels) + ) / 4 + else: + audio_weight = (audio_features@audio_features.T).detach() + audio_weight = (torch.exp(torch.sum(audio_weight, axis=1)/(self.weight_loss_kappa*len(audio_weight)))).detach() + text_weight = (text_features@text_features.T).detach() + text_weight = (torch.exp(torch.sum(text_weight, axis=1)/(self.weight_loss_kappa*len(text_features)))).detach() + total_loss = ( + F.cross_entropy(a_logits_per_audio, labels, weight=audio_weight) + + F.cross_entropy(a_logits_per_text, labels, weight=audio_weight) + + F.cross_entropy(t_logits_per_audio, labels, weight=text_weight) + + F.cross_entropy(t_logits_per_text, labels, weight=text_weight) + ) / 4 + else: + if self.world_size > 1: + all_audio_features, all_text_features = gather_features( + audio_features=audio_features,text_features=text_features, + local_loss=self.local_loss,gather_with_grad=self.gather_with_grad, + rank=self.rank,world_size=self.world_size,use_horovod=self.use_horovod, + mlp_loss=self.mlp_loss + ) + + if self.local_loss: + logits_per_audio = logit_scale_a * audio_features @ all_text_features.T + logits_per_text = logit_scale_a * text_features @ all_audio_features.T + else: + logits_per_audio = logit_scale_a * all_audio_features @ all_text_features.T + logits_per_text = logits_per_audio.T + else: + logits_per_audio = logit_scale_a * audio_features @ text_features.T + logits_per_text = logit_scale_a * text_features @ audio_features.T + + # calculated ground-truth and cache if enabled + num_logits = logits_per_audio.shape[0] + if self.prev_num_logits != num_logits or device not in self.labels: + labels = torch.arange(num_logits, device=device, dtype=torch.long) + if self.world_size > 1 and self.local_loss: + labels = labels + num_logits * self.rank + if self.cache_labels: + self.labels[device] = labels + self.prev_num_logits = num_logits + else: + labels = self.labels[device] + if not self.weighted_loss: + total_loss = ( + F.cross_entropy(logits_per_audio, labels) + + F.cross_entropy(logits_per_text, labels) + ) / 2 + else: + audio_weight = (all_audio_features@all_audio_features.T).detach() + audio_weight = (torch.exp(torch.sum(audio_weight, axis=1)/(self.weight_loss_kappa*len(all_audio_features)))).detach() + text_weight = (all_text_features@all_text_features.T).detach() + text_weight = (torch.exp(torch.sum(text_weight, axis=1)/(self.weight_loss_kappa*len(all_text_features)))).detach() + total_loss = ( + F.cross_entropy(logits_per_audio, labels, weight=text_weight) + + F.cross_entropy(logits_per_text, labels, weight=audio_weight) + ) / 2 + return total_loss + +def lp_gather_features( + pred, + target, + world_size=1, + use_horovod=False +): + if use_horovod: + assert hvd is not None, 'Please install horovod' + with torch.no_grad(): + all_preds = hvd.allgather(pred) + all_targets = hvd.allgath(target) + else: + gathered_preds = [torch.zeros_like(pred) for _ in range(world_size)] + gathered_targets = [torch.zeros_like(target) for _ in range(world_size)] + + dist.all_gather(gathered_preds, pred) + dist.all_gather(gathered_targets, target) + all_preds = torch.cat(gathered_preds, dim=0) + all_targets = torch.cat(gathered_targets, dim=0) + + return all_preds, all_targets + + +def get_map(pred, target): + pred = torch.sigmoid(pred).numpy() + target = target.numpy() + return np.mean(average_precision_score(target, pred, average=None)) + +def get_acc(pred, target): + pred = torch.argmax(pred,1).numpy() + target = torch.argmax(target,1).numpy() + return accuracy_score(target, pred) + +def get_mauc(pred, target): + pred = torch.sigmoid(pred).numpy() + target = target.numpy() + return np.mean(roc_auc_score(target, pred, average=None)) + + +class LPMetrics(object): + def __init__(self, metric_names = ['map','acc','mauc']): + self.metrics = [] + for name in metric_names: + self.metrics.append(self.get_metric(name)) + self.metric_names = metric_names + + def get_metric(self,name): + if name == 'map': + return get_map + elif name == 'acc': + return get_acc + elif name == 'mauc': + return get_mauc + else: + raise ValueError(f'the metric should be at least one of [map, acc, mauc]') + + def evaluate_mertics(self, pred, target): + metric_dict = {} + for i in range(len(self.metric_names)): + metric_dict[self.metric_names[i]] = self.metrics[i](pred, target) + return metric_dict + + +def calc_celoss(pred, target): + target = torch.argmax(target, 1).long() + return nn.CrossEntropyLoss()(pred, target) + + +class LPLoss(nn.Module): + + def __init__(self, loss_name): + super().__init__() + if loss_name == 'bce': + self.loss_func = nn.BCEWithLogitsLoss() + elif loss_name == 'ce': + self.loss_func = calc_celoss + elif loss_name == 'mse': + self.loss_func = nn.MSELoss() + else: + raise ValueError(f'the loss func should be at least one of [bce, ce, mse]') + + def forward(self, pred, target): + loss = self.loss_func(pred, target) + return loss + \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model.py b/utils/laion_clap/clap_module/model.py new file mode 100644 index 0000000000000000000000000000000000000000..60663ec2658cb302f093625c5ce02fc843e6a5bc --- /dev/null +++ b/utils/laion_clap/clap_module/model.py @@ -0,0 +1,892 @@ +""" CLAP Model + +Adapted from CLIP: https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI. +Adapted to the Audio Task. +""" + +from collections import OrderedDict +from dataclasses import dataclass +from email.mime import audio +from typing import Tuple, Union, Callable, Optional + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from .timm_model import TimmModel +import logging +from .utils import freeze_batch_norm_2d + +from .pann_model import create_pann_model +from .htsat import create_htsat_model +from transformers import BertModel, RobertaModel, BartModel +from transformers.tokenization_utils_base import BatchEncoding + + +class MLPLayers(nn.Module): + def __init__(self, units=[512, 512, 512], nonlin=nn.ReLU(), dropout=0.1): + super(MLPLayers, self).__init__() + self.nonlin = nonlin + self.dropout = dropout + + sequence = [] + for u0, u1 in zip(units[:-1], units[1:]): + sequence.append(nn.Linear(u0, u1)) + sequence.append(self.nonlin) + sequence.append(nn.Dropout(self.dropout)) + sequence = sequence[:-2] + + self.sequential = nn.Sequential(*sequence) + + def forward(self, X): + X = self.sequential(X) + return X + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict( + [ + ("-1", nn.AvgPool2d(stride)), + ( + "0", + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False, + ), + ), + ("1", nn.BatchNorm2d(planes * self.expansion)), + ] + ) + ) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + def __init__( + self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None + ): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5 + ) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute( + 2, 0, 1 + ) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias] + ), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False, + ) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, layers, output_dim, heads, image_size=224, width=64): + super().__init__() + self.output_dim = output_dim + self.image_size = image_size + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False + ) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False + ) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(image_size // 32, embed_dim, heads, output_dim) + + self.init_parameters() + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def init_parameters(self): + if self.attnpool is not None: + std = self.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.attnpool.c_proj.weight, std=std) + + for resnet_block in [self.layer1, self.layer2, self.layer3, self.layer4]: + for name, param in resnet_block.named_parameters(): + if name.endswith("bn3.weight"): + nn.init.zeros_(param) + + def lock(self, unlocked_groups=0, freeze_bn_stats=False): + assert ( + unlocked_groups == 0 + ), "partial locking not currently supported for this model" + for param in self.parameters(): + param.requires_grad = False + if freeze_bn_stats: + freeze_batch_norm_2d(self) + + def stem(self, x): + for conv, bn in [ + (self.conv1, self.bn1), + (self.conv2, self.bn2), + (self.conv3, self.bn3), + ]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + def forward(self, x): + x = self.stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps) + return x.to(orig_type) + + +class QuickGELU(nn.Module): + # NOTE This is slower than nn.GELU or nn.SiLU and uses more GPU memory + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + def __init__(self, d_model: int, n_head: int, act_layer: Callable = nn.GELU): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict( + [ + ("c_fc", nn.Linear(d_model, d_model * 4)), + ("gelu", act_layer()), + ("c_proj", nn.Linear(d_model * 4, d_model)), + ] + ) + ) + self.ln_2 = LayerNorm(d_model) + + def attention(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None): + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask)[0] + + def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None): + x = x + self.attention(self.ln_1(x), attn_mask=attn_mask) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + def __init__( + self, width: int, layers: int, heads: int, act_layer: Callable = nn.GELU + ): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.ModuleList( + [ + ResidualAttentionBlock(width, heads, act_layer=act_layer) + for _ in range(layers) + ] + ) + + def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None): + for r in self.resblocks: + x = r(x, attn_mask=attn_mask) + return x + + +class VisualTransformer(nn.Module): + def __init__( + self, + image_size: int, + patch_size: int, + width: int, + layers: int, + heads: int, + output_dim: int, + act_layer: Callable = nn.GELU, + ): + super().__init__() + self.image_size = image_size + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False, + ) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter( + scale * torch.randn((image_size // patch_size) ** 2 + 1, width) + ) + self.ln_pre = LayerNorm(width) + + self.text_branch = Transformer(width, layers, heads, act_layer=act_layer) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def lock(self, unlocked_groups=0, freeze_bn_stats=False): + assert ( + unlocked_groups == 0 + ), "partial locking not currently supported for this model" + for param in self.parameters(): + param.requires_grad = False + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + x = torch.cat( + [ + self.class_embedding.to(x.dtype) + + torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device + ), + x, + ], + dim=1, + ) # shape = [*, grid ** 2 + 1, width] + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.text_branch(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + +@dataclass +class CLAPVisionCfg: + layers: Union[Tuple[int, int, int, int], int] = 12 + width: int = 768 + patch_size: int = 16 + image_size: Union[Tuple[int, int], int] = 224 + timm_model_name: str = ( + None # a valid model name overrides layers, width, patch_size + ) + timm_model_pretrained: bool = ( + False # use (imagenet) pretrained weights for named model + ) + timm_pool: str = ( + "avg" # feature pooling for timm model ('abs_attn', 'rot_attn', 'avg', '') + ) + timm_proj: str = ( + "linear" # linear projection for timm model output ('linear', 'mlp', '') + ) + + +# Audio Config Class +@dataclass +class CLAPAudioCfp: + model_type: str = "PANN" + model_name: str = "Cnn14" + sample_rate: int = 48000 + # Param + audio_length: int = 1024 + window_size: int = 1024 + hop_size: int = 1024 + fmin: int = 50 + fmax: int = 14000 + class_num: int = 527 + mel_bins: int = 64 + clip_samples: int = 480000 + + +@dataclass +class CLAPTextCfg: + context_length: int + vocab_size: int + width: int + heads: int + layers: int + model_type: str + + +class CLAP(nn.Module): + def __init__( + self, + embed_dim: int, + audio_cfg: CLAPAudioCfp, + text_cfg: CLAPTextCfg, + quick_gelu: bool = False, + enable_fusion: bool = False, + fusion_type: str = 'None', + joint_embed_shape: int = 512, + mlp_act: str = 'relu', + ): + super().__init__() + if isinstance(audio_cfg, dict): + audio_cfg = CLAPAudioCfp(**audio_cfg) + if isinstance(text_cfg, dict): + text_cfg = CLAPTextCfg(**text_cfg) + + self.audio_cfg = audio_cfg + self.text_cfg = text_cfg + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + self.joint_embed_shape = joint_embed_shape + self.mlp_act = mlp_act + + + self.context_length = text_cfg.context_length + + # OpenAI models are pretrained w/ QuickGELU but native nn.GELU is both faster and more + # memory efficient in recent PyTorch releases (>= 1.10). + # NOTE: timm models always use native GELU regardless of quick_gelu flag. + act_layer = QuickGELU if quick_gelu else nn.GELU + + if mlp_act == 'relu': + mlp_act_layer = nn.ReLU() + elif mlp_act == 'gelu': + mlp_act_layer = nn.GELU() + else: + raise NotImplementedError + + # audio branch + # audio branch parameters + if audio_cfg.model_type == "PANN": + self.audio_branch = create_pann_model(audio_cfg, enable_fusion, fusion_type) + elif audio_cfg.model_type == "HTSAT": + self.audio_branch = create_htsat_model(audio_cfg, enable_fusion, fusion_type) + else: + logging.error(f"Model config for {audio_cfg.model_type} not found") + raise RuntimeError(f"Model config for {audio_cfg.model_type} not found.") + + # text branch + # text branch parameters + if text_cfg.model_type == "transformer": + self.text_branch = Transformer( + width=text_cfg.width, + layers=text_cfg.layers, + heads=text_cfg.heads, + act_layer=act_layer, + ) + self.vocab_size = text_cfg.vocab_size + self.token_embedding = nn.Embedding(text_cfg.vocab_size, text_cfg.width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, text_cfg.width) + ) + self.ln_final = LayerNorm(text_cfg.width) + self.text_transform = MLPLayers(units=[self.joint_embed_shape, + self.joint_embed_shape, + self.joint_embed_shape], dropout=0.1) + self.text_projection = nn.Sequential( + nn.Linear(text_cfg.width, self.joint_embed_shape), + mlp_act_layer, + nn.Linear(self.joint_embed_shape, self.joint_embed_shape) + ) + elif text_cfg.model_type == "bert": + self.text_branch = BertModel.from_pretrained("bert-base-uncased") + self.text_transform = MLPLayers(units=[self.joint_embed_shape, + self.joint_embed_shape, + self.joint_embed_shape], dropout=0.1) + self.text_projection = nn.Sequential( + nn.Linear(768, self.joint_embed_shape), + mlp_act_layer, + nn.Linear(self.joint_embed_shape, self.joint_embed_shape) + ) + elif text_cfg.model_type == "roberta": + self.text_branch = RobertaModel.from_pretrained('roberta-base') + self.text_transform = MLPLayers(units=[self.joint_embed_shape, + self.joint_embed_shape, + self.joint_embed_shape], dropout=0.1) + self.text_projection = nn.Sequential( + nn.Linear(768, self.joint_embed_shape), + mlp_act_layer, + nn.Linear(self.joint_embed_shape, self.joint_embed_shape) + ) + elif text_cfg.model_type == "bart": + self.text_branch = BartModel.from_pretrained('facebook/bart-base') + self.text_transform = MLPLayers(units=[self.joint_embed_shape, + self.joint_embed_shape, + self.joint_embed_shape], dropout=0.1) + self.text_projection = nn.Sequential( + nn.Linear(768, self.joint_embed_shape), + mlp_act_layer, + nn.Linear(self.joint_embed_shape, self.joint_embed_shape) + ) + else: + logging.error(f"Model config for {text_cfg.model_type} not found") + raise RuntimeError(f"Model config for {text_cfg.model_type} not found.") + self.text_branch_type = text_cfg.model_type + # text branch parameters + + # audio branch parameters + self.audio_transform = MLPLayers(units=[self.joint_embed_shape, + self.joint_embed_shape, + self.joint_embed_shape], dropout=0.1) + + # below here is text branch parameters + + # ============================================================================================================ + self.audio_projection = nn.Sequential( + nn.Linear(embed_dim, self.joint_embed_shape), + mlp_act_layer, + nn.Linear(self.joint_embed_shape, self.joint_embed_shape) + ) + + self.logit_scale_a = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + self.logit_scale_t = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + self.register_buffer("attn_mask", self.build_attention_mask(), persistent=False) + + self.init_text_branch_parameters() + + def init_text_branch_parameters(self): + if self.text_branch_type == "transformer": + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + proj_std = (self.text_branch.width**-0.5) * ( + (2 * self.text_branch.layers) ** -0.5 + ) + attn_std = self.text_branch.width**-0.5 + fc_std = (2 * self.text_branch.width) ** -0.5 + for block in self.text_branch.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + if self.text_branch_type == "bert" or self.text_branch_type == "roberta": + width = self.text_branch.embeddings.word_embeddings.weight.shape[-1] + elif self.text_branch_type == "bart": + width = self.text_branch.shared.weight.shape[-1] + else: + width = self.text_branch.width + nn.init.constant_(self.logit_scale_a, np.log(1 / 0.07)) + nn.init.constant_(self.logit_scale_t, np.log(1 / 0.07)) + + # deprecated + # if hasattr(self.visual, 'init_parameters'): + # self.visual.init_parameters() + + # if self.text_projection is not None: + # nn.init.normal_(self.text_projection, std=width**-0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float("-inf")) + mask.triu_(1) # zero out the lower diagonal + return mask + + def encode_audio(self, audio, device): + return self.audio_branch(audio, mixup_lambda=None, device=device) # mix lambda needs to add + + # def list_of_dict_of_tensor2dict_of_tensor(self, x, device): + # tmp = {} + # for k in x[0].keys(): + # tmp[k] = [] + # for i in range(len(x)): + # tmp[k].append(x[i][k][:77]) + # for k in x[0].keys(): + # tmp[k] = torch.tensor(tmp[k]).to(device=device, non_blocking=True) + # return tmp + + def encode_text(self, text, device): + if self.text_branch_type == "transformer": + text = text.to(device=device, non_blocking=True) + x = self.token_embedding(text) # [batch_size, n_ctx, d_model] + + x = x + self.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.text_branch(x, attn_mask=self.attn_mask) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = self.text_projection(x[torch.arange(x.shape[0]), text.argmax(dim=-1)]) + elif self.text_branch_type == "bert": + # text = self.list_of_dict_of_tensor2dict_of_tensor(text, device) + # text = BatchEncoding(text) + x = self.text_branch( + input_ids=text["input_ids"].to(device=device, non_blocking=True), + attention_mask=text["attention_mask"].to( + device=device, non_blocking=True + ), + token_type_ids=text["token_type_ids"].to( + device=device, non_blocking=True + ), + )["pooler_output"] + x = self.text_projection(x) + elif self.text_branch_type == "roberta": + x = self.text_branch( + input_ids=text["input_ids"].to(device=device, non_blocking=True), + attention_mask=text["attention_mask"].to( + device=device, non_blocking=True + ), + )["pooler_output"] + x = self.text_projection(x) + elif self.text_branch_type == "bart": + x = torch.mean(self.text_branch( + input_ids=text["input_ids"].to(device=device, non_blocking=True), + attention_mask=text["attention_mask"].to( + device=device, non_blocking=True + ), + )["encoder_last_hidden_state"],axis=1) + x = self.text_projection(x) + else: + logging.error(f"Model type {self.text_branch_type} not found") + raise RuntimeError(f"Model type {self.text_branch_type} not found.") + return x + + def forward(self, audio, text, device=None): + """Forward audio and text into the CLAP + + Parameters + ---------- + audio: torch.Tensor (batch_size, audio_length) + the time-domain audio input / the batch of mel_spec and longer list. + text: torch.Tensor () // need to add + the text token input + """ + if device is None: + if audio is not None: + device = audio.device + elif text is not None: + device = text.device + if audio is None and text is None: + # a hack to get the logit scale + return self.logit_scale_a.exp(), self.logit_scale_t.exp() + elif audio is None: + return self.encode_text(text, device=device) + elif text is None: + return self.audio_projection(self.encode_audio(audio, device=device)["embedding"]) + audio_features = self.audio_projection(self.encode_audio(audio, device=device)["embedding"]) + audio_features = F.normalize(audio_features, dim=-1) + + text_features = self.encode_text( + text, device=device + ) + # print("text_features", text_features) + # print("text_features.shape", text_features.shape) + # print("text_features.type", type(text_features)) + text_features = F.normalize(text_features, dim=-1) + + audio_features_mlp = self.audio_transform(audio_features) + text_features_mlp = self.text_transform(text_features) + # Four outputs: audio features (basic & MLP), text features (basic & MLP) + return ( + audio_features, + text_features, + audio_features_mlp, + text_features_mlp, + self.logit_scale_a.exp(), + self.logit_scale_t.exp(), + ) + + def get_logit_scale(self): + return self.logit_scale_a.exp(), self.logit_scale_t.exp() + + def get_text_embedding(self, data): + """Get the text embedding from the model + + Parameters + ---------- + data: torch.Tensor + a tensor of text embedding + + Returns + ---------- + text_embed: torch.Tensor + a tensor of text_embeds (N, D) + + """ + device = next(self.parameters()).device + for k in data: + data[k] = data[k].to(device) + text_embeds = self.encode_text(data, device=device) + text_embeds = F.normalize(text_embeds, dim=-1) + + return text_embeds + + def get_audio_embedding(self, data): + """Get the audio embedding from the model + + Parameters + ---------- + data: a list of dict + the audio input dict list from 'get_audio_feature' method + + Returns + ---------- + audio_embed: torch.Tensor + a tensor of audio_embeds (N, D) + + """ + device = next(self.parameters()).device + input_dict = {} + keys = data[0].keys() + for k in keys: + input_dict[k] = torch.cat([d[k].unsqueeze(0) for d in data], dim=0).to(device) + audio_embeds = self.encode_audio(input_dict, device=device)["embedding"] + audio_embeds = self.audio_projection(audio_embeds) + audio_embeds = F.normalize(audio_embeds, dim=-1) + return audio_embeds + + + + def audio_infer(self, audio, hopsize=None, device=None): + """Forward one audio and produce the audio embedding + + Parameters + ---------- + audio: (audio_length) + the time-domain audio input, notice that it must be only one input + hopsize: int + the overlap hopsize as the sliding window + + Returns + ---------- + output_dict: { + key: [n, (embedding_shape)] if "HTS-AT" + or + key: [(embedding_shape)] if "PANN" + } + the list of key values of the audio branch + + """ + + assert not self.training, "the inference mode must be run at eval stage" + output_dict = {} + # PANN + if self.audio_cfg.model_type == "PANN": + audio_input = audio.unsqueeze(dim=0) + output_dict[key] = self.encode_audio(audio_input, device=device)[key].squeeze(dim=0) + elif self.audio_cfg.model_type == "HTSAT": + # repeat + audio_len = len(audio) + k = self.audio_cfg.clip_samples // audio_len + if k > 1: + audio = audio.repeat(k) + audio_len = len(audio) + + if hopsize is None: + hopsize = min(hopsize, audio_len) + + if audio_len > self.audio_cfg.clip_samples: + audio_input = [ + audio[pos : pos + self.audio_cfg.clip_samples].clone() + for pos in range( + 0, audio_len - self.audio_cfg.clip_samples, hopsize + ) + ] + audio_input.append(audio[-self.audio_cfg.clip_samples :].clone()) + audio_input = torch.stack(audio_input) + output_dict[key] = self.encode_audio(audio_input, device=device)[key] + else: + audio_input = audio.unsqueeze(dim=0) + output_dict[key] = self.encode_audio(audio_input, device=device)[key].squeeze(dim=0) + + return output_dict + + +def convert_weights_to_fp16(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(l): + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)): + l.weight.data = l.weight.data.half() + if l.bias is not None: + l.bias.data = l.bias.data.half() + + if isinstance(l, nn.MultiheadAttention): + for attr in [ + *[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], + "in_proj_bias", + "bias_k", + "bias_v", + ]: + tensor = getattr(l, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ["text_projection", "proj"]: + if hasattr(l, name): + attr = getattr(l, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) + + +# Ignore the state dict of the vision part +def build_model_from_openai_state_dict(state_dict: dict, model_cfg, enable_fusion: bool = False, fusion_type: str = 'None'): + + embed_dim = model_cfg["embed_dim"] + audio_cfg = model_cfg["audio_cfg"] + text_cfg = model_cfg["text_cfg"] + context_length = state_dict["positional_embedding"].shape[0] + vocab_size = state_dict["token_embedding.weight"].shape[0] + transformer_width = state_dict["ln_final.weight"].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set( + k.split(".")[2] + for k in state_dict + if k.startswith(f"transformer.resblocks") + ) + ) + + audio_cfg = CLAPAudioCfp(**audio_cfg) + text_cfg = CLAPTextCfg(**text_cfg) + + model = CLAP( + embed_dim, + audio_cfg=audio_cfg, + text_cfg=text_cfg, + quick_gelu=True, # OpenAI models were trained with QuickGELU + enable_fusion=enable_fusion, + fusion_type=fusion_type + ) + state_dict["logit_scale_a"] = state_dict["logit_scale"] + state_dict["logit_scale_t"] = state_dict["logit_scale"] + pop_keys = list(state_dict.keys())[::] + # pop the visual branch saved weights + for key in pop_keys: + if key.startswith("visual."): + state_dict.pop(key, None) + + for key in ["logit_scale", "input_resolution", "context_length", "vocab_size"]: + state_dict.pop(key, None) + + # not use fp16 + # convert_weights_to_fp16(model) + model.load_state_dict(state_dict, strict=False) + return model.eval() + + +def trace_model(model, batch_size=256, device=torch.device("cpu")): + model.eval() + audio_length = model.audio_cfg.audio_length + example_audio = torch.ones((batch_size, audio_length), device=device) + example_text = torch.zeros( + (batch_size, model.context_length), dtype=torch.int, device=device + ) + model = torch.jit.trace_module( + model, + inputs=dict( + forward=(example_audio, example_text), + encode_text=(example_text,), + encode_image=(example_audio,), + ), + ) + model.audio_cfg.audio_length = audio_length # Question: what does this do? + return model diff --git a/utils/laion_clap/clap_module/model_configs/HTSAT-base.json b/utils/laion_clap/clap_module/model_configs/HTSAT-base.json new file mode 100644 index 0000000000000000000000000000000000000000..6cef625a89daf4431f1c9f72e10bc9640eef2ba8 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/HTSAT-base.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 1024, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "HTSAT", + "model_name": "base" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/HTSAT-large.json b/utils/laion_clap/clap_module/model_configs/HTSAT-large.json new file mode 100644 index 0000000000000000000000000000000000000000..699cdb1b16855582606551e4196b24aba2ffd871 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/HTSAT-large.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 2048, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "HTSAT", + "model_name": "large" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/HTSAT-tiny-win-1536.json b/utils/laion_clap/clap_module/model_configs/HTSAT-tiny-win-1536.json new file mode 100644 index 0000000000000000000000000000000000000000..73e42990fe8361a0df502e7f93d29f19f58c9ecb --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/HTSAT-tiny-win-1536.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 768, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1536, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "HTSAT", + "model_name": "tiny" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/HTSAT-tiny.json b/utils/laion_clap/clap_module/model_configs/HTSAT-tiny.json new file mode 100644 index 0000000000000000000000000000000000000000..a6e7821163d9afa81c27345a1e472475b92af169 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/HTSAT-tiny.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 768, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "HTSAT", + "model_name": "tiny" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-10.json b/utils/laion_clap/clap_module/model_configs/PANN-10.json new file mode 100644 index 0000000000000000000000000000000000000000..954ddf62921aed7dde9c37ffffec98a2e96a4ee7 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-10.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 1024, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn10" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-14-fmax-18k.json b/utils/laion_clap/clap_module/model_configs/PANN-14-fmax-18k.json new file mode 100644 index 0000000000000000000000000000000000000000..b7989bc0cd95d0d39049b7524eba508b3e386439 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-14-fmax-18k.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 2048, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 18000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn14" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-14-fmax-8k-20s.json b/utils/laion_clap/clap_module/model_configs/PANN-14-fmax-8k-20s.json new file mode 100644 index 0000000000000000000000000000000000000000..56bdb56bedc304ffa52d8bf5988cea2c1d82d14e --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-14-fmax-8k-20s.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 2048, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 960000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 360, + "fmin": 50, + "fmax": 8000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn14" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-14-tiny-transformer.json b/utils/laion_clap/clap_module/model_configs/PANN-14-tiny-transformer.json new file mode 100644 index 0000000000000000000000000000000000000000..5756e3bebc97cc985f512cb081930fee4e49bec1 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-14-tiny-transformer.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 2048, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn14" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 4 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-14-win-1536.json b/utils/laion_clap/clap_module/model_configs/PANN-14-win-1536.json new file mode 100644 index 0000000000000000000000000000000000000000..5a9e7e208b661619d5e26625e849da1adda8a475 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-14-win-1536.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 2048, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1536, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn14" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-14.json b/utils/laion_clap/clap_module/model_configs/PANN-14.json new file mode 100644 index 0000000000000000000000000000000000000000..39a5134cde1d8c50f4758377c952ef22f07bab41 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-14.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 2048, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn14" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/PANN-6.json b/utils/laion_clap/clap_module/model_configs/PANN-6.json new file mode 100644 index 0000000000000000000000000000000000000000..21ebc344326de260c386ba77e0ad63cf9b04febf --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/PANN-6.json @@ -0,0 +1,23 @@ +{ + "embed_dim": 512, + "audio_cfg": { + "audio_length": 1024, + "clip_samples": 480000, + "mel_bins": 64, + "sample_rate": 48000, + "window_size": 1024, + "hop_size": 480, + "fmin": 50, + "fmax": 14000, + "class_num": 527, + "model_type": "PANN", + "model_name": "Cnn6" + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/RN101-quickgelu.json b/utils/laion_clap/clap_module/model_configs/RN101-quickgelu.json new file mode 100644 index 0000000000000000000000000000000000000000..d0db2c161d13138788c4609d373b023b8454d624 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/RN101-quickgelu.json @@ -0,0 +1,22 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "vision_cfg": { + "image_size": 224, + "layers": [ + 3, + 4, + 23, + 3 + ], + "width": 64, + "patch_size": null + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/RN101.json b/utils/laion_clap/clap_module/model_configs/RN101.json new file mode 100644 index 0000000000000000000000000000000000000000..b88b4d3acbaa701c614ab0ea65fc88fcfe289c32 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/RN101.json @@ -0,0 +1,21 @@ +{ + "embed_dim": 512, + "vision_cfg": { + "image_size": 224, + "layers": [ + 3, + 4, + 23, + 3 + ], + "width": 64, + "patch_size": null + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/RN50-quickgelu.json b/utils/laion_clap/clap_module/model_configs/RN50-quickgelu.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2f91260cdeb043434dc1e893cce81d4ce7f0d1 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/RN50-quickgelu.json @@ -0,0 +1,22 @@ +{ + "embed_dim": 1024, + "quick_gelu": true, + "vision_cfg": { + "image_size": 224, + "layers": [ + 3, + 4, + 6, + 3 + ], + "width": 64, + "patch_size": null + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} diff --git a/utils/laion_clap/clap_module/model_configs/RN50.json b/utils/laion_clap/clap_module/model_configs/RN50.json new file mode 100644 index 0000000000000000000000000000000000000000..33aa884d54fee0076c33676831e49d5e1ffcb8f2 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/RN50.json @@ -0,0 +1,21 @@ +{ + "embed_dim": 1024, + "vision_cfg": { + "image_size": 224, + "layers": [ + 3, + 4, + 6, + 3 + ], + "width": 64, + "patch_size": null + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/RN50x16.json b/utils/laion_clap/clap_module/model_configs/RN50x16.json new file mode 100644 index 0000000000000000000000000000000000000000..3161e1a2c9a839161e652a4d729c2cdc971161db --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/RN50x16.json @@ -0,0 +1,21 @@ +{ + "embed_dim": 768, + "vision_cfg": { + "image_size": 384, + "layers": [ + 6, + 8, + 18, + 8 + ], + "width": 96, + "patch_size": null + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 768, + "heads": 12, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/RN50x4.json b/utils/laion_clap/clap_module/model_configs/RN50x4.json new file mode 100644 index 0000000000000000000000000000000000000000..e155237f8ce1026aaaeecc80751eabe6f329f0bb --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/RN50x4.json @@ -0,0 +1,21 @@ +{ + "embed_dim": 640, + "vision_cfg": { + "image_size": 288, + "layers": [ + 4, + 6, + 10, + 6 + ], + "width": 80, + "patch_size": null + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 640, + "heads": 10, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/ViT-B-16.json b/utils/laion_clap/clap_module/model_configs/ViT-B-16.json new file mode 100644 index 0000000000000000000000000000000000000000..395eea77ec3907c0611531aba63459b193e67b9c --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/ViT-B-16.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "vision_cfg": { + "image_size": 224, + "layers": 12, + "width": 768, + "patch_size": 16 + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/ViT-B-32-quickgelu.json b/utils/laion_clap/clap_module/model_configs/ViT-B-32-quickgelu.json new file mode 100644 index 0000000000000000000000000000000000000000..ce6bd923593293ed50dfcfb28b73ca7403bcf3c5 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/ViT-B-32-quickgelu.json @@ -0,0 +1,17 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "vision_cfg": { + "image_size": 224, + "layers": 12, + "width": 768, + "patch_size": 32 + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/ViT-B-32.json b/utils/laion_clap/clap_module/model_configs/ViT-B-32.json new file mode 100644 index 0000000000000000000000000000000000000000..07c8e28eb06fa1813ba932fe4eec668262d1c47f --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/ViT-B-32.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "vision_cfg": { + "image_size": 224, + "layers": 12, + "width": 768, + "patch_size": 32 + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 512, + "heads": 8, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/model_configs/ViT-L-14.json b/utils/laion_clap/clap_module/model_configs/ViT-L-14.json new file mode 100644 index 0000000000000000000000000000000000000000..d4a4bbb1dd4ed4edb317d3ace4f3ad13b211c241 --- /dev/null +++ b/utils/laion_clap/clap_module/model_configs/ViT-L-14.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 768, + "vision_cfg": { + "image_size": 224, + "layers": 24, + "width": 1024, + "patch_size": 14 + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "width": 768, + "heads": 12, + "layers": 12 + } +} \ No newline at end of file diff --git a/utils/laion_clap/clap_module/openai.py b/utils/laion_clap/clap_module/openai.py new file mode 100644 index 0000000000000000000000000000000000000000..9911b6e135e51970177fcac067c12192b0b57c1c --- /dev/null +++ b/utils/laion_clap/clap_module/openai.py @@ -0,0 +1,129 @@ +""" OpenAI pretrained model functions + +Adapted from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI. +""" + +import os +import warnings +from typing import Union, List + +import torch + +from .model import build_model_from_openai_state_dict +from .pretrained import get_pretrained_url, list_pretrained_tag_models, download_pretrained + +__all__ = ["list_openai_models", "load_openai_model"] + + +def list_openai_models() -> List[str]: + """Returns the names of available CLIP models""" + return list_pretrained_tag_models('openai') + + +def load_openai_model( + name: str, + model_cfg, + device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", + jit=True, + cache_dir=os.path.expanduser("~/.cache/clip"), + enable_fusion: bool = False, + fusion_type: str = 'None' +): + """Load a CLIP model, preserve its text pretrained part, and set in the CLAP model + + Parameters + ---------- + name : str + A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict + device : Union[str, torch.device] + The device to put the loaded model + jit : bool + Whether to load the optimized JIT model (default) or more hackable non-JIT model. + + Returns + ------- + model : torch.nn.Module + The CLAP model + preprocess : Callable[[PIL.Image], torch.Tensor] + A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input + """ + if get_pretrained_url(name, 'openai'): + model_path = download_pretrained(get_pretrained_url(name, 'openai'), root=cache_dir) + elif os.path.isfile(name): + model_path = name + else: + raise RuntimeError(f"Model {name} not found; available models = {list_openai_models()}") + + try: + # loading JIT archive + model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval() + state_dict = None + except RuntimeError: + # loading saved state dict + if jit: + warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead") + jit = False + state_dict = torch.load(model_path, map_location="cpu") + + if not jit: + try: + model = build_model_from_openai_state_dict(state_dict or model.state_dict(), model_cfg, enable_fusion, fusion_type).to(device) + except KeyError: + sd = {k[7:]: v for k, v in state_dict["state_dict"].items()} + model = build_model_from_openai_state_dict(sd, model_cfg, enable_fusion, fusion_type).to(device) + + if str(device) == "cpu": + model.float() + return model + + # patch the device names + device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[]) + device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1] + + def patch_device(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("prim::Constant"): + if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"): + node.copyAttributes(device_node) + + model.apply(patch_device) + patch_device(model.encode_audio) + patch_device(model.encode_text) + + # patch dtype to float32 on CPU + if str(device) == "cpu": + float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[]) + float_input = list(float_holder.graph.findNode("aten::to").inputs())[1] + float_node = float_input.node() + + def patch_float(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("aten::to"): + inputs = list(node.inputs()) + for i in [1, 2]: # dtype can be the second or third argument to aten::to() + if inputs[i].node()["value"] == 5: + inputs[i].node().copyAttributes(float_node) + + model.apply(patch_float) + patch_float(model.encode_audio) + patch_float(model.encode_text) + model.float() + + model.audio_branch.audio_length = model.audio_cfg.audio_length + return model diff --git a/utils/laion_clap/clap_module/pann_model.py b/utils/laion_clap/clap_module/pann_model.py new file mode 100644 index 0000000000000000000000000000000000000000..109db5f418a0bad32cae2452742589ff52a19b85 --- /dev/null +++ b/utils/laion_clap/clap_module/pann_model.py @@ -0,0 +1,543 @@ +# PANNs: Large-Scale Pretrained Audio Neural Networks for Audio Pattern Recognition +# Reference from https://github.com/qiuqiangkong/audioset_tagging_cnn +# Some layers are re-designed for CLAP +import os +os.environ['NUMBA_CACHE_DIR'] = '/tmp/' + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchlibrosa.stft import Spectrogram, LogmelFilterBank +from torchlibrosa.augmentation import SpecAugmentation + +from .utils import do_mixup, interpolate, pad_framewise_output +from .feature_fusion import iAFF, AFF, DAF + + +def init_layer(layer): + """Initialize a Linear or Convolutional layer. """ + nn.init.xavier_uniform_(layer.weight) + + if hasattr(layer, 'bias'): + if layer.bias is not None: + layer.bias.data.fill_(0.) + + +def init_bn(bn): + """Initialize a Batchnorm layer. """ + bn.bias.data.fill_(0.) + bn.weight.data.fill_(1.) + + +class ConvBlock(nn.Module): + def __init__(self, in_channels, out_channels): + + super(ConvBlock, self).__init__() + + self.conv1 = nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=(3, 3), stride=(1, 1), + padding=(1, 1), bias=False) + + self.conv2 = nn.Conv2d(in_channels=out_channels, + out_channels=out_channels, + kernel_size=(3, 3), stride=(1, 1), + padding=(1, 1), bias=False) + + self.bn1 = nn.BatchNorm2d(out_channels) + self.bn2 = nn.BatchNorm2d(out_channels) + + self.init_weight() + + def init_weight(self): + init_layer(self.conv1) + init_layer(self.conv2) + init_bn(self.bn1) + init_bn(self.bn2) + + + def forward(self, input, pool_size=(2, 2), pool_type='avg'): + + x = input + x = F.relu_(self.bn1(self.conv1(x))) + x = F.relu_(self.bn2(self.conv2(x))) + if pool_type == 'max': + x = F.max_pool2d(x, kernel_size=pool_size) + elif pool_type == 'avg': + x = F.avg_pool2d(x, kernel_size=pool_size) + elif pool_type == 'avg+max': + x1 = F.avg_pool2d(x, kernel_size=pool_size) + x2 = F.max_pool2d(x, kernel_size=pool_size) + x = x1 + x2 + else: + raise Exception('Incorrect argument!') + + return x + + +class ConvBlock5x5(nn.Module): + def __init__(self, in_channels, out_channels): + + super(ConvBlock5x5, self).__init__() + + self.conv1 = nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=(5, 5), stride=(1, 1), + padding=(2, 2), bias=False) + + self.bn1 = nn.BatchNorm2d(out_channels) + + self.init_weight() + + def init_weight(self): + init_layer(self.conv1) + init_bn(self.bn1) + + + def forward(self, input, pool_size=(2, 2), pool_type='avg'): + + x = input + x = F.relu_(self.bn1(self.conv1(x))) + if pool_type == 'max': + x = F.max_pool2d(x, kernel_size=pool_size) + elif pool_type == 'avg': + x = F.avg_pool2d(x, kernel_size=pool_size) + elif pool_type == 'avg+max': + x1 = F.avg_pool2d(x, kernel_size=pool_size) + x2 = F.max_pool2d(x, kernel_size=pool_size) + x = x1 + x2 + else: + raise Exception('Incorrect argument!') + + return x + + +class AttBlock(nn.Module): + def __init__(self, n_in, n_out, activation='linear', temperature=1.): + super(AttBlock, self).__init__() + + self.activation = activation + self.temperature = temperature + self.att = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True) + self.cla = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True) + + self.bn_att = nn.BatchNorm1d(n_out) + self.init_weights() + + def init_weights(self): + init_layer(self.att) + init_layer(self.cla) + init_bn(self.bn_att) + + def forward(self, x): + # x: (n_samples, n_in, n_time) + norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1) + cla = self.nonlinear_transform(self.cla(x)) + x = torch.sum(norm_att * cla, dim=2) + return x, norm_att, cla + + def nonlinear_transform(self, x): + if self.activation == 'linear': + return x + elif self.activation == 'sigmoid': + return torch.sigmoid(x) + + +class Cnn14(nn.Module): + def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, + fmax, classes_num, enable_fusion=False, fusion_type='None'): + + super(Cnn14, self).__init__() + + window = 'hann' + center = True + pad_mode = 'reflect' + ref = 1.0 + amin = 1e-10 + top_db = None + + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, + win_length=window_size, window=window, center=center, pad_mode=pad_mode, + freeze_parameters=True) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, + n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, + freeze_parameters=True) + + # Spec augmenter + self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, + freq_drop_width=8, freq_stripes_num=2) + + self.bn0 = nn.BatchNorm2d(64) + + if (self.enable_fusion) and (self.fusion_type == 'channel_map'): + self.conv_block1 = ConvBlock(in_channels=4, out_channels=64) + else: + self.conv_block1 = ConvBlock(in_channels=1, out_channels=64) + self.conv_block2 = ConvBlock(in_channels=64, out_channels=128) + self.conv_block3 = ConvBlock(in_channels=128, out_channels=256) + self.conv_block4 = ConvBlock(in_channels=256, out_channels=512) + self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024) + self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048) + + self.fc1 = nn.Linear(2048, 2048, bias=True) + self.fc_audioset = nn.Linear(2048, classes_num, bias=True) + + if (self.enable_fusion) and (self.fusion_type in ['daf_1d','aff_1d','iaff_1d']): + self.mel_conv1d = nn.Sequential( + nn.Conv1d(64, 64, kernel_size=5, stride=3, padding=2), + nn.BatchNorm1d(64) # No Relu + ) + if self.fusion_type == 'daf_1d': + self.fusion_model = DAF() + elif self.fusion_type == 'aff_1d': + self.fusion_model = AFF(channels=64, type='1D') + elif self.fusion_type == 'iaff_1d': + self.fusion_model = iAFF(channels=64, type='1D') + + if (self.enable_fusion) and (self.fusion_type in ['daf_2d','aff_2d','iaff_2d']): + self.mel_conv2d = nn.Sequential( + nn.Conv2d(1, 64, kernel_size=(5,5), stride=(6, 2), padding=(2,2)), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + if self.fusion_type == 'daf_2d': + self.fusion_model = DAF() + elif self.fusion_type == 'aff_2d': + self.fusion_model = AFF(channels=64, type='2D') + elif self.fusion_type == 'iaff_2d': + self.fusion_model = iAFF(channels=64, type='2D') + self.init_weight() + + def init_weight(self): + init_bn(self.bn0) + init_layer(self.fc1) + init_layer(self.fc_audioset) + + def forward(self, input, mixup_lambda=None, device=None): + """ + Input: (batch_size, data_length)""" + + if self.enable_fusion and input["longer"].sum() == 0: + # if no audio is longer than 10s, then randomly select one audio to be longer + input["longer"][torch.randint(0, input["longer"].shape[0], (1,))] = True + + if not self.enable_fusion: + x = self.spectrogram_extractor(input['waveform'].to(device=device, non_blocking=True)) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + else: + longer_list = input["longer"].to(device=device, non_blocking=True) + x = input["mel_fusion"].to(device=device, non_blocking=True) + longer_list_idx = torch.where(longer_list)[0] + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + if self.fusion_type in ['daf_1d','aff_1d','iaff_1d']: + new_x = x[:,0:1,:,:].clone().contiguous() + # local processing + if len(longer_list_idx) > 0: + fusion_x_local = x[longer_list_idx,1:,:,:].clone().contiguous() + FB,FC,FT,FF = fusion_x_local.size() + fusion_x_local = fusion_x_local.view(FB * FC, FT, FF) + fusion_x_local = torch.permute(fusion_x_local, (0,2,1)).contiguous() + fusion_x_local = self.mel_conv1d(fusion_x_local) + fusion_x_local = fusion_x_local.view(FB,FC,FF,fusion_x_local.size(-1)) + fusion_x_local = torch.permute(fusion_x_local, (0,2,1,3)).contiguous().flatten(2) + if fusion_x_local.size(-1) < FT: + fusion_x_local = torch.cat([fusion_x_local, torch.zeros((FB,FF,FT- fusion_x_local.size(-1)), device=device)], dim=-1) + else: + fusion_x_local = fusion_x_local[:,:,:FT] + # 1D fusion + new_x = new_x.squeeze(1).permute((0,2,1)).contiguous() + new_x[longer_list_idx] = self.fusion_model(new_x[longer_list_idx], fusion_x_local) + x = new_x.permute((0,2,1)).contiguous()[:,None,:,:] + else: + x = new_x + elif self.fusion_type in ['daf_2d','aff_2d','iaff_2d','channel_map']: + x = x # no change + + if self.training: + x = self.spec_augmenter(x) + # Mixup on spectrogram + if self.training and mixup_lambda is not None: + x = do_mixup(x, mixup_lambda) + if (self.enable_fusion) and (self.fusion_type in ['daf_2d','aff_2d','iaff_2d']): + global_x = x[:,0:1,:,:] + + # global processing + B, C, H, W = global_x.shape + global_x = self.conv_block1(global_x, pool_size=(2, 2), pool_type='avg') + if len(longer_list_idx) > 0: + local_x = x[longer_list_idx,1:,:,:].contiguous() + TH = global_x.size(-2) + # local processing + B, C, H, W = local_x.shape + local_x = local_x.view(B*C,1,H,W) + local_x = self.mel_conv2d(local_x) + local_x = local_x.view(B,C,local_x.size(1),local_x.size(2),local_x.size(3)) + local_x = local_x.permute((0,2,1,3,4)).contiguous().flatten(2,3) + TB,TC,_,TW = local_x.size() + if local_x.size(-2) < TH: + local_x = torch.cat([local_x, torch.zeros((TB,TC,TH-local_x.size(-2),TW), device=global_x.device)], dim=-2) + else: + local_x = local_x[:,:,:TH,:] + + global_x[longer_list_idx] = self.fusion_model(global_x[longer_list_idx],local_x) + x = global_x + else: + x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg') + + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = torch.mean(x, dim=3) + + latent_x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1) + latent_x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1) + latent_x = latent_x1 + latent_x2 + latent_x = latent_x.transpose(1, 2) + latent_x = F.relu_(self.fc1(latent_x)) + latent_output = interpolate(latent_x, 32) + + + (x1, _) = torch.max(x, dim=2) + x2 = torch.mean(x, dim=2) + x = x1 + x2 + x = F.dropout(x, p=0.5, training=self.training) + x = F.relu_(self.fc1(x)) + embedding = F.dropout(x, p=0.5, training=self.training) + clipwise_output = torch.sigmoid(self.fc_audioset(x)) + + output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding, 'fine_grained_embedding': latent_output} + return output_dict + + +class Cnn6(nn.Module): + def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, + fmax, classes_num, enable_fusion=False, fusion_type='None'): + + super(Cnn6, self).__init__() + + window = 'hann' + center = True + pad_mode = 'reflect' + ref = 1.0 + amin = 1e-10 + top_db = None + + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, + win_length=window_size, window=window, center=center, pad_mode=pad_mode, + freeze_parameters=True) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, + n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, + freeze_parameters=True) + + # Spec augmenter + self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, + freq_drop_width=8, freq_stripes_num=2) + + self.bn0 = nn.BatchNorm2d(64) + + self.conv_block1 = ConvBlock5x5(in_channels=1, out_channels=64) + self.conv_block2 = ConvBlock5x5(in_channels=64, out_channels=128) + self.conv_block3 = ConvBlock5x5(in_channels=128, out_channels=256) + self.conv_block4 = ConvBlock5x5(in_channels=256, out_channels=512) + + self.fc1 = nn.Linear(512, 512, bias=True) + self.fc_audioset = nn.Linear(512, classes_num, bias=True) + + self.init_weight() + + def init_weight(self): + init_bn(self.bn0) + init_layer(self.fc1) + init_layer(self.fc_audioset) + + def forward(self, input, mixup_lambda=None, device=None): + """ + Input: (batch_size, data_length)""" + + x = self.spectrogram_extractor(input) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + + if self.training: + x = self.spec_augmenter(x) + + # Mixup on spectrogram + if self.training and mixup_lambda is not None: + x = do_mixup(x, mixup_lambda) + + x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = torch.mean(x, dim=3) + + latent_x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1) + latent_x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1) + latent_x = latent_x1 + latent_x2 + latent_x = latent_x.transpose(1, 2) + latent_x = F.relu_(self.fc1(latent_x)) + latent_output = interpolate(latent_x, 16) + + (x1, _) = torch.max(x, dim=2) + x2 = torch.mean(x, dim=2) + x = x1 + x2 + x = F.dropout(x, p=0.5, training=self.training) + x = F.relu_(self.fc1(x)) + embedding = F.dropout(x, p=0.5, training=self.training) + clipwise_output = torch.sigmoid(self.fc_audioset(x)) + + output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding, 'fine_grained_embedding': latent_output} + + return output_dict + + +class Cnn10(nn.Module): + def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, + fmax, classes_num, enable_fusion=False, fusion_type='None'): + + super(Cnn10, self).__init__() + + window = 'hann' + center = True + pad_mode = 'reflect' + ref = 1.0 + amin = 1e-10 + top_db = None + + self.enable_fusion = enable_fusion + self.fusion_type = fusion_type + + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, + win_length=window_size, window=window, center=center, pad_mode=pad_mode, + freeze_parameters=True) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, + n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, + freeze_parameters=True) + + # Spec augmenter + self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, + freq_drop_width=8, freq_stripes_num=2) + + self.bn0 = nn.BatchNorm2d(64) + + self.conv_block1 = ConvBlock(in_channels=1, out_channels=64) + self.conv_block2 = ConvBlock(in_channels=64, out_channels=128) + self.conv_block3 = ConvBlock(in_channels=128, out_channels=256) + self.conv_block4 = ConvBlock(in_channels=256, out_channels=512) + self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024) + + self.fc1 = nn.Linear(1024, 1024, bias=True) + self.fc_audioset = nn.Linear(1024, classes_num, bias=True) + + self.init_weight() + + def init_weight(self): + init_bn(self.bn0) + init_layer(self.fc1) + init_layer(self.fc_audioset) + + def forward(self, input, mixup_lambda=None, device=None): + """ + Input: (batch_size, data_length)""" + + x = self.spectrogram_extractor(input) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + + if self.training: + x = self.spec_augmenter(x) + + # Mixup on spectrogram + if self.training and mixup_lambda is not None: + x = do_mixup(x, mixup_lambda) + + x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg') + x = F.dropout(x, p=0.2, training=self.training) + x = torch.mean(x, dim=3) + + latent_x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1) + latent_x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1) + latent_x = latent_x1 + latent_x2 + latent_x = latent_x.transpose(1, 2) + latent_x = F.relu_(self.fc1(latent_x)) + latent_output = interpolate(latent_x, 32) + + (x1, _) = torch.max(x, dim=2) + x2 = torch.mean(x, dim=2) + x = x1 + x2 + x = F.dropout(x, p=0.5, training=self.training) + x = F.relu_(self.fc1(x)) + embedding = F.dropout(x, p=0.5, training=self.training) + clipwise_output = torch.sigmoid(self.fc_audioset(x)) + + output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding, 'fine_grained_embedding': latent_output} + + return output_dict + + +def create_pann_model(audio_cfg, enable_fusion=False, fusion_type='None'): + try: + ModelProto = eval(audio_cfg.model_name) + model = ModelProto( + sample_rate = audio_cfg.sample_rate, + window_size = audio_cfg.window_size, + hop_size =audio_cfg.hop_size, + mel_bins = audio_cfg.mel_bins, + fmin = audio_cfg.fmin, + fmax = audio_cfg.fmax, + classes_num = audio_cfg.class_num, + enable_fusion = enable_fusion, + fusion_type = fusion_type + ) + return model + except: + raise RuntimeError(f'Import Model for {audio_cfg.model_name} not found, or the audio cfg parameters are not enough.') + diff --git a/utils/laion_clap/clap_module/pretrained.py b/utils/laion_clap/clap_module/pretrained.py new file mode 100644 index 0000000000000000000000000000000000000000..723619a9fd511cf8619def49c4631ec701891b93 --- /dev/null +++ b/utils/laion_clap/clap_module/pretrained.py @@ -0,0 +1,147 @@ +import hashlib +import os +import urllib +import warnings + +from tqdm import tqdm + +_RN50 = dict( + openai="https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt", + yfcc15m="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/rn50-quickgelu-yfcc15m-455df137.pt", + cc12m="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/rn50-quickgelu-cc12m-f000538c.pt" +) + +_RN50_quickgelu = dict( + openai="https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt", + yfcc15m="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/rn50-quickgelu-yfcc15m-455df137.pt", + cc12m="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/rn50-quickgelu-cc12m-f000538c.pt" +) + +_RN101 = dict( + openai="https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt", + yfcc15m="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/rn101-quickgelu-yfcc15m-3e04b30e.pt" +) + +_RN101_quickgelu = dict( + openai="https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt", + yfcc15m="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/rn101-quickgelu-yfcc15m-3e04b30e.pt" +) + +_RN50x4 = dict( + openai="https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt", +) + +_RN50x16 = dict( + openai="https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt", +) + +_RN50x64 = dict( + openai="https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt", +) + +_VITB32 = dict( + openai="https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt", + laion400m_e31="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/vit_b_32-quickgelu-laion400m_e31-d867053b.pt", + laion400m_e32="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/vit_b_32-quickgelu-laion400m_e32-46683a32.pt", + laion400m_avg="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/vit_b_32-quickgelu-laion400m_avg-8a00ab3c.pt", +) + +_VITB32_quickgelu = dict( + openai="https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt", + laion400m_e31="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/vit_b_32-quickgelu-laion400m_e31-d867053b.pt", + laion400m_e32="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/vit_b_32-quickgelu-laion400m_e32-46683a32.pt", + laion400m_avg="https://github.com/mlfoundations/open_clip/releases/download/v0.2-weights/vit_b_32-quickgelu-laion400m_avg-8a00ab3c.pt", +) + +_VITB16 = dict( + openai="https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt", +) + +_VITL14 = dict( + openai="https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt", +) + +_PRETRAINED = { + "RN50": _RN50, + "RN50-quickgelu": _RN50_quickgelu, + "RN101": _RN101, + "RN101-quickgelu": _RN101_quickgelu, + "RN50x4": _RN50x4, + "RN50x16": _RN50x16, + "ViT-B-32": _VITB32, + "ViT-B-32-quickgelu": _VITB32_quickgelu, + "ViT-B-16": _VITB16, + "ViT-L-14": _VITL14, +} + + +def list_pretrained(as_str: bool = False): + """ returns list of pretrained models + Returns a tuple (model_name, pretrain_tag) by default or 'name:tag' if as_str == True + """ + return [':'.join([k, t]) if as_str else (k, t) for k in _PRETRAINED.keys() for t in _PRETRAINED[k].keys()] + + +def list_pretrained_tag_models(tag: str): + """ return all models having the specified pretrain tag """ + models = [] + for k in _PRETRAINED.keys(): + if tag in _PRETRAINED[k]: + models.append(k) + return models + + +def list_pretrained_model_tags(model: str): + """ return all pretrain tags for the specified model architecture """ + tags = [] + if model in _PRETRAINED: + tags.extend(_PRETRAINED[model].keys()) + return tags + + +def get_pretrained_url(model: str, tag: str): + if model not in _PRETRAINED: + return '' + model_pretrained = _PRETRAINED[model] + if tag not in model_pretrained: + return '' + return model_pretrained[tag] + + +def download_pretrained(url: str, root: str = os.path.expanduser("~/.cache/clip")): + os.makedirs(root, exist_ok=True) + filename = os.path.basename(url) + + if 'openaipublic' in url: + expected_sha256 = url.split("/")[-2] + else: + expected_sha256 = '' + + download_target = os.path.join(root, filename) + + if os.path.exists(download_target) and not os.path.isfile(download_target): + raise RuntimeError(f"{download_target} exists and is not a regular file") + + if os.path.isfile(download_target): + if expected_sha256: + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256: + return download_target + else: + warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file") + else: + return download_target + + with urllib.request.urlopen(url) as source, open(download_target, "wb") as output: + with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True) as loop: + while True: + buffer = source.read(8192) + if not buffer: + break + + output.write(buffer) + loop.update(len(buffer)) + + if expected_sha256 and hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256: + raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match") + + return download_target diff --git a/utils/laion_clap/clap_module/timm_model.py b/utils/laion_clap/clap_module/timm_model.py new file mode 100644 index 0000000000000000000000000000000000000000..071dd148c772f398e87ecbfc836dcfa4a3ae01af --- /dev/null +++ b/utils/laion_clap/clap_module/timm_model.py @@ -0,0 +1,106 @@ +""" timm model adapter + +Wraps timm (https://github.com/rwightman/pytorch-image-models) models for use as a vision tower in CLIP model. +""" +from collections import OrderedDict + +import torch.nn as nn + +try: + import timm + from timm.models.layers import Mlp, to_2tuple + from timm.models.layers.attention_pool2d import RotAttentionPool2d + from timm.models.layers.attention_pool2d import AttentionPool2d as AbsAttentionPool2d +except ImportError as e: + timm = None + +from .utils import freeze_batch_norm_2d + + +class TimmModel(nn.Module): + """ timm model adapter + # FIXME this adapter is a work in progress, may change in ways that break weight compat + """ + + def __init__( + self, + model_name, + embed_dim, + image_size=224, + pool='avg', + proj='linear', + drop=0., + pretrained=False): + super().__init__() + if timm is None: + raise RuntimeError("Please `pip install timm` to use timm models.") + + self.image_size = to_2tuple(image_size) + self.trunk = timm.create_model(model_name, pretrained=pretrained) + feat_size = self.trunk.default_cfg.get('pool_size', None) + feature_ndim = 1 if not feat_size else 2 + if pool in ('abs_attn', 'rot_attn'): + assert feature_ndim == 2 + # if attn pooling used, remove both classifier and default pool + self.trunk.reset_classifier(0, global_pool='') + else: + # reset global pool if pool config set, otherwise leave as network default + reset_kwargs = dict(global_pool=pool) if pool else {} + self.trunk.reset_classifier(0, **reset_kwargs) + prev_chs = self.trunk.num_features + + head_layers = OrderedDict() + if pool == 'abs_attn': + head_layers['pool'] = AbsAttentionPool2d(prev_chs, feat_size=feat_size, out_features=embed_dim) + prev_chs = embed_dim + elif pool == 'rot_attn': + head_layers['pool'] = RotAttentionPool2d(prev_chs, out_features=embed_dim) + prev_chs = embed_dim + else: + assert proj, 'projection layer needed if non-attention pooling is used.' + + # NOTE attention pool ends with a projection layer, so proj should usually be set to '' if such pooling is used + if proj == 'linear': + head_layers['drop'] = nn.Dropout(drop) + head_layers['proj'] = nn.Linear(prev_chs, embed_dim) + elif proj == 'mlp': + head_layers['mlp'] = Mlp(prev_chs, 2 * embed_dim, embed_dim, drop=drop) + + self.head = nn.Sequential(head_layers) + + def lock(self, unlocked_groups=0, freeze_bn_stats=False): + """ lock modules + Args: + unlocked_groups (int): leave last n layer groups unlocked (default: 0) + """ + if not unlocked_groups: + # lock full model + for param in self.trunk.parameters(): + param.requires_grad = False + if freeze_bn_stats: + freeze_batch_norm_2d(self.trunk) + else: + # NOTE: partial freeze requires latest timm (master) branch and is subject to change + try: + # FIXME import here until API stable and in an official release + from timm.models.helpers import group_parameters, group_modules + except ImportError: + raise RuntimeError( + 'Please install latest timm `pip install git+https://github.com/rwightman/pytorch-image-models`') + matcher = self.trunk.group_matcher() + gparams = group_parameters(self.trunk, matcher) + max_layer_id = max(gparams.keys()) + max_layer_id = max_layer_id - unlocked_groups + for group_idx in range(max_layer_id + 1): + group = gparams[group_idx] + for param in group: + self.trunk.get_parameter(param).requires_grad = False + if freeze_bn_stats: + gmodules = group_modules(self.trunk, matcher, reverse=True) + gmodules = {k for k, v in gmodules.items() if v <= max_layer_id} + freeze_batch_norm_2d(self.trunk, gmodules) + + def forward(self, x): + x = self.trunk(x) + x = self.head(x) + return x diff --git a/utils/laion_clap/clap_module/tokenizer.py b/utils/laion_clap/clap_module/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..5b4a238b987ce66f2932b11451d916e40816b8a3 --- /dev/null +++ b/utils/laion_clap/clap_module/tokenizer.py @@ -0,0 +1,180 @@ +""" CLIP tokenizer + +Copied from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI. +""" +import gzip +import html +import os +from functools import lru_cache +from typing import Union, List + +import ftfy +import regex as re +import torch + + +@lru_cache() +def default_bpe(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz") + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path: str = default_bpe(), special_tokens=None): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') + merges = merges[1:49152-256-2+1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v+'' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + if not special_tokens: + special_tokens = ['', ''] + else: + special_tokens = ['', ''] + special_tokens + vocab.extend(special_tokens) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {t:t for t in special_tokens} + special = "|".join(special_tokens) + self.pat = re.compile(special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) + + self.vocab_size = len(self.encoder) + self.all_special_ids = [self.encoder[t] for t in special_tokens] + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + ( token[-1] + '',) + pairs = get_pairs(word) + + if not pairs: + return token+'' + + while True: + bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word)-1 and word[i+1] == second: + new_word.append(first+second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') + return text + + +_tokenizer = SimpleTokenizer() + + +def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor: + """ + Returns the tokenized representation of given input string(s) + + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + context_length : int + The context length to use; all CLIP models use 77 as the context length + + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = _tokenizer.encoder[""] + eot_token = _tokenizer.encoder[""] + all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts] + result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + tokens = tokens[:context_length] # Truncate + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/utils/laion_clap/clap_module/transform.py b/utils/laion_clap/clap_module/transform.py new file mode 100644 index 0000000000000000000000000000000000000000..7014c926f153a351d2256c869c67c02d57b30913 --- /dev/null +++ b/utils/laion_clap/clap_module/transform.py @@ -0,0 +1,30 @@ +from torchvision.transforms import Normalize, Compose, RandomResizedCrop, InterpolationMode, ToTensor, Resize, \ + CenterCrop + + +def _convert_to_rgb(image): + return image.convert('RGB') + + +def image_transform( + image_size: int, + is_train: bool, + mean=(0.48145466, 0.4578275, 0.40821073), + std=(0.26862954, 0.26130258, 0.27577711) +): + normalize = Normalize(mean=mean, std=std) + if is_train: + return Compose([ + RandomResizedCrop(image_size, scale=(0.9, 1.0), interpolation=InterpolationMode.BICUBIC), + _convert_to_rgb, + ToTensor(), + normalize, + ]) + else: + return Compose([ + Resize(image_size, interpolation=InterpolationMode.BICUBIC), + CenterCrop(image_size), + _convert_to_rgb, + ToTensor(), + normalize, + ]) diff --git a/utils/laion_clap/clap_module/utils.py b/utils/laion_clap/clap_module/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ee10b9310944e9de9f8e1db1a4104defd0423744 --- /dev/null +++ b/utils/laion_clap/clap_module/utils.py @@ -0,0 +1,389 @@ +import numpy as np +import torch +from torch import nn as nn +from torchvision.ops.misc import FrozenBatchNorm2d +import logging +import h5py +from tqdm import tqdm +import random +import json +import os +import pathlib + +# TODO: (yusong) this not a good place to store those information and does not scale. Need to be fixed later. +dataset_split = { + "audiocaps": ["train", "valid", "test"], + "audioset": ["balanced_train", "unbalanced_train", "eval"], + "BBCSoundEffects": ["train", "test"], + "Clotho": ["train", "test", "valid"], + "free_to_use_sounds": ["train", "test"], + "paramount_motion": ["train", "test"], + "sonniss_game_effects": ["train", "test"], + "wesoundeffects": ["train", "test"], + "MACS": ["train", "test"], + "freesound": ["train", "test"], + "FSD50K": ["train", "test", "valid"], + "fsd50k_class_label": ["train", "test", "valid"], + "esc50": ["train", "test"], + "ESC50_1": ["train", "test"], + "ESC50_2": ["train", "test"], + "ESC50_3": ["train", "test"], + "ESC50_4": ["train", "test"], + "ESC50_5": ["train", "test"], + "audiostock": ["train", "test"], + "freesound_no_overlap_noesc50": ["train", "test"], + "epidemic_sound_effects": ["train", "test"], + "VGGSound": ["train", "test"], + "urbansound8k_class_label": ["train", "test"], + "audioset_t5": ["balanced_train", "unbalanced_train", "eval"], + "audioset_t5_debiased": ["balanced_train", "unbalanced_train", "eval"], + "epidemic_sound_effects_t5": ["train", "test"], + "epidemic_sound_effects_t5_debiased": ["train", "test"], + "WavText5K": ["train", "test"], + "esc50_no_overlap": ["train", "test"], + "usd8k_no_overlap": ["train", "test"], + "fsd50k_200_class_label": ["train", "test", "valid"], + "fma_full": ["train", "test"], + "Genius": ["train", "test"], + "Jamendo": ["train", "test"], + "juno": ["train", "test"], + "CMU_Arctic": ["train", "test"], + "ravdess": ["train", "test"], + "Europarl-st": ["train", "test"], + "common_voice": ["train", "test"], + "Jamendo_16bit": ["train", "test"], + "genius_16bit_128": ["train", "test"], + "juno_16bit": ["train", "test"], + "fma_full_16bit_128": ["train", "test"], + "GTZAN": ["train", "test"], + } + + +def freeze_batch_norm_2d(module, module_match={}, name=""): + """ + Converts all `BatchNorm2d` and `SyncBatchNorm` layers of provided module into `FrozenBatchNorm2d`. If `module` is + itself an instance of either `BatchNorm2d` or `SyncBatchNorm`, it is converted into `FrozenBatchNorm2d` and + returned. Otherwise, the module is walked recursively and submodules are converted in place. + + Args: + module (torch.nn.Module): Any PyTorch module. + module_match (dict): Dictionary of full module names to freeze (all if empty) + name (str): Full module name (prefix) + + Returns: + torch.nn.Module: Resulting module + + Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762 + """ + res = module + is_match = True + if module_match: + is_match = name in module_match + if is_match and isinstance( + module, (nn.modules.batchnorm.BatchNorm2d, nn.modules.batchnorm.SyncBatchNorm) + ): + res = FrozenBatchNorm2d(module.num_features) + res.num_features = module.num_features + res.affine = module.affine + if module.affine: + res.weight.data = module.weight.data.clone().detach() + res.bias.data = module.bias.data.clone().detach() + res.running_mean.data = module.running_mean.data + res.running_var.data = module.running_var.data + res.eps = module.eps + else: + for child_name, child in module.named_children(): + full_child_name = ".".join([name, child_name]) if name else child_name + new_child = freeze_batch_norm_2d(child, module_match, full_child_name) + if new_child is not child: + res.add_module(child_name, new_child) + return res + + +def exist(dataset_name, dataset_type): + """ + Check if dataset exists + """ + if dataset_type in dataset_split[dataset_name]: + return True + else: + return False + + +def get_tar_path_from_dataset_name( + dataset_names, + dataset_types, + islocal, + dataset_path, + proportion=1, + full_dataset=None +): + """ + Get tar path from dataset name and type + """ + output = [] + for n in dataset_names: + if full_dataset is not None and n in full_dataset: + current_dataset_types = dataset_split[n] + else: + current_dataset_types = dataset_types + for s in current_dataset_types: + tmp = [] + if islocal: + sizefilepath_ = f"{dataset_path}/{n}/{s}/sizes.json" + if not os.path.exists(sizefilepath_): + sizefilepath_ = f"./json_files/{n}/{s}/sizes.json" + else: + sizefilepath_ = f"./json_files/{n}/{s}/sizes.json" + if not os.path.exists(sizefilepath_): + continue + sizes = json.load(open(sizefilepath_, "r")) + for k in sizes.keys(): + if islocal: + tmp.append(f"{dataset_path}/{n}/{s}/{k}") + else: + tmp.append( + f"pipe:aws s3 --cli-connect-timeout 0 cp s3://s-laion-audio/webdataset_tar/{n}/{s}/{k} -" + ) + if proportion != 1: + tmp = random.sample(tmp, int(proportion * len(tmp))) + output.append(tmp) + return sum(output, []) + + +def get_tar_path_from_txts(txt_path, islocal, proportion=1): + """ + Get tar path from txt path + """ + if isinstance(txt_path, (list, tuple)): + return sum( + [ + get_tar_path_from_txts( + txt_path[i], islocal=islocal, proportion=proportion + ) + for i in range(len(txt_path)) + ], + [], + ) + if isinstance(txt_path, str): + with open(txt_path) as f: + lines = f.readlines() + if islocal: + lines = [ + lines[i] + .split("\n")[0] + .replace("pipe:aws s3 cp s3://s-laion-audio/", "/mnt/audio_clip/") + for i in range(len(lines)) + ] + else: + lines = [ + lines[i].split("\n")[0].replace(".tar", ".tar -") + for i in range(len(lines)) + ] + if proportion != 1: + print("Sampling tars with proportion of {}".format(proportion)) + lines = random.sample(lines, int(proportion * len(lines))) + return lines + + +def get_mix_lambda(mixup_alpha, batch_size): + mixup_lambdas = [ + np.random.beta(mixup_alpha, mixup_alpha, 1)[0] for _ in range(batch_size) + ] + return np.array(mixup_lambdas).astype(np.float32) + + +def do_mixup(x, mixup_lambda): + """ + Args: + x: (batch_size , ...) + mixup_lambda: (batch_size,) + Returns: + out: (batch_size, ...) + """ + out = ( + x.transpose(0, -1) * mixup_lambda + + torch.flip(x, dims=[0]).transpose(0, -1) * (1 - mixup_lambda) + ).transpose(0, -1) + return out + + +def interpolate(x, ratio): + """Interpolate data in time domain. This is used to compensate the + resolution reduction in downsampling of a CNN. + + Args: + x: (batch_size, time_steps, classes_num) + ratio: int, ratio to interpolate + Returns: + upsampled: (batch_size, time_steps * ratio, classes_num) + """ + (batch_size, time_steps, classes_num) = x.shape + upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1) + upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num) + return upsampled + + +def pad_framewise_output(framewise_output, frames_num): + """Pad framewise_output to the same length as input frames. The pad value + is the same as the value of the last frame. + Args: + framewise_output: (batch_size, frames_num, classes_num) + frames_num: int, number of frames to pad + Outputs: + output: (batch_size, frames_num, classes_num) + """ + pad = framewise_output[:, -1:, :].repeat( + 1, frames_num - framewise_output.shape[1], 1 + ) + """tensor for padding""" + + output = torch.cat((framewise_output, pad), dim=1) + """(batch_size, frames_num, classes_num)""" + + +def process_ipc(index_path, classes_num, filename): + # load data + logging.info("Load Data...............") + ipc = [[] for _ in range(classes_num)] + with h5py.File(index_path, "r") as f: + for i in tqdm(range(len(f["target"]))): + t_class = np.where(f["target"][i])[0] + for t in t_class: + ipc[t].append(i) + print(ipc) + np.save(filename, ipc) + logging.info("Load Data Succeed...............") + + +def save_to_dict(s, o_={}): + sp = s.split(": ") + o_.update({sp[0]: float(sp[1])}) + return o_ + + +def get_data_from_log(txt_path): + """ + Output dictionary from out.txt log file + """ + with open(txt_path) as f: + lines = f.readlines() + val_data = {} + train_data = {} + train_losses = [] + train_losses_epoch = [] + for i in range(len(lines)): + if "| INFO |" in lines[i]: + if "Eval Epoch" in lines[i]: + if "val_loss" in lines[i]: + # float(regex.sub("", lines[310].split(" ")[-1]).replace(" ", "")) + line = lines[i].split("Eval Epoch: ")[-1] + num_epoch = int(line.split(" ")[0].split(" ")[0]) + d = { + line.split(" ")[0] + .split(" ")[1] + .replace(":", ""): float(line.split(" ")[0].split(" ")[-1]) + } + for i in range(1, len(line.split(" "))): + d = save_to_dict(line.split(" ")[i], d) + val_data[num_epoch] = d + elif "Train Epoch" in lines[i]: + num_epoch = int(lines[i].split("Train Epoch: ")[1][0]) + loss = float(lines[i].split("Loss: ")[-1].split(" (")[0]) + train_losses.append(loss) + train_losses_epoch.append(num_epoch) + for i in range(len(train_losses)): + train_data[i] = { + "num_epoch": train_losses_epoch[i], + "train_loss": train_losses[i], + } + return train_data, val_data + + +def save_p(obj, filename): + import pickle + + try: + from deepdiff import DeepDiff + except: + os.system("pip install deepdiff") + from deepdiff import DeepDiff + with open(filename, "wb") as file: + pickle.dump(obj, file, protocol=pickle.HIGHEST_PROTOCOL) # highest protocol + with open(filename, "rb") as file: + z = pickle.load(file) + assert ( + DeepDiff(obj, z, ignore_string_case=True) == {} + ), "there is something wrong with the saving process" + return + + +def load_p(filename): + import pickle + + with open(filename, "rb") as file: + z = pickle.load(file) + return z + + +def save_json(data, name="data.json"): + import json + with open(name, 'w') as fp: + json.dump(data, fp) + return + + +def load_json(name): + import json + with open(name, 'r') as fp: + data = json.load(fp) + return data + + +from multiprocessing import Process, Manager +from multiprocessing import Process, Value, Array +from ctypes import c_wchar + + +def load_class_label(path): + # https://stackoverflow.com/questions/48004243/how-to-share-large-read-only-dictionary-list-across-processes-in-multiprocessing + # https://stackoverflow.com/questions/45693949/storing-strings-in-a-multiprocessing-sharedctypes-array + out = None + if path is not None: + if pathlib.Path(path).suffix in [".pkl", ".pickle"]: + out = load_p(path) + elif pathlib.Path(path).suffix in [".json", ".txt"]: + out = load_json(path) + elif pathlib.Path(path).suffix in [".npy", ".npz"]: + out = np.load(path) + elif pathlib.Path(path).suffix in [".csv"]: + import pandas as pd + out = pd.read_csv(path) + return out + # if out is None: + # return None + # else: + # key = Array(c_wchar, '\n'.join(list(out.keys())), lock=False) + # val = Array('i', out.values(), lock=False) + # return (key, val) + + +from torch import optim + + +def get_optimizer(params, lr, betas, eps, momentum, optimizer_name): + if optimizer_name.lower() == "adamw": + optimizer = optim.AdamW( + params, lr=lr, betas=betas, eps=eps + ) + elif optimizer_name.lower() == "sgd": + optimizer = optim.SGD( + params, lr=lr, momentum=momentum + ) + elif optimizer_name.lower() == "adam": + optimizer = optim.Adam( + params, lr=lr, betas=betas, eps=eps + ) + else: + raise ValueError("optimizer name is not correct") + return optimizer diff --git a/utils/laion_clap/clap_module/version.py b/utils/laion_clap/clap_module/version.py new file mode 100644 index 0000000000000000000000000000000000000000..fc79d63d5430b972ac6ec1c4bfea9af80922da4d --- /dev/null +++ b/utils/laion_clap/clap_module/version.py @@ -0,0 +1 @@ +__version__ = '0.2.1' diff --git a/utils/laion_clap/evaluate/__init__.py b/utils/laion_clap/evaluate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/laion_clap/evaluate/eval_dcase.py b/utils/laion_clap/evaluate/eval_dcase.py new file mode 100644 index 0000000000000000000000000000000000000000..c615651f2d96f7e34d109e9c3dbb8abc7275065f --- /dev/null +++ b/utils/laion_clap/evaluate/eval_dcase.py @@ -0,0 +1,150 @@ +import torch +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +from open_clip import create_model +from open_clip import tokenize +import glob +import json +import librosa +from tqdm import tqdm +import numpy as np +import os +from laion_clap.training.params import parse_args + + +def get_output_from_single_audio(audio, text, model, device): + + # audio_embedding = model.audio_infer(audio, hopsize=5 * 48000, key="embedding", device=device)['embedding'] + # if audio_embedding.ndim > 1: + # audio_embedding = audio_embedding.mean(dim=0, keepdim=True) + # else: + # audio_embedding = audio_embedding.unsqueeze(0) + audio_features = model(audio, None, device) + audio_features = F.normalize(audio_features, dim=-1) + text_features = model(None, text, device=device) + text_features = F.normalize(text_features, dim=-1) + + # CHANGE: before normalize or after + audio_features_mlp = model.audio_transform(audio_features) + text_features_mlp = model.text_transform(text_features) + return audio_features, text_features, audio_features_mlp, text_features_mlp, model.logit_scale_a.exp(), model.logit_scale_t.exp() + + +def get_metrics(text_to_audio_logits): + metrics = {} + + # repeat ground truth 5 times because Clotho has 5 text for 1 audio + ground_truth = torch.repeat_interleave(torch.arange(len(text_features) // 5), 5).view(-1, 1) + + ranking = torch.argsort(text_to_audio_logits, descending=True) + preds = torch.where(ranking == ground_truth)[1] # (yusong) this line is slow because it uses single thread + preds = preds.detach().cpu().numpy() + metrics[f"mean_rank"] = preds.mean() + 1 + metrics[f"median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"R@{k}"] = np.mean(preds < k) + # map@10 + metrics[f"mAP@10"] = np.mean(np.where(preds < 10, 1 / (preds + 1), 0.0)) + return metrics + + +if __name__ == '__main__': + args = parse_args() + + model_path = args.pretrained + + clotho_test_preprocessed_dir = "/fsx/yusong/clotho_test_set/test" + + cudnn.benchmark = True + cudnn.deterministic = False + + audio_features_ensemble_all = [] + text_features_ensemble_all = [] + audio_features_mlp_ensemble_all = [] + text_features_mlp_ensemble_all = [] + logit_scale_a_ensemble_all = [] + logit_scale_t_ensemble_all = [] + + + device = torch.device('cuda') + model, clap_model_cfg = create_model( + args.amodel, + args.tmodel, + args.pretrained, + precision=args.precision, + device=device, + jit=args.torchscript, + force_quick_gelu=args.force_quick_gelu, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False + ) + + # load model + checkpoint = torch.load(model_path, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module."):]: v for k, v in sd.items()} + model.load_state_dict(sd) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + + model.to(device) + model.eval() + for param in model.parameters(): + param.requires_grad = False + + # take every 5th file because clotho has 5 texts for 1 audio + test_file_list = sorted(glob.glob(f"{clotho_test_preprocessed_dir}/*.flac")) + + audio_features_all = [] + text_features_all = [] + audio_features_mlp_all = [] + text_features_mlp_all = [] + logit_scale_a_all = [] + logit_scale_t_all = [] + + with torch.no_grad(): + for file_path in tqdm(test_file_list): + json_path = file_path.replace(".flac", ".json") + with open(json_path, "r") as f: + json_data = json.load(f) + audio, sr = librosa.load(file_path, sr=48000, mono=True) + audio = torch.from_numpy(audio).to(device) + audio = {'waveform': audio.unsqueeze(0), 'sample_rate': sr} + text = json_data["text"] + + if args.tmodel == "transformer": + from open_clip import tokenize + text = tokenize(text) + else: + from laion_clap.training.data import tokenizer + text = tokenizer(text, tmodel=args.tmodel) # 5 texts for each audio + + audio_features, text_features, audio_features_mlp, text_features_mlp, logit_scale_a, logit_scale_t = \ + get_output_from_single_audio(audio, text, model, device) + + audio_features_all.append(audio_features.detach().cpu()) + text_features_all.append(text_features.detach().cpu()) + audio_features_mlp_all.append(audio_features_mlp.detach().cpu()) + text_features_mlp_all.append(text_features_mlp.detach().cpu()) + logit_scale_a_all.append(logit_scale_a.detach().cpu()) + logit_scale_t_all.append(logit_scale_t.detach().cpu()) + + audio_features = torch.cat(audio_features_all) + text_features = torch.cat(text_features_all) + logit_scale_a = logit_scale_a_all[0] + + logits_per_audio = (logit_scale_a * audio_features @ text_features.t()).detach().cpu() + logits_per_text = logits_per_audio.t().detach().cpu() + + metrics = get_metrics( + logits_per_text + ) + + print(metrics) diff --git a/utils/laion_clap/evaluate/eval_linear_probe.py b/utils/laion_clap/evaluate/eval_linear_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..e5123451b503de7c54fac58e99b3507439992171 --- /dev/null +++ b/utils/laion_clap/evaluate/eval_linear_probe.py @@ -0,0 +1,515 @@ +''' +Evalute the linear probe performance on different checkpoints +''' +import logging +import os +import random +from datetime import datetime +import copy +import numpy as np +import torch +import torch.backends.cudnn as cudnn +from torch.cuda.amp import GradScaler +import glob + +try: + import wandb +except ImportError: + wandb = None + +try: + import torch.utils.tensorboard as tensorboard +except ImportError: + tensorboard = None + +try: + import horovod.torch as hvd +except ImportError: + hvd = None + +from clap_module import create_model_and_transforms, trace_model, create_model +from training.data import get_data +from training.params import parse_args +from training.distributed import is_master, init_distributed_device, world_info_from_env +from training.logger import setup_logging +from training.scheduler import cosine_lr +from training.lp_main import config_lp_optimizer +from training.lp_train import train_one_epoch, evaluate +from clap_module.utils import get_tar_path_from_dataset_name, dataset_split +from clap_module.utils import load_p, load_class_label +from clap_module.linear_probe import LinearProbe + +def maintain_ckpts(args, startidx, all_idx_len): + for i in reversed(range(startidx, all_idx_len)): + if os.path.exists(os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt")): + os.rename( + os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt"), + os.path.join(args.checkpoint_path, f"epoch_top_{i+1}.pt"), + ) + if os.path.exists( + os.path.join(args.checkpoint_path, f"epoch_top_{all_idx_len}.pt") + ): + os.remove(os.path.join(args.checkpoint_path, f"epoch_top_{all_idx_len}.pt")) + return + + +def update_top_k_performance( + new_metrics_inputs, current_top_k_ckpt_metrics, args, ckpt, bignumbetter=True, pretrain_epoch=0 +): + """ + Record the top-k performance of the current epoch. + current_top_k_metrics is a dictionary of the form: {1: top_1_ckpt_measure, 2: top_2_ckpt_measure, ...} + """ + if isinstance(new_metrics_inputs, (list, tuple)): + new_metrics_inputs = np.mean(new_metrics_inputs) + return update_top_k_performance( + new_metrics_inputs, + current_top_k_ckpt_metrics, + args=args, + ckpt=ckpt, + bignumbetter=bignumbetter, + pretrain_epoch=pretrain_epoch + ) + elif isinstance(new_metrics_inputs, dict): + new_metrics_inputs = np.mean(list(new_metrics_inputs.values())) + return update_top_k_performance( + new_metrics_inputs, + current_top_k_ckpt_metrics, + args=args, + ckpt=ckpt, + bignumbetter=bignumbetter, + pretrain_epoch=pretrain_epoch + ) + elif isinstance(new_metrics_inputs, (float, int)): + update_flag = {k: False for k in current_top_k_ckpt_metrics.keys()} + sorted_keys = sorted(current_top_k_ckpt_metrics.keys()) + sorted_values = sorted( + current_top_k_ckpt_metrics.values(), reverse=bignumbetter + ) + sorted_values_ = copy.deepcopy(sorted_values) + sorted_values.append(new_metrics_inputs) + sorted_values = sorted(sorted_values, reverse=bignumbetter) + sorted_values = sorted_values[:-1] + + if sorted_values == sorted_values_: + return current_top_k_ckpt_metrics, new_metrics_inputs + else: + for i in range(len(sorted_keys)): + if current_top_k_ckpt_metrics[sorted_keys[i]] != sorted_values[i]: + current_top_k_ckpt_metrics[sorted_keys[i]] = sorted_values[i] + update_flag[sorted_keys[i]] = True + for i in range(len(update_flag)): + if update_flag[i]: + maintain_ckpts(args, i, len(sorted_keys)) + torch.save( + ckpt, + os.path.join(args.checkpoint_path, f"pretrain_epoch_{pretrain_epoch}_lp_epoch_top_{i}.pt"), + ) + break + return current_top_k_ckpt_metrics, new_metrics_inputs + + +# def updateifNone(a, b): +# a = b if None else a +# return a + + +def is_pretrained_params(n): + return ( + n.startswith("clap_model.transformer") + or n in ["clap_model.positional_embedding", "clap_model.text_projection"] + or n.startswith("clap_model.token_embedding") + or n.startswith("clap_model.ln_final") + or n.startswith("clap_model.logit_scale_t") + ) + + +def random_seed(seed=42, rank=0): + torch.manual_seed(seed + rank) + np.random.seed(seed + rank) + random.seed(seed + rank) + +def main(): + args = parse_args() + # sanitize model name for filesystem / uri use, easier if we don't use / in name as a rule? + args.amodel = args.amodel.replace("/", "-") + + pretrained_ckpts = sorted(glob.glob(os.path.join(args.pretrained, "*.pt")), key=os.path.getmtime) + + if args.name is None: + args.name = "-".join( + [ + datetime.now().strftime("%Y_%m_%d-%H_%M_%S"), + f"linear_probe" + f"model_{args.amodel}", + f"lr_{args.lr}", + f"b_{args.batch_size}", + f"j_{args.workers}", + f"p_{args.precision}", + ] + ) + + # discover initial world args early so we can log properly + args.distributed = False + args.local_rank, args.rank, args.world_size = world_info_from_env() + + if args.remotedata and is_master(args): + for dataset_name in args.datasetnames: + for split in dataset_split[dataset_name]: + if not os.path.exists(f"./json_files/{dataset_name}/{split}"): + os.makedirs(f"./json_files/{dataset_name}/{split}") + os.system( + f"aws s3 cp s3://s-laion-audio/webdataset_tar/{dataset_name}/{split}/sizes.json ./json_files/{dataset_name}/{split}/sizes.json" + ) + args.log_path = None + if is_master(args, local=args.log_local): + log_base_path = os.path.join(args.logs, args.name) + os.makedirs(log_base_path, exist_ok=True) + log_filename = f"out-{args.rank}" if args.log_local else "out.log" + args.log_path = os.path.join(log_base_path, log_filename) + + # avoid log dir in same name: + postfix = 0 + while os.path.exists(args.log_path): + postfix += 1 + log_base_path_new = log_base_path+'-'+str(postfix) + os.makedirs(log_base_path_new, exist_ok=True) + log_filename = f"out-{args.rank}" if args.log_local else "out.log" + args.log_path = os.path.join(log_base_path_new, log_filename) + # print( + # "Error. Experiment already exists. Use --name {} to specify a new experiment." + # ) + # return -1 + + # Set logger + args.log_level = logging.DEBUG if args.debug else logging.INFO + setup_logging(args.log_path, args.log_level) + + # fully initialize distributed device environment + device = init_distributed_device(args) + + args.wandb = "wandb" in args.report_to or "all" in args.report_to + args.tensorboard = "tensorboard" in args.report_to or "all" in args.report_to + if is_master(args): + args.tensorboard_path = ( + os.path.join(args.logs, args.name, "tensorboard") + if args.tensorboard + else "" + ) + args.checkpoint_path = os.path.join(args.logs, args.name, "checkpoints") + for dirname in [args.tensorboard_path, args.checkpoint_path]: + if dirname: + os.makedirs(dirname, exist_ok=True) + else: + args.tensorboard_path = "" + args.checkpoint_path = "" + + if args.copy_codebase: + copy_codebase(args) + + assert args.precision in ["amp", "fp16", "fp32"] + if args.precision == "fp16": + logging.warning( + "It is recommended to use AMP mixed-precision instead of FP16. " + "FP16 support needs further verification and tuning, especially for train." + ) + + if args.horovod: + logging.info( + f"Running in horovod mode with multiple processes / nodes. Device: {args.device}." + f"Process (global: {args.rank}, local {args.local_rank}), total {args.world_size}." + ) + elif args.distributed: + logging.info( + f"Running in distributed mode with multiple processes. Device: {args.device}." + f"Process (global: {args.rank}, local {args.local_rank}), total {args.world_size}." + ) + else: + logging.info(f"Running with a single process. Device {args.device}.") + + logging.info(f'openai cache dir: {os.path.expanduser(args.openai_model_cache_dir)}') + + # determine if this worker should save logs and checkpoints. only do so if it is rank == 0 + args.save_logs = args.logs and args.logs.lower() != "none" and is_master(args) + writer = None + if args.save_logs and args.tensorboard: + assert tensorboard is not None, "Please install tensorboard." + writer = tensorboard.SummaryWriter(args.tensorboard_path) + + if args.wandb and is_master(args): + assert wandb is not None, "Please install wandb." + logging.debug("Starting wandb.") + # you will have to configure this for your project! + wandb.init( + project="clap", + notes=args.wandb_notes, + name=args.wandb_notes, + tags=[], + config=vars(args), + ) + logging.debug("Finished loading wandb.") + + for idx, f in enumerate(pretrained_ckpts): + logging.info(f"pretrained on {f}") + args.pretrained = f + ckpt = torch.load(f, map_location='cpu') + pretrain_epoch = 0 + if 'epoch' in ckpt: + pretrain_epoch = ckpt['epoch'] + # train + best_metrics = lp_main(args, device, writer, pretrain_epoch, idx) + + if args.wandb and is_master(args): + assert wandb is not None, "Please install wandb." + for name, val in best_metrics.items(): + wandb.log({f"val/summary/{name}": val, "epoch": pretrain_epoch}) + + if args.wandb and is_master(args): + wandb.finish() + +def update_metric(best_metric, new_metric): + for key in new_metric: + if key not in best_metric: + best_metric[key] = new_metric[key] + else: + best_metric[key] = max(best_metric[key], new_metric[key]) + return best_metric + +def lp_main(args, device, writer, pretrain_epoch, idx): + + random.seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + np.random.seed(args.seed) + args.class_index_dict = load_class_label(args.class_label_path) + + + # Create CLAP model + clap_model, clap_model_cfg = create_model( + args.amodel, + args.tmodel, + args.pretrained, + precision=args.precision, + device=device, + jit=args.torchscript, + force_quick_gelu=args.force_quick_gelu, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) + + args.lp_out_ch = len(list(args.class_index_dict.keys())) + # Linear Probe + if idx == 0: + logging.info(f"linear probe using mlp: {args.lp_mlp}") + logging.info(f"linear probe using freeze: {args.lp_freeze}") + logging.info(f"linear probe act layer: {args.lp_act}") + logging.info(f"linear probe out ch: {args.lp_out_ch}") + logging.info(f"linear probe learning rate (if applicable): {args.lp_lr}") + logging.info(f"linear probe loss func: {args.lp_loss}") + logging.info(f"linear probe lp_metrics: {args.lp_metrics}") + + model = LinearProbe( + clap_model, + mlp=args.lp_mlp, freeze=args.lp_freeze, + in_ch=512, out_ch=args.lp_out_ch, + act=args.lp_act + ) # in_ch is fixed (i.e., 512) + model = model.to(device) + + if args.horovod: + with torch.no_grad(): + for param in model.parameters(): + param.set_(param.contiguous()) + + if args.trace: + model = trace_model(model, batch_size=args.batch_size, device=device) + + if is_master(args) and idx == 0: + logging.info("Linear Probe CLAP Model:") + logging.info(f"{str(clap_model)}") + logging.info("Params:") + params_file = os.path.join(args.logs, args.name, "params.txt") + with open(params_file, "w") as f: + for name in sorted(vars(args)): + val = getattr(args, name) + logging.info(f" {name}: {val}") + f.write(f"{name}: {val}\n") + + + if args.distributed and not args.horovod: + if args.use_bn_sync: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + ddp_args = {} + if args.ddp_static_graph: + # this doesn't exist in older PyTorch, arg only added if enabled + ddp_args["static_graph"] = True + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[device], find_unused_parameters=True, **ddp_args + ) + + data = get_data(args, clap_model_cfg) + assert len(data), "At least one train or eval dataset must be specified." + if args.trace: + assert "train" not in data, "Cannot train with traced model" + + optimizer, scheduler, text_freeze_parameters = config_lp_optimizer(model, data, args) + + scaler = GradScaler() if args.precision == "amp" else None + + # optionally resume from a checkpoint + start_epoch = 0 + if args.resume is not None: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if not args.distributed and next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module.") :]: v for k, v in sd.items()} + model.load_state_dict(sd) + if args.split_opt: + if optimizer is not None: + for k, o_ in optimizer.items(): + o_.load_state_dict(checkpoint[k + "_" + "optimizer"]) + if optimizer is not None: + optimizer.load_state_dict(checkpoint["optimizer"]) + if scaler is not None and "scaler" in checkpoint: + scaler.load_state_dict(checkpoint["scaler"]) + logging.info( + f"=> resuming checkpoint '{args.resume}' (epoch {start_epoch})" + ) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + logging.info( + f"=> loaded checkpoint '{args.resume}' (epoch {start_epoch})" + ) + if args.freeze_text: + print("Freeze Text!!!!") + for k in text_freeze_parameters: + k.requires_grad = False + else: + logging.info("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + cudnn.deterministic = False + + if args.wandb and is_master(args): + args.train_sz = data["train"].dataloader.num_samples + if args.val_data is not None: + args.val_sz = data["val"].dataloader.num_samples + if args.debug: + wandb.watch(model, log="all") + if idx == 0: + wandb.save(params_file) + + best_metrics = {} + + if "train" not in data: + metric = evaluate(model, data, start_epoch, args, writer, extra_suffix="_pe@" + str(pretrain_epoch)) + if is_master(args): + best_metrics = update_metric(best_metrics, metric) + return + elif start_epoch == 0 and "val" in data and not args.no_eval: + metric = evaluate(model, data, 0, args, writer, extra_suffix="_pe@" + str(pretrain_epoch)) + if is_master(args): + best_metrics = update_metric(best_metrics, metric) + if args.save_top_performance: + current_top_k_ckpt_metrics = { + i: 0 for i in range(args.save_top_performance) + } # initialize the top-k metric for ckpts to 0 + + for epoch in range(start_epoch, args.epochs): + # freeze the text param after (include) args.freeze_text_after, this is -1 by default + if epoch == args.freeze_text_after: + print("Text pretrained parameters are freezed since this epoch.") + for k in text_freeze_parameters: + k.requires_grad = False + if is_master(args): + logging.info(f"Start epoch {epoch}") + + train_one_epoch(model, data, epoch, optimizer, scaler, scheduler, args, writer, extra_suffix="_pe@" + str(pretrain_epoch)) + completed_epoch = epoch + 1 + + if any(v in data for v in ("val", "imagenet-val", "imagenet-v2")) and not args.no_eval: + metric = evaluate(model, data, completed_epoch, args, writer, extra_suffix="_pe@" + str(pretrain_epoch)) + if is_master(args): + best_metrics = update_metric(best_metrics, metric) + if args.save_top_performance: + top_k_dataset = args.top_k_checkpoint_select_dataset + top_k_metric = args.top_k_checkpoint_select_metric + filtered_metrics = [ + v + for k, v in metric.items() + if top_k_metric in k and top_k_dataset in k + ] # check all R@10 metrics (all dataset) and use it to update the ckpt + # Saving checkpoints. + if args.save_logs: + opt_dict = { + k + "_" + "optimizer": v.state_dict() for k, v in optimizer.items() + } + checkpoint_dict = { + "epoch": completed_epoch, + "pretrain_epoch": pretrain_epoch, + "name": args.name, + "state_dict": model.state_dict(), + } + checkpoint_dict.update(opt_dict) + if scaler is not None: + checkpoint_dict["scaler"] = scaler.state_dict() + + if completed_epoch == args.epochs or ( + args.save_frequency > 0 and (completed_epoch % args.save_frequency) == 0 + ): + torch.save( + checkpoint_dict, + os.path.join(args.checkpoint_path, f"pretrain_epoch_{pretrain_epoch}_lp_epoch_{completed_epoch}.pt"), + ) + if args.save_most_recent: + torch.save( + checkpoint_dict, + os.path.join(args.checkpoint_path, f"pretrain_epoch_{pretrain_epoch}_lp_epoch_latest.pt"), + ) + if args.save_top_performance and not args.no_eval: + update_top_k_performance( + filtered_metrics, + current_top_k_ckpt_metrics, + args, + checkpoint_dict, + bignumbetter=True, + pretrain_epoch=pretrain_epoch + ) + del clap_model + return best_metrics + + +def copy_codebase(args): + from shutil import copytree, ignore_patterns + + new_code_path = os.path.join(args.logs, args.name, "code") + if os.path.exists(new_code_path): + print( + f"Error. Experiment already exists at {new_code_path}. Use --name to specify a new experiment." + ) + return -1 + print(f"Copying codebase to {new_code_path}") + current_code_path = os.path.realpath(__file__) + for _ in range(3): + current_code_path = os.path.dirname(current_code_path) + copytree( + current_code_path, new_code_path, ignore=ignore_patterns("log", "logs", "wandb") + ) + print("Done copying code.") + return 1 + + +if __name__ == "__main__": + main() + + diff --git a/utils/laion_clap/evaluate/eval_retrieval.py b/utils/laion_clap/evaluate/eval_retrieval.py new file mode 100644 index 0000000000000000000000000000000000000000..739734cee63588ec647dcb9189520af6294764f2 --- /dev/null +++ b/utils/laion_clap/evaluate/eval_retrieval.py @@ -0,0 +1,192 @@ +import os.path +import glob +import random +import numpy as np +import logging +import wandb +import torch +import torch.backends.cudnn as cudnn +from laion_clap import create_model +from laion_clap.training.logger import setup_logging +from laion_clap.training.data import get_data +from laion_clap.training.train import evaluate +from laion_clap.utils import get_tar_path_from_dataset_name, dataset_split +from laion_clap.training.params import parse_args + + +def find_params_value(file, key): + # find value of params in params_file + with open(file, 'r') as f: + for line in f: + if key + ': ' in line: + return line.split(': ')[1].strip() + return None + + +if __name__ == '__main__': + # (yusong) repeated run might have different metric results. + # This is because we randomly select crop 10s for each audio. + args = parse_args() + + if os.path.isdir(args.pretrained): + log_dir = os.path.dirname(args.pretrained) + else: + log_dir = os.path.dirname(os.path.dirname(args.pretrained)) + + args.log_level = logging.DEBUG if args.debug else logging.INFO + log_path = os.path.join(log_dir, 'out.log') + setup_logging(log_path, args.log_level) + params_file = os.path.join(log_dir, 'params.txt') + + seed = 3407 + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + + cudnn.benchmark = True + cudnn.deterministic = False + pretrained = 'openai' + amodel = find_params_value(params_file, 'amodel') + tmodel = find_params_value(params_file, 'tmodel') + + if amodel is None or tmodel is None: + raise ValueError('model type not found in params file') + + # set up dummy values for args + args.parallel_eval = False + args.rank = 0 + args.local_rank = 0 + args.world_size = 1 + args.val_frequency = 1 + args.epochs = 1 + args.precision = 'fp32' + args.save_logs = True + args.wandb = True + args.class_index_dict = None + + device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + args.device = device + + if args.remotedata: + for dataset_name in args.datasetnames: + for split in dataset_split[dataset_name]: + if not os.path.exists(f"./json_files/{dataset_name}/{split}"): + os.makedirs(f"./json_files/{dataset_name}/{split}") + os.system( + f"aws s3 cp s3://s-laion-audio/webdataset_tar/{dataset_name}/{split}/sizes.json ./json_files/{dataset_name}/{split}/sizes.json" + ) + + if args.datasetinfos is None: + args.datasetinfos = ["train", "unbalanced_train", "balanced_train"] + if args.dataset_type == "webdataset": + args.train_data = get_tar_path_from_dataset_name( + args.datasetnames, + args.datasetinfos, + islocal=not args.remotedata, + proportion=args.dataset_proportion, + dataset_path=args.datasetpath, + ) + args.val_data = get_tar_path_from_dataset_name( + args.datasetnames, + ["valid", "test", "eval"], + islocal=not args.remotedata, + proportion=1, + dataset_path=args.datasetpath, + ) + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision='fp32', + device=device, + jit=False, + force_quick_gelu=False, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) # a hack to get model_cfg + + data = get_data(args, model_cfg=model_cfg) # (yusong): hack: no model_cfg needed to get data + + writer = None # if use tensorboard, initalize writer here + + if args.wandb: + assert wandb is not None, "Please install wandb." + + # # find the line with "wandb_notes" and get the value + # wandb_notes = find_params_value(params_file, 'wandb_notes') + # if wandb_notes is None: + # print(f'wandb_notes not found in params file: {params_file}, set to timestamp.') + # wandb_notes = f'experiment_{time.strftime("%Y%m%d-%H%M%S")}' + # wandb_notes = wandb_notes + '-eval-retrieval' + wandb_notes = args.wandb_notes + + logging.debug("Starting wandb.") + args.train_sz = data["train"].dataloader.num_samples + if args.val_data is not None: + args.val_sz = data["val"].dataloader.num_samples + # you will have to configure this for your project! + if args.wandb_id is not None: + wandb.init( + project="clap", + id=args.wandb_id, + resume=True + ) + else: + wandb.init( + project="clap", + notes=wandb_notes, + name=wandb_notes, + tags=[], + config=vars(args), + ) + logging.debug("Finished loading wandb.") + + if os.path.isdir(args.pretrained): + all_model_checkpoints = sorted(glob.glob(os.path.join(log_dir, 'checkpoints', '*.pt')), key=os.path.getmtime) + else: + all_model_checkpoints = [args.pretrained] + for model_path in all_model_checkpoints: + args.checkpoint_path = os.path.dirname(model_path) + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision='fp32', + device=device, + jit=False, + force_quick_gelu=False, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) + + # load model + checkpoint = torch.load(model_path, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module."):]: v for k, v in sd.items()} + model.load_state_dict(sd) + logging.info( + f"=> resuming checkpoint '{model_path}' (epoch {start_epoch})" + ) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + start_epoch = 0 + + model.to(device) + model.eval() + for param in model.parameters(): + param.requires_grad = False + + evaluate(model, data, start_epoch, args, writer) diff --git a/utils/laion_clap/evaluate/eval_retrieval_main.py b/utils/laion_clap/evaluate/eval_retrieval_main.py new file mode 100644 index 0000000000000000000000000000000000000000..edfa65fdbf19377c1a0ba5c2e8c4fdc6f0d64e96 --- /dev/null +++ b/utils/laion_clap/evaluate/eval_retrieval_main.py @@ -0,0 +1,257 @@ +import os.path +import glob +import random +import numpy as np +import logging +import wandb +import torch +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +from clap_module import create_model +from clap_module import tokenize +from training.logger import setup_logging +from training.data import get_data +from training.train import evaluate +from clap_module.utils import get_tar_path_from_dataset_name, dataset_split +from training.params import parse_args + + +def find_params_value(file, key): + # find value of params in params_file + with open(file, 'r') as f: + for line in f: + if key + ': ' in line: + return line.split(': ')[1].strip() + return None + + +def evaluate_zeroshot(model, data, start_epoch, args, writer): + dataloader = data["val"].dataloader + metrics = {} + device = torch.device(args.device) + model.eval() + metrics.update({"epoch": start_epoch}) + + all_audio_features = [] + all_class_labels = [] + with torch.no_grad(): + for i, batch in enumerate(dataloader): + audios = batch # contains mel_spec, wavform, and longer list + audio_features = model(audios, None, device) + audio_features = F.normalize(audio_features, dim=-1) + all_audio_features.append(audio_features.detach().cpu()) + all_class_labels.append(torch.argmax(batch["class_label"], 1).long()) + all_audio_features = torch.cat(all_audio_features, dim=0) + all_class_labels = torch.cat(all_class_labels, dim=0) + metrics["num_samples"] = all_audio_features.shape[0] + + # get text features + all_texts = ["This is a sound of " + t for t in args.class_index_dict.keys()] + # (yusong): a hack, can make it better + if args.tmodel == "transformer": + from clap_module.tokenizer import tokenize + all_texts = tokenize(all_texts) + else: + from training.data import tokenizer + all_texts = tokenizer(all_texts) + all_text_features = model(None, all_texts, device) + all_text_features = F.normalize(all_text_features, dim=-1).detach().cpu() + + # compute similarity + logit_scale_a, logit_scale_t = model(None, None, device) + logit_scale_a = logit_scale_a.cpu() + + logits_per_audio = (logit_scale_a * all_audio_features @ all_text_features.t()).detach().cpu() + logits_per_text = logits_per_audio.t().detach().cpu() + + ground_truth = all_class_labels.view(-1, 1) + logit = logits_per_audio + + ranking = torch.argsort(logit, descending=True) + preds = torch.where(ranking == ground_truth)[1] # (yusong) this line is slow because it uses single thread + preds = preds.detach().cpu().numpy() + metrics[f"{args.datasetnames[0]}_mean_rank"] = preds.mean() + 1 + metrics[f"{args.datasetnames[0]}_median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"{args.datasetnames[0]}_R@{k}"] = np.mean(preds < k) + # map@10 + metrics[f"{args.datasetnames[0]}_mAP@10"] = np.mean(np.where(preds < 10, 1 / (preds + 1), 0.0)) + + logging.info( + f"Eval Epoch: {start_epoch} " + + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in metrics.items()]) + ) + + if args.wandb: + assert wandb is not None, "Please install wandb." + for name, val in metrics.items(): + wandb.log({f"val/{name}": val, "epoch": start_epoch}) + + +if __name__ == '__main__': + # (yusong) repeated run might have different metric results. + # This is because we randomly select crop 10s for each audio. + args = parse_args() + + if os.path.isdir(args.pretrained): + log_dir = os.path.dirname(args.pretrained) + else: + log_dir = os.path.dirname(os.path.dirname(args.pretrained)) + + args.log_level = logging.DEBUG if args.debug else logging.INFO + log_path = os.path.join(log_dir, 'out.log') + setup_logging(log_path, args.log_level) + params_file = os.path.join(log_dir, 'params.txt') + + seed = 3407 + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + + cudnn.benchmark = True + cudnn.deterministic = False + pretrained = 'openai' + amodel = find_params_value(params_file, 'amodel') + tmodel = find_params_value(params_file, 'tmodel') + + if amodel is None or tmodel is None: + raise ValueError('model type not found in params file') + + # set up dummy values for args + args.parallel_eval = False + args.rank = 0 + args.local_rank = 0 + args.world_size = 1 + args.val_frequency = 1 + args.epochs = 1 + args.precision = 'fp32' + args.save_logs = True + args.wandb = args.report_to == 'wandb' + args.class_index_dict = None + + device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + args.device = device + + if args.remotedata: + for dataset_name in args.datasetnames: + for split in dataset_split[dataset_name]: + if not os.path.exists(f"./json_files/{dataset_name}/{split}"): + os.makedirs(f"./json_files/{dataset_name}/{split}") + os.system( + f"aws s3 cp s3://s-laion-audio/webdataset_tar/{dataset_name}/{split}/sizes.json ./json_files/{dataset_name}/{split}/sizes.json" + ) + + if args.datasetinfos is None: + args.datasetinfos = ["train", "unbalanced_train", "balanced_train"] + if args.dataset_type == "webdataset": + args.train_data = get_tar_path_from_dataset_name( + args.datasetnames, + args.datasetinfos, + islocal=not args.remotedata, + proportion=args.dataset_proportion, + dataset_path=args.datasetpath, + ) + args.val_data = get_tar_path_from_dataset_name( + args.datasetnames, + ["valid", "test", "eval"], + islocal=not args.remotedata, + proportion=1, + dataset_path=args.datasetpath, + ) + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision='fp32', + device=device, + jit=False, + force_quick_gelu=False, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) # a hack to get model_cfg + + data = get_data(args, model_cfg=model_cfg) # (yusong): hack: no model_cfg needed to get data + + writer = None # if use tensorboard, initalize writer here + + if args.wandb: + assert wandb is not None, "Please install wandb." + + # # find the line with "wandb_notes" and get the value + # wandb_notes = find_params_value(params_file, 'wandb_notes') + # if wandb_notes is None: + # print(f'wandb_notes not found in params file: {params_file}, set to timestamp.') + # wandb_notes = f'experiment_{time.strftime("%Y%m%d-%H%M%S")}' + # wandb_notes = wandb_notes + '-eval-retrieval' + wandb_notes = args.wandb_notes + + logging.debug("Starting wandb.") + args.train_sz = data["train"].dataloader.num_samples + if args.val_data is not None: + args.val_sz = data["val"].dataloader.num_samples + # you will have to configure this for your project! + if args.wandb_id is not None: + wandb.init( + project="clap", + id=args.wandb_id, + resume=True + ) + else: + wandb.init( + project="clap", + notes=wandb_notes, + name=wandb_notes, + tags=[], + config=vars(args), + ) + logging.debug("Finished loading wandb.") + + if os.path.isdir(args.pretrained): + all_model_checkpoints = sorted(glob.glob(os.path.join(log_dir, 'checkpoints', '*.pt')), key=os.path.getmtime) + else: + all_model_checkpoints = [args.pretrained] + for model_path in all_model_checkpoints: + args.checkpoint_path = os.path.dirname(model_path) + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision='fp32', + device=device, + jit=False, + force_quick_gelu=False, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) + + # load model + checkpoint = torch.load(model_path, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module."):]: v for k, v in sd.items()} + model.load_state_dict(sd) + logging.info( + f"=> resuming checkpoint '{model_path}' (epoch {start_epoch})" + ) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + start_epoch = 0 + + model.to(device) + model.eval() + for param in model.parameters(): + param.requires_grad = False + + evaluate_zeroshot(model, data, start_epoch, args, writer) diff --git a/utils/laion_clap/evaluate/eval_zeroshot_classification.py b/utils/laion_clap/evaluate/eval_zeroshot_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..577cb91125bacb7f7dc7fb9841c2cd819478a736 --- /dev/null +++ b/utils/laion_clap/evaluate/eval_zeroshot_classification.py @@ -0,0 +1,261 @@ +import os.path +import glob +import random +import numpy as np +import logging +import wandb +import torch +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +from clap_module import create_model +from clap_module import tokenize +from training.logger import setup_logging +from training.data import get_data +from training.train import evaluate +from clap_module.utils import get_tar_path_from_dataset_name, dataset_split +from training.params import parse_args + + +def find_params_value(file, key): + # find value of params in params_file + with open(file, 'r') as f: + for line in f: + if key + ': ' in line: + return line.split(': ')[1].strip() + return None + + +def evaluate_zeroshot(model, data, start_epoch, args, writer): + dataloader = data["val"].dataloader + metrics = {} + device = torch.device(args.device) + model.eval() + metrics.update({"epoch": start_epoch}) + + all_audio_features = [] + all_class_labels = [] + with torch.no_grad(): + for i, batch in enumerate(dataloader): + audios = batch # contains mel_spec, wavform, and longer list + audio_features = model(audios, None, device) + audio_features = F.normalize(audio_features, dim=-1) + all_audio_features.append(audio_features.detach().cpu()) + all_class_labels.append(torch.argmax(batch["class_label"], 1).long()) + all_audio_features = torch.cat(all_audio_features, dim=0) + all_class_labels = torch.cat(all_class_labels, dim=0) + metrics["num_samples"] = all_audio_features.shape[0] + + # get text features + if args.val_dataset_names == ['GTZAN']: + all_texts = [f"This is a {t} song." for t in args.class_index_dict.keys()] + else: + all_texts = [f"This is a sound of {t}." for t in args.class_index_dict.keys()] + logging.info(f'class label prompts: {all_texts}') + # (yusong): a hack, can make it better + if args.tmodel == "transformer": + from clap_module.tokenizer import tokenize + all_texts = tokenize(all_texts) + else: + from training.data import tokenizer + all_texts = tokenizer(all_texts) + all_text_features = model(None, all_texts, device) + all_text_features = F.normalize(all_text_features, dim=-1).detach().cpu() + + # compute similarity + logit_scale_a, logit_scale_t = model(None, None, device) + logit_scale_a = logit_scale_a.cpu() + + logits_per_audio = (logit_scale_a * all_audio_features @ all_text_features.t()).detach().cpu() + logits_per_text = logits_per_audio.t().detach().cpu() + + ground_truth = all_class_labels.view(-1, 1) + logit = logits_per_audio + + ranking = torch.argsort(logit, descending=True) + preds = torch.where(ranking == ground_truth)[1] # (yusong) this line is slow because it uses single thread + preds = preds.detach().cpu().numpy() + metrics[f"{args.datasetnames[0]}_mean_rank"] = preds.mean() + 1 + metrics[f"{args.datasetnames[0]}_median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"{args.datasetnames[0]}_R@{k}"] = np.mean(preds < k) + # map@10 + metrics[f"{args.datasetnames[0]}_mAP@10"] = np.mean(np.where(preds < 10, 1 / (preds + 1), 0.0)) + + logging.info( + f"Eval Epoch: {start_epoch} " + + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in metrics.items()]) + ) + + if args.wandb: + assert wandb is not None, "Please install wandb." + for name, val in metrics.items(): + wandb.log({f"val/{name}": val, "epoch": start_epoch}) + + +if __name__ == '__main__': + # (yusong) repeated run might have different metric results. + # This is because we randomly select crop 10s for each audio. + args = parse_args() + + if os.path.isdir(args.pretrained): + log_dir = os.path.dirname(args.pretrained) + else: + log_dir = os.path.dirname(os.path.dirname(args.pretrained)) + + args.log_level = logging.DEBUG if args.debug else logging.INFO + log_path = os.path.join(log_dir, 'out.log') + setup_logging(log_path, args.log_level) + params_file = os.path.join(log_dir, 'params.txt') + + seed = 3407 + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + + cudnn.benchmark = True + cudnn.deterministic = False + pretrained = 'openai' + amodel = find_params_value(params_file, 'amodel') + tmodel = find_params_value(params_file, 'tmodel') + + if amodel is None or tmodel is None: + raise ValueError('model type not found in params file') + + # set up dummy values for args + args.parallel_eval = False + args.rank = 0 + args.local_rank = 0 + args.world_size = 1 + args.val_frequency = 1 + args.epochs = 1 + args.precision = 'fp32' + args.save_logs = True + args.wandb = args.report_to == 'wandb' + args.class_index_dict = None + + device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + args.device = device + + if args.remotedata: + for dataset_name in args.datasetnames: + for split in dataset_split[dataset_name]: + if not os.path.exists(f"./json_files/{dataset_name}/{split}"): + os.makedirs(f"./json_files/{dataset_name}/{split}") + os.system( + f"aws s3 cp s3://s-laion-audio/webdataset_tar/{dataset_name}/{split}/sizes.json ./json_files/{dataset_name}/{split}/sizes.json" + ) + + if args.datasetinfos is None: + args.datasetinfos = ["train", "unbalanced_train", "balanced_train"] + if args.dataset_type == "webdataset": + args.train_data = get_tar_path_from_dataset_name( + args.datasetnames, + args.datasetinfos, + islocal=not args.remotedata, + proportion=args.dataset_proportion, + dataset_path=args.datasetpath, + ) + args.val_data = get_tar_path_from_dataset_name( + args.datasetnames, + ["valid", "test", "eval"], + islocal=not args.remotedata, + proportion=1, + dataset_path=args.datasetpath, + ) + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision='fp32', + device=device, + jit=False, + force_quick_gelu=False, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) # a hack to get model_cfg + + data = get_data(args, model_cfg=model_cfg) # (yusong): hack: no model_cfg needed to get data + + writer = None # if use tensorboard, initalize writer here + + if args.wandb: + assert wandb is not None, "Please install wandb." + + # # find the line with "wandb_notes" and get the value + # wandb_notes = find_params_value(params_file, 'wandb_notes') + # if wandb_notes is None: + # print(f'wandb_notes not found in params file: {params_file}, set to timestamp.') + # wandb_notes = f'experiment_{time.strftime("%Y%m%d-%H%M%S")}' + # wandb_notes = wandb_notes + '-eval-retrieval' + wandb_notes = args.wandb_notes + + logging.debug("Starting wandb.") + args.train_sz = data["train"].dataloader.num_samples + if args.val_data is not None: + args.val_sz = data["val"].dataloader.num_samples + # you will have to configure this for your project! + if args.wandb_id is not None: + wandb.init( + project="clap", + id=args.wandb_id, + resume=True + ) + else: + wandb.init( + project="clap", + notes=wandb_notes, + name=wandb_notes, + tags=[], + config=vars(args), + ) + logging.debug("Finished loading wandb.") + + if os.path.isdir(args.pretrained): + all_model_checkpoints = sorted(glob.glob(os.path.join(log_dir, 'checkpoints', '*.pt')), key=os.path.getmtime) + else: + all_model_checkpoints = [args.pretrained] + for model_path in all_model_checkpoints: + args.checkpoint_path = os.path.dirname(model_path) + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision='fp32', + device=device, + jit=False, + force_quick_gelu=False, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) + + # load model + checkpoint = torch.load(model_path, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module."):]: v for k, v in sd.items()} + model.load_state_dict(sd) + logging.info( + f"=> resuming checkpoint '{model_path}' (epoch {start_epoch})" + ) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + start_epoch = 0 + + model.to(device) + model.eval() + for param in model.parameters(): + param.requires_grad = False + + evaluate_zeroshot(model, data, start_epoch, args, writer) diff --git a/utils/laion_clap/hook.py b/utils/laion_clap/hook.py new file mode 100644 index 0000000000000000000000000000000000000000..f857590cdaba16bdec6bc6918729a5b6f719480c --- /dev/null +++ b/utils/laion_clap/hook.py @@ -0,0 +1,220 @@ +""" +Contrastive Language-Audio Pretraining Model from LAION +-------------------------------------------------------- +Paper: https://arxiv.org/abs/2211.06687 +Authors (equal contributions): Ke Chen, Yusong Wu, Tianyu Zhang, Yuchen Hui +Support: LAION +""" +import os +import sys +import os +import torch +import librosa +from utils.laion_clap.clap_module import create_model +from utils.laion_clap.training.data import get_audio_features +from utils.laion_clap.training.data import int16_to_float32, float32_to_int16 + +from transformers import RobertaTokenizer +import wget +from utils.laion_clap.clap_module.factory import load_state_dict + + +class CLAP_Module(torch.nn.Module): + def __init__(self, enable_fusion=False, device=None, amodel= 'HTSAT-tiny', tmodel='roberta') -> None: + """Initialize CLAP Model + + Parameters + ---------- + enable_fusion: bool + if true, it will create the fusion clap model, otherwise non-fusion clap model (default: false) + device: str + if None, it will automatically detect the device (gpu or cpu) + amodel: str + audio encoder architecture, default: HTSAT-tiny + tmodel: str + text encoder architecture, default: roberta + """ + super(CLAP_Module, self).__init__() + if device is None: + device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + + precision = 'fp32' + + if enable_fusion: + fusion_type = 'aff_2d' + model, model_cfg = create_model( + amodel, + tmodel, + precision=precision, + device=device, + enable_fusion=enable_fusion, + fusion_type=fusion_type + ) + else: + model, model_cfg = create_model( + amodel, + tmodel, + precision=precision, + device=device, + enable_fusion=enable_fusion + ) + self.enable_fusion = enable_fusion + self.model = model + self.model_cfg = model_cfg + self.tokenize = RobertaTokenizer.from_pretrained('roberta-base') + + def tokenizer(self, text): + result = self.tokenize( + text, + padding="max_length", + truncation=True, + max_length=77, + return_tensors="pt", + ) + return {k: v.squeeze(0) for k, v in result.items()} + + def load_ckpt(self, ckpt = None, model_id = -1): + """Load the pretrained checkpoint of CLAP model + + Parameters + ---------- + ckpt: str + if ckpt is specified, the model will load this ckpt, otherwise the model will download the ckpt from zenodo. \n + For fusion model, it will download the 630k+audioset fusion model (id=3). For non-fusion model, it will download the 630k+audioset model (id=1). + model_id: + if model_id is specified, you can download our best ckpt, as: + id = 0 --> 630k non-fusion ckpt \n + id = 1 --> 630k+audioset non-fusion ckpt \n + id = 2 --> 630k fusion ckpt \n + id = 3 --> 630k+audioset fusion ckpt \n + Note that if your model is specied as non-fusion model but you download a fusion model ckpt, you will face an error. + """ + download_link = 'https://huggingface.co/lukewys/laion_clap/resolve/main/' + download_names = [ + '630k-best.pt', + '630k-audioset-best.pt', + '630k-fusion-best.pt', + '630k-audioset-fusion-best.pt' + ] + if ckpt is not None: + print(f'Load the specified checkpoint {ckpt} from users.') + else: + print(f'Load our best checkpoint in the paper.') + if model_id == -1: + model_id = 3 if self.enable_fusion else 1 + package_dir = os.path.dirname(os.path.realpath(__file__)) + weight_file_name = download_names[model_id] + ckpt = os.path.join(package_dir, weight_file_name) + if os.path.exists(ckpt): + print(f'The checkpoint is already downloaded') + else: + print('Downloading laion_clap weight files...') + ckpt = wget.download(download_link + weight_file_name, os.path.dirname(ckpt)) + print('Download completed!') + #print('Load Checkpoint...') + ckpt = load_state_dict(ckpt, skip_params=True) + self.model.load_state_dict(ckpt) + #param_names = [n for n, p in self.model.named_parameters()] + #for n in param_names: + # print(n, "\t", "Loaded" if n in ckpt else "Unloaded") + + def get_audio_embedding_from_filelist(self, x, use_tensor=False): + """get audio embeddings from the audio file list + + Parameters + ---------- + x: List[str] (N,): + an audio file list to extract features, audio files can have different lengths (as we have the feature fusion machanism) + use_tensor: boolean: + if True, it will return the torch tensor, preserving the gradient (default: False). + Returns + ---------- + audio_embed : numpy.darray | torch.Tensor (N,D): + audio embeddings that extracted from audio files + """ + self.model.eval() + audio_input = [] + for f in x: + # load the waveform of the shape (T,), should resample to 48000 + audio_waveform, _ = librosa.load(f, sr=48000) + # quantize + audio_waveform = int16_to_float32(float32_to_int16(audio_waveform)) + audio_waveform = torch.from_numpy(audio_waveform).float() + temp_dict = {} + temp_dict = get_audio_features( + temp_dict, audio_waveform, 480000, + data_truncating='fusion' if self.enable_fusion else 'rand_trunc', + data_filling='repeatpad', + audio_cfg=self.model_cfg['audio_cfg'], + require_grad=audio_waveform.requires_grad + ) + audio_input.append(temp_dict) + audio_embed = self.model.get_audio_embedding(audio_input) + if not use_tensor: + audio_embed = audio_embed.detach().cpu().numpy() + return audio_embed + + + def get_audio_embedding_from_data(self, x, use_tensor=False): + """get audio embeddings from the audio data + + Parameters + ---------- + x: np.darray | torch.Tensor (N,T): + audio data, must be mono audio tracks. + use_tensor: boolean: + if True, x should be the tensor input and the output will be the tesnor, preserving the gradient (default: False). + Note that if 'use tensor' is set to True, it will not do the quantize of the audio waveform (otherwise the gradient will not be preserved). + Returns + ---------- + audio embed: numpy.darray | torch.Tensor (N,D): + audio embeddings that extracted from audio files + """ + self.model.eval() + audio_input = [] + for audio_waveform in x: + # quantize + if not use_tensor: + audio_waveform = int16_to_float32(float32_to_int16(audio_waveform)) + audio_waveform = torch.from_numpy(audio_waveform).float() + temp_dict = {} + temp_dict = get_audio_features( + temp_dict, audio_waveform, 480000, + data_truncating='fusion' if self.enable_fusion else 'rand_trunc', + data_filling='repeatpad', + audio_cfg=self.model_cfg['audio_cfg'], + require_grad=audio_waveform.requires_grad + ) + audio_input.append(temp_dict) + audio_embed = self.model.get_audio_embedding(audio_input) + if not use_tensor: + audio_embed = audio_embed.detach().cpu().numpy() + return audio_embed + + def get_text_embedding(self, x, tokenizer = None, use_tensor = False): + """get text embeddings from texts + + Parameters + ---------- + x: List[str] (N,): + text list + tokenizer: func: + the tokenizer function, if not provided (None), will use the default Roberta tokenizer. + use_tensor: boolean: + if True, the output will be the tesnor, preserving the gradient (default: False). + Returns + ---------- + text_embed : numpy.darray | torch.Tensor (N,D): + text embeddings that extracted from texts + """ + self.model.eval() + if tokenizer is not None: + text_input = tokenizer(x) + else: + text_input = self.tokenizer(x) + text_embed = self.model.get_text_embedding(text_input) + if not use_tensor: + text_embed = text_embed.detach().cpu().numpy() + return text_embed + + diff --git a/utils/laion_clap/training/__init__.py b/utils/laion_clap/training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/laion_clap/training/audioset_textmap.npy b/utils/laion_clap/training/audioset_textmap.npy new file mode 100644 index 0000000000000000000000000000000000000000..3da4c92d3819aaec11e5f576464a9973a6df811b --- /dev/null +++ b/utils/laion_clap/training/audioset_textmap.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bada103070d92f9eadd33e1b4f45ec8583f59080ef218c966b43294bd4c86d5b +size 84448 diff --git a/utils/laion_clap/training/data.py b/utils/laion_clap/training/data.py new file mode 100644 index 0000000000000000000000000000000000000000..dc1730f067fbf3ea16b2d2c2939c9347105e0a2f --- /dev/null +++ b/utils/laion_clap/training/data.py @@ -0,0 +1,895 @@ +import ast +import json +import logging +import math +import os +import random +import h5py +from dataclasses import dataclass +import braceexpand +import numpy as np +import pandas as pd +import torch +import torch.nn.functional as F +import torchvision.datasets as datasets +import torchvision.transforms +import webdataset as wds +from PIL import Image +from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler +from torch.utils.data.distributed import DistributedSampler +from functools import partial +from pathlib import Path +import wget +import tempfile +import copy +from contextlib import suppress + +from utils.laion_clap.clap_module.utils import get_tar_path_from_dataset_name, dataset_split +from utils.laion_clap.clap_module.utils import load_p, load_class_label +from utils.laion_clap.clap_module import tokenize as clip_tokenizer +from transformers import BertTokenizer +from transformers import RobertaTokenizer +from transformers import BartTokenizer + +try: + import horovod.torch as hvd +except ImportError: + hvd = None + +try: + import torchaudio +except ImportError: + torchaudio = None + +bert_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") +roberta_tokenizer = RobertaTokenizer.from_pretrained("roberta-base") +bart_tokenizer = BartTokenizer.from_pretrained("facebook/bart-base") + +def tokenizer(text, tmodel="roberta", max_length=77): + """tokenizer for different models + tmodel is default to roberta as it is the best model for our task + max_length is default to 77 from the OpenAI CLIP parameters + We assume text to be a single string, but it can also be a list of strings + """ + if tmodel == "transformer": + return clip_tokenizer(text).squeeze(0) + + elif tmodel == "bert": + result = bert_tokenizer( + text, + padding="max_length", + truncation=True, + max_length=max_length, + return_tensors="pt", + ) + return {k: v.squeeze(0) for k, v in result.items()} + + elif tmodel == "roberta": + result = roberta_tokenizer( + text, + padding="max_length", + truncation=True, + max_length=max_length, + return_tensors="pt", + ) + return {k: v.squeeze(0) for k, v in result.items()} + + elif tmodel == "bart": + result = bart_tokenizer( + text, + padding="max_length", + truncation=True, + max_length=max_length, + return_tensors="pt", + ) + return {k: v.squeeze(0) for k, v in result.items()} + + +# initizlied the audioset map +_AUDIOSET_MAP_PATH = os.path.join(Path(__file__).parent, "audioset_textmap.npy") +_AUDIOSET_MAP = np.load(_AUDIOSET_MAP_PATH, allow_pickle=True) + + +def int16_to_float32(x): + return (x / 32767.0).astype(np.float32) + + +def float32_to_int16(x): + x = np.clip(x, a_min=-1., a_max=1.) + return (x * 32767.).astype(np.int16) + + +def int16_to_float32_torch(x): + return (x / 32767.0).type(torch.float32) + + +def float32_to_int16_torch(x): + x = torch.clamp(x, min=-1., max=1.) + return (x * 32767.).type(torch.int16) + + +# For Toy Dataset +class ToyDataset(Dataset): + def __init__(self, index_path, ipc, config, eval_mode=False): + """Toy Dataset for testing the audioset input with text labels + Parameters + ---------- + index_path: str + the link to the h5 file of each audio + idc: str + the link to the npy file, the number of samples in each class + config: dict + the audio cfg file + eval_model (bool): to indicate if the dataset is a testing dataset + """ + self.audio_cfg = config["audio_cfg"] + self.text_cfg = config["text_cfg"] + self.fp = h5py.File(index_path, "r") + self.ipc = np.load(ipc, allow_pickle=True) + self.total_size = len(self.fp["audio_name"]) + self.classes_num = self.audio_cfg["class_num"] + self.eval_mode = eval_mode + + if not eval_mode: + self.generate_queue() + else: + self.queue = [] + for i in range(self.total_size): + target = self.fp["target"][i] + if np.sum(target) > 0: + self.queue.append(i) + self.total_size = len(self.queue) + logging.info("total dataset size: %d" % (self.total_size)) + logging.info("class num: %d" % (self.classes_num)) + + def time_shifting(self, x): + frame_num = len(x) + shift_len = random.randint(0, frame_num - 1) + new_sample = np.concatenate([x[shift_len:], x[:shift_len]], axis=0) + return new_sample + + def generate_queue(self): + self.queue = [] + while len(self.queue) < self.total_size: + class_set = [*range(self.classes_num)] + random.shuffle(class_set) + self.queue += [ + self.ipc[d][random.randint(0, len(self.ipc[d]) - 1)] for d in class_set + ] + self.queue = self.queue[: self.total_size] + + logging.info("queue regenerated:%s" % (self.queue[-5:])) + + def crop_wav(self, x): + crop_size = self.audio_cfg["crop_size"] + crop_pos = random.randint(0, len(x) - crop_size - 1) + return x[crop_pos: crop_pos + crop_size] + + def prompt_text(self, target): + events = _AUDIOSET_MAP[np.where(target > 0)] + event_text = "The sounds of " + ", ".join(events[:-1]) + " and " + events[-1] + text = tokenizer(event_text)[0] + return text + + def __getitem__(self, index): + """Load waveform, text, and target of an audio clip + + Parameters + ---------- + index: int + the index number + Return + ------ + output: dict { + "hdf5_path": str, + "index_in_hdf5": int, + "audio_name": str, + "waveform": list (audio_length,), + "target": list (class_num, ), + "text": torch.tensor (context_length,) + } + the output dictionary + """ + s_index = self.queue[index] + + audio_name = self.fp["audio_name"][s_index].decode() + # Hardcode here CHANGE + hdf5_path = ( + self.fp["hdf5_path"][s_index] + .decode() + .replace( + "../workspace", + "/home/la/kechen/Research/ke_zsasp/workspace", + ) + ) + r_idx = self.fp["index_in_hdf5"][s_index] + target = self.fp["target"][s_index].astype(np.float32) + text = self.prompt_text(target) + with h5py.File(hdf5_path, "r") as f: + waveform = int16_to_float32(f["waveform"][r_idx])[ + : self.audio_cfg["clip_samples"] + ] + assert ( + len(waveform) == self.audio_cfg["clip_samples"] + ), "The sample length is not match" + # Time shift + # if (self.config.enable_time_shift) and (not self.eval_mode): + # waveform = self.time_shifting(waveform) + # # Label Enhance + # if (self.config.crop_size is not None) and (not self.eval_mode): + # waveform = self.crop_wav(waveform) + # # the label enhance rate is fixed 0.5 + # if (self.config.enable_label_enhance) and (not self.eval_mode) and random.random() < 0.5: + # kidx = np.where(target)[0] + # for k in kidx: + # for add_key in self.class_map[k][1]: + # target[add_key] = 1.0 + # if len(self.class_map[k][2]) > 0: + # add_key = random.choice(self.class_map[k][2]) + # target[add_key] = 1.0 + + # missing the text input + mel_spec = get_mel(torch.from_numpy(waveform), self.audio_cfg)[None, :, :] + mel_spec = torch.cat([mel_spec, mel_spec.clone(), mel_spec.clone(), mel_spec.clone()], dim=0).cpu().numpy() + longer = random.choice([True, False]) + if longer == False: + mel_spec[1:, :, :] = 0.0 + data_dict = { + "hdf5_path": hdf5_path, + "index_in_hdf5": r_idx, + "audio_name": audio_name, + "waveform": waveform, + "class_label": target, + "text": text, + "longer": longer, + "mel_fusion": mel_spec + } + return data_dict + + def __len__(self): + return self.total_size + +@dataclass +class DataInfo: + dataloader: DataLoader + sampler: DistributedSampler + + +def get_dataset_size(shards, sizefilepath_=None, is_local=True): + if isinstance(shards, list): + size_list = [] + for s in shards: + size_list.append( + get_dataset_size(s, sizefilepath_=sizefilepath_, is_local=is_local)[0] + ) + else: + if not is_local: + for n in dataset_split.keys(): + if n in shards.split("/"): + break + for s in dataset_split[n]: + if s in shards.split("/"): + break + sizefilepath_ = f"./json_files/{n}/{s}/sizes.json" + shards_list = list(braceexpand.braceexpand(shards)) + dir_path = os.path.dirname(shards) + if sizefilepath_ is not None: + sizes = json.load(open(sizefilepath_, "r")) + total_size = sum( + [ + int(sizes[os.path.basename(shard.replace(".tar -", ".tar"))]) + for shard in shards_list + ] + ) + else: + sizes_filename = os.path.join(dir_path, "sizes.json") + len_filename = os.path.join(dir_path, "__len__") + if os.path.exists(sizes_filename): + sizes = json.load(open(sizes_filename, "r")) + total_size = sum( + [int(sizes[os.path.basename(shard)]) for shard in shards_list] + ) + elif os.path.exists(len_filename): + # FIXME this used to be eval(open(...)) but that seemed rather unsafe + total_size = ast.literal_eval(open(len_filename, "r").read()) + else: + raise Exception( + f"Cannot find sizes file for dataset {shards}. Please specify the path to the file." + ) + # total_size = None # num samples undefined + # some common dataset sizes (at time of authors last download) + # cc3m-train: 2905954 + # cc12m: 10968539 + # LAION-400m: 407332084 + num_shards = len(shards_list) + if isinstance(shards, list): + return sum(size_list), len(shards) + else: + return total_size, num_shards + + +def count_samples(dataloader): + os.environ["WDS_EPOCH"] = "0" + n_elements, n_batches = 0, 0 + for images, texts in dataloader: + n_batches += 1 + n_elements += len(images) + assert len(images) == len(texts) + return n_elements, n_batches + + +def log_and_continue(exn): + """Call in an exception handler to ignore any exception, isssue a warning, and continue.""" + logging.warning(f"Handling webdataset error ({repr(exn)}). Ignoring.") + return True + + +_SHARD_SHUFFLE_SIZE = 2000 +_SHARD_SHUFFLE_INITIAL = 500 +_SAMPLE_SHUFFLE_SIZE = 5000 +_SAMPLE_SHUFFLE_INITIAL = 1000 + + +def sample_prop(sizefile, inputs, proportion, is_local=True): + """ + Sample a proportion of the data. + """ + file_path_dict = { + os.path.split(inputs[i])[1]: os.path.split(inputs[i])[0] + for i in range(len(inputs)) + } + sampled_filepath_dict = {} + sampled_size_dict = {} + if not is_local: + if os.path.exists("sizes.json"): + os.remove("sizes.json") + wget.download(sizefile, "sizes.json") + sizefile = "sizes.json" + with open(sizefile, "r", encoding="UTF-8") as f: + load_dict = json.load(f) + L = int(len(file_path_dict) * proportion) + subkeys = random.sample(file_path_dict.keys(), L) + for k in subkeys: + sampled_size_dict[k] = load_dict[k] + sampled_filepath_dict[k] = file_path_dict[k] + return ( + sum(sampled_size_dict.values()), + L, + [os.path.join(v, k) for k, v in sampled_filepath_dict.items()], + sampled_size_dict, + ) + + +def get_mel(audio_data, audio_cfg): + # mel shape: (n_mels, T) + mel_tf = torchaudio.transforms.MelSpectrogram( + sample_rate=audio_cfg['sample_rate'], + n_fft=audio_cfg['window_size'], + win_length=audio_cfg['window_size'], + hop_length=audio_cfg['hop_size'], + center=True, + pad_mode="reflect", + power=2.0, + norm=None, + onesided=True, + n_mels=audio_cfg['mel_bins'], + f_min=audio_cfg['fmin'], + f_max=audio_cfg['fmax'] + ).to(audio_data.device) + + mel = mel_tf(audio_data) + # Align to librosa: + # librosa_melspec = librosa.feature.melspectrogram( + # waveform, + # sr=audio_cfg['sample_rate'], + # n_fft=audio_cfg['window_size'], + # hop_length=audio_cfg['hop_size'], + # win_length=audio_cfg['window_size'], + # center=True, + # pad_mode="reflect", + # power=2.0, + # n_mels=audio_cfg['mel_bins'], + # norm=None, + # htk=True, + # f_min=audio_cfg['fmin'], + # f_max=audio_cfg['fmax'] + # ) + # we use log mel spectrogram as input + mel = torchaudio.transforms.AmplitudeToDB(top_db=None)(mel) + return mel.T # (T, n_mels) + + +def get_audio_features(sample, audio_data, max_len, data_truncating, data_filling, audio_cfg, require_grad=False): + """ + Calculate and add audio features to sample. + Sample: a dict containing all the data of current sample. + audio_data: a tensor of shape (T) containing audio data. + max_len: the maximum length of audio data. + data_truncating: the method of truncating data. + data_filling: the method of filling data. + audio_cfg: a dict containing audio configuration. Comes from model_cfg['audio_cfg']. + require_grad: whether to require gradient for audio data. + This is useful when we want to apply gradient-based classifier-guidance. + """ + grad_fn = suppress if require_grad else torch.no_grad + with grad_fn(): + if len(audio_data) > max_len: + if data_truncating == "rand_trunc": + longer = torch.tensor([True]) + elif data_truncating == "fusion": + # fusion + mel = get_mel(audio_data, audio_cfg) + # split to three parts + chunk_frames = max_len // audio_cfg['hop_size'] + 1 # the +1 related to how the spectrogram is computed + total_frames = mel.shape[0] + if chunk_frames == total_frames: + # there is a corner case where the audio length is + # larger than max_len but smaller than max_len+hop_size. + # In this case, we just use the whole audio. + mel_fusion = torch.stack([mel, mel, mel, mel], dim=0) + sample["mel_fusion"] = mel_fusion + longer = torch.tensor([False]) + else: + ranges = np.array_split(list(range(0, total_frames - chunk_frames + 1)), 3) + # print('total_frames-chunk_frames:', total_frames-chunk_frames, + # 'len(audio_data):', len(audio_data), + # 'chunk_frames:', chunk_frames, + # 'total_frames:', total_frames) + if len(ranges[1]) == 0: + # if the audio is too short, we just use the first chunk + ranges[1] = [0] + if len(ranges[2]) == 0: + # if the audio is too short, we just use the first chunk + ranges[2] = [0] + # randomly choose index for each part + idx_front = np.random.choice(ranges[0]) + idx_middle = np.random.choice(ranges[1]) + idx_back = np.random.choice(ranges[2]) + # select mel + mel_chunk_front = mel[idx_front:idx_front + chunk_frames, :] + mel_chunk_middle = mel[idx_middle:idx_middle + chunk_frames, :] + mel_chunk_back = mel[idx_back:idx_back + chunk_frames, :] + + # shrink the mel + mel_shrink = torchvision.transforms.Resize(size=[chunk_frames, audio_cfg['mel_bins']])(mel[None])[0] + # logging.info(f"mel_shrink.shape: {mel_shrink.shape}") + + # stack + mel_fusion = torch.stack([mel_shrink, mel_chunk_front, mel_chunk_middle, mel_chunk_back], dim=0) + sample["mel_fusion"] = mel_fusion + longer = torch.tensor([True]) + else: + raise NotImplementedError( + f"data_truncating {data_truncating} not implemented" + ) + # random crop to max_len (for compatibility) + overflow = len(audio_data) - max_len + idx = np.random.randint(0, overflow + 1) + audio_data = audio_data[idx: idx + max_len] + + else: # padding if too short + if len(audio_data) < max_len: # do nothing if equal + if data_filling == "repeatpad": + n_repeat = int(max_len / len(audio_data)) + audio_data = audio_data.repeat(n_repeat) + # audio_data = audio_data.unsqueeze(0).unsqueeze(0).unsqueeze(0) + # audio_data = F.interpolate(audio_data,size=max_len,mode="bicubic")[0,0,0] + audio_data = F.pad( + audio_data, + (0, max_len - len(audio_data)), + mode="constant", + value=0, + ) + elif data_filling == "pad": + audio_data = F.pad( + audio_data, + (0, max_len - len(audio_data)), + mode="constant", + value=0, + ) + elif data_filling == "repeat": + n_repeat = int(max_len / len(audio_data)) + audio_data = audio_data.repeat(n_repeat + 1)[:max_len] + else: + raise NotImplementedError( + f"data_filling {data_filling} not implemented" + ) + if data_truncating == 'fusion': + mel = get_mel(audio_data, audio_cfg) + mel_fusion = torch.stack([mel, mel, mel, mel], dim=0) + sample["mel_fusion"] = mel_fusion + longer = torch.tensor([False]) + + sample["longer"] = longer + sample["waveform"] = audio_data + + return sample + + +def select_text(json_dict_raw, text_augment_selection): + # For selecting augmented text from dataset + if text_augment_selection is None or text_augment_selection == "none": + texts = json_dict_raw["text"] + elif text_augment_selection == "all": + if "text_augment_all" in json_dict_raw.keys(): + texts = json_dict_raw["text_augment_all"] + else: + texts = json_dict_raw["text"] + elif text_augment_selection == "augment_only": + if "text_augment_all" in json_dict_raw.keys(): + if json_dict_raw["text_augment_t5"] is None: + texts = json_dict_raw["text"] + else: + texts = json_dict_raw["text_augment_t5"] + else: + texts = json_dict_raw["text"] + else: + raise NotImplementedError( + f"text_augment_selection {text_augment_selection} not implemented" + ) + return texts + + +def preprocess_single( + sample, + audio_ext, + text_ext, + max_len, + audio_cfg, + tmodel, + class_index_dict, + data_filling, + data_truncating, + text_augment_selection, +): + """ + Preprocess a single sample for wdsdataloader. + """ + audio_data, orig_sr = sample[audio_ext] + audio_data = int16_to_float32_torch(float32_to_int16_torch(audio_data[0])) + + sample = get_audio_features(sample, audio_data, max_len, data_truncating, data_filling, audio_cfg) + del sample[audio_ext] + + json_dict_raw = sample[text_ext] + + texts = select_text(json_dict_raw, text_augment_selection) + sample["full_text"] = texts + + if isinstance(texts, list) and isinstance(texts[0], str) and len(texts) > 1: + texts = random.choice(texts) + sample["raw_text"] = texts + sample["text"] = tokenizer(texts, tmodel=tmodel) # text shape: [num_token] + if class_index_dict is not None: + # https://stackoverflow.com/questions/48004243/how-to-share-large-read-only-dictionary-list-across-processes-in-multiprocessing + # https://stackoverflow.com/questions/45693949/storing-strings-in-a-multiprocessing-sharedctypes-array + + # in case the re-written version is wrong, here is the old version: + # sample["class_label"] = np.zeros(len(class_index_dict.keys())) + # for x in json_dict_raw["tag"]: + # sample["class_label"][class_index_dict[x]] = 1 + # sample["class_label"] = torch.tensor(sample["class_label"]).float() + + class_labels = np.zeros(len(class_index_dict)) + class_labels[np.in1d(list(class_index_dict.keys()), json_dict_raw["tag"])] = 1 + sample["class_label"] = torch.tensor(class_labels).float() + + del sample[text_ext] + sample["audio_name"] = sample["__key__"].split("/")[-1] + "." + audio_ext + sample["text_name"] = sample["__key__"].split("/")[-1] + "." + text_ext + sample["audio_orig_sr"] = orig_sr + return sample + + +def collate_fn_with_preprocess(batch, + audio_ext, + text_ext, + max_len, + audio_cfg, + args, + ): + """ + Collate function for wdsdataloader. + batch: a list of dict, each dict is a sample + """ + + class_index_dict = copy.deepcopy(args.class_index_dict) # To avoid deadlock in multiprocessing + data_filling = args.data_filling + data_truncating = args.data_truncating + text_augment_selection = args.text_augment_selection + tmodel = args.tmodel + + # concatenate values in each dictionary. if it is a tensor, concatenate. if it is a list, extend. + data_preprocessed = [] + + for sample in batch: + data_preprocessed.append( + preprocess_single(sample, audio_ext, text_ext, max_len, audio_cfg, tmodel, class_index_dict, data_filling, + data_truncating, text_augment_selection)) + + batch_dict = {} + for k in data_preprocessed[0].keys(): + if isinstance(data_preprocessed[0][k], dict): # dealwith bert tokenizer output + batch_dict[k] = {} + for kk in data_preprocessed[0][k].keys(): + tmp = [] + for i in range(len(data_preprocessed)): + tmp.append(data_preprocessed[i][k][kk]) + batch_dict[k][kk] = torch.vstack(tmp) + elif isinstance(data_preprocessed[0][k], torch.Tensor): + batch_dict[k] = torch.stack([sample[k] for sample in data_preprocessed]) + elif isinstance(data_preprocessed[0][k], np.ndarray): + batch_dict[k] = torch.tensor(np.stack([sample[k] for sample in data_preprocessed])) + else: + batch_dict[k] = [sample[k] for sample in data_preprocessed] + del data_preprocessed + return batch_dict + + +def get_wds_dataset( + args, + model_cfg, + is_train, + audio_ext="flac", + text_ext="json", + max_len=480000, + proportion=1.0, + sizefilepath_=None, + is_local=None, +): + """ + Get a dataset for wdsdataloader. + """ + if is_local is None and (not args.remotedata is None): + is_local = not args.remotedata + + input_shards = args.train_data if is_train else args.val_data + assert input_shards is not None + + if not sizefilepath_ is None: + sizefilepath = sizefilepath_ + else: + sizefilepath = os.path.join(os.path.dirname(input_shards[0]), "sizes.json") + + if proportion != 1.0: + num_samples, num_shards, input_shards, _ = sample_prop( + sizefilepath, input_shards, proportion, is_local=is_local + ) + else: + num_samples, num_shards = get_dataset_size( + input_shards, sizefilepath_=sizefilepath_, is_local=is_local + ) + + if not num_samples: + if is_train: + num_samples = args.train_num_samples + if not num_samples: + raise RuntimeError( + "Currently, number of dataset samples must be specified for training dataset. " + "Please specify via `--train-num-samples` if no dataset length info present." + ) + else: + num_samples = ( + args.val_num_samples or 0 + ) # eval will just exhaust the iterator if not specified + + pipeline = [wds.SimpleShardList(input_shards)] + # at this point we have an iterator over all the shards + # TODO: (yusong): add a if statement of distributed. If not, we don't need to split_by_node + if is_train or args.parallel_eval: + pipeline.extend( + [ + wds.detshuffle( + bufsize=_SHARD_SHUFFLE_SIZE, + initial=_SHARD_SHUFFLE_INITIAL, + seed=args.seed, + ), + wds.split_by_node, + wds.split_by_worker, + # at this point, we have an iterator over the shards assigned to each worker at each node + wds.tarfile_to_samples(handler=log_and_continue), + wds.shuffle( + bufsize=_SAMPLE_SHUFFLE_SIZE, + initial=_SAMPLE_SHUFFLE_INITIAL, + rng=random.Random(args.seed), + ), + # wds.repeatedly, # FIXME determine if this is beneficial + ] + ) + else: + pipeline.extend( + [ + wds.split_by_worker, + # at this point, we have an iterator over the shards assigned to each worker + wds.tarfile_to_samples(handler=log_and_continue), + ] + ) + + pipeline.append( + wds.decode(wds.torch_audio), + ) + + pipeline.append( + wds.batched( + args.batch_size, + partial=not (is_train or args.parallel_eval), + collation_fn=partial(collate_fn_with_preprocess, + audio_ext=audio_ext, + text_ext=text_ext, + max_len=max_len, + audio_cfg=model_cfg['audio_cfg'], + args=args, + ), + + ) + ) + + dataset = wds.DataPipeline(*pipeline) + if is_train or args.parallel_eval: + # (yusong): Currently parallel evaluation will be not precise as we are repeat the last few samples. + # (yusong): See comments below. + # roll over and repeat a few samples to get same number of full batches on each node + global_batch_size = args.batch_size * args.world_size + num_batches = math.ceil(num_samples / global_batch_size) + num_workers = max(1, args.workers) + num_worker_batches = math.ceil( + num_batches / num_workers + ) # per dataloader worker + num_batches = num_worker_batches * num_workers + num_samples = num_batches * global_batch_size + dataset = dataset.with_epoch( + num_worker_batches + ) # each worker is iterating over this + else: + # last batches are partial, eval is done on single (master) node + num_batches = math.ceil(num_samples / args.batch_size) + + kwargs = {} + if args.horovod: # multi-node training on summit + kwargs["multiprocessing_context"] = "forkserver" + + if is_train: + if args.prefetch_factor: + prefetch_factor = args.prefetch_factor + else: + prefetch_factor = max(2, args.batch_size // args.workers) + else: + prefetch_factor = 2 + + dataloader = wds.WebLoader( + dataset, + batch_size=None, + shuffle=False, + num_workers=args.workers, + pin_memory=True, + prefetch_factor=prefetch_factor, + **kwargs + ) + + # FIXME not clear which approach is better, with_epoch before vs after dataloader? + # hoping to resolve via https://github.com/webdataset/webdataset/issues/169 + # if is_train: + # # roll over and repeat a few samples to get same number of full batches on each node + # global_batch_size = args.batch_size * args.world_size + # num_batches = math.ceil(num_samples / global_batch_size) + # num_workers = max(1, args.workers) + # num_batches = math.ceil(num_batches / num_workers) * num_workers + # num_samples = num_batches * global_batch_size + # dataloader = dataloader.with_epoch(num_batches) + # else: + # # last batches are partial, eval is done on single (master) node + # num_batches = math.ceil(num_samples / args.batch_size) + + # add meta-data to dataloader instance for convenience + dataloader.num_batches = num_batches + dataloader.num_samples = num_samples + + return DataInfo(dataloader, None) + + +def wds_batch_list2dict( + batch, + keys=[ + "__url__", + "__key__", + "waveform", + "text", + "raw_text", + "audio_name", + "text_name", + "audio_orig_sr", + ], +): + """ + Return a dictionary of the batch, with keys as the names of the fields. + """ + assert len(keys) == len( + batch + ), "batch must have same number of keys as keys argument" + return {keys[i]: batch[i] for i in range(len(batch))} + + + +def get_toy_dataset(args, model_cfg, is_train): + index_path = args.train_data if is_train else args.val_data + ipc_path = args.train_ipc if is_train else args.val_ipc + assert index_path and ipc_path + eval_mode = not is_train + dataset = ToyDataset(index_path, ipc_path, model_cfg, eval_mode=eval_mode) + + num_samples = len(dataset) + sampler = ( + DistributedSampler(dataset, shuffle=False) + if args.distributed and is_train + else None + ) + + dataloader = DataLoader( + dataset, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.workers, + sampler=sampler, + drop_last=is_train, + ) + dataloader.num_samples = num_samples + dataloader.num_batches = len(dataloader) + + return DataInfo(dataloader, sampler) + + +def get_dataset_fn(dataset_type): + if dataset_type == "webdataset": + return get_wds_dataset + elif dataset_type == "toy": + return get_toy_dataset + else: + raise ValueError(f"Unsupported dataset type: {dataset_type}") + + +def get_data(args, model_cfg): + data = {} + + args.class_index_dict = load_class_label(args.class_label_path) + + if args.datasetinfos is None: + args.datasetinfos = ["train", "unbalanced_train", "balanced_train"] + if args.dataset_type == "webdataset": + args.train_data = get_tar_path_from_dataset_name( + args.datasetnames, + args.datasetinfos, + islocal=not args.remotedata, + proportion=args.dataset_proportion, + dataset_path=args.datasetpath, + full_dataset=args.full_train_dataset, + ) + + if args.full_train_dataset is None: + args.full_train_dataset = [] + if args.exclude_eval_dataset is None: + args.exclude_eval_dataset = [] + excluded_eval_datasets = args.full_train_dataset + args.exclude_eval_dataset + + val_dataset_names = [n for n in args.datasetnames if n not in excluded_eval_datasets] \ + if excluded_eval_datasets else args.datasetnames + args.val_dataset_names = val_dataset_names + args.val_data = get_tar_path_from_dataset_name( + val_dataset_names, + ["valid", "test", "eval"], + islocal=not args.remotedata, + proportion=1, + dataset_path=args.datasetpath, + full_dataset=None, + ) + + if args.train_data: + data["train"] = get_dataset_fn(args.dataset_type)( + args, model_cfg, is_train=True + ) + + if args.val_data: + data["val"] = get_dataset_fn(args.dataset_type)( + args, model_cfg, is_train=False + ) + + return data diff --git a/utils/laion_clap/training/distributed.py b/utils/laion_clap/training/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..adb0e927a64dbe7fc83fecf65be054ac6bd28a94 --- /dev/null +++ b/utils/laion_clap/training/distributed.py @@ -0,0 +1,139 @@ +import os + +import torch +import socket + +try: + import horovod.torch as hvd +except ImportError: + hvd = None + + +def is_global_master(args): + return args.rank == 0 + + +def is_local_master(args): + return args.local_rank == 0 + + +def is_master(args, local=False): + return is_local_master(args) if local else is_global_master(args) + + +def is_using_horovod(): + # NOTE w/ horovod run, OMPI vars should be set, but w/ SLURM PMI vars will be set + # Differentiating between horovod and DDP use via SLURM may not be possible, so horovod arg still required... + ompi_vars = ["OMPI_COMM_WORLD_RANK", "OMPI_COMM_WORLD_SIZE"] + pmi_vars = ["PMI_RANK", "PMI_SIZE"] + if all([var in os.environ for var in ompi_vars]) or all([var in os.environ for var in pmi_vars]): + return True + else: + return False + + +def is_using_distributed(): + if 'WORLD_SIZE' in os.environ: + return int(os.environ['WORLD_SIZE']) > 1 + if 'SLURM_NTASKS' in os.environ: + return int(os.environ['SLURM_NTASKS']) > 1 + return False + + +def world_info_from_env(): + local_rank = 0 + for v in ('SLURM_LOCALID', 'MPI_LOCALRANKID', 'OMPI_COMM_WORLD_LOCAL_RANK', 'LOCAL_RANK'): + if v in os.environ: + local_rank = int(os.environ[v]) + break + global_rank = 0 + for v in ('SLURM_PROCID', 'PMI_RANK', 'OMPI_COMM_WORLD_RANK', 'RANK'): + if v in os.environ: + global_rank = int(os.environ[v]) + break + world_size = 1 + for v in ('SLURM_NTASKS', 'PMI_SIZE', 'OMPI_COMM_WORLD_SIZE', 'WORLD_SIZE'): + if v in os.environ: + world_size = int(os.environ[v]) + break + + return local_rank, global_rank, world_size + + +def init_distributed_device(args): + # Distributed training = training on more than one GPU. + # Works in both single and multi-node scenarios. + args.distributed = False + args.world_size = 1 + args.rank = 0 # global rank + args.local_rank = 0 + if args.horovod: + assert hvd is not None, "Horovod is not installed" + hvd.init() + world_size = int(os.environ['OMPI_COMM_WORLD_SIZE']) + world_rank = int(os.environ['OMPI_COMM_WORLD_RANK']) + local_rank = int(os.environ['OMPI_COMM_WORLD_LOCAL_RANK']) + args.local_rank = local_rank + args.rank = world_rank + args.world_size = world_size + # args.local_rank = int(hvd.local_rank()) + # args.rank = hvd.rank() + # args.world_size = hvd.size() + args.distributed = True + os.environ['LOCAL_RANK'] = str(args.local_rank) + os.environ['RANK'] = str(args.rank) + os.environ['WORLD_SIZE'] = str(args.world_size) + print(f"Distributed training: local_rank={args.local_rank}, " + f"rank={args.rank}, world_size={args.world_size}, " + f"hostname={socket.gethostname()}, pid={os.getpid()}") + elif is_using_distributed(): + if 'SLURM_PROCID' in os.environ: + # DDP via SLURM + args.local_rank, args.rank, args.world_size = world_info_from_env() + # SLURM var -> torch.distributed vars in case needed + os.environ['LOCAL_RANK'] = str(args.local_rank) + os.environ['RANK'] = str(args.rank) + os.environ['WORLD_SIZE'] = str(args.world_size) + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) + elif 'OMPI_COMM_WORLD_SIZE' in os.environ: # using Summit cluster + world_size = int(os.environ['OMPI_COMM_WORLD_SIZE']) + world_rank = int(os.environ['OMPI_COMM_WORLD_RANK']) + local_rank = int(os.environ['OMPI_COMM_WORLD_LOCAL_RANK']) + args.local_rank = local_rank + args.rank = world_rank + args.world_size = world_size + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) + else: + # DDP via torchrun, torch.distributed.launch + args.local_rank, _, _ = world_info_from_env() + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url) + args.world_size = torch.distributed.get_world_size() + args.rank = torch.distributed.get_rank() + args.distributed = True + print(f"Distributed training: local_rank={args.local_rank}, " + f"rank={args.rank}, world_size={args.world_size}, " + f"hostname={socket.gethostname()}, pid={os.getpid()}") + + if torch.cuda.is_available(): + if args.distributed and not args.no_set_device_rank: + device = 'cuda:%d' % args.local_rank + else: + device = 'cuda:0' + torch.cuda.set_device(device) + else: + device = 'cpu' + args.device = device + device = torch.device(device) + return device diff --git a/utils/laion_clap/training/imagenet_zeroshot_data.py b/utils/laion_clap/training/imagenet_zeroshot_data.py new file mode 100644 index 0000000000000000000000000000000000000000..a78987448805afc228b2941302a2894818cac497 --- /dev/null +++ b/utils/laion_clap/training/imagenet_zeroshot_data.py @@ -0,0 +1,254 @@ +# NOTE: This script is currently not supported for CLAP. + +imagenet_classnames = ["tench", "goldfish", "great white shark", "tiger shark", "hammerhead shark", "electric ray", + "stingray", "rooster", "hen", "ostrich", "brambling", "goldfinch", "house finch", "junco", + "indigo bunting", "American robin", "bulbul", "jay", "magpie", "chickadee", "American dipper", + "kite (bird of prey)", "bald eagle", "vulture", "great grey owl", "fire salamander", + "smooth newt", "newt", "spotted salamander", "axolotl", "American bullfrog", "tree frog", + "tailed frog", "loggerhead sea turtle", "leatherback sea turtle", "mud turtle", "terrapin", + "box turtle", "banded gecko", "green iguana", "Carolina anole", + "desert grassland whiptail lizard", "agama", "frilled-necked lizard", "alligator lizard", + "Gila monster", "European green lizard", "chameleon", "Komodo dragon", "Nile crocodile", + "American alligator", "triceratops", "worm snake", "ring-necked snake", + "eastern hog-nosed snake", "smooth green snake", "kingsnake", "garter snake", "water snake", + "vine snake", "night snake", "boa constrictor", "African rock python", "Indian cobra", + "green mamba", "sea snake", "Saharan horned viper", "eastern diamondback rattlesnake", + "sidewinder rattlesnake", "trilobite", "harvestman", "scorpion", "yellow garden spider", + "barn spider", "European garden spider", "southern black widow", "tarantula", "wolf spider", + "tick", "centipede", "black grouse", "ptarmigan", "ruffed grouse", "prairie grouse", "peafowl", + "quail", "partridge", "african grey parrot", "macaw", "sulphur-crested cockatoo", "lorikeet", + "coucal", "bee eater", "hornbill", "hummingbird", "jacamar", "toucan", "duck", + "red-breasted merganser", "goose", "black swan", "tusker", "echidna", "platypus", "wallaby", + "koala", "wombat", "jellyfish", "sea anemone", "brain coral", "flatworm", "nematode", "conch", + "snail", "slug", "sea slug", "chiton", "chambered nautilus", "Dungeness crab", "rock crab", + "fiddler crab", "red king crab", "American lobster", "spiny lobster", "crayfish", "hermit crab", + "isopod", "white stork", "black stork", "spoonbill", "flamingo", "little blue heron", + "great egret", "bittern bird", "crane bird", "limpkin", "common gallinule", "American coot", + "bustard", "ruddy turnstone", "dunlin", "common redshank", "dowitcher", "oystercatcher", + "pelican", "king penguin", "albatross", "grey whale", "killer whale", "dugong", "sea lion", + "Chihuahua", "Japanese Chin", "Maltese", "Pekingese", "Shih Tzu", "King Charles Spaniel", + "Papillon", "toy terrier", "Rhodesian Ridgeback", "Afghan Hound", "Basset Hound", "Beagle", + "Bloodhound", "Bluetick Coonhound", "Black and Tan Coonhound", "Treeing Walker Coonhound", + "English foxhound", "Redbone Coonhound", "borzoi", "Irish Wolfhound", "Italian Greyhound", + "Whippet", "Ibizan Hound", "Norwegian Elkhound", "Otterhound", "Saluki", "Scottish Deerhound", + "Weimaraner", "Staffordshire Bull Terrier", "American Staffordshire Terrier", + "Bedlington Terrier", "Border Terrier", "Kerry Blue Terrier", "Irish Terrier", + "Norfolk Terrier", "Norwich Terrier", "Yorkshire Terrier", "Wire Fox Terrier", + "Lakeland Terrier", "Sealyham Terrier", "Airedale Terrier", "Cairn Terrier", + "Australian Terrier", "Dandie Dinmont Terrier", "Boston Terrier", "Miniature Schnauzer", + "Giant Schnauzer", "Standard Schnauzer", "Scottish Terrier", "Tibetan Terrier", + "Australian Silky Terrier", "Soft-coated Wheaten Terrier", "West Highland White Terrier", + "Lhasa Apso", "Flat-Coated Retriever", "Curly-coated Retriever", "Golden Retriever", + "Labrador Retriever", "Chesapeake Bay Retriever", "German Shorthaired Pointer", "Vizsla", + "English Setter", "Irish Setter", "Gordon Setter", "Brittany dog", "Clumber Spaniel", + "English Springer Spaniel", "Welsh Springer Spaniel", "Cocker Spaniel", "Sussex Spaniel", + "Irish Water Spaniel", "Kuvasz", "Schipperke", "Groenendael dog", "Malinois", "Briard", + "Australian Kelpie", "Komondor", "Old English Sheepdog", "Shetland Sheepdog", "collie", + "Border Collie", "Bouvier des Flandres dog", "Rottweiler", "German Shepherd Dog", "Dobermann", + "Miniature Pinscher", "Greater Swiss Mountain Dog", "Bernese Mountain Dog", + "Appenzeller Sennenhund", "Entlebucher Sennenhund", "Boxer", "Bullmastiff", "Tibetan Mastiff", + "French Bulldog", "Great Dane", "St. Bernard", "husky", "Alaskan Malamute", "Siberian Husky", + "Dalmatian", "Affenpinscher", "Basenji", "pug", "Leonberger", "Newfoundland dog", + "Great Pyrenees dog", "Samoyed", "Pomeranian", "Chow Chow", "Keeshond", "brussels griffon", + "Pembroke Welsh Corgi", "Cardigan Welsh Corgi", "Toy Poodle", "Miniature Poodle", + "Standard Poodle", "Mexican hairless dog (xoloitzcuintli)", "grey wolf", "Alaskan tundra wolf", + "red wolf or maned wolf", "coyote", "dingo", "dhole", "African wild dog", "hyena", "red fox", + "kit fox", "Arctic fox", "grey fox", "tabby cat", "tiger cat", "Persian cat", "Siamese cat", + "Egyptian Mau", "cougar", "lynx", "leopard", "snow leopard", "jaguar", "lion", "tiger", + "cheetah", "brown bear", "American black bear", "polar bear", "sloth bear", "mongoose", + "meerkat", "tiger beetle", "ladybug", "ground beetle", "longhorn beetle", "leaf beetle", + "dung beetle", "rhinoceros beetle", "weevil", "fly", "bee", "ant", "grasshopper", + "cricket insect", "stick insect", "cockroach", "praying mantis", "cicada", "leafhopper", + "lacewing", "dragonfly", "damselfly", "red admiral butterfly", "ringlet butterfly", + "monarch butterfly", "small white butterfly", "sulphur butterfly", "gossamer-winged butterfly", + "starfish", "sea urchin", "sea cucumber", "cottontail rabbit", "hare", "Angora rabbit", + "hamster", "porcupine", "fox squirrel", "marmot", "beaver", "guinea pig", "common sorrel horse", + "zebra", "pig", "wild boar", "warthog", "hippopotamus", "ox", "water buffalo", "bison", + "ram (adult male sheep)", "bighorn sheep", "Alpine ibex", "hartebeest", "impala (antelope)", + "gazelle", "arabian camel", "llama", "weasel", "mink", "European polecat", + "black-footed ferret", "otter", "skunk", "badger", "armadillo", "three-toed sloth", "orangutan", + "gorilla", "chimpanzee", "gibbon", "siamang", "guenon", "patas monkey", "baboon", "macaque", + "langur", "black-and-white colobus", "proboscis monkey", "marmoset", "white-headed capuchin", + "howler monkey", "titi monkey", "Geoffroy's spider monkey", "common squirrel monkey", + "ring-tailed lemur", "indri", "Asian elephant", "African bush elephant", "red panda", + "giant panda", "snoek fish", "eel", "silver salmon", "rock beauty fish", "clownfish", + "sturgeon", "gar fish", "lionfish", "pufferfish", "abacus", "abaya", "academic gown", + "accordion", "acoustic guitar", "aircraft carrier", "airliner", "airship", "altar", "ambulance", + "amphibious vehicle", "analog clock", "apiary", "apron", "trash can", "assault rifle", + "backpack", "bakery", "balance beam", "balloon", "ballpoint pen", "Band-Aid", "banjo", + "baluster / handrail", "barbell", "barber chair", "barbershop", "barn", "barometer", "barrel", + "wheelbarrow", "baseball", "basketball", "bassinet", "bassoon", "swimming cap", "bath towel", + "bathtub", "station wagon", "lighthouse", "beaker", "military hat (bearskin or shako)", + "beer bottle", "beer glass", "bell tower", "baby bib", "tandem bicycle", "bikini", + "ring binder", "binoculars", "birdhouse", "boathouse", "bobsleigh", "bolo tie", "poke bonnet", + "bookcase", "bookstore", "bottle cap", "hunting bow", "bow tie", "brass memorial plaque", "bra", + "breakwater", "breastplate", "broom", "bucket", "buckle", "bulletproof vest", + "high-speed train", "butcher shop", "taxicab", "cauldron", "candle", "cannon", "canoe", + "can opener", "cardigan", "car mirror", "carousel", "tool kit", "cardboard box / carton", + "car wheel", "automated teller machine", "cassette", "cassette player", "castle", "catamaran", + "CD player", "cello", "mobile phone", "chain", "chain-link fence", "chain mail", "chainsaw", + "storage chest", "chiffonier", "bell or wind chime", "china cabinet", "Christmas stocking", + "church", "movie theater", "cleaver", "cliff dwelling", "cloak", "clogs", "cocktail shaker", + "coffee mug", "coffeemaker", "spiral or coil", "combination lock", "computer keyboard", + "candy store", "container ship", "convertible", "corkscrew", "cornet", "cowboy boot", + "cowboy hat", "cradle", "construction crane", "crash helmet", "crate", "infant bed", + "Crock Pot", "croquet ball", "crutch", "cuirass", "dam", "desk", "desktop computer", + "rotary dial telephone", "diaper", "digital clock", "digital watch", "dining table", + "dishcloth", "dishwasher", "disc brake", "dock", "dog sled", "dome", "doormat", "drilling rig", + "drum", "drumstick", "dumbbell", "Dutch oven", "electric fan", "electric guitar", + "electric locomotive", "entertainment center", "envelope", "espresso machine", "face powder", + "feather boa", "filing cabinet", "fireboat", "fire truck", "fire screen", "flagpole", "flute", + "folding chair", "football helmet", "forklift", "fountain", "fountain pen", "four-poster bed", + "freight car", "French horn", "frying pan", "fur coat", "garbage truck", + "gas mask or respirator", "gas pump", "goblet", "go-kart", "golf ball", "golf cart", "gondola", + "gong", "gown", "grand piano", "greenhouse", "radiator grille", "grocery store", "guillotine", + "hair clip", "hair spray", "half-track", "hammer", "hamper", "hair dryer", "hand-held computer", + "handkerchief", "hard disk drive", "harmonica", "harp", "combine harvester", "hatchet", + "holster", "home theater", "honeycomb", "hook", "hoop skirt", "gymnastic horizontal bar", + "horse-drawn vehicle", "hourglass", "iPod", "clothes iron", "carved pumpkin", "jeans", "jeep", + "T-shirt", "jigsaw puzzle", "rickshaw", "joystick", "kimono", "knee pad", "knot", "lab coat", + "ladle", "lampshade", "laptop computer", "lawn mower", "lens cap", "letter opener", "library", + "lifeboat", "lighter", "limousine", "ocean liner", "lipstick", "slip-on shoe", "lotion", + "music speaker", "loupe magnifying glass", "sawmill", "magnetic compass", "messenger bag", + "mailbox", "tights", "one-piece bathing suit", "manhole cover", "maraca", "marimba", "mask", + "matchstick", "maypole", "maze", "measuring cup", "medicine cabinet", "megalith", "microphone", + "microwave oven", "military uniform", "milk can", "minibus", "miniskirt", "minivan", "missile", + "mitten", "mixing bowl", "mobile home", "ford model t", "modem", "monastery", "monitor", + "moped", "mortar and pestle", "graduation cap", "mosque", "mosquito net", "vespa", + "mountain bike", "tent", "computer mouse", "mousetrap", "moving van", "muzzle", "metal nail", + "neck brace", "necklace", "baby pacifier", "notebook computer", "obelisk", "oboe", "ocarina", + "odometer", "oil filter", "pipe organ", "oscilloscope", "overskirt", "bullock cart", + "oxygen mask", "product packet / packaging", "paddle", "paddle wheel", "padlock", "paintbrush", + "pajamas", "palace", "pan flute", "paper towel", "parachute", "parallel bars", "park bench", + "parking meter", "railroad car", "patio", "payphone", "pedestal", "pencil case", + "pencil sharpener", "perfume", "Petri dish", "photocopier", "plectrum", "Pickelhaube", + "picket fence", "pickup truck", "pier", "piggy bank", "pill bottle", "pillow", "ping-pong ball", + "pinwheel", "pirate ship", "drink pitcher", "block plane", "planetarium", "plastic bag", + "plate rack", "farm plow", "plunger", "Polaroid camera", "pole", "police van", "poncho", + "pool table", "soda bottle", "plant pot", "potter's wheel", "power drill", "prayer rug", + "printer", "prison", "missile", "projector", "hockey puck", "punching bag", "purse", "quill", + "quilt", "race car", "racket", "radiator", "radio", "radio telescope", "rain barrel", + "recreational vehicle", "fishing casting reel", "reflex camera", "refrigerator", + "remote control", "restaurant", "revolver", "rifle", "rocking chair", "rotisserie", "eraser", + "rugby ball", "ruler measuring stick", "sneaker", "safe", "safety pin", "salt shaker", "sandal", + "sarong", "saxophone", "scabbard", "weighing scale", "school bus", "schooner", "scoreboard", + "CRT monitor", "screw", "screwdriver", "seat belt", "sewing machine", "shield", "shoe store", + "shoji screen / room divider", "shopping basket", "shopping cart", "shovel", "shower cap", + "shower curtain", "ski", "balaclava ski mask", "sleeping bag", "slide rule", "sliding door", + "slot machine", "snorkel", "snowmobile", "snowplow", "soap dispenser", "soccer ball", "sock", + "solar thermal collector", "sombrero", "soup bowl", "keyboard space bar", "space heater", + "space shuttle", "spatula", "motorboat", "spider web", "spindle", "sports car", "spotlight", + "stage", "steam locomotive", "through arch bridge", "steel drum", "stethoscope", "scarf", + "stone wall", "stopwatch", "stove", "strainer", "tram", "stretcher", "couch", "stupa", + "submarine", "suit", "sundial", "sunglasses", "sunglasses", "sunscreen", "suspension bridge", + "mop", "sweatshirt", "swim trunks / shorts", "swing", "electrical switch", "syringe", + "table lamp", "tank", "tape player", "teapot", "teddy bear", "television", "tennis ball", + "thatched roof", "front curtain", "thimble", "threshing machine", "throne", "tile roof", + "toaster", "tobacco shop", "toilet seat", "torch", "totem pole", "tow truck", "toy store", + "tractor", "semi-trailer truck", "tray", "trench coat", "tricycle", "trimaran", "tripod", + "triumphal arch", "trolleybus", "trombone", "hot tub", "turnstile", "typewriter keyboard", + "umbrella", "unicycle", "upright piano", "vacuum cleaner", "vase", "vaulted or arched ceiling", + "velvet fabric", "vending machine", "vestment", "viaduct", "violin", "volleyball", + "waffle iron", "wall clock", "wallet", "wardrobe", "military aircraft", "sink", + "washing machine", "water bottle", "water jug", "water tower", "whiskey jug", "whistle", + "hair wig", "window screen", "window shade", "Windsor tie", "wine bottle", "airplane wing", + "wok", "wooden spoon", "wool", "split-rail fence", "shipwreck", "sailboat", "yurt", "website", + "comic book", "crossword", "traffic or street sign", "traffic light", "dust jacket", "menu", + "plate", "guacamole", "consomme", "hot pot", "trifle", "ice cream", "popsicle", "baguette", + "bagel", "pretzel", "cheeseburger", "hot dog", "mashed potatoes", "cabbage", "broccoli", + "cauliflower", "zucchini", "spaghetti squash", "acorn squash", "butternut squash", "cucumber", + "artichoke", "bell pepper", "cardoon", "mushroom", "Granny Smith apple", "strawberry", "orange", + "lemon", "fig", "pineapple", "banana", "jackfruit", "cherimoya (custard apple)", "pomegranate", + "hay", "carbonara", "chocolate syrup", "dough", "meatloaf", "pizza", "pot pie", "burrito", + "red wine", "espresso", "tea cup", "eggnog", "mountain", "bubble", "cliff", "coral reef", + "geyser", "lakeshore", "promontory", "sandbar", "beach", "valley", "volcano", "baseball player", + "bridegroom", "scuba diver", "rapeseed", "daisy", "yellow lady's slipper", "corn", "acorn", + "rose hip", "horse chestnut seed", "coral fungus", "agaric", "gyromitra", "stinkhorn mushroom", + "earth star fungus", "hen of the woods mushroom", "bolete", "corn cob", "toilet paper"] + + + + + +openai_imagenet_template = [ + lambda c: f'a bad photo of a {c}.', + lambda c: f'a photo of many {c}.', + lambda c: f'a sculpture of a {c}.', + lambda c: f'a photo of the hard to see {c}.', + lambda c: f'a low resolution photo of the {c}.', + lambda c: f'a rendering of a {c}.', + lambda c: f'graffiti of a {c}.', + lambda c: f'a bad photo of the {c}.', + lambda c: f'a cropped photo of the {c}.', + lambda c: f'a tattoo of a {c}.', + lambda c: f'the embroidered {c}.', + lambda c: f'a photo of a hard to see {c}.', + lambda c: f'a bright photo of a {c}.', + lambda c: f'a photo of a clean {c}.', + lambda c: f'a photo of a dirty {c}.', + lambda c: f'a dark photo of the {c}.', + lambda c: f'a drawing of a {c}.', + lambda c: f'a photo of my {c}.', + lambda c: f'the plastic {c}.', + lambda c: f'a photo of the cool {c}.', + lambda c: f'a close-up photo of a {c}.', + lambda c: f'a black and white photo of the {c}.', + lambda c: f'a painting of the {c}.', + lambda c: f'a painting of a {c}.', + lambda c: f'a pixelated photo of the {c}.', + lambda c: f'a sculpture of the {c}.', + lambda c: f'a bright photo of the {c}.', + lambda c: f'a cropped photo of a {c}.', + lambda c: f'a plastic {c}.', + lambda c: f'a photo of the dirty {c}.', + lambda c: f'a jpeg corrupted photo of a {c}.', + lambda c: f'a blurry photo of the {c}.', + lambda c: f'a photo of the {c}.', + lambda c: f'a good photo of the {c}.', + lambda c: f'a rendering of the {c}.', + lambda c: f'a {c} in a video game.', + lambda c: f'a photo of one {c}.', + lambda c: f'a doodle of a {c}.', + lambda c: f'a close-up photo of the {c}.', + lambda c: f'a photo of a {c}.', + lambda c: f'the origami {c}.', + lambda c: f'the {c} in a video game.', + lambda c: f'a sketch of a {c}.', + lambda c: f'a doodle of the {c}.', + lambda c: f'a origami {c}.', + lambda c: f'a low resolution photo of a {c}.', + lambda c: f'the toy {c}.', + lambda c: f'a rendition of the {c}.', + lambda c: f'a photo of the clean {c}.', + lambda c: f'a photo of a large {c}.', + lambda c: f'a rendition of a {c}.', + lambda c: f'a photo of a nice {c}.', + lambda c: f'a photo of a weird {c}.', + lambda c: f'a blurry photo of a {c}.', + lambda c: f'a cartoon {c}.', + lambda c: f'art of a {c}.', + lambda c: f'a sketch of the {c}.', + lambda c: f'a embroidered {c}.', + lambda c: f'a pixelated photo of a {c}.', + lambda c: f'itap of the {c}.', + lambda c: f'a jpeg corrupted photo of the {c}.', + lambda c: f'a good photo of a {c}.', + lambda c: f'a plushie {c}.', + lambda c: f'a photo of the nice {c}.', + lambda c: f'a photo of the small {c}.', + lambda c: f'a photo of the weird {c}.', + lambda c: f'the cartoon {c}.', + lambda c: f'art of the {c}.', + lambda c: f'a drawing of the {c}.', + lambda c: f'a photo of the large {c}.', + lambda c: f'a black and white photo of a {c}.', + lambda c: f'the plushie {c}.', + lambda c: f'a dark photo of a {c}.', + lambda c: f'itap of a {c}.', + lambda c: f'graffiti of the {c}.', + lambda c: f'a toy {c}.', + lambda c: f'itap of my {c}.', + lambda c: f'a photo of a cool {c}.', + lambda c: f'a photo of a small {c}.', + lambda c: f'a tattoo of the {c}.', +] diff --git a/utils/laion_clap/training/infer_demo.py b/utils/laion_clap/training/infer_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..b39f3f43fc9f295b244bc1ca2b444f4969488cba --- /dev/null +++ b/utils/laion_clap/training/infer_demo.py @@ -0,0 +1,92 @@ +import torch +import librosa +from clap_module import create_model +from training.data import get_audio_features +from training.data import int16_to_float32, float32_to_int16 +from transformers import RobertaTokenizer + +tokenize = RobertaTokenizer.from_pretrained('roberta-base') +def tokenizer(text): + result = tokenize( + text, + padding="max_length", + truncation=True, + max_length=77, + return_tensors="pt", + ) + return {k: v.squeeze(0) for k, v in result.items()} + +def infer_text(): + device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + precision = 'fp32' + amodel = 'HTSAT-tiny' # or 'PANN-14' + tmodel = 'roberta' # the best text encoder in our training + enable_fusion = True # False if you do not want to use the fusion model + fusion_type = 'aff_2d' + pretrained = "/home/la/kechen/Research/KE_CLAP/ckpt/fusion_best.pt" # the checkpoint name, the unfusion model can also be loaded. + + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision=precision, + device=device, + enable_fusion=enable_fusion, + fusion_type=fusion_type + ) + # load the text, can be a list (i.e. batch size) + text_data = ["I love the contrastive learning", "I love the pretrain model"] + # tokenize for roberta, if you want to tokenize for another text encoder, please refer to data.py#L43-90 + text_data = tokenizer(text_data) + model.eval() + text_embed = model.get_text_embedding(text_data) + text_embed = text_embed.detach().cpu().numpy() + print(text_embed) + print(text_embed.shape) + +def infer_audio(): + + device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + precision = 'fp32' + amodel = 'HTSAT-tiny' # or 'PANN-14' + tmodel = 'roberta' # the best text encoder in our training + enable_fusion = True # False if you do not want to use the fusion model + fusion_type = 'aff_2d' + pretrained = "/home/la/kechen/Research/KE_CLAP/ckpt/fusion_best.pt" # the checkpoint name, the unfusion model can also be loaded. + + model, model_cfg = create_model( + amodel, + tmodel, + pretrained, + precision=precision, + device=device, + enable_fusion=enable_fusion, + fusion_type=fusion_type + ) + + # load the waveform of the shape (T,), should resample to 48000 + audio_waveform, sr = librosa.load('/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_short.wav', sr=48000) + # quantize + audio_waveform = int16_to_float32(float32_to_int16(audio_waveform)) + audio_waveform = torch.from_numpy(audio_waveform).float() + audio_dict = {} + + # the 'fusion' truncate mode can be changed to 'rand_trunc' if run in unfusion mode + audio_dict = get_audio_features( + audio_dict, audio_waveform, 480000, + data_truncating='fusion', + data_filling='repeatpad', + audio_cfg=model_cfg['audio_cfg'] + ) + model.eval() + # can send a list to the model, to process many audio tracks in one time (i.e. batch size) + audio_embed = model.get_audio_embedding([audio_dict]) + audio_embed = audio_embed.detach().cpu().numpy() + print(audio_embed) + print(audio_embed.shape) + + + +if __name__ == "__main__": + infer_text() + # infer_audio() diff --git a/utils/laion_clap/training/logger.py b/utils/laion_clap/training/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..6d9abed92568d459cbc8d6094ae3901935d89621 --- /dev/null +++ b/utils/laion_clap/training/logger.py @@ -0,0 +1,26 @@ +import logging + + +def setup_logging(log_file, level, include_host=False): + if include_host: + import socket + hostname = socket.gethostname() + formatter = logging.Formatter( + f'%(asctime)s | {hostname} | %(levelname)s | %(message)s', datefmt='%Y-%m-%d,%H:%M:%S') + else: + formatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s', datefmt='%Y-%m-%d,%H:%M:%S') + + logging.root.setLevel(level) + loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] + for logger in loggers: + logger.setLevel(level) + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + logging.root.addHandler(stream_handler) + + if log_file: + file_handler = logging.FileHandler(filename=log_file) + file_handler.setFormatter(formatter) + logging.root.addHandler(file_handler) + diff --git a/utils/laion_clap/training/lp_main.py b/utils/laion_clap/training/lp_main.py new file mode 100644 index 0000000000000000000000000000000000000000..1e88f7356950ef03c78ca4d88681eb78ff1b4f6a --- /dev/null +++ b/utils/laion_clap/training/lp_main.py @@ -0,0 +1,643 @@ +import logging +import os +import random +from datetime import datetime +import copy +import numpy as np +import torch +import torch.backends.cudnn as cudnn +from torch.cuda.amp import GradScaler +import time + +try: + import wandb +except ImportError: + wandb = None + +try: + import torch.utils.tensorboard as tensorboard +except ImportError: + tensorboard = None + +try: + import horovod.torch as hvd +except ImportError: + hvd = None + +from clap_module import create_model_and_transforms, trace_model, create_model +from training.data import get_data +from training.params import parse_args +from training.distributed import is_master, init_distributed_device, world_info_from_env +from training.logger import setup_logging +from training.scheduler import cosine_lr +from training.lp_train import train_one_epoch, evaluate +from clap_module.utils import get_tar_path_from_dataset_name, dataset_split, get_optimizer +from clap_module.utils import load_p, load_class_label +from clap_module.linear_probe import LinearProbe + + +def maintain_ckpts(args, startidx, all_idx_len): + for i in reversed(range(startidx, all_idx_len)): + if os.path.exists(os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt")): + os.rename( + os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt"), + os.path.join(args.checkpoint_path, f"epoch_top_{i+1}.pt"), + ) + if os.path.exists( + os.path.join(args.checkpoint_path, f"epoch_top_{all_idx_len}.pt") + ): + os.remove(os.path.join(args.checkpoint_path, f"epoch_top_{all_idx_len}.pt")) + return + + +def update_top_k_performance( + new_metrics_inputs, current_top_k_ckpt_metrics, args, ckpt, bignumbetter=True +): + """ + Record the top-k performance of the current epoch. + current_top_k_metrics is a dictionary of the form: {1: top_1_ckpt_measure, 2: top_2_ckpt_measure, ...} + """ + if isinstance(new_metrics_inputs, (list, tuple)): + new_metrics_inputs = np.mean(new_metrics_inputs) + return update_top_k_performance( + new_metrics_inputs, + current_top_k_ckpt_metrics, + args=args, + ckpt=ckpt, + bignumbetter=bignumbetter, + ) + elif isinstance(new_metrics_inputs, dict): + new_metrics_inputs = np.mean(list(new_metrics_inputs.values())) + return update_top_k_performance( + new_metrics_inputs, + current_top_k_ckpt_metrics, + args=args, + ckpt=ckpt, + bignumbetter=bignumbetter, + ) + elif isinstance(new_metrics_inputs, (float, int)): + update_flag = {k: False for k in current_top_k_ckpt_metrics.keys()} + sorted_keys = sorted(current_top_k_ckpt_metrics.keys()) + sorted_values = sorted( + current_top_k_ckpt_metrics.values(), reverse=bignumbetter + ) + sorted_values_ = copy.deepcopy(sorted_values) + sorted_values.append(new_metrics_inputs) + sorted_values = sorted(sorted_values, reverse=bignumbetter) + sorted_values = sorted_values[:-1] + + if sorted_values == sorted_values_: + return current_top_k_ckpt_metrics, new_metrics_inputs + else: + for i in range(len(sorted_keys)): + if current_top_k_ckpt_metrics[sorted_keys[i]] != sorted_values[i]: + current_top_k_ckpt_metrics[sorted_keys[i]] = sorted_values[i] + update_flag[sorted_keys[i]] = True + for i in range(len(update_flag)): + if update_flag[i]: + maintain_ckpts(args, i, len(sorted_keys)) + torch.save( + ckpt, + os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt"), + ) + break + return current_top_k_ckpt_metrics, new_metrics_inputs + + +# def updateifNone(a, b): +# a = b if None else a +# return a + + +def is_pretrained_params(n): + return ( + n.startswith("clap_model.transformer") + or n in ["clap_model.positional_embedding", "clap_model.text_projection"] + or n.startswith("clap_model.token_embedding") + or n.startswith("clap_model.ln_final") + or n.startswith("clap_model.logit_scale_t") + ) + + +def random_seed(seed=42, rank=0): + torch.manual_seed(seed + rank) + np.random.seed(seed + rank) + random.seed(seed + rank) + +def config_lp_optimizer(model, data, args): + # set wd-related params to 0 if use adam optimizer + if args.optimizer == "adam": + args.wd = 0 + args.wd_pretrained = 0 + args.wd_new = 0 + + in_clap = ( + lambda n, p: n.startswith("clap_model") + ) + + named_parameters = list(model.named_parameters()) + + optimizer = {} + scheduler = {} + + # freeze text encoder + text_freeze_parameters = [ + p + for n, p in named_parameters + if n.startswith("clap_model.transformer") + or n in ["clap_model.positional_embedding", "clap_model.text_projection"] + or n.startswith("clap_model.token_embedding") + or n.startswith("clap_model.ln_final") + ] + + if args.freeze_text: + logging.info("Freeze Text!!!!") + for k in text_freeze_parameters: + k.requires_grad = False + + if not args.lp_freeze: + exclude = ( + lambda n, p: p.ndim < 2 + or "bn" in n + or "ln" in n + or "bias" in n + or "logit_scale" in n + ) + include = lambda n, p: not exclude(n, p) + + # (yusong): we do not split the learning rate anymore + # p for n, p in named_parameters if in_clap(n,p) and exclude(n, p) and p.requires_grad + gain_or_bias_params = [ + p for n, p in named_parameters if exclude(n, p) and p.requires_grad + ] + # rest_params = [p for n, p in named_parameters if in_clap(n,p) and include(n, p) and p.requires_grad] + rest_params = [p for n, p in named_parameters if include(n, p) and p.requires_grad] + + if args.train_data is None: + optimizer = None + scheduler = None + else: + total_steps = data["train"].dataloader.num_batches * args.epochs + + if args.split_opt: + for x in ["lr", "beta1", "beta2", "eps", "wd"]: + for y in ["_new", "_pretrained"]: + if getattr(args, x + y) is None: + setattr(args, x + y, getattr(args, x)) + + gain_or_bias_pretrained_params = [ + p + for n, p in named_parameters + if (exclude(n, p) and p.requires_grad) and is_pretrained_params(n) + ] + rest_pretrained_params = [ + p + for n, p in named_parameters + if (include(n, p) and p.requires_grad) and is_pretrained_params(n) + ] + gain_or_bias_new_params = [ + p + for n, p in named_parameters + if (exclude(n, p) and p.requires_grad) and (not is_pretrained_params(n)) + ] + rest_new_params = [ + p + for n, p in named_parameters + if (include(n, p) and p.requires_grad) and (not is_pretrained_params(n)) + ] + + pretrained_params_optimizer = get_optimizer( + [ + {"params": gain_or_bias_pretrained_params, "weight_decay": 0.0}, + { + "params": rest_pretrained_params, + "weight_decay": args.wd_pretrained, + }, + ], + lr=args.lr_pretrained, + betas=(args.beta1_pretrained, args.beta2_pretrained), + eps=args.eps_pretrained, + momentum=args.momentum_pretrained, + optimizer_name=args.optimizer, + ) + pretrained_params_scheduler = cosine_lr( + pretrained_params_optimizer, + args.lr_pretrained, + args.warmup, + total_steps, + ) + + new_params_optimizer = get_optimizer( + [ + {"params": gain_or_bias_new_params, "weight_decay": 0.0}, + {"params": rest_new_params, "weight_decay": args.wd_new}, + ], + lr=args.lr_new, + betas=(args.beta1_new, args.beta2_new), + eps=args.eps_new, + momentum=args.momentum_new, + optimizer_name=args.optimizer, + ) + new_params_scheduler = cosine_lr( + new_params_optimizer, args.lr_new, args.warmup, total_steps + ) + + optimizer["text"] = pretrained_params_optimizer + optimizer["audio"] = new_params_optimizer + scheduler["text"] = pretrained_params_scheduler + scheduler["audio"] = new_params_scheduler + + if args.horovod: + pretrained_params_optimizer = hvd.DistributedOptimizer( + pretrained_params_optimizer, + named_parameters=model.named_parameters(), + ) + new_params_optimizer = hvd.DistributedOptimizer( + new_params_optimizer, named_parameters=model.named_parameters() + ) + hvd.broadcast_parameters(model.state_dict(), root_rank=0) + hvd.broadcast_optimizer_state(pretrained_params_optimizer, root_rank=0) + hvd.broadcast_optimizer_state(new_params_optimizer, root_rank=0) + else: + + optimizer["clap"] = get_optimizer( + [ + {"params": gain_or_bias_params, "weight_decay": 0.0}, + {"params": rest_params, "weight_decay": args.wd}, + ], + lr=args.lr, + betas=(args.beta1, args.beta2), + eps=args.eps, + momentum=args.momentum, + optimizer_name=args.optimizer, + ) + scheduler["clap"] = cosine_lr(optimizer["clap"], args.lr, args.warmup, total_steps) + + if args.horovod: + optimizer["clap"] = hvd.DistributedOptimizer( + optimizer["clap"], named_parameters=model.named_parameters() + ) + hvd.broadcast_parameters(model.state_dict(), root_rank=0) + hvd.broadcast_optimizer_state(optimizer["clap"], root_rank=0) + + # linear probe optimizer + else: + lp_params = [p for n, p in named_parameters if (not in_clap(n, p)) and p.requires_grad] + lp_optim = get_optimizer(lp_params, lr=args.lp_lr, betas=(args.beta1, args.beta2), eps=args.eps, momentum=0.9, + optimizer_name=args.optimizer) + optimizer["lp"] = lp_optim + + return optimizer, scheduler, text_freeze_parameters + + +def main(): + args = parse_args() + + time.sleep(args.sleep) + + # sanitize model name for filesystem / uri use, easier if we don't use / in name as a rule? + args.amodel = args.amodel.replace("/", "-") + # download sizes.json file + + # (yusong): the below two lines are for debug + # print("setting up faulthandler") + # faulthandler.register(10) + + random.seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + np.random.seed(args.seed) + args.class_index_dict = load_class_label(args.class_label_path) + + # get the name of the experiments + if args.name is None: + args.name = "-".join( + [ + datetime.now().strftime("%Y_%m_%d-%H_%M_%S"), + f"linear_probe" + f"model_{args.amodel}", + f"lr_{args.lr}", + f"b_{args.batch_size}", + f"j_{args.workers}", + f"p_{args.precision}", + ] + ) + + # discover initial world args early so we can log properly + args.distributed = False + args.local_rank, args.rank, args.world_size = world_info_from_env() + + if args.remotedata and is_master(args): + for dataset_name in args.datasetnames: + for split in dataset_split[dataset_name]: + if not os.path.exists(f"./json_files/{dataset_name}/{split}"): + os.makedirs(f"./json_files/{dataset_name}/{split}") + os.system( + f"aws s3 cp s3://s-laion-audio/webdataset_tar/{dataset_name}/{split}/sizes.json ./json_files/{dataset_name}/{split}/sizes.json" + ) + + args.log_path = None + if is_master(args, local=args.log_local): + log_base_path = os.path.join(args.logs, args.name) + os.makedirs(log_base_path, exist_ok=True) + log_filename = f"out-{args.rank}" if args.log_local else "out.log" + args.log_path = os.path.join(log_base_path, log_filename) + + # avoid log dir in same name: + postfix = 0 + while os.path.exists(args.log_path): + postfix += 1 + log_base_path_new = log_base_path+'-'+str(postfix) + os.makedirs(log_base_path_new, exist_ok=True) + log_filename = f"out-{args.rank}" if args.log_local else "out.log" + args.log_path = os.path.join(log_base_path_new, log_filename) + # print( + # "Error. Experiment already exists. Use --name {} to specify a new experiment." + # ) + # return -1 + + # Set logger + args.log_level = logging.DEBUG if args.debug else logging.INFO + setup_logging(args.log_path, args.log_level) + + # fully initialize distributed device environment + device = init_distributed_device(args) + + args.wandb = "wandb" in args.report_to or "all" in args.report_to + args.tensorboard = "tensorboard" in args.report_to or "all" in args.report_to + if is_master(args): + args.tensorboard_path = ( + os.path.join(args.logs, args.name, "tensorboard") + if args.tensorboard + else "" + ) + args.checkpoint_path = os.path.join(args.logs, args.name, "checkpoints") + for dirname in [args.tensorboard_path, args.checkpoint_path]: + if dirname: + os.makedirs(dirname, exist_ok=True) + else: + args.tensorboard_path = "" + args.checkpoint_path = "" + + if args.copy_codebase: + copy_codebase(args) + + assert args.precision in ["amp", "fp16", "fp32"] + if args.precision == "fp16": + logging.warning( + "It is recommended to use AMP mixed-precision instead of FP16. " + "FP16 support needs further verification and tuning, especially for train." + ) + + if args.horovod: + logging.info( + f"Running in horovod mode with multiple processes / nodes. Device: {args.device}." + f"Process (global: {args.rank}, local {args.local_rank}), total {args.world_size}." + ) + elif args.distributed: + logging.info( + f"Running in distributed mode with multiple processes. Device: {args.device}." + f"Process (global: {args.rank}, local {args.local_rank}), total {args.world_size}." + ) + else: + logging.info(f"Running with a single process. Device {args.device}.") + + logging.info(f'openai cache dir: {os.path.expanduser(args.openai_model_cache_dir)}') + + # Create CLAP model + clap_model, clap_model_cfg = create_model( + args.amodel, + args.tmodel, + args.pretrained, + precision=args.precision, + device=device, + jit=args.torchscript, + force_quick_gelu=args.force_quick_gelu, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=False, + pretrained_audio=args.pretrained_audio, + pretrained_text=args.pretrained_text, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) + + args.lp_out_ch = len(list(args.class_index_dict.keys())) + # Linear Probe + logging.info(f"linear probe using mlp: {args.lp_mlp}") + logging.info(f"linear probe using freeze: {args.lp_freeze}") + logging.info(f"linear probe act layer: {args.lp_act}") + logging.info(f"linear probe out ch: {args.lp_out_ch}") + logging.info(f"linear probe learning rate (if applicable): {args.lp_lr}") + logging.info(f"linear probe loss func: {args.lp_loss}") + logging.info(f"linear probe lp_metrics: {args.lp_metrics}") + + model = LinearProbe( + clap_model, + mlp=args.lp_mlp, freeze=args.lp_freeze, + in_ch=512, out_ch=args.lp_out_ch, + act=args.lp_act + ) # in_ch is fixed (i.e., 512) + model = model.to(device) + + if args.horovod: + with torch.no_grad(): + for param in model.parameters(): + param.set_(param.contiguous()) + + if args.trace: + model = trace_model(model, batch_size=args.batch_size, device=device) + + if is_master(args): + logging.info("Linear Probe CLAP Model:") + logging.info(f"{str(clap_model)}") + logging.info("Params:") + params_file = os.path.join(args.logs, args.name, "params.txt") + with open(params_file, "w") as f: + for name in sorted(vars(args)): + val = getattr(args, name) + logging.info(f" {name}: {val}") + f.write(f"{name}: {val}\n") + + if args.distributed and not args.horovod: + if args.use_bn_sync: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + ddp_args = {} + if args.ddp_static_graph: + # this doesn't exist in older PyTorch, arg only added if enabled + ddp_args["static_graph"] = True + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[device], find_unused_parameters=True, **ddp_args + ) + + data = get_data(args, clap_model_cfg) + assert len(data), "At least one train or eval dataset must be specified." + if args.trace: + assert "train" not in data, "Cannot train with traced model" + + + optimizer, scheduler, text_freeze_parameters = config_lp_optimizer(model, data, args) + + + scaler = GradScaler() if args.precision == "amp" else None + + # optionally resume from a checkpoint + start_epoch = 0 + if args.resume is not None: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if not args.distributed and next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module.") :]: v for k, v in sd.items()} + model.load_state_dict(sd) + if args.split_opt: + if optimizer is not None: + for k, o_ in optimizer.items(): + o_.load_state_dict(checkpoint[k + "_" + "optimizer"]) + if optimizer is not None: + optimizer.load_state_dict(checkpoint["optimizer"]) + if scaler is not None and "scaler" in checkpoint: + scaler.load_state_dict(checkpoint["scaler"]) + logging.info( + f"=> resuming checkpoint '{args.resume}' (epoch {start_epoch})" + ) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + logging.info( + f"=> loaded checkpoint '{args.resume}' (epoch {start_epoch})" + ) + if args.freeze_text: + print("Freeze Text!!!!") + for k in text_freeze_parameters: + k.requires_grad = False + else: + logging.info("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + cudnn.deterministic = False + + # determine if this worker should save logs and checkpoints. only do so if it is rank == 0 + args.save_logs = args.logs and args.logs.lower() != "none" and is_master(args) + writer = None + if args.save_logs and args.tensorboard: + assert tensorboard is not None, "Please install tensorboard." + writer = tensorboard.SummaryWriter(args.tensorboard_path) + + if args.wandb and is_master(args): + assert wandb is not None, "Please install wandb." + logging.debug("Starting wandb.") + args.train_sz = data["train"].dataloader.num_samples + if args.val_data is not None: + args.val_sz = data["val"].dataloader.num_samples + # you will have to configure this for your project! + wandb.init( + project="clap", + notes=args.wandb_notes, + name=args.wandb_notes, + tags=[], + config=vars(args), + ) + if args.debug: + wandb.watch(model, log="all") + wandb.save(params_file) + logging.debug("Finished loading wandb.") + + if "train" not in data: + evaluate(model, data, start_epoch, args, writer) + return + elif start_epoch == 0 and "val" in data and not args.no_eval: + evaluate(model, data, 0, args, writer) + if args.save_top_performance: + current_top_k_ckpt_metrics = { + i: 0 for i in range(args.save_top_performance) + } # initialize the top-k metric for ckpts to 0 + + for epoch in range(start_epoch, args.epochs): + # freeze the text param after (include) args.freeze_text_after, this is -1 by default + if epoch == args.freeze_text_after: + print("Text pretrained parameters are freezed since this epoch.") + for k in text_freeze_parameters: + k.requires_grad = False + if is_master(args): + logging.info(f"Start epoch {epoch}") + + train_one_epoch(model, data, epoch, optimizer, scaler, scheduler, args, writer) + completed_epoch = epoch + 1 + + if any(v in data for v in ("val", "imagenet-val", "imagenet-v2")) and not args.no_eval: + metrics = evaluate(model, data, completed_epoch, args, writer) + if args.save_top_performance: + top_k_dataset = args.top_k_checkpoint_select_dataset + top_k_metric = args.top_k_checkpoint_select_metric + filtered_metrics = [ + v + for k, v in metrics.items() + if top_k_metric in k and top_k_dataset in k + ] # check all R@10 metrics (all dataset) and use it to update the ckpt + # Saving checkpoints. + if args.save_logs: + opt_dict = { + k + "_" + "optimizer": v.state_dict() for k, v in optimizer.items() + } + checkpoint_dict = { + "epoch": completed_epoch, + "name": args.name, + "state_dict": model.state_dict(), + } + checkpoint_dict.update(opt_dict) + if scaler is not None: + checkpoint_dict["scaler"] = scaler.state_dict() + + if completed_epoch == args.epochs or ( + args.save_frequency > 0 and (completed_epoch % args.save_frequency) == 0 + ): + torch.save( + checkpoint_dict, + os.path.join(args.checkpoint_path, f"epoch_{completed_epoch}.pt"), + ) + if args.save_most_recent: + torch.save( + checkpoint_dict, + os.path.join(args.checkpoint_path, f"epoch_latest.pt"), + ) + if args.save_top_performance and not args.no_eval: + update_top_k_performance( + filtered_metrics, + current_top_k_ckpt_metrics, + args, + checkpoint_dict, + bignumbetter=True, + ) + + if args.wandb and is_master(args): + wandb.finish() + + +def copy_codebase(args): + from shutil import copytree, ignore_patterns + + new_code_path = os.path.join(args.logs, args.name, "code") + if os.path.exists(new_code_path): + print( + f"Error. Experiment already exists at {new_code_path}. Use --name to specify a new experiment." + ) + return -1 + print(f"Copying codebase to {new_code_path}") + current_code_path = os.path.realpath(__file__) + for _ in range(3): + current_code_path = os.path.dirname(current_code_path) + copytree( + current_code_path, new_code_path, ignore=ignore_patterns("log", "logs", "wandb") + ) + print("Done copying code.") + return 1 + + +if __name__ == "__main__": + main() diff --git a/utils/laion_clap/training/lp_train.py b/utils/laion_clap/training/lp_train.py new file mode 100644 index 0000000000000000000000000000000000000000..d686336c824dd34b45f056c414d761540141a46f --- /dev/null +++ b/utils/laion_clap/training/lp_train.py @@ -0,0 +1,292 @@ +import json +import logging +import math +import os +import time +from contextlib import suppress + +import numpy as np +import torch +import torch.nn.functional as F + +try: + import wandb +except ImportError: + wandb = None + +from clap_module import LPLoss, LPMetrics, lp_gather_features +from clap_module.utils import do_mixup, get_mix_lambda +from .distributed import is_master +from .zero_shot import zero_shot_eval + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def unwrap_model(model): + if hasattr(model, "module"): + return model.module + else: + return model + + +def train_one_epoch( + model, data, epoch, optimizer, scaler, scheduler, args, tb_writer=None, extra_suffix="" +): + device = torch.device(args.device) + autocast = torch.cuda.amp.autocast if args.precision == "amp" else suppress + model.train() + loss = LPLoss(args.lp_loss) + + dataloader, sampler = data["train"].dataloader, data["train"].sampler + if args.distributed and sampler is not None: + sampler.set_epoch(epoch) + num_batches_per_epoch = dataloader.num_batches + sample_digits = math.ceil(math.log(dataloader.num_samples + 1, 10)) + + # for toy dataset + if args.dataset_type == "toy": + dataloader.dataset.generate_queue() + + loss_m = AverageMeter() + batch_time_m = AverageMeter() + data_time_m = AverageMeter() + end = time.time() + + for i, batch in enumerate(dataloader): + step = num_batches_per_epoch * epoch + i + + if isinstance(scheduler, dict): + for s in scheduler.values(): + s(step) + else: + scheduler(step) + + audio = batch # contains mel_spec, wavform, and longer list + class_label = batch['class_label'] + # audio = audio.to(device=device, non_blocking=True) + class_label = class_label.to(device=device, non_blocking=True) + + if args.mixup: + # https://github.com/RetroCirce/HTS-Audio-Transformer/blob/main/utils.py#L146 + mix_lambda = torch.from_numpy(get_mix_lambda(0.5, len(audio["waveform"]))).to(device) + class_label = do_mixup(class_label, mix_lambda) + else: + mix_lambda = None + + data_time_m.update(time.time() - end) + if isinstance(optimizer, dict): + for o_ in optimizer.values(): + o_.zero_grad() + else: + optimizer.zero_grad() + + with autocast(): + pred = model(audio, mix_lambda=mix_lambda, device=device) + total_loss = loss(pred, class_label) + + if isinstance(optimizer, dict): + if scaler is not None: + scaler.scale(total_loss).backward() + for o_ in optimizer.values(): + if args.horovod: + o_.synchronize() + scaler.unscale_(o_) + with o_.skip_synchronize(): + scaler.step(o_) + else: + scaler.step(o_) + scaler.update() + else: + total_loss.backward() + for o_ in optimizer.values(): + o_.step() + else: + if scaler is not None: + scaler.scale(total_loss).backward() + if args.horovod: + optimizer.synchronize() + scaler.unscale_(optimizer) + with optimizer.skip_synchronize(): + scaler.step(optimizer) + else: + scaler.step(optimizer) + scaler.update() + else: + total_loss.backward() + optimizer.step() + + # Note: we clamp to 4.6052 = ln(100), as in the original paper. + with torch.no_grad(): + unwrap_model(model).clap_model.logit_scale_a.clamp_(0, math.log(100)) + unwrap_model(model).clap_model.logit_scale_t.clamp_(0, math.log(100)) + + batch_time_m.update(time.time() - end) + end = time.time() + batch_count = i + 1 + + if is_master(args) and (i % 100 == 0 or batch_count == num_batches_per_epoch): + if isinstance(audio, dict): + batch_size = len(audio["waveform"]) + else: + batch_size = len(audio) + num_samples = batch_count * batch_size * args.world_size + samples_per_epoch = dataloader.num_samples + percent_complete = 100.0 * batch_count / num_batches_per_epoch + + # NOTE loss is coarsely sampled, just master node and per log update + loss_m.update(total_loss.item(), batch_size) + if isinstance(optimizer, dict): + logging.info( + f"Train Epoch: {epoch} [{num_samples:>{sample_digits}}/{samples_per_epoch} ({percent_complete:.0f}%)] " + f"Loss: {loss_m.val:#.5g} ({loss_m.avg:#.4g}) " + f"Data (t): {data_time_m.avg:.3f} " + f"Batch (t): {batch_time_m.avg:.3f} " + f"LR: {[o_.param_groups[0]['lr'] for o_ in optimizer.values()]}" + ) + log_data = { + "loss": loss_m.val, + "data_time": data_time_m.val, + "batch_time": batch_time_m.val, + "lr": [o_.param_groups[0]["lr"] for o_ in optimizer.values()], + } + else: + logging.info( + f"Train Epoch: {epoch} [{num_samples:>{sample_digits}}/{samples_per_epoch} ({percent_complete:.0f}%)] " + f"Loss: {loss_m.val:#.5g} ({loss_m.avg:#.4g}) " + f"Data (t): {data_time_m.avg:.3f} " + f"Batch (t): {batch_time_m.avg:.3f} " + f"LR: {optimizer.param_groups[0]['lr']:5f} " + ) + + # Save train loss / etc. Using non avg meter values as loggers have their own smoothing + log_data = { + "loss": loss_m.val, + "data_time": data_time_m.val, + "batch_time": batch_time_m.val, + "lr": optimizer.param_groups[0]["lr"], + } + for name, val in log_data.items(): + name = f"train{extra_suffix}/{name}" + if tb_writer is not None: + tb_writer.add_scalar(name, val, step) + if args.wandb: + assert wandb is not None, "Please install wandb." + wandb.log({name: val, "step": step}) + + # resetting batch / data time meters per log window + batch_time_m.reset() + data_time_m.reset() + # end for + +def evaluate(model, data, epoch, args, tb_writer=None, extra_suffix=""): + metrics = {} + if not args.parallel_eval: + if not is_master(args): + return metrics + device = torch.device(args.device) + model.eval() + + # CHANGE + # zero_shot_metrics = zero_shot_eval(model, data, epoch, args) + # metrics.update(zero_shot_metrics) + if is_master(args): + print('Evaluating...') + metric_names = args.lp_metrics.split(',') + eval_tool = LPMetrics(metric_names=metric_names) + + autocast = torch.cuda.amp.autocast if args.precision == "amp" else suppress + if "val" in data and ( + args.val_frequency + and ((epoch % args.val_frequency) == 0 or epoch == args.epochs) + ): + if args.parallel_eval: + dataloader, sampler = data["val"].dataloader, data["val"].sampler + if args.distributed and sampler is not None: + sampler.set_epoch(epoch) + samples_per_val = dataloader.num_samples + else: + dataloader = data["val"].dataloader + num_samples = 0 + samples_per_val = dataloader.num_samples + + eval_info = { + 'pred': [], + 'target': [] + } + with torch.no_grad(): + for i, batch in enumerate(dataloader): + audio = batch # contains mel_spec, wavform, and longer list + class_label = batch['class_label'] + + # audio = audio.to(device=device, non_blocking=True) + class_label = class_label.to(device=device, non_blocking=True) + + with autocast(): + pred = model(audio, device=device) + if args.parallel_eval: + pred, class_label = lp_gather_features(pred, class_label, args.world_size, args.horovod) + eval_info['pred'].append(pred) + eval_info['target'].append(class_label) + + num_samples += class_label.shape[0] + + if (i % 100) == 0: # and i != 0: + logging.info( + f"Eval Epoch: {epoch} [{num_samples} / {samples_per_val}]" + ) + + if is_master(args): + eval_info['pred'] = torch.cat(eval_info['pred'], 0).cpu() + eval_info['target'] = torch.cat(eval_info['target'], 0).cpu() + metric_dict = eval_tool.evaluate_mertics(eval_info['pred'], eval_info['target']) + metrics.update(metric_dict) + if "epoch" not in metrics.keys(): + metrics.update({"epoch": epoch}) + + if is_master(args): + if not metrics: + return metrics + + logging.info( + f"Eval Epoch: {epoch} " + + "\n".join( + [ + "\t".join([f"{m}: {round(metrics[m], 4):.4f}" ]) + for m in metrics + ] + ) + ) + if args.save_logs: + for name, val in metrics.items(): + if tb_writer is not None: + tb_writer.add_scalar(f"val{extra_suffix}/{name}", val, epoch) + + with open(os.path.join(args.checkpoint_path, "results.jsonl"), "a+") as f: + f.write(json.dumps(metrics)) + f.write("\n") + + if args.wandb: + assert wandb is not None, "Please install wandb." + for name, val in metrics.items(): + wandb.log({f"val{extra_suffix}/{name}": val, "epoch": epoch}) + + return metrics + else: + return metrics diff --git a/utils/laion_clap/training/main.py b/utils/laion_clap/training/main.py new file mode 100644 index 0000000000000000000000000000000000000000..7c48b66e2287785961501dfd75f2b6f5d331c245 --- /dev/null +++ b/utils/laion_clap/training/main.py @@ -0,0 +1,597 @@ +import logging +import os +import random +from datetime import datetime +import copy +import numpy as np +import torch +import torch.backends.cudnn as cudnn +from torch.cuda.amp import GradScaler + +try: + import wandb +except ImportError: + wandb = None + +try: + import torch.utils.tensorboard as tensorboard +except ImportError: + tensorboard = None + +try: + import horovod.torch as hvd +except ImportError: + hvd = None + +from clap_module import create_model_and_transforms, trace_model, create_model +from training.data import get_data +from training.distributed import is_master, init_distributed_device, world_info_from_env +from training.logger import setup_logging +from training.params import parse_args +from training.scheduler import cosine_lr +from training.train import train_one_epoch, evaluate +from clap_module.utils import dataset_split, get_optimizer + + +def maintain_ckpts(args, startidx, all_idx_len): + for i in reversed(range(startidx, all_idx_len)): + if os.path.exists(os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt")): + os.rename( + os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt"), + os.path.join(args.checkpoint_path, f"epoch_top_{i+1}.pt"), + ) + if os.path.exists( + os.path.join(args.checkpoint_path, f"epoch_top_{all_idx_len}.pt") + ): + os.remove(os.path.join(args.checkpoint_path, f"epoch_top_{all_idx_len}.pt")) + return + + +def update_top_k_performance( + new_metrics_inputs, current_top_k_ckpt_metrics, args, ckpt, bignumbetter=True +): + """ + Record the top-k performance of the current epoch. + current_top_k_metrics is a dictionary of the form: {1: top_1_ckpt_measure, 2: top_2_ckpt_measure, ...} + """ + if isinstance(new_metrics_inputs, (list, tuple)): + new_metrics_inputs = np.mean(new_metrics_inputs) + return update_top_k_performance( + new_metrics_inputs, + current_top_k_ckpt_metrics, + args=args, + ckpt=ckpt, + bignumbetter=bignumbetter, + ) + elif isinstance(new_metrics_inputs, dict): + new_metrics_inputs = np.mean(list(new_metrics_inputs.values())) + return update_top_k_performance( + new_metrics_inputs, + current_top_k_ckpt_metrics, + args=args, + ckpt=ckpt, + bignumbetter=bignumbetter, + ) + elif isinstance(new_metrics_inputs, (float, int)): + update_flag = {k: False for k in current_top_k_ckpt_metrics.keys()} + sorted_keys = sorted(current_top_k_ckpt_metrics.keys()) + sorted_values = sorted( + current_top_k_ckpt_metrics.values(), reverse=bignumbetter + ) + sorted_values_ = copy.deepcopy(sorted_values) + sorted_values.append(new_metrics_inputs) + sorted_values = sorted(sorted_values, reverse=bignumbetter) + sorted_values = sorted_values[:-1] + + if sorted_values == sorted_values_: + return current_top_k_ckpt_metrics, new_metrics_inputs + else: + for i in range(len(sorted_keys)): + if current_top_k_ckpt_metrics[sorted_keys[i]] != sorted_values[i]: + current_top_k_ckpt_metrics[sorted_keys[i]] = sorted_values[i] + update_flag[sorted_keys[i]] = True + for i in range(len(update_flag)): + if update_flag[i]: + maintain_ckpts(args, i, len(sorted_keys)) + torch.save( + ckpt, + os.path.join(args.checkpoint_path, f"epoch_top_{i}.pt"), + ) + break + return current_top_k_ckpt_metrics, new_metrics_inputs + + +# def updateifNone(a, b): +# a = b if None else a +# return a + + +def is_pretrained_params(n): + return ( + n.startswith("transformer") + or n in ["positional_embedding", "text_projection"] + or n.startswith("token_embedding") + or n.startswith("ln_final") + or n.startswith("logit_scale_t") + ) + + +def random_seed(seed=42, rank=0): + torch.manual_seed(seed + rank) + np.random.seed(seed + rank) + random.seed(seed + rank) + + +def main(): + args = parse_args() + # sanitize model name for filesystem / uri use, easier if we don't use / in name as a rule? + args.amodel = args.amodel.replace("/", "-") + # download sizes.json file + + # (yusong): the below two lines are for debug + # print("setting up faulthandler") + # faulthandler.register(10) + + random.seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + np.random.seed(args.seed) + if args.tmodel == "bert" or args.tmodel == "roberta" or args.tmodel == "bart": + assert ( + args.pretrained == "" or args.pretrained is None + ), "bert/roberta/bart text encoder does not support pretrained models." + + # get the name of the experiments + if args.name is None: + args.name = "-".join( + [ + datetime.now().strftime("%Y_%m_%d-%H_%M_%S"), + f"model_{args.amodel}", + f"lr_{args.lr}", + f"b_{args.batch_size}", + f"j_{args.workers}", + f"p_{args.precision}", + ] + ) + + # discover initial world args early so we can log properly + args.distributed = False + args.local_rank, args.rank, args.world_size = world_info_from_env() + + if args.remotedata and is_master(args): + for dataset_name in args.datasetnames: + for split in dataset_split[dataset_name]: + if not os.path.exists(f"./json_files/{dataset_name}/{split}"): + os.makedirs(f"./json_files/{dataset_name}/{split}") + os.system( + f"aws s3 cp s3://s-laion-audio/webdataset_tar/{dataset_name}/{split}/sizes.json ./json_files/{dataset_name}/{split}/sizes.json" + ) + + args.log_path = None + if is_master(args, local=args.log_local): + log_base_path = os.path.join(args.logs, args.name) + os.makedirs(log_base_path, exist_ok=True) + log_filename = f"out-{args.rank}" if args.log_local else "out.log" + args.log_path = os.path.join(log_base_path, log_filename) + if os.path.exists(args.log_path): + print( + "Error. Experiment already exists. Use --name {} to specify a new experiment." + ) + return -1 + + # Set logger + args.log_level = logging.DEBUG if args.debug else logging.INFO + setup_logging(args.log_path, args.log_level) + + # fully initialize distributed device environment + device = init_distributed_device(args) + + args.wandb = "wandb" in args.report_to or "all" in args.report_to + args.tensorboard = "tensorboard" in args.report_to or "all" in args.report_to + if is_master(args): + args.tensorboard_path = ( + os.path.join(args.logs, args.name, "tensorboard") + if args.tensorboard + else "" + ) + args.checkpoint_path = os.path.join(args.logs, args.name, "checkpoints") + for dirname in [args.tensorboard_path, args.checkpoint_path]: + if dirname: + os.makedirs(dirname, exist_ok=True) + else: + args.tensorboard_path = "" + args.checkpoint_path = "" + + if args.copy_codebase: + copy_codebase(args) + + assert args.precision in ["amp", "fp16", "fp32"] + if args.precision == "fp16": + logging.warning( + "It is recommended to use fp32 mixed-precision instead of FP16 and AMP in this model. " + "They will cause NaN loss and NaN gradients. " + "FP16 and AMP support needs further verification and tuning, especially for train." + ) + + if args.horovod: + logging.info( + f"Running in horovod mode with multiple processes / nodes. Device: {args.device}." + f"Process (global: {args.rank}, local {args.local_rank}), total {args.world_size}." + ) + elif args.distributed: + logging.info( + f"Running in distributed mode with multiple processes. Device: {args.device}." + f"Process (global: {args.rank}, local {args.local_rank}), total {args.world_size}." + ) + else: + logging.info(f"Running with a single process. Device {args.device}.") + + logging.info(f"openai cache dir: {os.path.expanduser(args.openai_model_cache_dir)}") + + model, model_cfg = create_model( + args.amodel, + args.tmodel, + args.pretrained, + precision=args.precision, + device=device, + jit=args.torchscript, + force_quick_gelu=args.force_quick_gelu, + openai_model_cache_dir=os.path.expanduser(args.openai_model_cache_dir), + skip_params=True, + pretrained_audio=args.pretrained_audio, + pretrained_text=args.pretrained_text, + enable_fusion=args.enable_fusion, + fusion_type=args.fusion_type + ) + + if args.horovod: + with torch.no_grad(): + for param in model.parameters(): + param.set_(param.contiguous()) + + if args.trace: + model = trace_model(model, batch_size=args.batch_size, device=device) + + if is_master(args): + logging.info("Model:") + logging.info(f"{str(model)}") + logging.info("Params:") + params_file = os.path.join(args.logs, args.name, "params.txt") + with open(params_file, "w") as f: + for name in sorted(vars(args)): + val = getattr(args, name) + logging.info(f" {name}: {val}") + f.write(f"{name}: {val}\n") + + if args.distributed and not args.horovod: + if args.use_bn_sync: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + ddp_args = {} + if args.ddp_static_graph: + # this doesn't exist in older PyTorch, arg only added if enabled + ddp_args["static_graph"] = True + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[device], find_unused_parameters=True, **ddp_args + ) + + data = get_data(args, model_cfg) + assert len(data), "At least one train or eval dataset must be specified." + if args.trace: + assert "train" not in data, "Cannot train with traced model" + + exclude = ( + lambda n, p: p.ndim < 2 + or "bn" in n + or "ln" in n + or "bias" in n + or "logit_scale" in n + ) + include = lambda n, p: not exclude(n, p) + + named_parameters = list(model.named_parameters()) + + # freeze text encoder + text_freeze_parameters = [ + p + for n, p in named_parameters + if 'text_branch' in n + ] + + if args.freeze_text: + print("Freeze Text!!!!") + for k in text_freeze_parameters: + k.requires_grad = False + + gain_or_bias_params = [ + p for n, p in named_parameters if exclude(n, p) and p.requires_grad + ] + rest_params = [p for n, p in named_parameters if include(n, p) and p.requires_grad] + + # set wd-related params to 0 if use adam optimizer + if args.optimizer == "adam": + args.wd = 0 + args.wd_pretrained = 0 + args.wd_new = 0 + + if args.train_data is None: + optimizer = None + scheduler = None + else: + total_steps = data["train"].dataloader.num_batches * args.epochs + + if args.split_opt: + for x in ["lr", "beta1", "beta2", "eps", "wd"]: + for y in ["_new", "_pretrained"]: + if getattr(args, x + y) is None: + setattr(args, x + y, getattr(args, x)) + + gain_or_bias_pretrained_params = [ + p + for n, p in named_parameters + if (exclude(n, p) and p.requires_grad) and is_pretrained_params(n) + ] + rest_pretrained_params = [ + p + for n, p in named_parameters + if (include(n, p) and p.requires_grad) and is_pretrained_params(n) + ] + gain_or_bias_new_params = [ + p + for n, p in named_parameters + if (exclude(n, p) and p.requires_grad) and (not is_pretrained_params(n)) + ] + rest_new_params = [ + p + for n, p in named_parameters + if (include(n, p) and p.requires_grad) and (not is_pretrained_params(n)) + ] + pretrained_params_optimizer = get_optimizer( + [ + {"params": gain_or_bias_pretrained_params, "weight_decay": 0.0}, + { + "params": rest_pretrained_params, + "weight_decay": args.wd_pretrained, + }, + ], + lr=args.lr_pretrained, + betas=(args.beta1_pretrained, args.beta2_pretrained), + eps=args.eps_pretrained, + momentum=args.momentum_pretrained, + optimizer_name=args.optimizer, + ) + pretrained_params_scheduler = cosine_lr( + pretrained_params_optimizer, + args.lr_pretrained, + args.warmup, + total_steps, + ) + new_params_optimizer = get_optimizer( + [ + {"params": gain_or_bias_new_params, "weight_decay": 0.0}, + {"params": rest_new_params, "weight_decay": args.wd_new}, + ], + lr=args.lr_new, + betas=(args.beta1_new, args.beta2_new), + eps=args.eps_new, + momentum=args.momentum_new, + optimizer_name=args.optimizer, + ) + + new_params_scheduler = cosine_lr( + new_params_optimizer, args.lr_new, args.warmup, total_steps + ) + + optimizer = { + "pretrained": pretrained_params_optimizer, + "new": new_params_optimizer, + } + scheduler = { + "pretrained": pretrained_params_scheduler, + "new": new_params_scheduler, + } + + if args.horovod: + pretrained_params_optimizer = hvd.DistributedOptimizer( + pretrained_params_optimizer, + named_parameters=model.named_parameters(), + ) + new_params_optimizer = hvd.DistributedOptimizer( + new_params_optimizer, named_parameters=model.named_parameters() + ) + hvd.broadcast_parameters(model.state_dict(), root_rank=0) + hvd.broadcast_optimizer_state(pretrained_params_optimizer, root_rank=0) + hvd.broadcast_optimizer_state(new_params_optimizer, root_rank=0) + else: + optimizer = get_optimizer( + [ + {"params": gain_or_bias_params, "weight_decay": 0.0}, + {"params": rest_params, "weight_decay": args.wd}, + ], + lr=args.lr, + betas=(args.beta1, args.beta2), + eps=args.eps, + momentum=args.momentum, + optimizer_name=args.optimizer, + ) + + scheduler = cosine_lr(optimizer, args.lr, args.warmup, total_steps) + + if args.horovod: + optimizer = hvd.DistributedOptimizer( + optimizer, named_parameters=model.named_parameters() + ) + hvd.broadcast_parameters(model.state_dict(), root_rank=0) + hvd.broadcast_optimizer_state(optimizer, root_rank=0) + + scaler = GradScaler() if args.precision == "amp" else None + + # optionally resume from a checkpoint + start_epoch = 0 + if args.resume is not None: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume, map_location=device) + if "epoch" in checkpoint: + # resuming a train checkpoint w/ epoch and optimizer state + start_epoch = checkpoint["epoch"] + sd = checkpoint["state_dict"] + if not args.distributed and next(iter(sd.items()))[0].startswith( + "module" + ): + sd = {k[len("module.") :]: v for k, v in sd.items()} + model.load_state_dict(sd) + if args.split_opt: + if optimizer is not None: + for k, o_ in optimizer.items(): + o_.load_state_dict(checkpoint[k + "_" + "optimizer"]) + if optimizer is not None: + optimizer.load_state_dict(checkpoint["optimizer"]) + if scaler is not None and "scaler" in checkpoint: + scaler.load_state_dict(checkpoint["scaler"]) + logging.info( + f"=> resuming checkpoint '{args.resume}' (epoch {start_epoch})" + ) + else: + # loading a bare (model only) checkpoint for fine-tune or evaluation + model.load_state_dict(checkpoint) + logging.info( + f"=> loaded checkpoint '{args.resume}' (epoch {start_epoch})" + ) + if args.freeze_text: + print("Freeze Text!!!!") + for k in text_freeze_parameters: + k.requires_grad = False + else: + logging.info("=> no checkpoint found at '{}'".format(args.resume)) + + cudnn.benchmark = True + cudnn.deterministic = False + + # determine if this worker should save logs and checkpoints. only do so if it is rank == 0 + args.save_logs = args.logs and args.logs.lower() != "none" and is_master(args) + writer = None + if args.save_logs and args.tensorboard: + assert tensorboard is not None, "Please install tensorboard." + writer = tensorboard.SummaryWriter(args.tensorboard_path) + + if args.wandb and is_master(args): + assert wandb is not None, "Please install wandb." + logging.debug("Starting wandb.") + args.train_sz = data["train"].dataloader.num_samples + if args.val_data is not None: + args.val_sz = data["val"].dataloader.num_samples + # you will have to configure this for your project! + wandb.init( + entity="clap", + project="clap", + notes=args.wandb_notes, + name=args.wandb_notes, + tags=[], + config=vars(args), + ) + if args.debug: + wandb.watch(model, log="all") + wandb.save(params_file) + logging.debug("Finished loading wandb.") + + if "train" not in data: + evaluate(model, data, start_epoch, args, writer) + return + elif start_epoch == 0 and "val" in data and not args.no_eval: + evaluate(model, data, 0, args, writer) + # print(f'rank {args.rank}, Start First Evaluation')# (yusong): for debug + if args.save_top_performance: + current_top_k_ckpt_metrics = { + i: 0 for i in range(args.save_top_performance) + } # initialize the top-k metric for ckpts to 0 + + # print(f'rank {args.rank}, Start Training') # (yusong): for debug + for epoch in range(start_epoch, args.epochs): + # freeze the text param after (include) args.freeze_text_after, this is -1 by default + if epoch == args.freeze_text_after: + print("Text pretrained parameters are freezed since this epoch.") + for k in text_freeze_parameters: + k.requires_grad = False + if is_master(args): + logging.info(f"Start epoch {epoch}") + + train_one_epoch(model, data, epoch, optimizer, scaler, scheduler, args, writer) + completed_epoch = epoch + 1 + + if ( + any(v in data for v in ("val", "imagenet-val", "imagenet-v2")) + and not args.no_eval + ): + metrics = evaluate(model, data, completed_epoch, args, writer) + if args.save_top_performance: + top_k_dataset = args.top_k_checkpoint_select_dataset + top_k_metric = args.top_k_checkpoint_select_metric + filtered_metrics = [ + v + for k, v in metrics.items() + if top_k_metric in k and top_k_dataset in k + ] # check all R@10 metrics (all dataset) and use it to update the ckpt + # Saving checkpoints. + if args.save_logs: + if args.split_opt: + opt_dict = { + k + "_" + "optimizer": v.state_dict() for k, v in optimizer.items() + } + else: + opt_dict = {"optimizer": optimizer.state_dict()} + checkpoint_dict = { + "epoch": completed_epoch, + "name": args.name, + "state_dict": model.state_dict(), + } + checkpoint_dict.update(opt_dict) + if scaler is not None: + checkpoint_dict["scaler"] = scaler.state_dict() + + if completed_epoch == args.epochs or ( + args.save_frequency > 0 and (completed_epoch % args.save_frequency) == 0 + ): + torch.save( + checkpoint_dict, + os.path.join(args.checkpoint_path, f"epoch_{completed_epoch}.pt"), + ) + if args.save_most_recent: + torch.save( + checkpoint_dict, + os.path.join(args.checkpoint_path, f"epoch_latest.pt"), + ) + if args.save_top_performance and not args.no_eval: + update_top_k_performance( + filtered_metrics, + current_top_k_ckpt_metrics, + args, + checkpoint_dict, + bignumbetter=True, + ) + + if args.wandb and is_master(args): + wandb.finish() + + +def copy_codebase(args): + from shutil import copytree, ignore_patterns + + new_code_path = os.path.join(args.logs, args.name, "code") + if os.path.exists(new_code_path): + print( + f"Error. Experiment already exists at {new_code_path}. Use --name to specify a new experiment." + ) + return -1 + print(f"Copying codebase to {new_code_path}") + current_code_path = os.path.realpath(__file__) + for _ in range(3): + current_code_path = os.path.dirname(current_code_path) + copytree( + current_code_path, new_code_path, ignore=ignore_patterns("log", "logs", "wandb") + ) + print("Done copying code.") + return 1 + + +if __name__ == "__main__": + main() diff --git a/utils/laion_clap/training/params.py b/utils/laion_clap/training/params.py new file mode 100644 index 0000000000000000000000000000000000000000..84cd3b43104007a14835e4de9cd4521899ba6345 --- /dev/null +++ b/utils/laion_clap/training/params.py @@ -0,0 +1,567 @@ +import argparse + + +def get_default_params(model_name): + # Params from paper (https://arxiv.org/pdf/2103.00020.pdf) + model_name = model_name.lower() + if "vit" in model_name: + return {"lr": 5.0e-4, "beta1": 0.9, "beta2": 0.98, "eps": 1.0e-6} + else: + return {"lr": 5.0e-4, "beta1": 0.9, "beta2": 0.999, "eps": 1.0e-8} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--train-data", + type=str, + default=None, + help="Path to h5 filewith training data", + ) + parser.add_argument( + "--val-data", + type=str, + default=None, + help="Path to h5 file with validation data", + ) + parser.add_argument( + "--freeze-text", + default=False, + action="store_true", + help="if you need to freeze the text encoder, make this True", + ) + parser.add_argument( + "--freeze-text-after", + type=int, + default=-1, + help="if you need to freeze the text encoder after (include) epoch x, set this param to x. Set -1 to disable it", + ) + parser.add_argument( + "--train-ipc", + type=str, + default=None, + help="Path to npy file of the number of instance per class in training data", + ) + parser.add_argument( + "--val-ipc", + type=str, + default=None, + help="Path to npy file of the number of instance per class in validation data", + ) + parser.add_argument( + "--train-num-samples", + type=int, + default=None, + help="Number of samples in dataset. Required for webdataset if not available in info file.", + ) + parser.add_argument( + "--val-num-samples", + type=int, + default=None, + help="Number of samples in dataset. Useful for webdataset if not available in info file.", + ) + parser.add_argument( + "--dataset-type", + choices=["webdataset", "csv", "auto", "toy"], + default="auto", + help="Which type of dataset to process.", + ) + parser.add_argument( + "--csv-separator", + type=str, + default="\t", + help="For csv-like datasets, which separator to use.", + ) + parser.add_argument( + "--csv-img-key", + type=str, + default="filepath", + help="For csv-like datasets, the name of the key for the image paths.", + ) + parser.add_argument( + "--csv-caption-key", + type=str, + default="title", + help="For csv-like datasets, the name of the key for the captions.", + ) + parser.add_argument( + "--imagenet-val", + type=str, + default=None, + help="Path to imagenet val set for conducting zero shot evaluation.", + ) + parser.add_argument( + "--imagenet-v2", + type=str, + default=None, + help="Path to imagenet v2 for conducting zero shot evaluation.", + ) + parser.add_argument( + "--datasetnames", + nargs="+", + default=None, + help="If loading webdataset, spedify the dataset names to load. Can be some of these: Clotho, audioset, audiocaps, BBCSoundEffects", + ) + parser.add_argument( + "--full-train-dataset", + nargs="+", + default=None, + help="Which dataset will be trained with all the subsets. (train+test)", + ) + parser.add_argument( + "--exclude-eval-dataset", + nargs="+", + default=None, + help="Which dataset will be excluded with evaluation", + ) + parser.add_argument( + "--datasetinfos", + nargs="+", + default=None, + help="If loading webdataset, spedify the dataset types to load. Can be some of these: train, test, valid, unbalanced_train, balanced_train, eval", + ) + parser.add_argument( + "--dataset-proportion", + type=float, + default=1.0, + help="How much proportion of dataset we want to train.", + ) + parser.add_argument( + "--remotedata", + default=False, + action="store_true", + help="if the dataset is remote, set this flag", + ) + parser.add_argument( + "--class-label-path", + type=str, + default=None, + help="The path of the class label pickle or csv.", + ) + parser.add_argument( + "--datasetpath", + type=str, + default="/mnt/audio_clip/webdataset_tar", + help="The path to the dataset", + ) + parser.add_argument( + "--logs", + type=str, + default="./logs/", + help="Where to store tensorboard logs. Use None to avoid storing logs.", + ) + parser.add_argument( + "--log-local", + action="store_true", + default=False, + help="log files on local master, otherwise global master only.", + ) + parser.add_argument( + "--name", + type=str, + default=None, + help="Optional identifier for the experiment when storing logs. Otherwise use current time.", + ) + parser.add_argument( + "--workers", type=int, default=1, help="Number of workers per GPU." + ) + parser.add_argument( + "--batch-size", type=int, default=64, help="Batch size per GPU." + ) + parser.add_argument( + "--epochs", type=int, default=32, help="Number of epochs to train for." + ) + parser.add_argument("--lr", type=float, default=None, help="Learning rate.") + parser.add_argument("--beta1", type=float, default=None, help="Adam beta 1.") + parser.add_argument("--beta2", type=float, default=None, help="Adam beta 2.") + parser.add_argument("--eps", type=float, default=None, help="Adam epsilon.") + parser.add_argument("--momentum", type=float, default=None, help="SGD epsilon.") + parser.add_argument("--wd", type=float, default=0.2, help="Weight decay.") + + parser.add_argument( + "--split-opt", + action="store_true", + default=False, + help="Use this flag to skip the learning rate decay.", + ) + parser.add_argument( + "--lr-pretrained", type=float, default=None, help="Learning rate for text." + ) + parser.add_argument( + "--beta1-pretrained", type=float, default=None, help="Adam beta 1 for text." + ) + parser.add_argument( + "--beta2-pretrained", type=float, default=None, help="Adam beta 2 for text." + ) + parser.add_argument( + "--eps-pretrained", type=float, default=None, help="Adam epsilon for text." + ) + parser.add_argument( + "--wd-pretrained", type=float, default=0.2, help="Weight decay for text." + ) + parser.add_argument( + "--momentum-pretrained", type=float, default=0.9, help="Momentum for text." + ) + parser.add_argument( + "--lr-new", type=float, default=None, help="Learning rate for audio." + ) + parser.add_argument( + "--beta1-new", type=float, default=None, help="Adam beta 1 for audio." + ) + parser.add_argument( + "--beta2-new", type=float, default=None, help="Adam beta 2 for audio." + ) + parser.add_argument( + "--eps-new", type=float, default=None, help="Adam epsilon for audio." + ) + parser.add_argument( + "--wd-new", type=float, default=0.2, help="Weight decay for audio." + ) + parser.add_argument( + "--momentum-new", type=float, default=0.9, help="Momentum for audio." + ) + parser.add_argument( + "--warmup", type=int, default=10000, help="Number of steps to warmup for." + ) + parser.add_argument( + "--use-bn-sync", + default=False, + action="store_true", + help="Whether to use batch norm sync.", + ) + parser.add_argument( + "--skip-scheduler", + action="store_true", + default=False, + help="Use this flag to skip the learning rate decay.", + ) + parser.add_argument( + "--save-frequency", type=int, default=1, help="How often to save checkpoints." + ) + parser.add_argument( + "--save-top-performance", + type=int, + default=0, + help="Save the top x performance weights if the value >0", + ) + parser.add_argument( + "--save-most-recent", + action="store_true", + default=False, + help="Always save the most recent model trained to epoch_latest.pt.", + ) + parser.add_argument( + "--zeroshot-frequency", type=int, default=2, help="How often to run zero shot." + ) + parser.add_argument( + "--val-frequency", + type=int, + default=1, + help="How often to run evaluation with val data.", + ) + parser.add_argument( + "--resume", + default=None, + type=str, + help="path to latest checkpoint (default: none)", + ) + parser.add_argument( + "--precision", + choices=["amp", "fp16", "fp32"], + default="amp", + help="Floating point precision.", + ) + parser.add_argument( + "--amodel", + type=str, + default="RN50", + help="Name of the audio backbone to use.", + ) + parser.add_argument( + "--tmodel", + type=str, + default="transformer", + help="Name of the text backbone to use. Can be [transformer, bert, roberta, bart]", + ) + parser.add_argument( + "--pretrained-audio", + default="", + type=str, + help="Use a pretrained audio model weights for the audio encoder of CLAP", + ) + parser.add_argument( + "--pretrained-text", + default="", + type=str, + help="Use a pretrained text model weights for the text encoder of CLAP", + ) + parser.add_argument( + "--pretrained", + default="", + type=str, + help="Use a pretrained CLIP model weights with the specified tag or file path.", + ) + parser.add_argument( + "--pretrained-image", + default=False, + action="store_true", + help="Load imagenet pretrained weights for image tower backbone if available.", + ) + parser.add_argument( + "--lock-image", + default=False, + action="store_true", + help="Lock full image tower by disabling gradients.", + ) + parser.add_argument( + "--lock-image-unlocked-groups", + type=int, + default=0, + help="Leave last n image tower layer groups unlocked.", + ) + parser.add_argument( + "--lock-image-freeze-bn-stats", + default=False, + action="store_true", + help="Freeze BatchNorm running stats in image tower for any locked layers.", + ) + parser.add_argument( + "--local-loss", + default=False, + action="store_true", + help="calculate loss w/ local features @ global (instead of realizing full global @ global matrix)", + ) + parser.add_argument( + "--gather-with-grad", + default=False, + action="store_true", + help="enable full distributed gradient for feature gather", + ) + parser.add_argument( + "--force-quick-gelu", + default=False, + action="store_true", + help="Force use of QuickGELU activation for non-OpenAI transformer models.", + ) + parser.add_argument( + "--torchscript", + default=False, + action="store_true", + help="torch.jit.script the model, also uses jit version of OpenAI models if pretrained=='openai'", + ) + parser.add_argument( + "--trace", + default=False, + action="store_true", + help="torch.jit.trace the model for inference / eval only", + ) + # arguments for distributed training + parser.add_argument( + "--dist-url", + default="env://", + type=str, + help="url used to set up distributed training", + ) + parser.add_argument( + "--dist-backend", default="nccl", type=str, help="distributed backend" + ) + parser.add_argument( + "--report-to", + default="", + type=str, + help="Options are ['wandb', 'tensorboard', 'wandb,tensorboard']", + ) + parser.add_argument( + "--wandb-notes", default="", type=str, help="Notes if logging with wandb" + ) + parser.add_argument( + "--C", type=float, default=3.16, help="inverse regularizer for logistic reg." + ) + parser.add_argument( + "--debug", + default=False, + action="store_true", + help="If true, more information is logged.", + ) + parser.add_argument( + "--copy-codebase", + default=False, + action="store_true", + help="If true, we copy the entire base on the log diretory, and execute from there.", + ) + parser.add_argument( + "--horovod", + default=False, + action="store_true", + help="Use horovod for distributed training.", + ) + parser.add_argument( + "--ddp-static-graph", + default=False, + action="store_true", + help="Enable static graph optimization for DDP in PyTorch >= 1.11.", + ) + parser.add_argument( + "--no-set-device-rank", + default=False, + action="store_true", + help="Don't set device index from local rank (when CUDA_VISIBLE_DEVICES restricted to one per proc).", + ) + parser.add_argument("--seed", type=int, default=4242, help="Default random seed.") + + parser.add_argument( + "--top-k-checkpoint-select-dataset", + type=str, + default="all", + help="The dataset of selecting top-k checkpoint.", + ) + + # @R10, @R@5, @R1, mAP@10 + parser.add_argument( + "--top-k-checkpoint-select-metric", + type=str, + default="_R@10", + help="The metric for selecting top-k checkpoint.", + ) + parser.add_argument( + "--openai-model-cache-dir", + type=str, + default="~/.cache/clip", + help="Directory to download OpenAI models.", + ) + parser.add_argument( + "--optimizer", + type=str, + default="adamw", + help="can be AdamW or SGD", + ) + parser.add_argument( + "--parallel-eval", + default=False, + action="store_true", + help="Eval in parallel (multi-GPU, multi-node).", + ) + + parser.add_argument( + "--no-eval", + default=False, + action="store_true", + help="Training without evaluation.", + ) + + parser.add_argument( + "--lp-mlp", + default=False, + action="store_true", + help="Linear Probe using MLP layer or not.", + ) + + parser.add_argument( + "--lp-freeze", + default=False, + action="store_true", + help="Linear Probe using Freeze CLAP or not", + ) + + parser.add_argument( + "--lp-act", + default="None", + type=str, + help="Options are ['relu','elu','prelu','softmax','sigmoid']", + ) + + parser.add_argument( + "--lp-loss", type=str, default="bce", help="Loss func of Linear Probe." + ) + + parser.add_argument( + "--lp-metrics", + type=str, + default="map,mauc,acc", + help="Metrics of Linear Probe.", + ) + + parser.add_argument( + "--lp-lr", type=float, default=1e-4, help="learning rate of linear probe" + ) + parser.add_argument( + "--kappa", type=float, default=0, + help="the kappa in the weighted contrastive loss, default is to turn off the weighted contrastive loss" + ) + + parser.add_argument( + "--data-filling", + type=str, + default="pad", + help="type of data filling when the audio length is shorter than the max length." + "Can be one of the following: repeat, repeatpad, pad", + ) + parser.add_argument( + "--data-truncating", + type=str, + default="rand_trunc", + help="type of data truncation when the audio length is longer than the max length." + "Can be one of the following: rand_trunc, fusion", + ) + + parser.add_argument( + "--clap-mlploss", + default=False, + action="store_true", + help="Using MLP loss for CLAP model or not", + ) + + parser.add_argument( + "--wandb-id", + type=str, + default=None, + help="the id of wandb experiment to restore.", + ) + + parser.add_argument( + "--sleep", type=float, default=0, help="sleep n seconds before start training" + ) + + # variable length processing + parser.add_argument( + "--enable-fusion", + default=False, + action="store_true", + help="Enable feature funsion for variable-length data", + ) + + parser.add_argument( + "--fusion-type", + type=str, + default='None', + help="Type is among ['channel_map', 'daf_1d','aff_1d','iaff_1d','daf_2d','aff_2d','iaff_2d']", + ) + + parser.add_argument( + "--mixup", + default=False, + action="store_true", + help="Enable mixup in finetuning training.", + ) + parser.add_argument( + "--text-augment-selection", + type=str, + default=None, + help="For selecting levels of augmented text. Type is among ['all', 'augment_only', 'none']", + ) + parser.add_argument( + "--prefetch-factor", + type=int, + default=None, + help="The prefetch factor for dataloader. Larger value will use more memory and CPU but faster.", + ) + + args = parser.parse_args() + + # If some params are not passed, we use the default values based on model name. + default_params = get_default_params(args.amodel) + for name, val in default_params.items(): + if getattr(args, name) is None: + setattr(args, name, val) + + return args diff --git a/utils/laion_clap/training/scheduler.py b/utils/laion_clap/training/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..e0bfdf796c95df003582ede43f0511bd9181c1e4 --- /dev/null +++ b/utils/laion_clap/training/scheduler.py @@ -0,0 +1,23 @@ +import numpy as np + + +def assign_learning_rate(optimizer, new_lr): + for param_group in optimizer.param_groups: + param_group["lr"] = new_lr + + +def _warmup_lr(base_lr, warmup_length, step): + return base_lr * (step + 1) / warmup_length + + +def cosine_lr(optimizer, base_lr, warmup_length, steps): + def _lr_adjuster(step): + if step < warmup_length: + lr = _warmup_lr(base_lr, warmup_length, step) + else: + e = step - warmup_length + es = steps - warmup_length + lr = 0.5 * (1 + np.cos(np.pi * e / es)) * base_lr + assign_learning_rate(optimizer, lr) + return lr + return _lr_adjuster \ No newline at end of file diff --git a/utils/laion_clap/training/train.py b/utils/laion_clap/training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..06db94158fe15c0a17c95babf60b75d31e8893c7 --- /dev/null +++ b/utils/laion_clap/training/train.py @@ -0,0 +1,781 @@ +import json +import logging +import math +import os +import time +from contextlib import suppress + +import numpy as np +import torch +import torch.nn.functional as F + +try: + import wandb +except ImportError: + wandb = None + +from clap_module import ClipLoss, gather_features +from .distributed import is_master + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def unwrap_model(model): + if hasattr(model, "module"): + return model.module + else: + return model + + +def train_one_epoch( + model, data, epoch, optimizer, scaler, scheduler, args, tb_writer=None +): + device = torch.device(args.device) + autocast = torch.cuda.amp.autocast if args.precision == "amp" else suppress + model.train() + loss = ClipLoss( + local_loss=args.local_loss, + gather_with_grad=args.gather_with_grad, + cache_labels=True, + rank=args.rank, + world_size=args.world_size, + use_horovod=args.horovod, + mlp_loss=args.clap_mlploss, + weight_loss_kappa=args.kappa, + ) + + dataloader, sampler = data["train"].dataloader, data["train"].sampler + if args.distributed and sampler is not None: + sampler.set_epoch(epoch) + num_batches_per_epoch = dataloader.num_batches + sample_digits = math.ceil(math.log(dataloader.num_samples + 1, 10)) + + # for toy dataset + if args.dataset_type == "toy": + dataloader.dataset.generate_queue() + + loss_m = AverageMeter() + batch_time_m = AverageMeter() + data_time_m = AverageMeter() + end = time.time() + + for i, batch in enumerate(dataloader): + # logging.info(f"batch {i} of {num_batches_per_epoch}") + step = num_batches_per_epoch * epoch + i + if isinstance(scheduler, dict): + for s in scheduler.values(): + s(step) + else: + scheduler(step) + audios = batch # contains mel_spec, wavform, and longer list + texts = batch['text'] + # audios = audios.to(device=device, non_blocking=True) + # texts = texts.to(device=device, non_blocking=True) + + data_time_m.update(time.time() - end) + if isinstance(optimizer, dict): + for o_ in optimizer.values(): + o_.zero_grad() + else: + optimizer.zero_grad() + + with autocast(): + ( + audio_features, + text_features, + audio_features_mlp, + text_features_mlp, + logit_scale_a, + logit_scale_t, + ) = model(audios, texts, device) + + if args.clap_mlploss: + total_loss = loss( + audio_features=audio_features, + text_features=text_features, + logit_scale_a=logit_scale_a, + logit_scale_t=logit_scale_t, + audio_features_mlp=audio_features_mlp, + text_features_mlp=text_features_mlp + ) + else: + total_loss = loss( + audio_features=audio_features, + text_features=text_features, + logit_scale_a=logit_scale_a + ) + if isinstance(optimizer, dict): + if scaler is not None: + scaler.scale(total_loss).backward() + for o_ in optimizer.values(): + if args.horovod: + o_.synchronize() + scaler.unscale_(o_) + with o_.skip_synchronize(): + scaler.step(o_) + else: + scaler.step(o_) + scaler.update() + else: + total_loss.backward() + for o_ in optimizer.values(): + o_.step() + else: + if scaler is not None: + scaler.scale(total_loss).backward() + if args.horovod: + optimizer.synchronize() + scaler.unscale_(optimizer) + with optimizer.skip_synchronize(): + scaler.step(optimizer) + else: + scaler.step(optimizer) + scaler.update() + else: + total_loss.backward() + optimizer.step() + + # Note: we clamp to 4.6052 = ln(100), as in the original paper. + with torch.no_grad(): + unwrap_model(model).logit_scale_a.clamp_(0, math.log(100)) + if args.clap_mlploss: + unwrap_model(model).logit_scale_t.clamp_(0, math.log(100)) + + batch_time_m.update(time.time() - end) + end = time.time() + batch_count = i + 1 + if is_master(args) and (i % 100 == 0 or batch_count == num_batches_per_epoch): + if isinstance(audios, dict): + batch_size = len(audios["waveform"]) + else: + batch_size = len(audios) + num_samples = batch_count * batch_size * args.world_size + samples_per_epoch = dataloader.num_samples + percent_complete = 100.0 * batch_count / num_batches_per_epoch + + # NOTE loss is coarsely sampled, just master node and per log update + loss_m.update(total_loss.item(), batch_size) + logit_scale_scalar_a = logit_scale_a.item() + logit_scale_scalar_t = logit_scale_t.item() + if isinstance(optimizer, dict): + if args.clap_mlploss: + logging.info( + f"Train Epoch: {epoch} [{num_samples:>{sample_digits}}/{samples_per_epoch} ({percent_complete:.0f}%)] " + f"Loss: {loss_m.val:#.5g} ({loss_m.avg:#.4g}) " + f"Data (t): {data_time_m.avg:.3f} " + f"Batch (t): {batch_time_m.avg:.3f} " + f"LR: {[o_.param_groups[0]['lr'] for o_ in optimizer.values()]} " + f"Logit Scale Audio: {logit_scale_scalar_a:.3f}" + f"Logit Scale Text: {logit_scale_scalar_t:.3f}" + ) + log_data = { + "loss": loss_m.val, + "data_time": data_time_m.val, + "batch_time": batch_time_m.val, + "scale_audio": logit_scale_scalar_a, + "scale_text": logit_scale_scalar_t, + "lr": [o_.param_groups[0]["lr"] for o_ in optimizer.values()], + } + else: + logging.info( + f"Train Epoch: {epoch} [{num_samples:>{sample_digits}}/{samples_per_epoch} ({percent_complete:.0f}%)] " + f"Loss: {loss_m.val:#.5g} ({loss_m.avg:#.4g}) " + f"Data (t): {data_time_m.avg:.3f} " + f"Batch (t): {batch_time_m.avg:.3f} " + f"LR: {[o_.param_groups[0]['lr'] for o_ in optimizer.values()]} " + f"Logit Scale Audio: {logit_scale_scalar_a:.3f}" + ) + log_data = { + "loss": loss_m.val, + "data_time": data_time_m.val, + "batch_time": batch_time_m.val, + "scale_audio": logit_scale_scalar_a, + "lr": [o_.param_groups[0]["lr"] for o_ in optimizer.values()], + } + + else: + if args.clap_mlploss: + logging.info( + f"Train Epoch: {epoch} [{num_samples:>{sample_digits}}/{samples_per_epoch} ({percent_complete:.0f}%)] " + f"Loss: {loss_m.val:#.5g} ({loss_m.avg:#.4g}) " + f"Data (t): {data_time_m.avg:.3f} " + f"Batch (t): {batch_time_m.avg:.3f} " + f"LR: {optimizer.param_groups[0]['lr']:5f} " + f"Logit Scale Audio: {logit_scale_scalar_a:.3f}" + f"Logit Scale Text: {logit_scale_scalar_t:.3f}" + ) + + # Save train loss / etc. Using non avg meter values as loggers have their own smoothing + log_data = { + "loss": loss_m.val, + "data_time": data_time_m.val, + "batch_time": batch_time_m.val, + "scale_audio": logit_scale_scalar_a, + "scale_text": logit_scale_scalar_t, + "lr": optimizer.param_groups[0]["lr"], + } + else: + logging.info( + f"Train Epoch: {epoch} [{num_samples:>{sample_digits}}/{samples_per_epoch} ({percent_complete:.0f}%)] " + f"Loss: {loss_m.val:#.5g} ({loss_m.avg:#.4g}) " + f"Data (t): {data_time_m.avg:.3f} " + f"Batch (t): {batch_time_m.avg:.3f} " + f"LR: {optimizer.param_groups[0]['lr']:5f} " + f"Logit Scale Audio: {logit_scale_scalar_a:.3f}" + ) + + # Save train loss / etc. Using non avg meter values as loggers have their own smoothing + log_data = { + "loss": loss_m.val, + "data_time": data_time_m.val, + "batch_time": batch_time_m.val, + "scale_audio": logit_scale_scalar_a, + "lr": optimizer.param_groups[0]["lr"], + } + for name, val in log_data.items(): + name = "train/" + name + if tb_writer is not None: + tb_writer.add_scalar(name, val, step) + if args.wandb: + assert wandb is not None, "Please install wandb." + wandb.log({name: val, "step": step}) + + # resetting batch / data time meters per log window + batch_time_m.reset() + data_time_m.reset() + # end for + + +def evaluate(model, data, epoch, args, tb_writer=None): + metrics = {} + if not args.parallel_eval: + if not is_master(args): + return metrics + device = torch.device(args.device) + model.eval() + + # CHANGE + # zero_shot_metrics = zero_shot_eval(model, data, epoch, args) + # metrics.update(zero_shot_metrics) + if is_master(args): + print('Evaluating...') + autocast = torch.cuda.amp.autocast if args.precision == "amp" else suppress + if args.val_dataset_names == ['Clotho', 'audiocaps']: + # if only clotho and audiocaps are used, then we will use a different evaluation function. + # This is because in the Clotho and audiocaps valid and test set, there are 5 text for 1 audio. + if args.parallel_eval: + # (yusong): just a hack here. Don't use parallel eval when evaluating only clotho and audiocaps. + raise NotImplementedError("Parallel evaluation not supported for eval only Clotho and audiocaps.") + val_metrics_per_dataset = evaluate_clotho_audiocaps(model, data, epoch, args, autocast, device, tb_writer) + for m in val_metrics_per_dataset.values(): + metrics.update(m) + if "epoch" not in metrics.keys(): + metrics.update({"epoch": epoch}) + metrics = select_top_metric_clotho_audiocaps(metrics, val_metrics_per_dataset, args) + elif "val" in data and ( + args.val_frequency + and ((epoch % args.val_frequency) == 0 or epoch == args.epochs) + ): + dataloader = data["val"].dataloader + num_samples = 0 + samples_per_val = dataloader.num_samples + + # FIXME this does not scale past small eval datasets + # all_audio_features @ all_text_features will blow up memory and compute very quickly + eval_info = {} + if args.clap_mlploss: + eval_info["all"] = { + "cumulative_loss": 0.0, + "num_samples": 0, + "all_audio_features": [], + "all_text_features": [], + "all_audio_features_mlp": [], + "all_text_features_mlp": [] + } # cumulative_loss = 0.0 + else: + eval_info["all"] = { + "cumulative_loss": 0.0, + "num_samples": 0, + "all_audio_features": [], + "all_text_features": [] + } # cumu + # all_audio_features, all_text_features, all_audio_features_mlp, all_text_features_mlp = [], [], [], [] + with torch.no_grad(): + for i, batch in enumerate(dataloader): + audios = batch # contains mel_spec, wavform, and longer list + texts = batch['text'] + # audios = audios.to(device=device, non_blocking=True) + + all_names = list(set(["-".join(b.split("/")[-3:-1]) for b in batch['__url__']])) + for name in all_names: + if name not in eval_info.keys(): + if args.clap_mlploss: + eval_info[name] = { + "cumulative_loss": 0.0, + "num_samples": 0, + "all_audio_features": [], + "all_text_features": [], + "all_audio_features_mlp": [], + "all_text_features_mlp": [], + } + else: + eval_info[name] = { + "cumulative_loss": 0.0, + "num_samples": 0, + "all_audio_features": [], + "all_text_features": [] + } + with autocast(): + ( + audio_features, + text_features, + audio_features_mlp, + text_features_mlp, + logit_scale_a, + logit_scale_t, + ) = model(audios, texts, device) + + if args.parallel_eval: + # multi-GPU eval + if args.clap_mlploss: + ( + audio_features, + text_features, + audio_features_mlp, + text_features_mlp, + ) = gather_features( + audio_features=audio_features, + text_features=text_features, + audio_features_mlp=audio_features_mlp, + text_features_mlp=text_features_mlp, + local_loss=False, + gather_with_grad=False, + rank=args.rank, + world_size=args.world_size, + use_horovod=args.horovod, + mlp_loss=args.clap_mlploss + ) + else: + ( + audio_features, + text_features, + ) = gather_features( + audio_features=audio_features, + text_features=text_features, + local_loss=False, + gather_with_grad=False, + rank=args.rank, + world_size=args.world_size, + use_horovod=args.horovod, + mlp_loss=args.clap_mlploss + ) + + if is_master(args): + num_samples += audio_features.shape[0] + for n in [*all_names, "all"]: + if n == "all": + eval_info[n]["all_audio_features"].append( + audio_features.cpu() + ) + eval_info[n]["all_text_features"].append( + text_features.cpu() + ) + if args.clap_mlploss: + eval_info[n]["all_audio_features_mlp"].append( + audio_features_mlp.cpu() + ) + eval_info[n]["all_text_features_mlp"].append( + text_features_mlp.cpu() + ) + else: + idx = np.where( + np.array( + ["-".join(b.split("/")[-3:-1]) for b in batch['__url__']] + ) + == n + )[0] + eval_info[n]["all_audio_features"].append( + audio_features.cpu().index_select( + 0, torch.tensor(idx).long() + ) + ) + eval_info[n]["all_text_features"].append( + text_features.cpu().index_select( + 0, torch.tensor(idx).long() + ) + ) + if args.clap_mlploss: + eval_info[n]["all_audio_features_mlp"].append( + audio_features_mlp.cpu().index_select( + 0, torch.tensor(idx).long() + ) + ) + eval_info[n]["all_text_features_mlp"].append( + text_features_mlp.cpu().index_select( + 0, torch.tensor(idx).long() + ) + ) + # print(f'eval step {i}') # (yusong): for debug + + # cumulative_loss += total_loss * batch_size + # num_samples += batch_size + if is_master(args) and (i % 100) == 0: # and i != 0: + logging.info( + f"Eval Epoch: {epoch} [{num_samples} / {samples_per_val}]" + ) + if is_master(args): + val_metrics_per_dataset = {} + for n in eval_info.keys(): + if args.clap_mlploss: + metrics_single_dataset = get_metrics( + audio_features=torch.cat(eval_info[n]["all_audio_features"]), + text_features=torch.cat(eval_info[n]["all_text_features"]), + logit_scale_a=logit_scale_a.cpu(), + audio_features_mlp=torch.cat( + eval_info[n]["all_audio_features_mlp"] + ), + text_features_mlp=torch.cat(eval_info[n]["all_text_features_mlp"]), + logit_scale_t=logit_scale_t.cpu(), + mlp_loss=args.clap_mlploss + ) + else: + metrics_single_dataset = get_metrics( + audio_features=torch.cat(eval_info[n]["all_audio_features"]), + text_features=torch.cat(eval_info[n]["all_text_features"]), + logit_scale_a=logit_scale_a.cpu(), + mlp_loss=args.clap_mlploss + ) + val_metrics_per_dataset[n] = { + n + "/" + k: v for k, v in metrics_single_dataset.items() + } + metrics.update(val_metrics_per_dataset[n]) + if "epoch" not in metrics.keys(): + metrics.update({"epoch": epoch}) + if is_master(args): + if not metrics: + return metrics + + logging.info( + f"Eval Epoch: {epoch} " + + "\n".join( + [ + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in m.items()]) + for m in val_metrics_per_dataset.values() + ] + ) + ) + + if args.save_logs: + for name, val in metrics.items(): + if tb_writer is not None: + tb_writer.add_scalar(f"val/{name}", val, epoch) + + with open(os.path.join(args.checkpoint_path, "results.jsonl"), "a+") as f: + f.write(json.dumps(metrics)) + f.write("\n") + + if args.wandb: + assert wandb is not None, "Please install wandb." + for name, val in metrics.items(): + wandb.log({f"val/{name}": val, "epoch": epoch}) + + return metrics + else: + return metrics + + +def get_metrics( + audio_features, + text_features, + logit_scale_a, + audio_features_mlp=None, + text_features_mlp=None, + logit_scale_t=None, + mlp_loss=False +): + metrics = {} + if mlp_loss: + # Set up audio to text & text to audio similary matrice + a_logits_per_audio = ( + (logit_scale_a * audio_features @ text_features_mlp.t()).detach().cpu() + ) + a_logits_per_text = a_logits_per_audio.t().detach().cpu() + t_logits_per_audio = ( + (logit_scale_t * audio_features_mlp @ text_features.t()).detach().cpu() + ) + t_logits_per_text = t_logits_per_audio.t().detach().cpu() + + labels = torch.arange(audio_features.shape[0]).long() + # Change the loss from two terms into four terms with 2x2 combined CE loss + total_loss = ( + F.cross_entropy(a_logits_per_audio, labels) + + F.cross_entropy(a_logits_per_text, labels) + + F.cross_entropy(t_logits_per_audio, labels) + + F.cross_entropy(t_logits_per_text, labels) + ) / 4 + + metrics[f"cumulative_loss"] = total_loss.item() + metrics[f"num_samples"] = audio_features.shape[0] + + logits = { + "audio_to_text": (a_logits_per_audio + t_logits_per_audio) / 2, + "text_to_audio": (a_logits_per_text + t_logits_per_text) / 2, + } + ground_truth = torch.arange(len(text_features)).view(-1, 1) + + else: + # print("text_features", text_features) + # print("text_features.shape", text_features.shape) + logits_per_audio = (logit_scale_a * audio_features @ text_features.t()).detach().cpu() + logits_per_text = logits_per_audio.t().detach().cpu() + + labels = torch.arange(audio_features.shape[0]).long() + # Change the loss from two terms into four terms with 2x2 combined CE loss + total_loss = ( + F.cross_entropy(logits_per_audio, labels) + + F.cross_entropy(logits_per_text, labels) + ) / 2 + + metrics[f"cumulative_loss"] = total_loss.item() + metrics[f"num_samples"] = audio_features.shape[0] + + logits = {"audio_to_text": logits_per_audio, "text_to_audio": logits_per_text} + + ground_truth = torch.arange(len(text_features)).view(-1, 1) + + for name, logit in logits.items(): + ranking = torch.argsort(logit, descending=True) + preds = torch.where(ranking == ground_truth)[1] # (yusong) this line is slow because it uses single thread + preds = preds.detach().cpu().numpy() + metrics[f"{name}_mean_rank"] = preds.mean() + 1 + metrics[f"{name}_median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"{name}_R@{k}"] = np.mean(preds < k) + # map@10 + metrics[f"{name}_mAP@10"] = np.mean(np.where(preds < 10, 1 / (preds + 1), 0.0)) + + return metrics + + +def evaluate_clotho_audiocaps( + model, data, epoch, args, autocast, device, tb_writer=None +): + """ + Adapted from https://github.com/XinhaoMei/audio-text_retrieval/blob/main/tools/utils.py. + 1. for text-to-audio retrieval, do 5 times and average the results + 2. for R@1, R@5, R@10 in audio-to-text retrieval, take the best rank among 5 text + 3. for map@10 in audio-to-text retrieval: + 3.1: sort the rank of 5 text + 3.2: exclude the rank >=10 (0-index) + 3.3: compute the map regarding the remaining ranks: np.mean(np.arange(1, len(ranks)+1) / ranks). + (3.3) That is, take the top ranks of 5 text that is < 10, and assign the descending number as ground truth. + (3.3) E.g.: the ground truth of first rank of the 5 text should be 1, the second rank should be 2, etc. + """ + # TODO: (yusong) only support single GPU evaluation and only support non-mlp case for now. + dataloader = data["val"].dataloader + with torch.no_grad(): + eval_info = {} + for i, batch in enumerate(dataloader): + audios = batch # contains mel_spec, wavform, and longer list + + # each item in the list has 5 texts + if args.tmodel == "transformer": + from clap_module import tokenize + texts = [tokenize(t) for t in batch['full_text']] + texts = torch.cat(texts) + else: + from .data import tokenizer + texts = [tokenizer(t, tmodel=args.tmodel) for t in batch['full_text']] # 5 texts for each audio + texts = {k: torch.cat([t[k] for t in texts]) for k in texts[0].keys()} # 5 x batch + + # audios = audios.to(device=device, non_blocking=True) + + # batch['__url__'] contains the path to the data tar this sample is from + # So, b.split("/")[-3:-1] will get you '-' + all_names = list(set(["-".join(b.split("/")[-3:-1]) for b in batch['__url__']])) + for name in all_names: + if name not in eval_info.keys(): + # we will not use mlp outputs even if args.clap_mlploss=True + eval_info[name] = { + "cumulative_loss": 0.0, + "num_samples": 0, + "all_audio_features": [], + "all_text_features": [] + } + with autocast(): + audio_features = model(audios, None, device) + text_features = model(None, texts, device) + audio_features = F.normalize(audio_features, dim=-1) + text_features = F.normalize(text_features, dim=-1) + + all_names = list(set(["-".join(b.split("/")[-3:-1]) for b in batch['__url__']])) + for n in all_names: + idx = np.where( + np.array( + ["-".join(b.split("/")[-3:-1]) for b in batch['__url__']] + ) + == n + )[0] + eval_info[n]["all_audio_features"].append( + audio_features.cpu().index_select( + 0, torch.tensor(idx).long() + ) + ) + # (yusong) please double-check. This is for selecting 5 text features at once. + # because idx is a list of indices in size of num_samples, + # and text_features is a tensor of size (5*num_samples, dim) + # so we need to select 5 consecutive indices at once for a single index in idx. + eval_info[n]["all_text_features"].append( + text_features.cpu().reshape([-1, 5, text_features.shape[1]]).index_select( + 0, torch.tensor(idx).long() + ).reshape([-1, text_features.shape[1]]) + ) + + val_metrics_all = {} + + for n in eval_info.keys(): + logit_scale_a, logit_scale_t = model(None, None, device) + logit_scale_a = logit_scale_a.cpu() + + audio_features = torch.cat(eval_info[n]["all_audio_features"], dim=0) + text_features = torch.cat(eval_info[n]["all_text_features"], dim=0) + + logits_per_audio = (logit_scale_a * audio_features @ text_features.t()).detach().cpu() + logits_per_text = logits_per_audio.t().detach().cpu() + + # logits_per_audio shape: [num_samples, num_samples*5] + # logits_per_text shape: [num_samples*5, num_samples] + + logging.info(f"dataset {n}, logits_per_audio shape: {logits_per_audio.shape}, " + f"logits_per_text shape: {logits_per_text.shape}") + + metrics = {} + num_samples = audio_features.shape[0] + metrics[f"num_samples"] = num_samples + + # (yusong) the following code is very important, please double-check: + # logits_per_audio.reshape(num_samples, num_samples, 5)[:, :, d] + # logits_per_text.reshape(num_samples, 5, num_samples)[:, d, :] + # Those two are retrieving one of the 5 text for each audio. + labels = torch.arange(audio_features.shape[0]).long() + audio_to_text_loss = [ + F.cross_entropy( + logits_per_audio.reshape(num_samples, num_samples, 5)[:, :, d], labels) for d in range(5) + ] + text_to_audio_loss = [ + F.cross_entropy( + logits_per_text.reshape(num_samples, 5, num_samples)[:, d, :], labels) for d in range(5) + ] + total_loss = ( + np.mean(audio_to_text_loss) + np.mean(text_to_audio_loss) + ) / 2 + + metrics[f"cumulative_loss"] = total_loss.item() + + # text to audio: do 5 times + pred_text = [] + for d in range(5): + logit = logits_per_text.reshape(num_samples, 5, num_samples)[:, d, :] + ground_truth = torch.arange(len(logit)).view(-1, 1) + ranking = torch.argsort(logit, descending=True) # [num_samples, num_samples] + preds = torch.where(ranking == ground_truth)[1] + pred_text.append(preds.detach().cpu().numpy()) + pred_text_concat = np.concatenate(pred_text, axis=0) # [5*num_samples] + metrics[f"text_to_audio_mean_rank"] = pred_text_concat.mean() + 1 + metrics[f"text_to_audio_median_rank"] = np.floor(np.median(pred_text_concat)) + 1 + for k in [1, 5, 10]: + metrics[f"text_to_audio_R@{k}"] = np.mean(pred_text_concat < k) + # map@10 + metrics[f"text_to_audio_mAP@10"] = np.mean(np.where(pred_text_concat < 10, 1 / (pred_text_concat + 1), 0.0)) + + # audio to text: take the best result + # for audio to text map 10, sort and assign descending ground truth. + # see https://github.com/XinhaoMei/audio-text_retrieval/blob/main/tools/utils.py#L103 + # map@10 + map_all = [] + pred_audio_all = [] + for d in range(num_samples): + # logits_per_audio: [num_samples, num_samples*5] + logit_single = logits_per_audio[d, :] # [5*num_samples] + # Ground-truth index: [d*5, d*5+1, d*5+2, d*5+3, d*5+4] + ranking = torch.argsort(logit_single, descending=True) # [5*num_samples] + # ranking: the index of first match, second match, ... + ground_truth = torch.arange(d * 5, d * 5 + 5)[None] + all_pred = torch.where(torch.stack([ranking] * 5) == ground_truth.view(-1, 1))[1] + min_pred = torch.min(all_pred) + pred_audio_all.append(min_pred.detach().cpu().numpy()) + all_pred_filter = all_pred[all_pred < 10].detach().cpu().numpy() + # /5 because we have 5 text, so it means for the text rank >=10 we count as 0. + map_single = np.sum((np.arange(1, len(all_pred_filter) + 1) / (all_pred_filter + 1))) / 5 + map_all.append(map_single) + metrics[f"audio_to_text_mAP@10"] = np.mean(map_all) + for k in [1, 5, 10]: + metrics[f"audio_to_text_R@{k}"] = np.mean(np.array(pred_audio_all) < k) + + val_metrics_all[n] = { + n + "/" + k: v for k, v in metrics.items() + } + return val_metrics_all + + +def calculate_selection_performance_clotho_audiocaps(val_metrics_per_dataset): + """ + Calculate performance for Clotho+AudioCaps for model selection. + """ + selection_performance_all = [] + for n in val_metrics_per_dataset.keys(): + selection_performance = (val_metrics_per_dataset[n][f"{n}/audio_to_text_mAP@10"] + + val_metrics_per_dataset[n][f"{n}/text_to_audio_mAP@10"]) / 2 + selection_performance_all.append(selection_performance) + return np.mean(selection_performance_all) + + +def select_top_metric_clotho_audiocaps(metrics, val_metrics_per_dataset, args): + # val_metrics_per_dataset: dict, key: dataset name, value: dict, key: metric name, value: metric value + # metrics: dict, key: metric name, value: metric value + # Hack: use args to save the top performance + if not hasattr(args, "top_selection_performance"): + selection_performance = calculate_selection_performance_clotho_audiocaps(val_metrics_per_dataset) + # TODO: write the if and else together + metric_update = {} + for n in val_metrics_per_dataset.keys(): + for k in val_metrics_per_dataset[n].keys(): + metric_update[k.split('/')[0] + '-top' + '/' + k.split('/')[1]] = val_metrics_per_dataset[n][k] + metric_update['top_selection_performance'] = selection_performance + metric_update['top-selection-epoch'] = metrics['epoch'] + metrics.update(metric_update) + args.top_metric = metric_update + args.top_selection_performance = selection_performance + else: + selection_performance_new = calculate_selection_performance_clotho_audiocaps(val_metrics_per_dataset) + selection_performance_old = args.top_selection_performance + if selection_performance_new > selection_performance_old: + metric_update = {} + for n in val_metrics_per_dataset.keys(): + for k in val_metrics_per_dataset[n].keys(): + metric_update[k.split('/')[0] + '-top' + '/' + k.split('/')[1]] = val_metrics_per_dataset[n][k] + metric_update['top_selection_performance'] = selection_performance_new + metric_update['top-selection-epoch'] = metrics['epoch'] + metrics.update(metric_update) + args.top_metric = metric_update + args.top_selection_performance = selection_performance_new + else: + metrics.update(args.top_metric) + return metrics diff --git a/utils/laion_clap/training/zero_shot.py b/utils/laion_clap/training/zero_shot.py new file mode 100644 index 0000000000000000000000000000000000000000..04472c16e36041f90c8f229c5e026dcc394fb977 --- /dev/null +++ b/utils/laion_clap/training/zero_shot.py @@ -0,0 +1,90 @@ +# NOTE: This script is currently not supported for CLAP. +import logging +from contextlib import suppress + +import torch +import torch.nn.functional as F +from tqdm import tqdm + +from clap_module import tokenize +from .imagenet_zeroshot_data import imagenet_classnames, openai_imagenet_template + + +def zero_shot_classifier(model, classnames, templates, args): + with torch.no_grad(): + zeroshot_weights = [] + for classname in tqdm(classnames): + texts = [template(classname) for template in templates] # format with class + texts = tokenize(texts).to(args.device) # tokenize + if args.distributed and not args.horovod: + class_embeddings = model.module.encode_text(texts) + else: + class_embeddings = model.encode_text(texts) + class_embedding = F.normalize(class_embeddings, dim=-1).mean(dim=0) + class_embedding /= class_embedding.norm() + zeroshot_weights.append(class_embedding) + zeroshot_weights = torch.stack(zeroshot_weights, dim=1).to(args.device) + return zeroshot_weights + + +def accuracy(output, target, topk=(1,)): + pred = output.topk(max(topk), 1, True, True)[1].t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk] + + +def run(model, classifier, dataloader, args): + autocast = torch.cuda.amp.autocast if args.precision == 'amp' else suppress + with torch.no_grad(): + top1, top5, n = 0., 0., 0. + for images, target in tqdm(dataloader, unit_scale=args.batch_size): + images = images.to(args.device) + target = target.to(args.device) + + with autocast(): + # predict + if args.distributed and not args.horovod: + image_features = model.module.encode_image(images) + else: + image_features = model.encode_image(images) + image_features = F.normalize(image_features, dim=-1) + logits = 100. * image_features @ classifier + + # measure accuracy + acc1, acc5 = accuracy(logits, target, topk=(1, 5)) + top1 += acc1 + top5 += acc5 + n += images.size(0) + + top1 = (top1 / n) + top5 = (top5 / n) + return top1, top5 + + +def zero_shot_eval(model, data, epoch, args): + if 'imagenet-val' not in data and 'imagenet-v2' not in data: + return {} + if args.zeroshot_frequency == 0: + return {} + if (epoch % args.zeroshot_frequency) != 0 and epoch != args.epochs: + return {} + + logging.info('Starting zero-shot imagenet.') + + logging.info('Building zero-shot classifier') + classifier = zero_shot_classifier(model, imagenet_classnames, openai_imagenet_template, args) + + logging.info('Using classifier') + results = {} + if 'imagenet-val' in data: + top1, top5 = run(model, classifier, data['imagenet-val'].dataloader, args) + results['imagenet-zeroshot-val-top1'] = top1 + results['imagenet-zeroshot-val-top5'] = top5 + if 'imagenet-v2' in data: + top1, top5 = run(model, classifier, data['imagenet-v2'].dataloader, args) + results['imagenetv2-zeroshot-val-top1'] = top1 + results['imagenetv2-zeroshot-val-top5'] = top5 + + logging.info('Finished zero-shot imagenet.') + + return results diff --git a/utils/laion_clap/unit_test.py b/utils/laion_clap/unit_test.py new file mode 100644 index 0000000000000000000000000000000000000000..f138c627b43bb70b45bdd9364841ac0f113e32dc --- /dev/null +++ b/utils/laion_clap/unit_test.py @@ -0,0 +1,75 @@ +""" +Contrastive Language-Audio Pretraining Model from LAION +-------------------------------------------------------- +Paper: https://arxiv.org/abs/2211.06687 +Authors (equal contributions): Ke Chen, Yusong Wu, Tianyu Zhang, Yuchen Hui +Support: LAION +""" + +import numpy as np +import librosa +import torch +import laion_clap + +# quantization +def int16_to_float32(x): + return (x / 32767.0).astype(np.float32) + + +def float32_to_int16(x): + x = np.clip(x, a_min=-1., a_max=1.) + return (x * 32767.).astype(np.int16) + +model = laion_clap.CLAP_Module(enable_fusion=False) +model.load_ckpt() + +# Directly get audio embeddings from audio files +audio_file = [ + '/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_short.wav', + '/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_long.wav' +] +audio_embed = model.get_audio_embedding_from_filelist(x = audio_file, use_tensor=False) +print(audio_embed[:,-20:]) +print(audio_embed.shape) + +# Get audio embeddings from audio data +audio_data, _ = librosa.load('/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_short.wav', sr=48000) # sample rate should be 48000 +audio_data = audio_data.reshape(1, -1) # Make it (1,T) or (N,T) +audio_embed = model.get_audio_embedding_from_data(x = audio_data, use_tensor=False) +print(audio_embed[:,-20:]) +print(audio_embed.shape) + +# Directly get audio embeddings from audio files, but return torch tensor +audio_file = [ + '/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_short.wav', + '/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_long.wav' +] +audio_embed = model.get_audio_embedding_from_filelist(x = audio_file, use_tensor=True) +print(audio_embed[:,-20:]) +print(audio_embed.shape) + +# Get audio embeddings from audio data +audio_data, _ = librosa.load('/home/la/kechen/Research/KE_CLAP/ckpt/test_clap_short.wav', sr=48000) # sample rate should be 48000 +audio_data = audio_data.reshape(1, -1) # Make it (1,T) or (N,T) +audio_data = torch.from_numpy(int16_to_float32(float32_to_int16(audio_data))).float() # quantize before send it in to the model +audio_embed = model.get_audio_embedding_from_data(x = audio_data, use_tensor=True) +print(audio_embed[:,-20:]) +print(audio_embed.shape) + +# Get text embedings from texts: +text_data = ["I love the contrastive learning", "I love the pretrain model"] +text_embed = model.get_text_embedding(text_data) +print(text_embed) +print(text_embed.shape) + +# Get text embedings from texts, but return torch tensor: +text_data = ["I love the contrastive learning", "I love the pretrain model"] +text_embed = model.get_text_embedding(text_data, use_tensor=True) +print(text_embed) +print(text_embed.shape) + + + + + + diff --git a/utils/log.py b/utils/log.py new file mode 100644 index 0000000000000000000000000000000000000000..684aaa8eddb5c8cb0f67459b78994ca35ec15a1c --- /dev/null +++ b/utils/log.py @@ -0,0 +1,976 @@ +import os +import torch +import time +import numpy as np +#import torchaudio +import plotly.express as px +import soundfile as sf +#import plotly.graph_objects as go +import pandas as pd +import plotly +import plotly.graph_objects as go +import matplotlib.pyplot as plt + +#from notebooks.shared_log_filter import frequencies + +""" +Logging related functions that I wrote for my own use +This is quite a mess, but I'm too lazy to clean it up +""" +#from src.CQT_nsgt import CQT_cpx + +def do_stft(noisy, clean=None, win_size=2048, hop_size=512, device="cpu", DC=True): + """ + applies the stft, this an ugly old function, but I'm using it for logging and I'm to lazy to modify it + """ + + #window_fn = tf.signal.hamming_window + + #win_size=args.stft.win_size + #hop_size=args.stft.hop_size + window=torch.hamming_window(window_length=win_size) + window=window.to(noisy.device) + noisy=torch.cat((noisy, torch.zeros(noisy.shape[0],win_size).to(noisy.device)),1) + stft_signal_noisy=torch.stft(noisy, win_size, hop_length=hop_size,window=window,center=False,return_complex=False) + stft_signal_noisy=stft_signal_noisy.permute(0,3,2,1) + #stft_signal_noisy=tf.signal.stft(noisy,frame_length=win_size, window_fn=window_fn, frame_step=hop_size) + #stft_noisy_stacked=tf.stack( values=[tf.math.real(stft_signal_noisy), tf.math.imag(stft_signal_noisy)], axis=-1) + + if clean!=None: + + # stft_signal_clean=tf.signal.stft(clean,frame_length=win_size, window_fn=window_fn, frame_step=hop_size) + clean=torch.cat((clean, torch.zeros(clean.shape[0],win_size).to(device)),1) + stft_signal_clean=torch.stft(clean, win_size, hop_length=hop_size,window=window, center=False,return_complex=False) + stft_signal_clean=stft_signal_clean.permute(0,3,2,1) + #stft_clean_stacked=tf.stack( values=[tf.math.real(stft_signal_clean), tf.math.imag(stft_signal_clean)], axis=-1) + + + if DC: + return stft_signal_noisy, stft_signal_clean + else: + return stft_signal_noisy[...,1:], stft_signal_clean[...,1:] + else: + + if DC: + return stft_signal_noisy + else: + return stft_signal_noisy[...,1:] + +def plot_norms(path, normsscores, normsguides, t, name): + values=t.cpu().numpy() + + df=pd.DataFrame.from_dict( + {"sigma": values[0:-1], "score": normsscores.cpu().numpy(), "guidance": normsguides.cpu().numpy()} + ) + fig= px.line(df, x="sigma", y=["score", "guidance"],log_x=True, log_y=True, markers=True) + + path_to_plotly_html = path+"/"+name+".html" + + fig.write_html(path_to_plotly_html, auto_play = False) + return fig + + +def plot_spectral_analysis_sampling( avgspecNF,avgspecDEN, ts, fs=22050, nfft=1024): + T,F=avgspecNF.shape + f=torch.arange(0, F)*fs/nfft + f=f.unsqueeze(1).repeat(1,T).view(-1) + avgspecNF=avgspecNF.permute(1,0).reshape(-1) + avgspecDEN=avgspecDEN.permute(1,0).reshape(-1) + ts=ts.cpu().numpy().tolist() + ts=513*ts + #ts=np.repeat(ts,513) + print(f.shape, avgspecDEN.shape, avgspecNF.shape, len(ts)) + df=pd.DataFrame.from_dict( + {"f": f, "noisy": avgspecNF, "denoised": avgspecDEN, "sigma":ts + } + ) + + fig= px.line(df, x="f", y=["noisy", "denoised"], animation_frame="sigma", log_x=False,log_y=False, markers=False) + return fig + +def plot_spectral_analysis(avgspecY, avgspecNF,avgspecDEN, ts, fs=22050, nfft=1024): + T,F=avgspecNF.shape + f=torch.arange(0, F)*fs/nfft + f=f.unsqueeze(1).repeat(1,T).view(-1) + avgspecY=avgspecY.squeeze(0).unsqueeze(1).repeat(1,T).view(-1) + avgspecNF=avgspecNF.permute(1,0).reshape(-1) + avgspecDEN=avgspecDEN.permute(1,0).reshape(-1) + ts=ts.cpu().numpy().tolist() + ts=513*ts + #ts=np.repeat(ts,513) + print(f.shape, avgspecY.shape, avgspecNF.shape, len(ts)) + df=pd.DataFrame.from_dict( + {"f": f, "y": avgspecY, "noisy": avgspecNF, "denoised": avgspecDEN, "sigma":ts + } + ) + + fig= px.line(df, x="f", y=["y", "noisy", "denoised"], animation_frame="sigma", log_x=False,log_y=False, markers=False) + return fig + +def plot_loss_by_sigma_test_snr(average_snr, average_snr_out, t): + #write a fancy plot to log in wandb + values=np.array(t) + df=pd.DataFrame.from_dict( + {"sigma": values, "SNR": average_snr, "SNR_denoised": average_snr_out + } + ) + + fig= px.line(df, x="sigma", y=["SNR", "SNR_denoised"],log_x=True, markers=True) + return fig + + + + +# Create and style traces + +def plot_loss_by_sigma(sigma_means, sigma_stds, sigma_bins, log_scale=True): + df=pd.DataFrame.from_dict( + {"sigma": sigma_bins, "loss": sigma_means, "std": sigma_stds + } + ) + + fig= error_line('bar', data_frame=df, x="sigma", y="loss", error_y="std", log_x=log_scale, markers=True, range_y=[0, 2]) + + return fig + +def plot_loss_by_sigma_and_freq(sigma_freq_means, sigma_freq_stds, sigma_bins, freq_bins): + df=pd.DataFrame.from_dict( + {"sigma": np.tile(sigma_bins, len(freq_bins))}) + names=[] + means=[] + stds=[] + for i in range(len(freq_bins)): + name=[str(freq_bins[i])+"Hz" for j in range(len(sigma_bins))] + names.extend(name) + means.extend(sigma_freq_means[i]) + stds.extend(sigma_freq_stds[i]) + df['freq']=names + df['means']=means + df['stds']=stds + print(df) + + + #basically, we want to plot the loss as a function of sigma and freq. We do it by plotting a line for each freq. + fig= error_line('bar', data_frame=df, x="sigma", y="means", error_y="stds", color="freq", log_x=True, markers=True, range_y=[0, 2]) + + return fig + + + +def plot_melspectrogram(X, refr=1): + X=X.squeeze(1) #?? + X=X.cpu().numpy() + if refr==None: + refr=np.max(np.abs(X))+1e-8 + + S_db = 10*np.log10(np.abs(X)/refr) + + S_db=np.transpose(S_db, (0,2,1)) + S_db=np.flip(S_db, axis=1) + + for i in range(X.shape[0]): #iterate over batch size, shity way of ploting all the batched spectrograms + o=S_db[i] + if i==0: + res=o + else: + res=np.concatenate((res,o), axis=1) + + fig=px.imshow( res, zmin=-40, zmax=20) + + fig.update_layout(coloraxis_showscale=False) + + return fig + +def plot_cpxspectrogram(X): + X=X.squeeze(1) + X=X.cpu().numpy() + fig=px.imshow(X, facet_col=3, animation_frame=0) + fig.update_layout(coloraxis_showscale=False) + return fig +def print_cuda_memory(): + t = torch.cuda.get_device_properties(0).total_memory + r = torch.cuda.memory_reserved(0) + a = torch.cuda.memory_allocated(0) + f = r-a # free inside reservedk + print("memrylog",t,r,a,f) + + + +def plot_mag_spectrogram(X, refr=None, path=None,name="spec"): + X=X.cpu().numpy() + if refr==None: + refr=np.max(np.abs(X))+1e-8 + + S_db = 10*np.log10(np.abs(X)/refr) + S_db=np.flip(S_db, axis=1) + + for i in range(X.shape[0]): #iterate over batch size, shity way of ploting all the batched spectrograms + o=S_db[i] + if i==0: + res=o + else: + res=np.concatenate((res,o), axis=1) + + fig = px.imshow(res, zmin=-40, zmax=20) + + fig.update_layout(coloraxis_showscale=False) + + path_to_plotly_png = path+"/"+name+".png" + plotly.io.write_image(fig, path_to_plotly_png) + + return fig + + +def plot_spectrogram(X, refr=None, cplx=True, freqs=None, time=None): + X=X.squeeze(1) + X=X.cpu().numpy() + if cplx: + X=np.sqrt(X[:,:,:,0]**2 + X[:,:,:,1]**2) + + if refr==None: + refr=np.max(np.abs(X))+1e-8 + + S_db = 10*np.log10(np.abs(X)/refr) + + S_db=np.transpose(S_db, (0,2,1)) + S_db=np.flip(S_db, axis=1) + + for i in range(X.shape[0]): #iterate over batch size, shity way of ploting all the batched spectrograms + o=S_db[i] + if i==0: + res=o + else: + res=np.concatenate((res,o), axis=1) + #freqs ia a tensor of shape (F,) + #time is a tensor of shape (T,) + #add the frequency and time axis correctly + + + fig=px.imshow( res, zmin=-40, zmax=20) + fig.update_layout(coloraxis_showscale=False) + + return fig + +def write_audio_file(x, sr, string: str, path='tmp', stereo=False, normalize=False): + if normalize: + x=x/torch.max(torch.abs(x)) + if not(os.path.exists(path)): + os.makedirs(path) + + path=os.path.join(path,string+".wav") + if stereo: + ''' + x has shape (B,2,T) + ''' + x=x.permute(0,2,1) #B,T,2 + x=x.flatten(0,1) #B*T,2 + x=x.cpu().numpy() + #if np.abs(np.max(x))>=1: + # #normalize to avoid clipping + # x=x/np.abs(np.max(x)) + else: + x=x.flatten() + x=x.unsqueeze(1) + x=x.cpu().numpy() + #if np.abs(np.max(x))>=1: + # #normalize to avoid clipping + # x=x/np.abs(np.max(x)) + sf.write(path,x,sr) + return path + +def plot_trajectories(x,sr, sigma_data): + """ + x: (B,T) Batch of trajectories + sr: sampling rate + args: args object + """ + x=x.cpu() + times=np.arange(x.shape[1])/sr + + fig=plot_batch_of_lines(x/sigma_data, times, log_x=False) + return fig + +def plot_trajectories_fft(x, sr, sigma_data): + """ + x: (B,T) Batch of trajectories + sr: sampling rate + args: args object + + Apply fft to the trajectories and plot the magnitude of the fft + """ + x=x.cpu() + + times=np.arange(x.shape[1])/sr + x=x/sigma_data + x_fft=np.fft.rfft(x, axis=1) + x_fft=np.abs(x_fft) + freqs=np.fft.rfftfreq(x.shape[1], d=1/sr) + #now do it on db scale + x_fft_db=10*np.log10(x_fft) + fig=plot_batch_of_lines(x_fft_db, freqs, log_x=False) + return fig + return fig + + +def plot_cpxCQT_from_raw_audio(x, args, refr=None ): + #shape of input spectrogram: ( ,T,F, ) + fmax=args.sample_rate/2 + fmin=fmax/(2**args.cqt.numocts) + fbins=int(args.cqt.binsoct*args.cqt.numocts) + device=x.device + CQTransform=CQT_cpx(fmin,fbins, args.sample_rate, args.audio_len, device=device, split_0_nyq=False) + + x=x + X=CQTransform.fwd(x) + return plot_cpxspectrogram(X) + +def plot_CQT_from_raw_audio(x, args, refr=None ): + #shape of input spectrogram: ( ,T,F, ) + fmax=args.sample_rate/2 + fmin=fmax/(2**args.cqt.numocts) + fbins=int(args.cqt.binsoct*args.cqt.numocts) + device=x.device + CQTransform=CQT_cpx(fmin,fbins, args.sample_rate, args.audio_len, device=device, split_0_nyq=False) + + refr=3 + x=x + X=CQTransform.fwd(x) + return plot_spectrogram(X, refr) + +def get_spectrogram_from_raw_audio(x, stft, refr=1): + X=do_stft(x, win_size=stft.win_size, hop_size=stft.hop_size) + X=X.permute(0,2,3,1) + + X=X.squeeze(1) + #X=X.cpu().numpy() + X=torch.sqrt(X[:,:,:,0]**2 + X[:,:,:,1]**2) + + #if refr==None: + # refr=np.max(np.abs(X))+1e-8 + + S_db = 10*torch.log10(torch.abs(X)/refr) + + S_db=S_db.permute(0,2,1) + #np.transpose(S_db, (0,2,1)) + S_db=torch.flip(S_db, [1]) + + for i in range(X.shape[0]): #iterate over batch size, shity way of ploting all the batched spectrograms + o=S_db[i] + if i==0: + res=o + else: + res=torch.cat((res,o), 1) + return res + + + +def diffusion_CQT_animation(path, x ,t, args, refr=1, name="animation_diffusion", resample_factor=1 ): + """ + Utility for creating an animation of the cqt diffusion process + """ + #shape of input spectrograms: (Nsteps,B,Time,Freq) + #print(noisy.shape) + Nsteps=x.shape[0] + numsteps=10 + tt=torch.linspace(0, Nsteps-1, numsteps) + i_s=[] + allX=None + device=x.device + + fmax=args.sample_rate/2 + fmin=fmax/(2**args.cqt.numocts) + fbins=int(args.cqt.binsoct*args.cqt.numocts) + CQTransform=CQT_cpx(fmin,fbins, args.sample_rate, args.audio_len, device=device, split_0_nyq=False) + + for i in tt: + i=int(torch.floor(i)) + i_s.append(i) + xx=x[i] + X=CQTransform.fwd(xx) + + X=torch.sqrt(X[...,0]**2 + X[...,1]**2) + + S_db = 10*torch.log10(torch.abs(X)/refr) + S_db = S_db[:,1:-1,1:-1] + + S_db=S_db.permute(0,2,1) + #np.transpose(S_db, (0,2,1)) + S_db=torch.flip(S_db, [1]) + S_db=S_db.unsqueeze(0) + S_db=torch.nn.functional.interpolate(S_db, size= (S_db.shape[2]//resample_factor, S_db.shape[3]//resample_factor), mode="bilinear") + S_db=S_db.squeeze(0) + S_db=S_db.cpu().numpy() + #S_db=S_db.squeeze(0) + + if i==0: + allX=S_db + else: + allX=np.concatenate((allX,S_db), 0) + + #fig=px.imshow(S_db, animation_frame=0, facet_col=3, binary_compression_level=0) #I need to incorporate t here!!! + fig=px.imshow(allX, animation_frame=0, zmin=-40, zmax=10, binary_compression_level=0) #I need to incorporate t here!!! + + fig.update_layout(coloraxis_showscale=False) + fig.update_xaxes(showticklabels=False) + fig.update_yaxes(showticklabels=False) + + t=t[i_s].cpu().numpy() + + assert len(t)==len(fig.frames), print(len(t), len(fig.frames)) + #print(fig) + for i, f in enumerate(fig.layout.sliders[0].steps): + #hacky way of changing the axis values + #f.args[0]=[str(t[i])] + f.label=str(t[i]) + + path_to_plotly_html = path+"/"+name+".html" + + fig.write_html(path_to_plotly_html, auto_play = False) + + + return fig + +def diffusion_joint_filter_animation(path, score, grads , f, t, refr=1, name="animation_diffusion", NT=15 ): + ''' + plot an animation of the reverse diffusion process of filters + args: + path: path to save the animation + x: filters (Nsteps, F) + f: frequencies (F) + t: timesteps (sigma) + name: name of the animation + ''' + #shape of input spectrograms: (Nsteps,B,Time,Freq) + #print(noisy.shape) + Nsteps=score.shape[0] + #numsteps=min(Nsteps,NT) #hardcoded, I'll probably need more! + + #tt=torch.linspace(0, Nsteps-1, numsteps) + #i_s=[] + #for i in tt: + # i=int(torch.floor(i)) + # i_s.append(i) + + print(f.shape, score.shape, grads.shape, t.shape) #19, (100,19) + #fig=px.line(x=f.unsqueeze(0).numpy(),y=x, animation_frame=0) #I need to incorporate t here!!! + sigma=t + s=score.squeeze(1)# (100,19) + g=grads.squeeze(1)# (100,19) + f=f# (19,) + f=f.unsqueeze(0).expand(s.shape[0], -1).reshape(-1) + sigma=sigma.unsqueeze(-1).expand(-1, s.shape[1]).reshape(-1) + s=s.reshape(-1) + g=g.reshape(-1) + max_score=torch.max(s).item() + df=pd.DataFrame( + { + "f": f.cpu().numpy(), + "score": s.cpu().numpy(), + "grads": g.cpu().numpy(), + "sigma": sigma.cpu().numpy() + } + ) + fig=px.line(df, x="f",y=["score","grads"], animation_frame="sigma", log_x=True) #I need to incorporate t here!!! + #set the range of th y axis + fig.update_yaxes(range=[-max_score,max_score]) + #t=t[i_s].cpu().numpy() + #t=t.cpu().numpy() + + #assert len(t)==len(fig.frames), print(len(t), len(fig.frames)) + #print(fig) + #for i, f in enumerate(fig.layout.sliders[0].steps): + #hacky way of changing the axis values + #f.args[0]=[str(t[i])] + # f.label=str(t[i]) + + path_to_plotly_html = path+"/"+name+".html" + + fig.write_html(path_to_plotly_html, auto_play = False) + + return fig + +def diffusion_filter_animation(path, x , f, t, refr=1, name="animation_diffusion", NT=15 ): + ''' + plot an animation of the reverse diffusion process of filters + args: + path: path to save the animation + x: filters (Nsteps, F) + f: frequencies (F) + t: timesteps (sigma) + name: name of the animation + ''' + #shape of input spectrograms: (Nsteps,B,Time,Freq) + #print(noisy.shape) + Nsteps=x.shape[0] + #numsteps=min(Nsteps,NT) #hardcoded, I'll probably need more! + + #tt=torch.linspace(0, Nsteps-1, numsteps) + #i_s=[] + #for i in tt: + # i=int(torch.floor(i)) + # i_s.append(i) + + print(f.shape, x.shape, t.shape) #19, (100,19) + #fig=px.line(x=f.unsqueeze(0).numpy(),y=x, animation_frame=0) #I need to incorporate t here!!! + sigma=t + x=x.squeeze(1)# (100,19) + f=f# (19,) + f=f.unsqueeze(0).expand(x.shape[0], -1).reshape(-1) + sigma=sigma.unsqueeze(-1).expand(-1, x.shape[1]).reshape(-1) + x=x.reshape(-1) + df=pd.DataFrame( + { + "f": f.cpu().numpy(), + "x": x.cpu().numpy(), + "sigma": sigma.cpu().numpy() + } + ) + fig=px.line(df, x="f",y="x", animation_frame="sigma", log_x=True) #I need to incorporate t here!!! + #t=t[i_s].cpu().numpy() + #t=t.cpu().numpy() + + #assert len(t)==len(fig.frames), print(len(t), len(fig.frames)) + #print(fig) + #for i, f in enumerate(fig.layout.sliders[0].steps): + #hacky way of changing the axis values + #f.args[0]=[str(t[i])] + # f.label=str(t[i]) + + path_to_plotly_html = path+"/"+name+".html" + + fig.write_html(path_to_plotly_html, auto_play = False) + + return fig + +def diffusion_spec_animation(path, x ,t, stft, refr=1, name="animation_diffusion",NT=15 ): + ''' + plot an animation of the reverse diffusion process of filters + args: + path: path to save the animation + x: input audio (N,T) + t: timesteps (sigma) + name: name of the animation + ''' + #print(noisy.shape) + Nsteps=x.shape[0] + numsteps=min(Nsteps,NT) #hardcoded, I'll probably need more! + tt=torch.linspace(0, Nsteps-1, numsteps) + i_s=[] + allX=None + for i in tt: + i=int(torch.floor(i)) + i_s.append(i) + X=get_spectrogram_from_raw_audio(x[i],stft, refr) + X=X.unsqueeze(0) + if allX==None: + allX=X + else: + allX=torch.cat((allX,X), 0) + + + allX=allX.cpu().numpy() + fig=px.imshow(allX, animation_frame=0, zmin=-40, zmax=20) #I need to incorporate t here!!! + + fig.update_layout(coloraxis_showscale=False) + + t=t[i_s].cpu().numpy() + + assert len(t)==len(fig.frames), print(len(t), len(fig.frames)) + #print(fig) + for i, f in enumerate(fig.layout.sliders[0].steps): + #hacky way of changing the axis values + #f.args[0]=[str(t[i])] + f.label=str(t[i]) + + path_to_plotly_html = path+"/"+name+".html" + + fig.write_html(path_to_plotly_html, auto_play = False) + + + return fig + +def plot_spectrogram_from_raw_audio(x, stft, refr=None ): + #shape of input spectrogram: ( ,T,F, ) + refr=3 + x=x + X=do_stft(x, win_size=stft.win_size, hop_size=stft.hop_size) + X=X.permute(0,2,3,1) + return plot_spectrogram(X, refr) + +def plot_spectrogram_from_cpxspec(X, refr=None): + #shape of input spectrogram: ( ,T,F, ) + return plot_spectrogram(X, refr) + + +def error_line(error_y_mode='band', **kwargs): + print("hello error line", error_y_mode) + """Extension of `plotly.express.line` to use error bands.""" + ERROR_MODES = {'bar','band','bars','bands',None} + if error_y_mode not in ERROR_MODES: + raise ValueError(f"'error_y_mode' must be one of {ERROR_MODES}, received {repr(error_y_mode)}.") + if error_y_mode in {'bar','bars',None}: + print("Using error bars", kwargs) + fig = px.line(**kwargs) + elif error_y_mode in {'band','bands'}: + if 'error_y' not in kwargs: + raise ValueError(f"If you provide argument 'error_y_mode' you must also provide 'error_y'.") + figure_with_error_bars = px.line(**kwargs) + fig = px.line(**{arg: val for arg,val in kwargs.items() if arg != 'error_y'}) + for data in figure_with_error_bars.data: + x = list(data['x']) + y_upper = list(data['y'] + data['error_y']['array']) + y_lower = list(data['y'] - data['error_y']['array'] if data['error_y']['arrayminus'] is None else data['y'] - data['error_y']['arrayminus']) + color = f"rgba({tuple(int(data['line']['color'].lstrip('#')[i:i+2], 16) for i in (0, 2, 4))},.3)".replace('((','(').replace('),',',').replace(' ','') + fig.add_trace( + go.Scatter( + x = x+x[::-1], + y = y_upper+y_lower[::-1], + fill = 'toself', + fillcolor = color, + line = dict( + color = 'rgba(255,255,255,0)' + ), + hoverinfo = "skip", + showlegend = False, + legendgroup = data['legendgroup'], + xaxis = data['xaxis'], + yaxis = data['yaxis'], + ) + ) + # Reorder data as said here: https://stackoverflow.com/a/66854398/8849755 + reordered_data = [] + for i in range(int(len(fig.data)/2)): + reordered_data.append(fig.data[i+int(len(fig.data)/2)]) + reordered_data.append(fig.data[i]) + fig.data = tuple(reordered_data) + return fig + + +def plot_filters( x, freqs): + ''' + This function plots a batch of lines using plotly + args: + x: (B, F) + ''' + fig = px.line(x=freqs, y=x[0,:], log_x=True) + + for i in range(1,x.shape[0]): + fig.add_trace(go.Scatter(x=freqs, y=x[i,:])) + return fig + +def plot_batch_of_lines( x, freqs, log_x=True): + ''' + This function plots a batch of lines using plotly + args: + x: (B, F) + ''' + fig = px.line(x=freqs, y=x[0,:], log_x=log_x) + + for i in range(1,x.shape[0]): + fig.add_trace(go.Scatter(x=freqs, y=x[i,:])) + return fig + + + + +def plot_rir(rir, T_separation, fs, path=None, name=None, save=True): + ''' + Plot the RIR and indicate the separation time between early and late reflections + ''' + rir = rir.cpu() + RIR = torch.stft(rir, n_fft=512, hop_length=128, window=torch.hann_window(512).to(rir.device), return_complex=True) + + fig, ax = plt.subplots(1, 2) + + ax[0].pcolormesh( + torch.linspace(0, RIR.size(-1)/128, RIR.size(-1)), + torch.linspace(0, fs/2, RIR.size(-2)), + 20*torch.log10(RIR.abs() + 1e-10)) + + ax[1].plot( + torch.linspace(0, rir.size(-1)/fs, rir.size(-1)), + rir) + + if save: + plt.savefig(os.path.join(path, name+".png")) + else: + return fig + + +def create_mag_phase_plot(): + # Create subplots + from plotly.subplots import make_subplots + + # fig = make_subplots(rows=2, cols=1, vertical_spacing=0.1, + # subplot_titles=('Magnitude Response', 'Phase Response')) + fig = make_subplots(specs=[[{"secondary_y": True}]]) + # Update layout + fig.update_layout(height=600, width=800, title_text="Magnitude and Phase of Filter") + # fig.update_xaxes(type='log',title_text="Frequency [Hz])", row=2, col=1) + fig.update_xaxes(type='log', title_text="Frequency [Hz])") + # fig.update_yaxes(title_text="Magnitude [dB]", row=1, col=1) + # fig.update_yaxes(title_text="Phase [rad]", row=2, col=1) + # Set y-axes titles + fig.update_yaxes(title_text="Magnitude [dB]", secondary_y=False) + fig.update_yaxes(title_text="Phase [rad]", secondary_y=True) + return fig + + +def plot_STFT(x, operator, type='mag_dB', min_freq=100): + + # Ensure that shape is (B, F, T) + fs = operator.sample_rate + nfft = operator.n_fft + hop_size = operator.hop_length + + print(x.device, operator.device) + X = operator.apply_stft(x.to(operator.device)) + + print(X.shape, x.shape, X.device) + + # , X.device) + if type == 'mag': + x_spec = X.abs() + if type == 'mag_dB': + x_spec = 20*torch.log10(X.abs()+1e-7) + if type == 'phase': + x_spec = X.angle() + # if type == 'phase_unwrap': + # x_spec = X.angle().grad() + + + if "mag" in type: + z_min=-100 + z_max= 10 + else: + z_min=-np.pi + z_max= np.pi + + + + + frequencies = torch.fft.rfftfreq(nfft) * fs + timestamps = torch.arange(X.shape[-1]) * (hop_size / fs) + fig = go.Figure(data=go.Heatmap( + z=x_spec.squeeze(0).cpu().numpy(), + x=timestamps.numpy(), + y=frequencies.numpy(), + colorscale='Magma', + zmax=z_max, + zmin=z_min + )) + + fig.update_layout( + yaxis=dict( + type='log', + title='Frequency(Hz)', + range= [np.log10(min_freq), np.log10(frequencies[-1].item())], + ), + xaxis_title='Time(s)', + title='Magnitude' + ) + + #define min and max for frequency axis + + return fig + +def lineplot(x=None, dict=None, xaxis="linear", yaxis="log"): + assert dict is not None + + y=dict.items() + y=torch.stack([v for k,v in y]) + + labels = [k for k,v in dict.items()] + + + + # Ensure y is a NumPy array for compatibility with Plotly + if isinstance(y, torch.Tensor): + y = y.detach().cpu().numpy() + + if x is None: + x = np.arange(y.shape[-1]) # Ensure x is NumPy array too + elif isinstance(x, torch.Tensor): + x = x.detach().cpu().numpy() + + # Validate labels + num_series = y.shape[0] + if labels is None: + labels = [f"Series {i}" for i in range(num_series)] + elif len(labels) != num_series: + raise ValueError(f"Expected {num_series} labels, but got {len(labels)}.") + + # Convert to DataFrame for Plotly + df = pd.DataFrame(y.T, index=x, columns=labels) # Use labels as column names + df.index.name = "x" + df = df.reset_index() + + # Melt DataFrame for Plotly (long format) + df_melted = df.melt(id_vars=["x"], var_name="series", value_name="y") + + fig = px.line(df_melted, x="x", y="y", color="series", labels={"y": "Value", "x": "t", "series": "Legend"}) + + # Set y-axis to log scale + if yaxis=="log": + fig.update_yaxes(type="log") + #fix range from 1e-4 to 10 + fig.update_yaxes(range=[-4, 1]) + if xaxis=="log": + fig.update_xaxes(type="log") + + + return fig + +def make_PCA_figure(data_dict, num_bins=20, title="PCA"): + + fig = plt.figure(figsize=(6, 5), dpi=200) + gs = fig.add_gridspec( + 2, + 2, + width_ratios=(4, 1), + height_ratios=(1, 4), + left=0.1, + right=0.9, + bottom=0.1, + top=0.9, + ) + + # Create the Axes. + ax = fig.add_subplot(gs[1, 0]) + ax_histx = fig.add_subplot(gs[0, 0], sharex=ax) + ax_histy = fig.add_subplot(gs[1, 1], sharey=ax) + # Draw the scatter plot and marginals. + # no labels + ax_histx.tick_params(axis="x", labelbottom=False) + ax_histy.tick_params(axis="y", labelleft=False) + + xymax=0 + xymin=999999 + markers= ["o", "x", "D", "^", "v", "x", "p", "*", "h", "+", "s"] + colors=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"] + + + for i,(k, v) in enumerate(data_dict.items()): + if v is None or len(v) == 0: + continue # Skip empty or None values + + # the scatter plot: + ax.scatter( + v[:, 0], + v[:, 1], + marker=markers[i % len(markers)], + c=colors[i % len(colors)], + linewidths=0.8, + s=24, + label=k, + alpha=0.7, + ) + + try: + xymax=max(xymax, np.max(v[:, :2])) + xymin=min(xymin, np.min(v[:, :2])) + except Exception as e: + print(f"Error processing data for {k}: {e}") + print("data:",v) + print(f"Data shape: {v.shape}") + continue + + ax.legend(loc="upper left") + ax.set_xlabel("PC 1") + ax.set_ylabel("PC 2") + + # now determine nice limits by hand: + data_range = xymax - xymin + binwidth = data_range / num_bins # Calculate bin width based on the range and number of bins + + lim = (int(xymax / binwidth) + 1) * binwidth + bins = np.arange(-lim, lim + binwidth, binwidth) + + style = { "edgecolor": "black", "linewidth": 0.8, "alpha": 0.5 } + + + + #linestyles=[":", "-", "--", "-.", ":", "-"] + facecolors=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"] + hatches = ['/', '\\', '.', '+', '*', 'o', 'O', '.', '-', '|', 'x'] + + for i, (k, v) in enumerate(data_dict.items()): + + hist_style = style.copy() + hist_style["hatch"] = hatches[i % len(hatches)] + + ax_histx.hist(v[:, 0], bins=bins, density=False, label=k, facecolor=facecolors[i], **hist_style) + + ax_histy.hist( + v[:, 1], + bins=bins, + orientation="horizontal", + density=False, + facecolor=facecolors[i], + **hist_style + ) + + ax_histx.set_ylabel("Count") + ax_histy.set_xlabel("Count") + + #ax_histx.legend(loc="upper right") + + #add title + fig.suptitle(title, fontsize=16, fontweight='bold') + + return fig + +def make_histogram_figure(data_dict, num_bins=20): + + fig = plt.figure(figsize=(5, 3), dpi=100) + + # Create the Axes. + #ax = fig.add_subplot(gs[1, 0]) + #ax_histx = fig.add_subplot(gs[0, 0], sharex=ax) + #ax_histy = fig.add_subplot(gs[1, 1], sharey=ax) + # Draw the scatter plot and marginals. + # no labels + + #get ax from the figure + ax = fig.add_subplot(111) + + ax.tick_params(axis="x", labelbottom=True) + + + xymax=0 + xymin=999999 + + for i,(k, v) in enumerate(data_dict.items()): + + xymax=max(xymax, np.max(v)) + xymin=min(xymin, np.min(v)) + + + data_range = xymax - xymin + binwidth = data_range / num_bins # Calculate bin width based on the range and number of bins + + lim_up = (int(xymax / binwidth) + 1) * binwidth+binwidth + lim_down= (int(xymin / binwidth) - 1) * binwidth-binwidth + + bins = np.arange(lim_down, lim_up, binwidth) + + style = { "edgecolor": "black", "linewidth": 0.8, "alpha": 0.5 } + + #linestyles=[":", "-", "--", "-.", ":", "-"] + facecolors=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"] + hatches = ['/', '\\', '.', '+', '*', 'o', 'O', '.', '-', '|', 'x'] + + for i, (k, v) in enumerate(data_dict.items()): + + hist_style = style.copy() + hist_style["hatch"] = hatches[i % len(hatches)] + + ax.hist(v[:], bins=bins, density=False, label=k, facecolor=facecolors[i], **hist_style) + + ax.set_ylabel("Count") + + ax.legend(loc="upper right") + + + return fig + diff --git a/utils/multitrack_utils.py b/utils/multitrack_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f92a5a9b4911598200b2c84de620f605f29f8290 --- /dev/null +++ b/utils/multitrack_utils.py @@ -0,0 +1,94 @@ + +import einops +import time +import torch + + +def multitrack_batched_processing( x, taxonomy=None,function=None, class_dependent=False, masks=None, number_outputs=1): + """ + x: tensor of shape [B, N, C, T] where B is the batch size, N is the number of tracks, C is the number of channels and T is the length of the audio + taxonomy: list of lists of taxonomies. Outer list is the batch, inner list is + function: function to apply to each track, it should take a tensor of shape [B, C, T] and a list of taxonomies as input and return a tensor of the same shape + + This function reshapes the input tensor x to a 2D tensor of shape [B*N, C, T] and applies the function to each track independently. It then reshapes the output back to the original shape [B, N, C, T]. + """ + + assert not class_dependent, "this function needs an effect randomizer that is not class dependent (although it needs the taxonomy list), use simulate_effects instead" + + if masks is None: + masks = torch.ones((x.shape[0], x.shape[1]), dtype=torch.bool, device=x.device) + + + original_shape=x.shape + + x_masked, taxonomy_reshaped = forward_reshaping(x, taxonomy, masks) + + func_out=function(x_masked, taxonomy_reshaped) + + if number_outputs==1: + + output_shape = (original_shape[0], original_shape[1], func_out.shape[-2],func_out.shape[-1]) + + out = torch.zeros(output_shape, dtype=func_out.dtype, device=func_out.device) + + # Create a counter to keep track of where we are in x_emb + emb_idx = 0 + + for b in range(original_shape[0]): + for n in range(original_shape[1]): + if masks[b, n]: + out[b, n] = func_out[emb_idx] + emb_idx += 1 + + return out + elif number_outputs>1: + + outs=() + + for i in range(number_outputs): + func_out_i = func_out[i] + output_shape= (original_shape[0], original_shape[1], func_out_i.shape[-2],func_out_i.shape[-1]) + + out = torch.zeros(output_shape, dtype=func_out_i.dtype, device=func_out_i.device) + + # Create a counter to keep track of where we are in x_emb + emb_idx = 0 + for b in range(original_shape[0]): + for n in range(original_shape[1]): + if masks[b, n]: + out[b, n] = func_out_i[emb_idx] + emb_idx += 1 + + outs += (out,) + + return outs + + + +def forward_reshaping( x, taxonomy, masks=None): + """ + x: tensor of shape [B, N, C, T] where B is the batch size, N is the number of tracks, C is the number of channels and T is the length of the audio + taxonomy: list of lists of taxonomies. Outer list is the batch, inner list is + function: function to apply to each track, it should take a tensor of shape [B, C, T] and a list of taxonomies as input and return a tensor of the same shape + + This function reshapes the input tensor x to a 2D tensor of shape [B*N, C, T] and applies the function to each track independently. It then reshapes the output back to the original shape [B, N, C, T]. + """ + + if masks is None: + masks = torch.ones((x.shape[0], x.shape[1]), dtype=torch.bool, device=x.device) + + original_shape=x.shape + x_reshaped=einops.rearrange(x, "b n c t -> (b n) c t") #flatten the batch and number of tracks + + mask_reshaped=einops.rearrange(masks, "b n -> (b n)") if masks is not None else None + + x_masked=x_reshaped[mask_reshaped] + + taxonomy_reshaped=[] + if taxonomy is not None: + for b in range(original_shape[0]): + for n in range(original_shape[1]): + if masks[b, n]: + taxonomy_reshaped.append(taxonomy[b][n]) + + return x_masked, taxonomy_reshaped diff --git a/utils/st_ito/__init__.py b/utils/st_ito/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/st_ito/callbacks.py b/utils/st_ito/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..4517f57ba452f4eb67cd8bab54a9efd367579b07 --- /dev/null +++ b/utils/st_ito/callbacks.py @@ -0,0 +1,164 @@ +import os +import wandb +import torch +import shutil +import numpy as np +import pytorch_lightning as pl +import matplotlib.pyplot as plt + +from typing import Any +from sklearn.metrics import confusion_matrix +from pytorch_lightning.utilities.types import STEP_OUTPUT + + +# trainer.logger.experiment.dir: /import/c4dm-datasets-ext/wandb/run-20230826_192342-4zjz1inp/files + + +class LogAudioCallback(pl.Callback): + def __init__(self) -> None: + super().__init__() + + def on_validation_batch_end( + self, + trainer: pl.Trainer, + pl_module: pl.LightningModule, + outputs: STEP_OUTPUT, + batch: Any, + batch_idx: int, + dataloader_idx: int = 0, + ) -> None: + # Only log audio for the first batch + if batch_idx > 0: + return + # Log audio to wandb + else: + data_dict = outputs + silence_gap = 48000 + + # check the batch size + batch_size = data_dict["input_audio_A"].shape[0] + + for bidx in range(min(4, batch_size)): + audio_tensors = [] + for key in ["input_audio_A", "target_audio_A", "output_audio_A"]: + audio_segment = data_dict[key][bidx, ...] + chs, seq_len = audio_segment.shape + + if chs == 1: # make it stereo + audio_segment = audio_segment.repeat(2, 1) + + audio_tensors.append(audio_segment) + + # check the length of the audio tensors + total_audio_length = sum( + [audio_tensor.shape[-1] for audio_tensor in audio_tensors] + ) + + # create a single tensor to hold the audio + output_tensor = torch.zeros((total_audio_length + 2 * silence_gap, 2)) + + # copy the audio tensors into the single tensor + start_idx = 0 + for audio_tensor in audio_tensors: + end_idx = start_idx + audio_tensor.shape[-1] + output_tensor[start_idx:end_idx, :] = audio_tensor.permute(1, 0) + start_idx = end_idx + silence_gap + + trainer.logger.experiment.log( + { + f"{bidx}-audio": wandb.Audio( + output_tensor.cpu().numpy(), sample_rate=48000 + ), + } + ) + + +class MoveConfigCallback(pl.Callback): + def __init__(self, project_name: str) -> None: + super().__init__() + self.project_name = project_name + + def on_fit_start(self, trainer, pl_module): + full_run_dir = trainer.logger.experiment.dir + run_id = full_run_dir.split(os.sep)[-2].split("-")[-1] + src_dir = os.path.join(trainer.log_dir, "config.yaml") + dest_dir = os.path.join( + trainer.log_dir, self.project_name, run_id, "checkpoints", "config.yaml" + ) + + run_dir = os.path.join(trainer.log_dir, self.project_name, run_id) + ckpt_dir = os.path.join(run_dir, "checkpoints") + os.makedirs(run_dir, exist_ok=True) + os.makedirs(ckpt_dir, exist_ok=True) + + shutil.copyfile(src_dir, dest_dir) + + +class ConfusionMatrixCallback(pl.Callback): + def __init__(self) -> None: + super().__init__() + self.outputs = [] + + def on_validation_batch_end( + self, + trainer: pl.Trainer, + pl_module: pl.LightningModule, + outputs: STEP_OUTPUT, + batch: Any, + batch_idx: int, + dataloader_idx: int = 0, + ) -> None: + logits, labels = outputs + self.outputs.append(outputs) + + def on_validation_epoch_end( + self, trainer: pl.Trainer, pl_module: pl.LightningModule + ) -> None: + labels = list(trainer.datamodule.val_dataset.plugins.keys()) + + # Concatenate logits and labels from all batches in the validation set + all_logits = torch.cat([output[0] for output in self.outputs], dim=0) + all_labels = torch.cat([output[1] for output in self.outputs], dim=0) + + # Convert logits to predicted labels + predicted_labels = torch.argmax(all_logits, dim=-1) + + # Compute confusion matrix + cm = confusion_matrix(all_labels.cpu().numpy(), predicted_labels.cpu().numpy()) + + # Plot confusion matrix using matplotlib + num_classes = cm.shape[0] + class_labels = [str(i) for i in labels] + + if num_classes != len(class_labels): + print( + f"Number of classes ({num_classes}) does not match number of class labels ({len(class_labels)}). Skipping." + ) + return + + fig, ax = plt.subplots(figsize=(num_classes // 2, num_classes // 2)) + plt.imshow(cm, interpolation="nearest", cmap=plt.cm.Blues) + plt.title("Confusion Matrix") + plt.colorbar() + + tick_marks = np.arange(num_classes) + plt.xticks(tick_marks, class_labels, rotation=90) + ax.xaxis.set_tick_params(top=True) + plt.yticks(tick_marks, class_labels) + + plt.xlabel("Predicted") + plt.ylabel("True") + plt.tight_layout() + + for i in range(num_classes): + for j in range(num_classes): + plt.text( + j, i, str(cm[i, j]), horizontalalignment="center", color="black" + ) + + # log confusion matrix to wandb + trainer.logger.experiment.log({"confusion_matrix": plt}) + plt.close("all") + + # reset outputs + self.outputs = [] diff --git a/utils/st_ito/config.yaml b/utils/st_ito/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51c6c44b3082764c40f7ecd85a384c12d2219b14 --- /dev/null +++ b/utils/st_ito/config.yaml @@ -0,0 +1,151 @@ +# pytorch_lightning==2.1.2 +seed_everything: 4000152388 +trainer: + accelerator: gpu + strategy: ddp_find_unused_parameters_true + devices: -1 + num_nodes: 1 + precision: 32 + logger: + class_path: pytorch_lightning.loggers.WandbLogger + init_args: + name: null + save_dir: /import/c4dm-datasets-ext/lcap/param + version: null + offline: false + dir: null + id: null + anonymous: null + project: lcap-param + log_model: false + experiment: null + prefix: '' + checkpoint_name: null + job_type: null + config: null + entity: null + reinit: null + tags: null + group: null + notes: null + magic: null + config_exclude_keys: null + config_include_keys: null + mode: null + allow_val_change: null + resume: null + force: null + tensorboard: null + sync_tensorboard: null + monitor_gym: null + save_code: null + settings: null + callbacks: + - class_path: pytorch_lightning.callbacks.ModelCheckpoint + init_args: + dirpath: null + filename: '{epoch}-{step}' + monitor: val_loss + verbose: false + save_last: true + save_top_k: 1 + save_weights_only: false + mode: min + auto_insert_metric_name: true + every_n_train_steps: null + train_time_interval: null + every_n_epochs: null + save_on_train_epoch_end: null + enable_version_counter: true + - class_path: pytorch_lightning.callbacks.ModelSummary + init_args: + max_depth: 2 + - class_path: pytorch_lightning.callbacks.LearningRateMonitor + init_args: + logging_interval: null + log_momentum: false + log_weight_decay: false + - class_path: lcap.callbacks.MoveConfigCallback + init_args: + project_name: lcap-param + - class_path: lcap.callbacks.ConfusionMatrixCallback + fast_dev_run: false + max_epochs: 250 + min_epochs: null + max_steps: -1 + min_steps: null + max_time: null + limit_train_batches: null + limit_val_batches: null + limit_test_batches: null + limit_predict_batches: null + overfit_batches: 0.0 + val_check_interval: null + check_val_every_n_epoch: 5 + num_sanity_val_steps: 32 + log_every_n_steps: 100 + enable_checkpointing: null + enable_progress_bar: null + enable_model_summary: null + accumulate_grad_batches: 1 + gradient_clip_val: null + gradient_clip_algorithm: null + deterministic: null + benchmark: true + inference_mode: true + use_distributed_sampler: true + profiler: null + detect_anomaly: false + barebones: false + plugins: null + sync_batchnorm: true + reload_dataloaders_every_n_epochs: 0 + default_root_dir: /import/c4dm-datasets-ext/lcap/param +ckpt_path: null +model: + class_path: lcap.methods.param.ParameterEstimator + init_args: + encoder: + class_path: lcap.models.panns.Cnn14 + init_args: + embed_dim: 512 + sample_rate: 48000.0 + window_size: 2048 + hop_size: 1024 + mel_bins: 128 + fmin: 20.0 + fmax: 20000.0 + use_batchnorm: true + input_norm: minmax + lr: 0.0001 + num_instances: 63 + num_presets: 10 + weight_decay: 0.0001 + max_epochs: 250 + embed_mode: concat + norm: L2 +data: + class_path: lcap.dataset.dataset_param.PluginTarfileDataModule + init_args: + train_tarfiles: + - /import/c4dm-datasets-ext/lcap-datasets/20k/musdb18-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/mtg-jamendo-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/ursing-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/enst-drums-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/fsd50k-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/librispeech-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/medley-solos-db-vst3-presets.tar + - /import/c4dm-datasets-ext/lcap-datasets/20k/guitarset-vst3-presets.tar + val_tarfiles: + - /import/c4dm-datasets-ext/lcap-datasets/20k/guitarset-vst3-presets.tar + plugin_json: plugins/valid_plugins.json + batch_size: 64 + num_workers: 8 + length: 262144 + stereo: true + test_tarfiles: null + pin_memory: true + persistent_workers: true + num_train_examples: 140000 + num_val_examples: 2000 + num_test_examples: 2000 diff --git a/utils/st_ito/dataset/dataset_param.py b/utils/st_ito/dataset/dataset_param.py new file mode 100644 index 0000000000000000000000000000000000000000..dcc9957f544adb5ef2f364ed45b370c70cd90676 --- /dev/null +++ b/utils/st_ito/dataset/dataset_param.py @@ -0,0 +1,595 @@ +import os +import io +import sys +import glob +import json +import math +import torch +import tarfile +import torchaudio +import numpy as np +import pytorch_lightning as pl + +from tqdm import tqdm +from typing import List +from torch.utils.data import Dataset, IterableDataset + + +def tarfile_worker_init_fn(worker_id): + worker_info = torch.utils.data.get_worker_info() + dataset = worker_info.dataset # the dataset copy in this worker process + + # if there is more than one tar file in the dataset, then we need to + # select one tar file for this worker + print(f"Worker {worker_id} creating new tar file handles.") + + # reopen the tarfile handles for this worker + dataset.tar_handles = [] + for tar_file in dataset.tar_files: + tar_handle = tarfile.open(tar_file, "r:") + tar_handle.next() # skip the first root directory + tar_handle.next() # skip the first example directory + dataset.tar_handles.append(tar_handle) + + +def torchaudio_decode(b): + x, sr = torchaudio.load(b, format="FLAC") + return x + + +class PluginTarfileDataset(IterableDataset): + def __init__( + self, + tar_files: List[str], + plugin_json: str, + length: int = 262144, + stereo: bool = True, + num_examples: int = 20000, + label_files: List[str] = None, + ) -> None: + super().__init__() + """ Interable dataset that returns a single input/output pair from a tar file. + + When multiple tar files are provided, we open a file handle to each tar file. Then on each + iteration we select a tar file at random and then select a random example from that tar file. + We repeat this until each tar file has been read once. Then we start over again. + + Args: + tar_files (List[str]): List of tar files to read from. + plugin_json (str): Path to json file containing plugin instance details. + length (int): Length of audio to return. Defaults to 262144. + stereo (bool): whether to return stereo audio. Defaults to True. + When set to False, the mean of the two channels is returned. + num_examples (int): Number of examples to return. Defaults to 20000. + label_files (List[str]): List of label files to use for classification. Defaults to None. + Note: they must be int he same order as the associated tar_files. + + """ + self.tar_files = tar_files + self.length = length + self.stereo = stereo + self.num_examples = num_examples + self.label_files = label_files + + # first open plugin info + # this is used for indexing into the plugin list + with open(plugin_json, "r") as f: + self.plugins = json.load(f) + + # get the tar file handles + self.tar_handles = [] + for tar_file in tar_files: + tar_handle = tarfile.open(tar_file, "r:") + tar_handle.next() # skip the first root directory + tar_handle.next() # skip the first example directory + self.tar_handles.append(tar_handle) + + # check if we have label files + if label_files is not None: + assert len(label_files) == len(tar_files) # must be the same length + self.labels = [] + for label_file in label_files: + results = torch.load(label_file) + self.labels.append(results) + + def __len__(self): + return self.num_examples + + def next_example(self): + while True: + # select tar file at random + tar_index = np.random.choice(len(self.tar_handles)) + + # get the next member (should be root directory of this example) + members = {} + done = False + handle_reset = False + while not done: + # get next member + member = self.tar_handles[tar_index].next() + + if member is None: + # we have reached the end of this tar file + # so, we will close it and remove open tar handle + + # close this tar handle + self.tar_handles[tar_index].close() + + # reopen + tar_handle = tarfile.open(self.tar_files[tar_index], "r:") + tar_handle.next() # skip the first root directory + tar_handle.next() # skip the first example directory + self.tar_handles[tar_index] = tar_handle # replace handle + print(f"Resetting tar handle {self.tar_files[tar_index]}.") + handle_reset = True + break + + if member.isdir(): + done = True + break + + members[os.path.basename(member.name)] = member + + if handle_reset: + continue + + # get input file + input_file = self.tar_handles[tar_index].extractfile(members["input.flac"]) + input = torchaudio_decode(input_file) + + parent_dir = os.path.dirname(members["input.flac"].name).split("/")[-1] + + # select at random from the processed files + processed_members = [ + x for x in members.keys() if x != "input.flac" and ".json" not in x + ] + processed_member = np.random.choice(processed_members) + processed_fp = self.tar_handles[tar_index].extractfile( + members[processed_member] + ) + output = torchaudio_decode(processed_fp) + + # load the json file + json_member = members[processed_member.replace(".flac", ".json")] + json_fp = self.tar_handles[tar_index].extractfile(json_member) + details = json.load(json_fp) + + # if using labels, get the classification info + if self.label_files is not None: + print( + tar_index, members["input.flac"].name, self.label_files[tar_index] + ) + label_info = self.labels[tar_index][parent_dir] + predicted_class_id = label_info["predicted_class_ids"] + predicted_label = label_info["predicted_label"] + logits = label_info["logits"].squeeze() + else: + logits = torch.empty(0) + + # get the index of the plugin instance for classification + instance_index = self.plugins[details["instance"]] + instance_index -= 1 # zero index + instance_index = torch.tensor(instance_index).long() + preset_index = torch.tensor(details["preset"]).long() + + # fit the proper length + if input.shape[-1] < self.length: + input = torch.cat( + [ + input, + torch.zeros([input.shape[0], self.length - input.shape[-1]]), + ], + dim=-1, + ) + if output.shape[-1] < self.length: + output = torch.cat( + [ + output, + torch.zeros([output.shape[0], self.length - output.shape[-1]]), + ], + dim=-1, + ) + + # take a different random crop from each file + if input.shape[-1] > self.length: + start = torch.randint(0, (input.shape[-1] - self.length), [1]) + input = input[..., start : start + self.length] + + # take a random crop + if output.shape[-1] > self.length: + start = torch.randint(0, (output.shape[-1] - self.length), [1]) + output = output[..., start : start + self.length] + + # check if stereo + if self.stereo: + # convert mono to stereo + if input.shape[0] == 1: + input = input.repeat(2, 1) + if output.shape[0] == 1: + output = output.repeat(2, 1) + else: + # convert stereo to mono + if input.shape[0] == 2: + input = torch.mean(input, dim=0, keepdim=True) + if output.shape[0] == 2: + output = torch.mean(output, dim=0, keepdim=True) + + # randomize the gain of input and output + input /= torch.max(torch.abs(input)).clamp(1e-8) + output /= torch.max(torch.abs(output)).clamp(1e-8) + + # from 0 to -32 dB + gain_db = torch.rand(1) * -32.0 + input *= 10.0 ** (gain_db / 20.0) + + # from 0 to -32 dB + gain_db = torch.rand(1) * -32.0 + output *= 10.0 ** (gain_db / 20.0) + + # swap left and right channels at random + if torch.rand(1) > 0.5: + input = input.flip(0) + output = output.flip(0) + + yield input, output, instance_index, preset_index, tar_index, logits + + def __iter__(self): + return iter(self.next_example()) + + +class PluginTarfileDataModule(pl.LightningDataModule): + def __init__( + self, + train_tarfiles: List[str], + val_tarfiles: List[str], + plugin_json: str, + batch_size: int, + num_workers: int, + length: int = 262144, + stereo: bool = True, + test_tarfiles: List[str] = None, + train_labels: List[str] = None, + val_labels: List[str] = None, + test_labels: List[str] = None, + pin_memory: bool = False, + persistent_workers: bool = True, + num_train_examples: int = 20000, + num_val_examples: int = 2000, + num_test_examples: int = 2000, + ) -> None: + super().__init__() + """ Pytorch Lightning DataModule for loading data from tar files. + + Args: + train_tarfiles (List[str]): List of tar files to use for training. + val_tarfiles (List[str]): List of tar files to use for validation. + plugin_json (str): Path to json file containing plugin instance details. + batch_size (int): Batch size. + num_workers (int): Number of workers. + length (int): Length of audio to return. Defaults to 262144. + stereo (bool): whether to return stereo audio. Defaults to True. + When set to False, the mean of the two channels is returned. + test_tarfiles (List[str]): List of tar files to use for testing. + train_labels (List[str]): List of label files to use for training. Defaults to None. + val_labels (List[str]): List of label files to use for validation. Defaults to None. + test_labels (List[str]): List of label files to use for testing. Defaults to None. + pin_memory (bool): Whether to pin memory. Defaults to False. + persistent_workers (bool): Whether to use persistent workers. Defaults to True. + num_train_examples (int): Number of training examples to return. Defaults to 20000. + num_val_examples (int): Number of validation examples to return. Defaults to 2000. + num_test_examples (int): Number of testing examples to return. Defaults to 2000. + """ + self.save_hyperparameters() + + def setup(self, stage=None): + self.train_dataset = PluginTarfileDataset( + self.hparams.train_tarfiles, + self.hparams.plugin_json, + self.hparams.length, + self.hparams.stereo, + self.hparams.num_train_examples, + label_files=self.hparams.train_labels, + ) + + self.val_dataset = PluginTarfileDataset( + self.hparams.val_tarfiles, + self.hparams.plugin_json, + self.hparams.length, + self.hparams.stereo, + self.hparams.num_val_examples, + label_files=self.hparams.val_labels, + ) + + if self.hparams.test_tarfiles is not None: + self.test_dataset = PluginTarfileDataset( + self.hparams.test_tarfiles, + self.hparams.plugin_json, + self.hparams.length, + self.hparams.stereo, + self.hparams.num_test_examples, + label_files=self.hparams.test_labels, + ) + + def train_dataloader(self): + return torch.utils.data.DataLoader( + self.train_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + drop_last=True, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + worker_init_fn=tarfile_worker_init_fn, + ) + + def val_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + worker_init_fn=tarfile_worker_init_fn, + ) + + def test_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + persistent_workers=self.hparams.persistent_workers, + worker_init_fn=tarfile_worker_init_fn, + ) + + +# ------------------------------ File based dataset -------------------------------- + + +class PluginAudioFileWebDataset(Dataset): + def __init__( + self, + root_dirs: str, + plugin_json: str, + length: int = 524288, + stereo: bool = True, + ) -> None: + """ + + Args: + root_dirs (List[str]): List of directories containing dataset files. + plugin_json: Path to json file containing plugin instance details. + length (int): Length of audio to return. Defaults to 524288. + stereo (bool): whether to return stereo audio. Defaults to True. + When set to False, the mean of the two channels is returned. + + """ + super().__init__() + self.root_dirs = root_dirs + self.stereo = stereo + self.length = length + + self.examples = [] + + # find all segment directories + for root_dir in root_dirs: + examples = [] + # first, check if there is an examples.json file + examples_json = os.path.join(root_dir, "examples.json") + if os.path.exists(examples_json): + print(f"Found {examples_json}.") + with open(examples_json, "r") as f: + examples = json.load(f) + else: + print( + f"No examples.json file found. Crawling to find all examples in {root_dir}..." + ) + segment_dirs = glob.glob(os.path.join(root_dir, "*")) + segment_dirs = sorted([x for x in segment_dirs if os.path.isdir(x)]) + + for segment_dir in tqdm(segment_dirs): + # find the input file + input_file = os.path.join(segment_dir, "input.flac") + if not os.path.exists(input_file): + continue + + # find all processed audio files + processed_files = glob.glob(os.path.join(segment_dir, "*.flac")) + + for processed_file in processed_files: + if processed_file == input_file: + continue + # find the appropriate json file + json_file = processed_file.replace(".flac", ".json") + + # check if json file exists + if not os.path.exists(json_file): + print(f"Could not find {json_file}.") + continue + + # test open json file + # try: + # with open(json_file, "r") as f: + # details = json.load(f) + # except Exception: + # print(f"Error opening {json_file}.") + # continue + + example = (input_file, processed_file, json_file) + examples.append(example) + + self.examples += examples + print( + f"Found {len(examples)} examples in {root_dir}. Total: {len(self.examples)}" + ) + + with open(plugin_json, "r") as f: + self.plugins = json.load(f) + + # for testing, load one example into memory + # input_file, processed_file, json_file = self.examples[0] + + # open the json file + # with open(json_file, "r") as f: + # details = json.load(f) + + # input, sr = torchaudio.load(input_file, backend="soundfile") + # output, sr = torchaudio.load(processed_file, backend="soundfile") + + # self.input = input + # self.output = output + # self.details = details + + def __len__(self): + return len(self.examples) + + def __getitem__(self, idx): + """Get a single example from the dataset.""" + + if True: + input_file, processed_file, json_file = self.examples[idx] + + # open the json file + with open(json_file, "r") as f: + details = json.load(f) + + input, sr = torchaudio.load(input_file, backend="soundfile") + output, sr = torchaudio.load(processed_file, backend="soundfile") + else: + input = self.input + output = self.output + details = self.details + + # check length of input and output + # if input.shape[-1] != output.shape[-1]: + # raise RuntimeError( + # f"Input and output shapes do not match: {input.shape} vs {output.shape}" + # ) + + # fit the proper length + if input.shape[-1] < self.length: + input = torch.cat( + [input, torch.zeros([input.shape[0], self.length - input.shape[-1]])], + dim=-1, + ) + if output.shape[-1] < self.length: + output = torch.cat( + [ + output, + torch.zeros([output.shape[0], self.length - output.shape[-1]]), + ], + dim=-1, + ) + + # take a random crop + start = torch.randint( + 0, (min(input.shape[-1], output.shape[-1]) - self.length), [1] + ) + input = input[..., start : start + self.length] + output = output[..., start : start + self.length] + + # note: now we take a crop from the same position in both files + # however, it might be a good augmentation to take a different + # random crop from each file + + # get the index of the plugin instance for classification + instance_index = self.plugins[details["instance"]] + instance_index -= 1 # zero index + instance_index = torch.tensor(instance_index).long() + preset_index = torch.tensor(details["preset"]).long() + + # check if stereo + if self.stereo: + # convert mono to stereo + if input.shape[0] == 1: + input = input.repeat(2, 1) + if output.shape[0] == 1: + output = output.repeat(2, 1) + else: + # convert stereo to mono + if input.shape[0] == 2: + input = torch.mean(input, dim=0, keepdim=True) + if output.shape[0] == 2: + output = torch.mean(output, dim=0, keepdim=True) + + # randomize the gain of input and output + input /= torch.max(torch.abs(input)).clamp(1e-8) + output /= torch.max(torch.abs(output)).clamp(1e-8) + + # from 0 to -32 dB + gain_db = torch.rand(1) * -32.0 + input *= 10.0 ** (gain_db / 20.0) + + # from 0 to -32 dB + gain_db = torch.rand(1) * -32.0 + output *= 10.0 ** (gain_db / 20.0) + + return input, output, instance_index, preset_index + + +class PluginAudioFileWebDataModule(pl.LightningDataModule): + def __init__( + self, + train_root_dirs: List[str], + val_root_dirs: List[str], + plugin_json: str, + batch_size: int, + num_workers: int, + length: int = 262144, + test_root_dirs: List[str] = None, + pin_memory: bool = False, + persistent_workers: bool = False, + use_tarfile: bool = False, + ) -> None: + super().__init__() + self.save_hyperparameters() + + def setup(self, stage=None): + if not self.hparams.use_tarfile: + self.train_dataset = PluginAudioFileWebDataset( + self.hparams.train_root_dirs, + self.hparams.plugin_json, + self.hparams.length, + ) + + self.val_dataset = PluginAudioFileWebDataset( + self.hparams.val_root_dirs, + self.hparams.plugin_json, + self.hparams.length, + ) + + if self.hparams.test_root_dirs is not None: + self.test_dataset = PluginAudioFileWebDataset( + self.hparams.test_root_dirs, + self.hparams.plugin_json, + self.hparams.length, + ) + else: + raise NotImplementedError + + def train_dataloader(self): + return torch.utils.data.DataLoader( + self.train_dataset, + batch_size=self.hparams.batch_size, + shuffle=True, + num_workers=self.hparams.num_workers, + drop_last=True, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + prefetch_factor=8, + ) + + def val_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + ) + + def test_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + persistent_workers=self.hparams.persistent_workers, + ) diff --git a/utils/st_ito/dataset/dataset_sim.py b/utils/st_ito/dataset/dataset_sim.py new file mode 100644 index 0000000000000000000000000000000000000000..01dddeb87e9cc544bbb38ef7be5923757084c096 --- /dev/null +++ b/utils/st_ito/dataset/dataset_sim.py @@ -0,0 +1,320 @@ +import os +import glob +import torch +import torchaudio +import pedalboard +import numpy as np +import pytorch_lightning as pl + +from tqdm import tqdm +from typing import List, Optional, Tuple, Iterable +from st_ito.effects import ( + BasicChorus, + BasicCompressor, + BasicDelay, + BasicDistortion, + BasicReverb, +) + + +def worker_init_fn(worker_id): + worker_info = torch.utils.data.get_worker_info() + # re-init the vst_plugins for each worker + vst_plugins = load_plugins(worker_info.dataset.vst_dir) + worker_info.dataset.vst_plugins = vst_plugins + + +def find_files_with_glob( + directory: str, +): + """Find all files in the given directory with specific pattern.""" + audio_filepaths = [] + for ext in ["*.wav", "*.mp3", "*.ogg", "*.flac"]: + audio_filepaths += glob.glob(os.path.join(directory, "**", ext), recursive=True) + + return audio_filepaths + + +def find_files_with_extensions( + directory: str, + extensions: Tuple[str] = (".wav", ".mp3", ".ogg", ".flac"), +): + """Find all files in the given directory with specific extensions.""" + for entry in os.scandir(directory): + if entry.is_file() and entry.name.lower().endswith(extensions): + yield entry.path + elif entry.is_dir(): + yield from find_files_with_extensions(entry.path, extensions) + + +def find_files_lazy( + directory: str, + extensions: Tuple[str] = (".wav", ".mp3", ".ogg", ".flac"), +): + """Lazily find files with specific extensions in the given directory.""" + for root, dirs, files in os.walk(directory): + for file in files: + if file.lower().endswith(extensions): + yield os.path.join(root, file) + + +def get_nonsilent_region( + audio_filepath: str, + total_num_frames: int, + length: int, + threshold: float = 1e-8, + max_tires: int = 100, +): + silent = True + tries = 0 + while silent: + + if total_num_frames > length: + # load a random segment of the audio file + audio, sr = torchaudio.load( + audio_filepath, + num_frames=length, + frame_offset=np.random.randint(0, total_num_frames - length), + backend="soundfile", + ) + + else: + # load the entire audio file + audio, sr = torchaudio.load( + audio_filepath, + num_frames=total_num_frames, + backend="soundfile", + ) + + # repeat the audio to reach the desired length + audio = torch.cat([audio] * (length // total_num_frames + 1), axis=1) + audio = audio[:, :length] + + # check for silence + energy = torch.mean(audio**2) + print(energy) + if energy > threshold: + silent = False + else: + silent = False # TODO: remove this line + + tries += 1 + + if tries > max_tires: + raise ValueError( + f"Could not find a non-silent region in the audio: {audio_filepath}." + ) + + return audio, sr + + +def random_gain(audio: torch.Tensor): + # first peak normalize + audio = audio / torch.max(torch.abs(audio)) + # then apply random gain + gain_db = np.random.uniform(-18, 0) + gain_lin = 10 ** (gain_db / 20) + audio = audio * gain_lin + return audio + + +def conform_channels(audio_input: torch.Tensor, num_channels: int): + if audio_input.shape[0] == 1 and num_channels == 2: + audio_input = torch.cat((audio_input, audio_input), axis=0) + if audio_input.shape[0] == 2 and num_channels == 1: + audio_input = audio_input[0:1, :] + return audio_input + + +def load_plugins(vst_dir: str, sample_rate: int = 48000): + # find all vst3 files + vst_filepaths = [] + for ext in ["*.vst3"]: + vst_filepaths += glob.glob(os.path.join(vst_dir, ext), recursive=True) + + # vst_filepaths = vst_filepaths[:2] + + vst_filepaths = sorted(vst_filepaths) + print(f"Found {len(vst_filepaths)} VST3 files in {vst_dir}.") + + vst_plugins = {} + print("Loading plugins...") + for vst_filepath in tqdm(vst_filepaths): + instance_name = os.path.basename(vst_filepath).replace(".vst3", "") + print(f"Loading {instance_name}...") + # try to load each plugin + try: + plugin = pedalboard.load_plugin(vst_filepath) + except Exception: + print(f"Error loading {vst_filepath}.") + + # try to pass audio through the plugin + failed = True + try: + num_channels = 2 + audio_input = np.random.randn(2, sample_rate * 5) + audio_output = plugin(audio_input, sample_rate) + failed = False + except Exception as e: + num_channels = 1 + print(e) + print(f"Error processing audio with 2 channels.") + + try: + audio_input = np.random.randn(1, sample_rate * 5) + audio_output = plugin(audio_input, sample_rate) + failed = False + except Exception as e: + num_channels = 2 + print(e) + print(f"Failed processing audio with 1 channel.") + + if failed: + continue + + print(f"Loaded {os.path.basename(vst_filepath)} successfully.") + + vst_plugins[instance_name] = { + "filepath": vst_filepath, + "instance": plugin, + "num_channels": num_channels, + } + + for idx, (instance_name, vst_plugin) in enumerate(vst_plugins.items()): + print(idx + 1, instance_name) + + return vst_plugins + + +class PluginSimilarityDataset(torch.utils.data.Dataset): + def __init__( + self, + audio_filepaths: List[str], + vst_dir: str, + length: int = 262144, + num_examples_per_epoch: int = 10000, + ): + self.audio_filepaths = audio_filepaths + self.vst_dir = vst_dir + self.length = length + self.num_examples_per_epoch = num_examples_per_epoch + self.plugins = load_plugins(self.vst_dir) + + def __len__(self): + return self.num_examples_per_epoch + + def __getitem__(self, idx: int): + # sample the first filepath + filepath_a = np.random.choice(self.audio_filepaths) + # sample the second filepath + filepath_b = np.random.choice(self.audio_filepaths) + + # load the audio files + num_frames_a = torchaudio.info(filepath_a).num_frames + audio_a, sr_a = get_nonsilent_region(filepath_a, num_frames_a, self.length) + + num_frames_b = torchaudio.info(filepath_b).num_frames + audio_b, sr_b = get_nonsilent_region(filepath_b, num_frames_b, self.length) + + # sample randomly a plugin from the dictionary + plugin_name = np.random.choice(list(self.plugins.keys())) + plugin = self.plugins[plugin_name] + + # randomize the parameters of the plugin + for name, parameter in plugin["instance"].parameters.items(): + if "bypass" in name.lower(): + parameter.raw_value = 0 + elif name in ["buffer_size_frames", "sample_rate_frames"]: + pass + else: # set the parameter to the preset value + parameter.raw_value = np.random.rand() + + audio_a = conform_channels(audio_a, plugin["num_channels"]) + audio_b = conform_channels(audio_b, plugin["num_channels"]) + + # apply the effect to the two audio files + audio_a_out = torch.from_numpy( + plugin["instance"].process(audio_a.numpy(), sr_a) + ) + audio_b_out = torch.from_numpy( + plugin["instance"].process(audio_b.numpy(), sr_b) + ) + + # peak normalize and then apply random gain + audio_a = random_gain(audio_a) + audio_b = random_gain(audio_b) + audio_a_out = random_gain(audio_a_out) + audio_b_out = random_gain(audio_b_out) + + # ensure all outputs are stereo + audio_a = conform_channels(audio_a, 2) + audio_b = conform_channels(audio_b, 2) + audio_a_out = conform_channels(audio_a_out, 2) + audio_b_out = conform_channels(audio_b_out, 2) + + return audio_a, audio_b, audio_a_out, audio_b_out + + +class PluginSimilarityDataModule(pl.LightningDataModule): + def __init__( + self, + audio_dirs: List[str], + vst_dir: str, + length: int = 262144, + batch_size: int = 32, + num_train_examples: int = 10000, + num_val_examples: int = 1000, + num_workers: int = 4, + persistent_workers: bool = True, + pin_memory: bool = True, + ) -> None: + super().__init__() + self.save_hyperparameters() + + def prepare_data(self): + # download, split, etc... + pass + + def setup(self, stage: Optional[str] = None): + # find all audio files in the input directory based on file extensions + audio_filepaths = [] + for audio_dir in self.hparams.audio_dirs: + audio_filepaths += find_files_with_glob(audio_dir) + print("Found", len(audio_filepaths), "audio files in", audio_dir) + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + self.plugin_similarity_train = PluginSimilarityDataset( + audio_filepaths, + self.hparams.vst_dir, + self.hparams.length, + self.hparams.num_train_examples, + ) + self.plugin_similarity_val = PluginSimilarityDataset( + audio_filepaths, + self.hparams.vst_dir, + self.hparams.length, + self.hparams.num_val_examples, + ) + + def train_dataloader(self): + return torch.utils.data.DataLoader( + self.plugin_similarity_train, + batch_size=self.hparams.batch_size, + shuffle=True, + num_workers=self.hparams.num_workers, + persistent_workers=self.hparams.persistent_workers, + pin_memory=self.hparams.pin_memory, + # worker_init_fn=worker_init_fn, + ) + + def val_dataloader(self): + return torch.utils.data.DataLoader( + self.plugin_similarity_val, + batch_size=self.hparams.batch_size, + shuffle=False, + num_workers=self.hparams.num_workers, + persistent_workers=self.hparams.persistent_workers, + pin_memory=self.hparams.pin_memory, + # worker_init_fn=worker_init_fn, + ) diff --git a/utils/st_ito/dataset/dataset_style.py b/utils/st_ito/dataset/dataset_style.py new file mode 100644 index 0000000000000000000000000000000000000000..8ce96f5396621656ad8d64b72545ea127d564b38 --- /dev/null +++ b/utils/st_ito/dataset/dataset_style.py @@ -0,0 +1,351 @@ +import os +import glob +import json +import torch +import tarfile +import torchaudio +import numpy as np +import pytorch_lightning as pl + +from typing import List, Tuple, Dict, Optional +from st_ito.dataset.utils import tarfile_worker_init_fn, torchaudio_decode + + +class StyleTransferTarfileDataset(torch.utils.data.IterableDataset): + def __init__( + self, + tar_files: List[str], + length: int, + num_examples: int = 10000, + audio_format: str = "wav", + input_only: bool = False, + ): + super().__init__() + self.tar_files = tar_files + self.length = length + self.num_examples = num_examples + self.audio_format = audio_format + self.input_only = input_only + + # get the tar file handles + self.tar_handles = [] + for tar_file in tar_files: + tar_handle = tarfile.open(tar_file, "r:") + tar_handle.next() # skip the first root directory + tar_handle.next() # skip the first example directory + self.tar_handles.append(tar_handle) + + def __len__(self): + return self.num_examples + + def next_example(self): + while True: + # select tar file at random + tar_index = np.random.choice(len(self.tar_handles)) + + # get the next member (should be root directory of this example) + members = {} + done = False + handle_reset = False + while not done: + # get next member + member = self.tar_handles[tar_index].next() + + if member is None: + # we have reached the end of this tar file + # so, we will close it and remove open tar handle + + # close this tar handle + self.tar_handles[tar_index].close() + + # reopen + tar_handle = tarfile.open(self.tar_files[tar_index], "r:") + tar_handle.next() # skip the first root directory + tar_handle.next() # skip the first example directory + self.tar_handles[tar_index] = tar_handle # replace handle + print(f"Resetting tar handle {self.tar_files[tar_index]}.") + handle_reset = True + break + + if member.isdir(): + done = True + break + + members[os.path.basename(member.name)] = member + + if handle_reset: + continue + + # get input file + input_file = self.tar_handles[tar_index].extractfile( + members[f"input.{self.audio_format}"] + ) + audio_input = torchaudio_decode(input_file) + + if self.input_only: + + num_frames = audio_input.shape[1] + start_frame = torch.randint(0, num_frames - self.length, (1,)).item() + audio_input = audio_input[:, start_frame : start_frame + self.length] + + # we will generate the output in the system + audio_output = torch.empty_like(audio_input) + params = torch.empty(1) + + else: + # get the output file + output_file = self.tar_handles[tar_index].extractfile( + members[f"output.{self.audio_format}"] + ) + audio_output = torchaudio_decode(output_file) + + # get the params file + params_file = self.tar_handles[tar_index].extractfile( + members["params.json"] + ) + params = json.load(params_file) + # convert params to tensor + params = torch.tensor( + [val for val in params.values()], dtype=torch.float32 + ) + + # crop the audio to the desired length + num_frames = audio_input.shape[1] + start_frame = torch.randint(0, num_frames - self.length, (1,)).item() + audio_input = audio_input[:, start_frame : start_frame + self.length] + audio_output = audio_output[:, start_frame : start_frame + self.length] + + # check for silence in the output + input_energy = torch.mean(audio_input**2, dim=1) + output_energy = torch.mean(audio_output**2, dim=1) + + if output_energy < 1e-6 or input_energy < 1e-6: + # skip this example + continue + + yield audio_input, audio_output, params + + def __iter__(self): + return iter(self.next_example()) + + +class StyleTransferTarfileDataModule(pl.LightningDataModule): + def __init__( + self, + train_tarfiles: List[str], + val_tarfiles: List[str], + batch_size: int, + num_workers: int, + input_only: bool = False, + length: int = 262144, + test_tarfiles: List[str] = None, + pin_memory: bool = False, + persistent_workers: bool = True, + num_train_examples: int = 20000, + num_val_examples: int = 2000, + num_test_examples: int = 2000, + audio_format: str = "wav", + ) -> None: + super().__init__() + """ Pytorch Lightning DataModule for loading data from tar files. + + Args: + train_tarfiles (List[str]): List of tar files to use for training. + val_tarfiles (List[str]): List of tar files to use for validation. + batch_size (int): Batch size. + num_workers (int): Number of workers. + input_only (bool): Whether to use only the input audio. Defaults to False. + length (int): Length of audio to return. Defaults to 262144. + test_tarfiles (List[str]): List of tar files to use for testing. + pin_memory (bool): Whether to pin memory. Defaults to False. + persistent_workers (bool): Whether to use persistent workers. Defaults to True. + num_train_examples (int): Number of training examples to return. Defaults to 20000. + num_val_examples (int): Number of validation examples to return. Defaults to 2000. + num_test_examples (int): Number of testing examples to return. Defaults to 2000. + audio_format (str): Audio format to use. Defaults to "wav". + """ + self.save_hyperparameters() + + def setup(self, stage=None): + self.train_dataset = StyleTransferTarfileDataset( + self.hparams.train_tarfiles, + self.hparams.length, + self.hparams.num_train_examples, + self.hparams.audio_format, + self.hparams.input_only, + ) + + self.val_dataset = StyleTransferTarfileDataset( + self.hparams.val_tarfiles, + self.hparams.length, + self.hparams.num_val_examples, + self.hparams.audio_format, + self.hparams.input_only, + ) + + if self.hparams.test_tarfiles is not None: + self.test_dataset = StyleTransferTarfileDataset( + self.hparams.test_tarfiles, + self.hparams.length, + self.hparams.num_test_examples, + self.hparams.audio_format, + self.hparams.input_only, + ) + + def train_dataloader(self): + return torch.utils.data.DataLoader( + self.train_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + drop_last=True, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + worker_init_fn=tarfile_worker_init_fn, + ) + + def val_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + worker_init_fn=tarfile_worker_init_fn, + ) + + def test_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + persistent_workers=self.hparams.persistent_workers, + worker_init_fn=tarfile_worker_init_fn, + ) + + +# ------------------------------ File based dataset -------------------------------- + + +class StyleTransferDataset(torch.utils.data.Dataset): + def __init__( + self, + root_dirs: List[str], + length: int, + ): + super().__init__() + self.root_dirs = root_dirs + self.length = length + + self.examples = [] + for root_dir in root_dirs: + # get all examples in the root dir + print("Searching for examples in", root_dir) + examples = glob.glob(os.path.join(root_dir, "*")) + self.examples.extend(examples) + + # for testing use only the first 100 examples + # self.examples = self.examples[:100] + # repeat those examples 10 times + # self.examples = self.examples * 100 + + print(f"Found {len(self.examples)} examples.") + + def __len__(self): + return len(self.examples) + + def __getitem__(self, idx): + example = self.examples[idx] + + input_filepath = os.path.join(example, "input.wav") + output_filepath = os.path.join(example, "output.wav") + params_filepath = os.path.join(example, "params.json") + + # check the length of the audio + md = torchaudio.info(input_filepath, backend="soundfile") + num_frames = md.num_frames + + # pick random start frame + start_frame = torch.randint(0, num_frames - self.length, (1,)).item() + # start_frame = 0 # for now, just use the first frame + + # load the audio + input_audio, sr = torchaudio.load( + input_filepath, + frame_offset=start_frame, + num_frames=self.length, + backend="soundfile", + ) + output_audio, sr = torchaudio.load( + output_filepath, + frame_offset=start_frame, + num_frames=self.length, + backend="soundfile", + ) + + # load the params + with open(params_filepath, "r") as fp: + params = json.load(fp) + + # convert params to tensor + params = torch.tensor([val for val in params.values()], dtype=torch.float32) + + return input_audio, output_audio, params + + +class StyleTransferDataModule(pl.LightningDataModule): + def __init__( + self, + train_root_dirs: List[str], + val_root_dirs: List[str], + batch_size: int, + num_workers: int, + length: int = 262144, + test_root_dirs: List[str] = None, + pin_memory: bool = False, + persistent_workers: bool = False, + ) -> None: + super().__init__() + self.save_hyperparameters() + + def setup(self, stage=None): + self.train_dataset = StyleTransferDataset( + self.hparams.train_root_dirs, + self.hparams.length, + ) + self.val_dataset = StyleTransferDataset( + self.hparams.val_root_dirs, + self.hparams.length, + ) + if self.hparams.test_root_dirs is not None: + self.test_dataset = StyleTransferDataset( + self.hparams.test_root_dirs, + self.hparams.length, + ) + + def train_dataloader(self): + return torch.utils.data.DataLoader( + self.train_dataset, + batch_size=self.hparams.batch_size, + shuffle=True, + num_workers=self.hparams.num_workers, + drop_last=True, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + ) + + def val_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + pin_memory=self.hparams.pin_memory, + persistent_workers=self.hparams.persistent_workers, + ) + + def test_dataloader(self): + return torch.utils.data.DataLoader( + self.val_dataset, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + persistent_workers=self.hparams.persistent_workers, + ) diff --git a/utils/st_ito/dataset/utils.py b/utils/st_ito/dataset/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..71c8243d39f7d7cd8926adca711933928413de2b --- /dev/null +++ b/utils/st_ito/dataset/utils.py @@ -0,0 +1,25 @@ +import torch +import tarfile +import torchaudio + + +def tarfile_worker_init_fn(worker_id): + worker_info = torch.utils.data.get_worker_info() + dataset = worker_info.dataset # the dataset copy in this worker process + + # if there is more than one tar file in the dataset, then we need to + # select one tar file for this worker + print(f"Worker {worker_id} creating new tar file handles.") + + # reopen the tarfile handles for this worker + dataset.tar_handles = [] + for tar_file in dataset.tar_files: + tar_handle = tarfile.open(tar_file, "r:") + tar_handle.next() # skip the first root directory + tar_handle.next() # skip the first example directory + dataset.tar_handles.append(tar_handle) + + +def torchaudio_decode(b): + x, sr = torchaudio.load(b, format="FLAC") + return x diff --git a/utils/st_ito/dsp.py b/utils/st_ito/dsp.py new file mode 100644 index 0000000000000000000000000000000000000000..9d8cca32536c8cdc6673af08f5ab176a555ab402 --- /dev/null +++ b/utils/st_ito/dsp.py @@ -0,0 +1,87 @@ +import torch +import numpy as np +import dasp_pytorch +from dasp_pytorch.functional import ( + simple_distortion, + compressor, + noise_shaped_reverberation, +) +import pyloudnorm as pyln + + +def apply_random_simple_distortion( + x: torch.Tensor, sample_rate: float, use_gpu: bool = False +): + drive_db = np.random.uniform(0, 32) + drive_db_tensor = torch.tensor(drive_db, dtype=torch.float32) + + if use_gpu: + drive_db_tensor = drive_db_tensor.cuda() + x = x.cuda() + + y = simple_distortion(x, drive_db_tensor) + return y, drive_db + + +def apply_random_reverb(x: torch.Tensor, sample_rate: float, use_gpu: bool = False): + mix = np.random.uniform(0, 1.0) + band_decays = torch.tensor( + [0.6, 0.4, 0.4, 0.5, 0.2, 0.3, 0.3, 0.2, 0.1, 0.1, 0.2, 0.1] + ) + band_gains = torch.tensor( + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + ) + mix_tensor = torch.tensor(mix, dtype=torch.float32) + + if use_gpu: + band_decays = band_decays.cuda() + band_gains = band_gains.cuda() + mix_tensor = mix_tensor.cuda() + x = x.cuda() + + y = noise_shaped_reverberation( + x, sample_rate, *band_gains, *band_decays, mix_tensor + ).cpu() + + return y, mix + + +def apply_random_compressor(x: torch.Tensor, sample_rate: float, use_gpu: bool = False): + threshold_db = np.random.uniform(-48, 0) + threshold_db_tensor = torch.tensor(threshold_db, dtype=torch.float32) + ratio = torch.tensor([4.0]) + attack_ms = torch.tensor([100.0]) + release_ms = torch.tensor([100.0]) + knee_db = torch.tensor([24.0]) + makeup_gain_db = torch.tensor([0.0]) + + if use_gpu: + threshold_db_tensor = threshold_db_tensor.cuda() + ratio = ratio.cuda() + attack_ms = attack_ms.cuda() + release_ms = release_ms.cuda() + knee_db = knee_db.cuda() + makeup_gain_db = makeup_gain_db.cuda() + x = x.cuda() + + y = compressor( + x, + sample_rate, + threshold_db_tensor, + ratio, + attack_ms, + release_ms, + knee_db, + makeup_gain_db, + ).cpu() + + return y, threshold_db + + +def normalize_loudness(x: torch.Tensor, sr: float, target_lufs_db: float): + meter = pyln.Meter(sr) + x_lufs_db = meter.integrated_loudness(x.permute(1, 0).numpy()) + delta_lufs_db = torch.tensor([target_lufs_db - x_lufs_db]).float() + gain_lin = 10.0 ** (delta_lufs_db.clamp(-120, 40.0) / 20.0) + x = gain_lin * x + return x diff --git a/utils/st_ito/effects.py b/utils/st_ito/effects.py new file mode 100644 index 0000000000000000000000000000000000000000..e2e645e3317b981f8ae92f9fef3162e91eea6702 --- /dev/null +++ b/utils/st_ito/effects.py @@ -0,0 +1,1533 @@ +# Adapted from https://github.com/aim-qmul/sdx23-aimless/blob/master/data/augment.py +import torch +import random +import pedalboard +import torchaudio +import numpy as np +import scipy.stats +import scipy.signal +import dasp_pytorch +import pyloudnorm as pyln +from typing import List, Tuple + +from pedalboard import ( + Pedalboard, + Gain, + Chorus, + Reverb, + Compressor, + Phaser, + Delay, + Distortion, + Limiter, +) + +# from dasp_pytorch.functional import single_band_eq + + +__all__ = [ + "RandomSwapLR", + "RandomGain", + "RandomFlipPhase", + "LimitAug", + "CPUBase", + "RandomParametricEQ", + "RandomStereoWidener", + "RandomVolumeAutomation", + "RandomPedalboardCompressor", + "RandomPedalboardDelay", + "RandomPedalboardChorus", + "RandomPedalboardDistortion", + "RandomPedalboardPhaser", + "RandomPedalboardReverb", + "RandomPedalboardLimiter", + "RandomSoxReverb", + "LoudnessNormalize", + "RandomPan", + "Mono2Stereo", +] + + +def db2linear(x): + return 10 ** (x / 20) + + +def loguniform(low=0, high=1): + return scipy.stats.loguniform.rvs(low, high) + + +def rand(low=0, high=1): + return (random.random() * (high - low)) + low + + +def randint(low=0, high=1): + return random.randint(low, high) + + +def normalize_param(x: torch.Tensor, low: float, high: float): + return (x - low) / (high - low) + + +# effect functions (expect audio array and return audio array, parameters, and normalized parameters) +def pedalboard_distortion(x: np.ndarray, parameters: List[float], sample_rate: int): + drive_db = parameters[0] + board = pedalboard.Pedalboard([pedalboard.Distortion(drive_db=drive_db)]) + output = board(x, sample_rate) + return output + + +def pedalboard_reverb(x: np.ndarray, parameters: List[float], sample_rate: int): + room_size = parameters[0] + damping = parameters[1] + wet_level = parameters[2] + dry_level = 1 - wet_level + board = pedalboard.Pedalboard( + [ + pedalboard.Reverb( + room_size=room_size, + damping=damping, + wet_level=wet_level, + dry_level=dry_level, + ) + ] + ) + output = board(x, sample_rate) + return output + + +def flying_reverb(x: np.ndarray, parameters: List[float], sample_rate: int): + board = pedalboard.load_plugin("vst/FlyingReverb.vst3") + # Print all of the parameter names: + for parameter, value in board.parameters.items(): + print(parameter, value) + + +EFFECT_FUNCS = { + "pedalboard_distortion": { + "func": pedalboard_distortion, + "class": "distortion", + "instance": "pedalboard_distortion", + "parameters": { + "drive_db": {"min": 0, "max": 24, "scale": "linear"}, + }, + }, + "pedalboard_reverb": { + "func": pedalboard_reverb, + "class": "reverb", + "instance": "pedalboard_reverb", + "parameters": { + "room_size": {"min": 0, "max": 1, "scale": "linear"}, + "damping": {"min": 0, "max": 1, "scale": "linear"}, + "wetdry_mix": {"min": 0, "max": 1, "scale": "linear"}, + }, + }, +} + + +def apply_random_pedalboard_distortion( + x: torch.Tensor, + sample_rate: float, + min_drive_db: float = 0.0, + max_drive_db: float = 48.0, +): + board = Pedalboard() + drive_db = torch.rand(1) * (max_drive_db - min_drive_db) + min_drive_db + board.append(Distortion(drive_db=drive_db)) + + params = {"drive_db": normalize_param(drive_db, min_drive_db, max_drive_db)} + + return torch.from_numpy(board(x.numpy(), sample_rate)), params + + +def apply_random_pedalboard_delay( + x: torch.Tensor, + sample_rate: float, + min_delay_seconds: float = 0.01, + max_delay_seconds: float = 1.0, + min_feedback: float = 0.05, + max_feedback: float = 0.6, + min_mix: float = 0.0, + max_mix: float = 1.0, +): + board = Pedalboard() + delay_seconds = loguniform(min_delay_seconds, max_delay_seconds) + feedback = rand(min_feedback, max_feedback) + mix = rand(min_mix, max_mix) + board.append(Delay(delay_seconds=delay_seconds, feedback=feedback, mix=mix)) + y = torch.from_numpy(board(x.numpy(), sample_rate)) + + params = [ + normalize_param(delay_seconds, min_delay_seconds, max_delay_seconds), + normalize_param(feedback, min_feedback, max_feedback), + normalize_param(mix, min_mix, max_mix), + ] + + params = torch.tensor(params, dtype=torch.float32) + + return y, params + + +def apply_random_pedalboard_chorus( + x: torch.Tensor, + sample_rate: float, + min_rate_hz: float = 0.25, + max_rate_hz: float = 4.0, + min_depth: float = 0.0, + max_depth: float = 0.6, + min_centre_delay_ms: float = 5.0, + max_centre_delay_ms: float = 10.0, + min_feedback: float = 0.1, + max_feedback: float = 0.6, + min_mix: float = 0.0, + max_mix: float = 1.0, +): + board = Pedalboard() + rate_hz = rand(min_rate_hz, max_rate_hz) + depth = rand(min_depth, max_depth) + centre_delay_ms = rand(min_centre_delay_ms, max_centre_delay_ms) + feedback = rand(min_feedback, max_feedback) + mix = rand(min_mix, max_mix) + board.append( + Chorus( + rate_hz=rate_hz, + depth=depth, + centre_delay_ms=centre_delay_ms, + feedback=feedback, + mix=mix, + ) + ) + # process audio using the pedalboard + y = torch.from_numpy(board(x.numpy(), sample_rate)) + + params = [ + normalize_param(rate_hz, min_rate_hz, max_rate_hz), + normalize_param(depth, min_depth, max_depth), + normalize_param(centre_delay_ms, min_centre_delay_ms, max_centre_delay_ms), + normalize_param(feedback, min_feedback, max_feedback), + normalize_param(mix, min_mix, max_mix), + ] + + params = torch.tensor(params, dtype=torch.float32) + + return y, params + + +def apply_random_pedalboard_reverb( + x: torch.Tensor, + sample_rate: float, + min_room_size: float = 0.0, + max_room_size: float = 1.0, + min_damping: float = 0.0, + max_damping: float = 1.0, + min_wet_dry: float = 0.0, + max_wet_dry: float = 1.0, + min_width: float = 0.0, + max_width: float = 1.0, +): + board = Pedalboard() + room_size = rand(min_room_size, max_room_size) + damping = rand(min_damping, max_damping) + wet_dry = rand(min_wet_dry, max_wet_dry) + width = rand(min_width, max_width) + + board.append( + Reverb( + room_size=room_size, + damping=damping, + wet_level=wet_dry, + dry_level=(1 - wet_dry), + width=width, + ) + ) + + y = torch.from_numpy(board(x.numpy(), sample_rate)) + + params = [ + normalize_param(room_size, min_room_size, max_room_size), + normalize_param(damping, min_damping, max_damping), + normalize_param(wet_dry, min_wet_dry, max_wet_dry), + normalize_param(width, min_width, max_width), + ] + + params = torch.tensor(params, dtype=torch.float32) + + return y, params + + +def apply_random_pedalboard_compressor( + x: torch.Tensor, + sample_rate: float, + min_threshold_db: float = -42.0, + max_threshold_db: float = 0.0, + min_ratio: float = 1.0, + max_ratio: float = 10.0, + min_attack_ms: float = 0.1, + max_attack_ms: float = 100.0, + min_release_ms: float = 10.0, + max_release_ms: float = 1000.0, +): + board = Pedalboard() + threshold_db = torch.tensor([rand(min_threshold_db, max_threshold_db)]) + ratio = torch.tensor([rand(min_ratio, max_ratio)]) + attack_ms = torch.tensor([rand(min_attack_ms, max_attack_ms)]) + release_ms = torch.tensor([rand(min_release_ms, max_release_ms)]) + + board.append( + Compressor( + threshold_db=threshold_db, + ratio=ratio, + attack_ms=attack_ms, + release_ms=release_ms, + ) + ) + + y = torch.from_numpy(board(x.numpy(), sample_rate)) + + params = { + "threshold_db": normalize_param( + threshold_db, min_threshold_db, max_threshold_db + ).float(), + "ratio": normalize_param(ratio, min_ratio, max_ratio).float(), + "attack_ms": normalize_param(attack_ms, min_attack_ms, max_attack_ms).float(), + "release_ms": normalize_param( + release_ms, min_release_ms, max_release_ms + ).float(), + } + + # process audio using the pedalboard + return y, params + + +def apply_random_equalizer( + x: torch.Tensor, + sample_rate: float, + min_gain_db: float = -48.0, + max_gain_db: float = 48.0, + min_q_factor: float = 0.1, + max_q_factor: float = 10.0, + min_cutoff_freq: float = 100.0, + max_cutoff_freq: float = 10000.0, +): + gain_db = torch.tensor([rand(min_gain_db, max_gain_db)]) + cutoff_freq = torch.tensor([loguniform(min_cutoff_freq, max_cutoff_freq)]) + q_factor = torch.tensor([rand(min_q_factor, max_q_factor)]) + + y = single_band_eq( + x.unsqueeze(0), + sample_rate, + gain_db, + cutoff_freq, + q_factor, + ) + + params = { + "gain_db": normalize_param(gain_db, min_gain_db, max_gain_db).float(), + "cutoff_freq": normalize_param( + cutoff_freq, min_cutoff_freq, max_cutoff_freq + ).float(), + "q_factor": normalize_param(q_factor, min_q_factor, max_q_factor).float(), + } + + return y.squeeze(0), params + + +class BaseTransform: + def __init__(self, p: float = 1.0): + assert 0 <= p <= 1, "invalid probability value" + self.p = p + + def __call__(self, inputs: Tuple[torch.Tensor, torch.Tensor]): + """ + Args: + inputs (Tuple[torch.Tensor, torch.Tensor]): + x_a (torch.Tensor): (chs, seq_len) + x_b (torch.Tensor): (chs, seq_len) + Return: + outputs (Tuple[torch.Tensor, torch.Tensor]): + x_a_out (torch.Tensor): (chs, seq_len) + x_b_out (torch.Tensor): (chs, seq_len) + """ + x_a, x_b = inputs + seed = random.randint(0, 2**32 - 1) # Random seed from 0 to 2^32-1 + + if random.random() < self.p: + random.seed(seed) + x_a_out = self._transform(x_a) + + random.seed(seed) # reset seed to get same effect combination + x_b_out = self._transform(x_b) + else: + x_a_out, x_b_out = x_a, x_b + + return (x_a_out, x_b_out) + + def _transform(self, stems: torch.Tensor) -> torch.Tensor: + raise NotImplementedError + + +class RandomSwapLR(BaseTransform): + def __init__(self, p=0.5) -> None: + super().__init__(p=p) + + def _transform(self, stems: torch.Tensor): + return torch.flip(stems, [0]) + + +class RandomGain(BaseTransform): + def __init__(self, low=0.25, high=1.25, **kwargs) -> None: + super().__init__(**kwargs) + self.low = low + self.high = high + + def _transform(self, stems): + gain = random.random() * (self.high - self.low) + self.low + return stems * gain + + +class RandomFlipPhase(RandomSwapLR): + def __init__(self, p=0.5) -> None: + super().__init__(p=p) + + def _transform(self, stems: torch.Tensor): + return -stems + + +def biqaud( + gain_db: float, + cutoff_freq: float, + q_factor: float, + sample_rate: float, + filter_type: str, +): + """Use design parameters to generate coeffieicnets for a specific filter type. + Args: + gain_db (float): Shelving filter gain in dB. + cutoff_freq (float): Cutoff frequency in Hz. + q_factor (float): Q factor. + sample_rate (float): Sample rate in Hz. + filter_type (str): Filter type. + One of "low_shelf", "high_shelf", or "peaking" + Returns: + b (np.ndarray): Numerator filter coefficients stored as [b0, b1, b2] + a (np.ndarray): Denominator filter coefficients stored as [a0, a1, a2] + """ + + A = 10 ** (gain_db / 40.0) + w0 = 2.0 * np.pi * (cutoff_freq / sample_rate) + alpha = np.sin(w0) / (2.0 * q_factor) + + cos_w0 = np.cos(w0) + sqrt_A = np.sqrt(A) + + if filter_type == "high_shelf": + b0 = A * ((A + 1) + (A - 1) * cos_w0 + 2 * sqrt_A * alpha) + b1 = -2 * A * ((A - 1) + (A + 1) * cos_w0) + b2 = A * ((A + 1) + (A - 1) * cos_w0 - 2 * sqrt_A * alpha) + a0 = (A + 1) - (A - 1) * cos_w0 + 2 * sqrt_A * alpha + a1 = 2 * ((A - 1) - (A + 1) * cos_w0) + a2 = (A + 1) - (A - 1) * cos_w0 - 2 * sqrt_A * alpha + elif filter_type == "low_shelf": + b0 = A * ((A + 1) - (A - 1) * cos_w0 + 2 * sqrt_A * alpha) + b1 = 2 * A * ((A - 1) - (A + 1) * cos_w0) + b2 = A * ((A + 1) - (A - 1) * cos_w0 - 2 * sqrt_A * alpha) + a0 = (A + 1) + (A - 1) * cos_w0 + 2 * sqrt_A * alpha + a1 = -2 * ((A - 1) + (A + 1) * cos_w0) + a2 = (A + 1) + (A - 1) * cos_w0 - 2 * sqrt_A * alpha + elif filter_type == "peaking": + b0 = 1 + alpha * A + b1 = -2 * cos_w0 + b2 = 1 - alpha * A + a0 = 1 + alpha / A + a1 = -2 * cos_w0 + a2 = 1 - alpha / A + else: + pass + # raise ValueError(f"Invalid filter_type: {filter_type}.") + + b = np.array([b0, b1, b2]) / a0 + a = np.array([a0, a1, a2]) / a0 + + return b, a + + +def parametric_eq( + x: np.ndarray, + sample_rate: float, + low_shelf_gain_db: float = 0.0, + low_shelf_cutoff_freq: float = 80.0, + low_shelf_q_factor: float = 0.707, + band_gains_db: List[float] = [0.0], + band_cutoff_freqs: List[float] = [300.0], + band_q_factors: List[float] = [0.707], + high_shelf_gain_db: float = 0.0, + high_shelf_cutoff_freq: float = 1000.0, + high_shelf_q_factor: float = 0.707, + dtype=np.float32, +): + """Multiband parametric EQ. + + Low-shelf -> Band 1 -> ... -> Band N -> High-shelf + + Args: + + """ + assert ( + len(band_gains_db) == len(band_cutoff_freqs) == len(band_q_factors) + ) # must define for all bands + + # -------- apply low-shelf filter -------- + b, a = biqaud( + low_shelf_gain_db, + low_shelf_cutoff_freq, + low_shelf_q_factor, + sample_rate, + "low_shelf", + ) + x = scipy.signal.lfilter(b, a, x) + + # -------- apply peaking filters -------- + for gain_db, cutoff_freq, q_factor in zip( + band_gains_db, band_cutoff_freqs, band_q_factors + ): + b, a = biqaud( + gain_db, + cutoff_freq, + q_factor, + sample_rate, + "peaking", + ) + x = scipy.signal.lfilter(b, a, x) + + # -------- apply high-shelf filter -------- + b, a = biqaud( + high_shelf_gain_db, + high_shelf_cutoff_freq, + high_shelf_q_factor, + sample_rate, + "high_shelf", + ) + sos5 = np.concatenate((b, a)) + x = scipy.signal.lfilter(b, a, x) + + return x.astype(dtype) + + +# ------------- Differentiable audio effects ---------------------------- + + +def denormalize(p: torch.Tensor, min_val: float, max_val: float): + """When p is on (0,1) restore the original range of the parameter values. + + Args: + p (torch.Tensor): [bs, num_params] + min_val (float): minimum value of the parameter + max_val (float): maximum value of the parameter + + Returns: + torch.Tensor: [bs, num_params] + """ + return p * (max_val - min_val) + min_val + + +def apply_gain(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 1, "Number of parameters must be 1" + + gain_db = denormalize(params[:, 0], -48, 48) + + audio = dasp_pytorch.gain(audio, sample_rate, gain_db) + + return audio + + +def apply_distortion(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 1, "Number of parameters must be 1" + + gain_db = denormalize(params[:, 0], 0, 48) + + audio = dasp_pytorch.distortion(audio, sample_rate, gain_db) + + return audio + + +def apply_reverb(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 25, "Number of parameters must be 25" + + band0_gain = params[:, 0] + band1_gain = params[:, 1] + band2_gain = params[:, 2] + band3_gain = params[:, 3] + band4_gain = params[:, 4] + band5_gain = params[:, 5] + band6_gain = params[:, 6] + band7_gain = params[:, 7] + band8_gain = params[:, 8] + band9_gain = params[:, 9] + band10_gain = params[:, 10] + band11_gain = params[:, 11] + band0_decay = params[:, 12] + band1_decay = params[:, 13] + band2_decay = params[:, 14] + band3_decay = params[:, 15] + band4_decay = params[:, 16] + band5_decay = params[:, 17] + band6_decay = params[:, 18] + band7_decay = params[:, 19] + band8_decay = params[:, 20] + band9_decay = params[:, 21] + band10_decay = params[:, 22] + band11_decay = params[:, 23] + mix = params[:, 24] + + audio = dasp_pytorch.noise_shaped_reverberation( + audio, + sample_rate, + band0_gain, + band1_gain, + band2_gain, + band3_gain, + band4_gain, + band5_gain, + band6_gain, + band7_gain, + band8_gain, + band9_gain, + band10_gain, + band11_gain, + band0_decay, + band1_decay, + band2_decay, + band3_decay, + band4_decay, + band5_decay, + band6_decay, + band7_decay, + band8_decay, + band9_decay, + band10_decay, + band11_decay, + mix, + ) + + return audio + + +def apply_compressor(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 6, "Number of parameters must be 6" + + threshold_db = denormalize(params[:, 0], -60, 0) + ratio = denormalize(params[:, 1], 1, 20) + attack_ms = denormalize(params[:, 2], 0.1, 250) + release_ms = denormalize(params[:, 3], 10, 2000) + knee_db = denormalize(params[:, 4], 1.0, 24.0) + makeup_gain_db = denormalize(params[:, 5], 0, 24.0) + + audio = dasp_pytorch.compressor( + audio, + sample_rate, + threshold_db, + ratio, + attack_ms, + release_ms, + knee_db, + makeup_gain_db, + lookahead_samples=512, + ) + + return audio + + +def apply_parametric_eq(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert sample_rate >= 44100, "Sample rate must be at least 44100 Hz" + assert num_params == 18, "Number of parameters must be 18" + + # convert params to correct range + low_shelf_gain_db = denormalize(params[:, 0], -18, 18) + low_shelf_cutoff_freq = denormalize(params[:, 1], 20, 20000) + low_shelf_q_factor = denormalize(params[:, 2], 0.1, 10) + + band0_gain_db = denormalize(params[:, 3], -18, 18) + band0_center_freq = denormalize(params[:, 4], 20, 20000) + band0_q_factor = denormalize(params[:, 5], 0.1, 10) + + band1_gain_db = denormalize(params[:, 6], -18, 18) + band1_center_freq = denormalize(params[:, 7], 20, 20000) + band1_q_factor = denormalize(params[:, 8], 0.1, 10) + + band2_gain_db = denormalize(params[:, 9], -18, 18) + band2_center_freq = denormalize(params[:, 10], 20, 20000) + band2_q_factor = denormalize(params[:, 11], 0.1, 10) + + band3_gain_db = denormalize(params[:, 12], -18, 18) + band3_center_freq = denormalize(params[:, 13], 20, 20000) + band3_q_factor = denormalize(params[:, 14], 0.1, 10) + + high_shelf_gain_db = denormalize(params[:, 15], -18, 18) + high_shelf_cutoff_freq = denormalize(params[:, 16], 20, 20000) + high_shelf_q_factor = denormalize(params[:, 17], 0.1, 10) + + audio = dasp_pytorch.parametric_eq( + audio, + sample_rate, + low_shelf_gain_db, + low_shelf_cutoff_freq, + low_shelf_q_factor, + band0_gain_db, + band0_center_freq, + band0_q_factor, + band1_gain_db, + band1_center_freq, + band1_q_factor, + band2_gain_db, + band2_center_freq, + band2_q_factor, + band3_gain_db, + band3_center_freq, + band3_q_factor, + high_shelf_gain_db, + high_shelf_cutoff_freq, + high_shelf_q_factor, + ) + + return audio + + +def apply_simple_autodiff_processor( + audio: torch.Tensor, params: torch.Tensor, sample_rate: int, *args +): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + num_parametric_eq_params = 15 + num_compressor_params = 6 + + assert num_params == num_parametric_eq_params + num_compressor_params + + parametric_eq_params = params[:, :num_parametric_eq_params] + compressor_params = params[:, num_parametric_eq_params:] + + audio = apply_parametric_eq(audio, parametric_eq_params, sample_rate) + audio = apply_compressor(audio, compressor_params, sample_rate) + + return audio + + +def apply_complex_autodiff_processor( + audio: torch.Tensor, params: torch.Tensor, sample_rate: int, *args +): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + num_parametric_eq_params = 18 + num_compressor_params = 6 + num_distortion_params = 1 + num_reverb_params = 25 + num_gain_params = 1 + + assert ( + num_params + == num_parametric_eq_params + + num_compressor_params + + num_distortion_params + + num_reverb_params + + num_gain_params + ) + + assert torch.all(params >= 0) and torch.all(params <= 1) + + # extract parameters for each effect + eq_param_start = 0 + eq_param_end = num_parametric_eq_params + parametric_eq_params = params[:, :num_parametric_eq_params] + + compressor_param_start = eq_param_end + compressor_param_end = compressor_param_start + num_compressor_params + compressor_params = params[:, compressor_param_start:compressor_param_end] + + distortion_param_start = compressor_param_end + distortion_param_end = distortion_param_start + num_distortion_params + distortion_params = params[:, distortion_param_start:distortion_param_end] + + reverb_param_start = distortion_param_end + reverb_param_end = reverb_param_start + num_reverb_params + reverb_params = params[:, reverb_param_start:reverb_param_end] + + gain_param_start = reverb_param_end + gain_param_end = gain_param_start + num_gain_params + gain_params = params[:, gain_param_start:gain_param_end] + + # apply effects in sequence + audio = apply_parametric_eq(audio, parametric_eq_params, sample_rate) + audio = apply_compressor(audio, compressor_params, sample_rate) + audio = apply_distortion(audio, distortion_params, sample_rate) + audio = apply_reverb(audio, reverb_params, sample_rate) + audio = apply_gain(audio, gain_params, sample_rate) + + return audio + + +# ------------- VST-like effect wrapper classes ---------------------------- +class Parameter: + def __init__(self, init_value: float, min_value: float, max_value: float): + self.min_value = min_value + self.max_value = max_value + self.set_value(init_value) + + def set_value(self, value: float): + """Normalize the value to the range [0, 1] and store it as raw_value.""" + assert self.min_value <= value <= self.max_value + self.raw_value = (value - self.min_value) / (self.max_value - self.min_value) + + def get_value(self): + """Denormalize the value to the range [min_value, max_value].""" + return self.raw_value * (self.max_value - self.min_value) + self.min_value + + +class BasicParametricEQ: + def __init__( + self, + low_shelf_gain_db: float = 0.0, + low_shelf_cutoff_freq: float = 80.0, + low_shelf_q_factor: float = 0.707, + band0_gain_db: float = 0.0, + band0_cutoff_freq: float = 300.0, + band0_q_factor: float = 0.707, + band1_gain_db: float = 0.0, + band1_cutoff_freq: float = 1000.0, + band1_q_factor: float = 0.707, + band2_gain_db: float = 0.0, + band2_cutoff_freq: float = 3000.0, + band2_q_factor: float = 0.707, + band3_gain_db: float = 0.0, + band3_cutoff_freq: float = 10000.0, + band3_q_factor: float = 0.707, + high_shelf_gain_db: float = 0.0, + high_shelf_cutoff_freq: float = 1000.0, + high_shelf_q_factor: float = 0.707, + ): + self.parameters = { + "low_shelf_gain_db": Parameter(low_shelf_gain_db, -24.0, 24.0), + "low_shelf_cutoff_freq": Parameter(low_shelf_cutoff_freq, 20.0, 4000.0), + "low_shelf_q_factor": Parameter(low_shelf_q_factor, 0.1, 4.0), + "band0_gain_db": Parameter(band0_gain_db, -24.0, 24.0), + "band0_cutoff_freq": Parameter(band0_cutoff_freq, 20.0, 10000.0), + "band0_q_factor": Parameter(band0_q_factor, 0.1, 4.0), + "band1_gain_db": Parameter(band1_gain_db, -24.0, 24.0), + "band1_cutoff_freq": Parameter(band1_cutoff_freq, 20.0, 10000.0), + "band1_q_factor": Parameter(band1_q_factor, 0.1, 4.0), + "band2_gain_db": Parameter(band2_gain_db, -24.0, 24.0), + "band2_cutoff_freq": Parameter(band2_cutoff_freq, 20.0, 10000.0), + "band2_q_factor": Parameter(band2_q_factor, 0.1, 4.0), + "band3_gain_db": Parameter(band3_gain_db, -24.0, 24.0), + "band3_cutoff_freq": Parameter(band3_cutoff_freq, 20.0, 10000.0), + "band3_q_factor": Parameter(band3_q_factor, 0.1, 4.0), + "high_shelf_gain_db": Parameter(high_shelf_gain_db, -24.0, 24.0), + "high_shelf_cutoff_freq": Parameter(high_shelf_cutoff_freq, 200.0, 18000.0), + "high_shelf_q_factor": Parameter(high_shelf_q_factor, 0.1, 4.0), + } + + def process(self, x: np.ndarray, sample_rate: float): + return parametric_eq( + x, + sample_rate, + low_shelf_gain_db=self.parameters["low_shelf_gain_db"].get_value(), + low_shelf_cutoff_freq=self.parameters["low_shelf_cutoff_freq"].get_value(), + low_shelf_q_factor=self.parameters["low_shelf_q_factor"].get_value(), + band_gains_db=[ + self.parameters["band0_gain_db"].get_value(), + self.parameters["band1_gain_db"].get_value(), + self.parameters["band2_gain_db"].get_value(), + self.parameters["band3_gain_db"].get_value(), + ], + band_cutoff_freqs=[ + self.parameters["band0_cutoff_freq"].get_value(), + self.parameters["band1_cutoff_freq"].get_value(), + self.parameters["band2_cutoff_freq"].get_value(), + self.parameters["band3_cutoff_freq"].get_value(), + ], + band_q_factors=[ + self.parameters["band0_q_factor"].get_value(), + self.parameters["band1_q_factor"].get_value(), + self.parameters["band2_q_factor"].get_value(), + self.parameters["band3_q_factor"].get_value(), + ], + high_shelf_gain_db=self.parameters["high_shelf_gain_db"].get_value(), + high_shelf_cutoff_freq=self.parameters[ + "high_shelf_cutoff_freq" + ].get_value(), + high_shelf_q_factor=self.parameters["high_shelf_q_factor"].get_value(), + ) + + +class BasicCompressor: + def __init__( + self, + threshold_db: float = 0.0, + ratio: float = 4.0, + attack_ms: float = 1.0, + release_ms: float = 100.0, + ): + self.parameters = { + "threshold_db": Parameter(threshold_db, -80.0, 0.0), + "ratio": Parameter(ratio, 1.0, 20.0), + "attack_ms": Parameter(attack_ms, 0.1, 100.0), + "release_ms": Parameter(release_ms, 10.0, 1000.0), + } + + def process(self, x: np.ndarray, sample_rate: float): + return pedalboard.Compressor( + threshold_db=self.parameters["threshold_db"].get_value(), + ratio=self.parameters["ratio"].get_value(), + attack_ms=self.parameters["attack_ms"].get_value(), + release_ms=self.parameters["release_ms"].get_value(), + ).process(x, sample_rate) + + +class BasicDistortion: + def __init__(self, drive_db: float = 0.0, output_gain_db: float = 0.0): + self.parameters = { + "drive_db": Parameter(0.0, -48.0, 48.0), + "output_gain_db": Parameter(0.0, -24.0, 24.0), + } + + def process(self, x: np.ndarray, sample_rate: float): + out = pedalboard.Distortion( + drive_db=self.parameters["drive_db"].get_value() + ).process(x, sample_rate) + + out = pedalboard.Gain( + gain_db=self.parameters["output_gain_db"].get_value() + ).process(out, sample_rate) + + return out + + +class BasicDelay: + def __init__( + self, delay_seconds: float = 0.5, feedback: float = 0.5, mix: float = 0.5 + ): + self.parameters = { + "delay_seconds": Parameter(delay_seconds, 0.01, 1.0), + "feedback": Parameter(feedback, 0.05, 1.0), + "mix": Parameter(mix, 0.0, 1.0), + } + + def process(self, x: np.ndarray, sample_rate: float): + return pedalboard.Delay( + delay_seconds=self.parameters["delay_seconds"].get_value(), + feedback=self.parameters["feedback"].get_value(), + mix=self.parameters["mix"].get_value(), + ).process(x, sample_rate) + + +class BasicReverb: + def __init__( + self, + room_size: float = 0.5, + damping: float = 0.5, + wet_dry: float = 0.5, + width: float = 0.5, + ): + self.parameters = { + "room_size": Parameter(room_size, 0.0, 1.0), + "damping": Parameter(damping, 0.0, 1.0), + "wet_dry": Parameter(wet_dry, 0.0, 1.0), + "width": Parameter(width, 0.0, 1.0), + } + + def process(self, x: np.ndarray, sample_rate: float): + return pedalboard.Reverb( + room_size=self.parameters["room_size"].get_value(), + damping=self.parameters["damping"].get_value(), + wet_level=self.parameters["wet_dry"].get_value(), + dry_level=1 - self.parameters["wet_dry"].get_value(), + width=self.parameters["width"].get_value(), + ).process(x, sample_rate) + + +class BasicChorus: + def __init__( + self, + rate_hz: float = 1.0, + centre_delay_ms: float = 7.0, + depth: float = 0.1, + feedback: float = 0.5, + mix: float = 0.5, + ): + self.parameters = { + "rate_hz": Parameter(rate_hz, 0.1, 10.0), + "centre_delay_ms": Parameter(centre_delay_ms, 0.1, 20.0), + "depth": Parameter(depth, 0.0, 1.0), + "feedback": Parameter(feedback, 0.0, 1.0), + "mix": Parameter(mix, 0.0, 1.0), + } + + def process(self, x: np.ndarray, sample_rate: float): + return pedalboard.Chorus( + centre_delay_ms=self.parameters["centre_delay_ms"].get_value(), + depth=self.parameters["depth"].get_value(), + feedback=self.parameters["feedback"].get_value(), + mix=self.parameters["mix"].get_value(), + ).process(x, sample_rate) + + +# ----------------------------------------------------------------------- + + +class RandomParametricEQ(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + num_bands: int = 3, + min_gain_db: float = -6.0, + max_gain_db: float = +6.0, + min_cutoff_freq: float = 1000.0, + max_cutoff_freq: float = 10000.0, + min_q_factor: float = 0.1, + max_q_factor: float = 4.0, + **kwargs, + ): + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.num_bands = num_bands + self.min_gain_db = min_gain_db + self.max_gain_db = max_gain_db + self.min_cutoff_freq = min_cutoff_freq + self.max_cutoff_freq = max_cutoff_freq + self.min_q_factor = min_q_factor + self.max_q_factor = max_q_factor + + def _transform(self, x: torch.Tensor): + """ + Args: + x: (torch.Tensor): Array of audio samples with shape (chs, seq_leq). + The filter will be applied the final dimension, and by default the same + filter will be applied to all channels. + """ + low_shelf_gain_db = rand(self.min_gain_db, self.max_gain_db) + low_shelf_cutoff_freq = loguniform(20.0, 200.0) + low_shelf_q_factor = rand(self.min_q_factor, self.max_q_factor) + + high_shelf_gain_db = rand(self.min_gain_db, self.max_gain_db) + high_shelf_cutoff_freq = loguniform(8000.0, 16000.0) + high_shelf_q_factor = rand(self.min_q_factor, self.max_q_factor) + + band_gain_dbs = [] + band_cutoff_freqs = [] + band_q_factors = [] + for _ in range(self.num_bands): + band_gain_dbs.append(rand(self.min_gain_db, self.max_gain_db)) + band_cutoff_freqs.append( + loguniform(self.min_cutoff_freq, self.max_cutoff_freq) + ) + band_q_factors.append(rand(self.min_q_factor, self.max_q_factor)) + + y = parametric_eq( + x.numpy(), + self.sample_rate, + low_shelf_gain_db=low_shelf_gain_db, + low_shelf_cutoff_freq=low_shelf_cutoff_freq, + low_shelf_q_factor=low_shelf_q_factor, + band_gains_db=band_gain_dbs, + band_cutoff_freqs=band_cutoff_freqs, + band_q_factors=band_q_factors, + high_shelf_gain_db=high_shelf_gain_db, + high_shelf_cutoff_freq=high_shelf_cutoff_freq, + high_shelf_q_factor=high_shelf_q_factor, + ) + + return torch.from_numpy(y) + + +def stereo_widener(x: torch.Tensor, width: torch.Tensor): + sqrt2 = np.sqrt(2) + + left = x[0, ...] + right = x[1, ...] + + mid = (left + right) / sqrt2 + side = (left - right) / sqrt2 + + # amplify mid and side signal seperately: + mid *= 2 * (1 - width) + side *= 2 * width + + left = (mid + side) / sqrt2 + right = (mid - side) / sqrt2 + + x = torch.stack((left, right), dim=0) + + return x + + +class RandomStereoWidener(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_width: float = 0.0, + max_width: float = 1.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_width = min_width + self.max_width = max_width + + def _transform(self, x: torch.Tensor): + width = rand(self.min_width, self.max_width) + return stereo_widener(x, width) + + +class RandomVolumeAutomation(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_segment_seconds: float = 3.0, + min_gain_db: float = -6.0, + max_gain_db: float = 6.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_segment_seconds = min_segment_seconds + self.min_gain_db = min_gain_db + self.max_gain_db = max_gain_db + + def _transform(self, x: torch.Tensor): + gain_db = torch.zeros(x.shape[-1]).type_as(x) + + seconds = x.shape[-1] / self.sample_rate + max_num_segments = max(1, int(seconds // self.min_segment_seconds)) + + num_segments = randint(1, max_num_segments) + segment_lengths = ( + x.shape[-1] + * np.random.dirichlet( + [rand(0, 10) for _ in range(num_segments)], 1 + ) # TODO(cm): this can crash training + ).astype("int")[0] + + segment_lengths = np.maximum(segment_lengths, 1) + + # check the sum is equal to the length of the signal + diff = segment_lengths.sum() - x.shape[-1] + if diff < 0: + segment_lengths[-1] -= diff + elif diff > 0: + for idx in range(num_segments): + if segment_lengths[idx] > diff + 1: + segment_lengths[idx] -= diff + break + + samples_filled = 0 + start_gain_db = 0 + for idx in range(num_segments): + segment_samples = segment_lengths[idx] + if idx != 0: + start_gain_db = end_gain_db + + # sample random end gain + end_gain_db = rand(self.min_gain_db, self.max_gain_db) + fade = torch.linspace(start_gain_db, end_gain_db, steps=segment_samples) + gain_db[samples_filled : samples_filled + segment_samples] = fade + samples_filled = samples_filled + segment_samples + + # print(gain_db) + x *= 10 ** (gain_db / 20.0) + return x + + +class RandomPedalboardCompressor(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_threshold_db: float = -42.0, + max_threshold_db: float = -6.0, + min_ratio: float = 1.5, + max_ratio: float = 4.0, + min_attack_ms: float = 1.0, + max_attack_ms: float = 50.0, + min_release_ms: float = 10.0, + max_release_ms: float = 250.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_threshold_db = min_threshold_db + self.max_threshold_db = max_threshold_db + self.min_ratio = min_ratio + self.max_ratio = max_ratio + self.min_attack_ms = min_attack_ms + self.max_attack_ms = max_attack_ms + self.min_release_ms = min_release_ms + self.max_release_ms = max_release_ms + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + threshold_db = rand(self.min_threshold_db, self.max_threshold_db) + ratio = rand(self.min_ratio, self.max_ratio) + attack_ms = rand(self.min_attack_ms, self.max_attack_ms) + release_ms = rand(self.min_release_ms, self.max_release_ms) + + board.append( + Compressor( + threshold_db=threshold_db, + ratio=ratio, + attack_ms=attack_ms, + release_ms=release_ms, + ) + ) + + # process audio using the pedalboard + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class RandomPedalboardDelay(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_delay_seconds: float = 0.1, + max_delay_sconds: float = 1.0, + min_feedback: float = 0.05, + max_feedback: float = 0.6, + min_mix: float = 0.0, + max_mix: float = 0.7, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_delay_seconds = min_delay_seconds + self.max_delay_seconds = max_delay_sconds + self.min_feedback = min_feedback + self.max_feedback = max_feedback + self.min_mix = min_mix + self.max_mix = max_mix + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + delay_seconds = loguniform(self.min_delay_seconds, self.max_delay_seconds) + feedback = rand(self.min_feedback, self.max_feedback) + mix = rand(self.min_mix, self.max_mix) + board.append(Delay(delay_seconds=delay_seconds, feedback=feedback, mix=mix)) + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class RandomPedalboardChorus(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_rate_hz: float = 0.25, + max_rate_hz: float = 4.0, + min_depth: float = 0.0, + max_depth: float = 0.6, + min_centre_delay_ms: float = 5.0, + max_centre_delay_ms: float = 10.0, + min_feedback: float = 0.1, + max_feedback: float = 0.6, + min_mix: float = 0.1, + max_mix: float = 0.7, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_rate_hz = min_rate_hz + self.max_rate_hz = max_rate_hz + self.min_depth = min_depth + self.max_depth = max_depth + self.min_centre_delay_ms = min_centre_delay_ms + self.max_centre_delay_ms = max_centre_delay_ms + self.min_feedback = min_feedback + self.max_feedback = max_feedback + self.min_mix = min_mix + self.max_mix = max_mix + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + rate_hz = rand(self.min_rate_hz, self.max_rate_hz) + depth = rand(self.min_depth, self.max_depth) + centre_delay_ms = rand(self.min_centre_delay_ms, self.max_centre_delay_ms) + feedback = rand(self.min_feedback, self.max_feedback) + mix = rand(self.min_mix, self.max_mix) + board.append( + Chorus( + rate_hz=rate_hz, + depth=depth, + centre_delay_ms=centre_delay_ms, + feedback=feedback, + mix=mix, + ) + ) + # process audio using the pedalboard + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class RandomPedalboardPhaser(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_rate_hz: float = 0.25, + max_rate_hz: float = 5.0, + min_depth: float = 0.1, + max_depth: float = 0.6, + min_centre_frequency_hz: float = 200.0, + max_centre_frequency_hz: float = 600.0, + min_feedback: float = 0.1, + max_feedback: float = 0.6, + min_mix: float = 0.1, + max_mix: float = 0.7, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_rate_hz = min_rate_hz + self.max_rate_hz = max_rate_hz + self.min_depth = min_depth + self.max_depth = max_depth + self.min_centre_frequency_hz = min_centre_frequency_hz + self.max_centre_frequency_hz = max_centre_frequency_hz + self.min_feedback = min_feedback + self.max_feedback = max_feedback + self.min_mix = min_mix + self.max_mix = max_mix + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + rate_hz = rand(self.min_rate_hz, self.max_rate_hz) + depth = rand(self.min_depth, self.max_depth) + centre_frequency_hz = rand( + self.min_centre_frequency_hz, self.min_centre_frequency_hz + ) + feedback = rand(self.min_feedback, self.max_feedback) + mix = rand(self.min_mix, self.max_mix) + board.append( + Phaser( + rate_hz=rate_hz, + depth=depth, + centre_frequency_hz=centre_frequency_hz, + feedback=feedback, + mix=mix, + ) + ) + # process audio using the pedalboard + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class RandomPedalboardLimiter(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_threshold_db: float = -32.0, + max_threshold_db: float = -6.0, + min_release_ms: float = 10.0, + max_release_ms: float = 300.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_threshold_db = min_threshold_db + self.max_threshold_db = max_threshold_db + self.min_release_ms = min_release_ms + self.max_release_ms = max_release_ms + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + threshold_db = rand(self.min_threshold_db, self.max_threshold_db) + release_ms = rand(self.min_release_ms, self.max_release_ms) + board.append( + Limiter( + threshold_db=threshold_db, + release_ms=release_ms, + ) + ) + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class RandomPedalboardDistortion(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_drive_db: float = -12.0, + max_drive_db: float = 48.0, + **kwargs, + ): + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_drive_db = min_drive_db + self.max_drive_db = max_drive_db + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + drive_db = rand(self.min_drive_db, self.max_drive_db) + board.append(Distortion(drive_db=drive_db)) + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class RandomSoxReverb(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_reverberance: float = 10.0, + max_reverberance: float = 100.0, + min_high_freq_damping: float = 0.0, + max_high_freq_damping: float = 100.0, + min_wet_dry: float = 0.0, + max_wet_dry: float = 1.0, + min_room_scale: float = 5.0, + max_room_scale: float = 100.0, + min_stereo_depth: float = 20.0, + max_stereo_depth: float = 100.0, + min_pre_delay: float = 0.0, + max_pre_delay: float = 100.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_reverberance = min_reverberance + self.max_reverberance = max_reverberance + self.min_high_freq_damping = min_high_freq_damping + self.max_high_freq_damping = max_high_freq_damping + self.min_wet_dry = min_wet_dry + self.max_wet_dry = max_wet_dry + self.min_room_scale = min_room_scale + self.max_room_scale = max_room_scale + self.min_stereo_depth = min_stereo_depth + self.max_stereo_depth = max_stereo_depth + self.min_pre_delay = min_pre_delay + self.max_pre_delay = max_pre_delay + + def _transform(self, x: torch.Tensor): + reverberance = rand(self.min_reverberance, self.max_reverberance) + high_freq_damping = rand(self.min_high_freq_damping, self.max_high_freq_damping) + room_scale = rand(self.min_room_scale, self.max_room_scale) + stereo_depth = rand(self.min_stereo_depth, self.max_stereo_depth) + wet_dry = rand(self.min_wet_dry, self.max_wet_dry) + pre_delay = rand(self.min_pre_delay, self.max_pre_delay) + + effects = [ + [ + "reverb", + f"{reverberance}", + f"{high_freq_damping}", + f"{room_scale}", + f"{stereo_depth}", + f"{pre_delay}", + "--wet-only", + ] + ] + y, _ = torchaudio.sox_effects.apply_effects_tensor( + x, self.sample_rate, effects, channels_first=True + ) + + # manual wet/dry mix + return (x * (1 - wet_dry)) + (y * wet_dry) + + +class RandomPedalboardReverb(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + min_room_size: float = 0.0, + max_room_size: float = 1.0, + min_damping: float = 0.0, + max_damping: float = 1.0, + min_wet_dry: float = 0.0, + max_wet_dry: float = 0.7, + min_width: float = 0.0, + max_width: float = 1.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.min_room_size = min_room_size + self.max_room_size = max_room_size + self.min_damping = min_damping + self.max_damping = max_damping + self.min_wet_dry = min_wet_dry + self.max_wet_dry = max_wet_dry + self.min_width = min_width + self.max_width = max_width + + def _transform(self, x: torch.Tensor): + board = Pedalboard() + room_size = rand(self.min_room_size, self.max_room_size) + damping = rand(self.min_damping, self.max_damping) + wet_dry = rand(self.min_wet_dry, self.max_wet_dry) + width = rand(self.min_width, self.max_width) + + board.append( + Reverb( + room_size=room_size, + damping=damping, + wet_level=wet_dry, + dry_level=(1 - wet_dry), + width=width, + ) + ) + + return torch.from_numpy(board(x.numpy(), self.sample_rate)) + + +class LoudnessNormalize(BaseTransform): + def __init__( + self, + sample_rate: float = 44100.0, + target_lufs_db: float = -32.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.sample_rate = sample_rate + self.meter = pyln.Meter(sample_rate) + self.target_lufs_db = target_lufs_db + + def _transform(self, x: torch.Tensor): + x_lufs_db = self.meter.integrated_loudness(x.permute(1, 0).numpy()) + delta_lufs_db = torch.tensor([self.target_lufs_db - x_lufs_db]).float() + gain_lin = 10.0 ** (delta_lufs_db.clamp(-120, 40.0) / 20.0) + return gain_lin * x + + +class Mono2Stereo(BaseTransform): + def __init__(self) -> None: + super().__init__(p=1.0) + + def _transform(self, x: torch.Tensor): + assert x.ndim == 2 and x.shape[0] == 1, "x must be mono" + return torch.cat([x, x], dim=0) + + +class RandomPan(BaseTransform): + def __init__( + self, + min_pan: float = -1.0, + max_pan: float = 1.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.min_pan = min_pan + self.max_pan = max_pan + + def _transform(self, x: torch.Tensor): + """Constant power panning""" + assert x.ndim == 2 and x.shape[0] == 2, "x must be stereo" + theta = rand(self.min_pan, self.max_pan) * np.pi / 4 + x = x * 0.707 # normalize to prevent clipping + left_x, right_x = x[0], x[1] + cos_theta = np.cos(theta) + sin_theta = np.sin(theta) + left_x = left_x * (cos_theta - sin_theta) + right_x = right_x * (cos_theta + sin_theta) + return torch.stack([left_x, right_x], dim=0) diff --git a/utils/st_ito/features.py b/utils/st_ito/features.py new file mode 100644 index 0000000000000000000000000000000000000000..c9a56f9b78240a748f43617b62b11a516471df77 --- /dev/null +++ b/utils/st_ito/features.py @@ -0,0 +1,333 @@ +import math +import torch +import warnings +import torchaudio +import pyloudnorm as pyln + +# https://github.com/pytorch/audio/blob/d9942bae249329bd8c8bf5c92f0f108595fcb84f/torchaudio/functional/functional.py#L495 + + +def _create_triangular_filterbank( + all_freqs: torch.Tensor, + f_pts: torch.Tensor, +) -> torch.Tensor: + """Create a triangular filter bank. + + Args: + all_freqs (Tensor): STFT freq points of size (`n_freqs`). + f_pts (Tensor): Filter mid points of size (`n_filter`). + + Returns: + fb (Tensor): The filter bank of size (`n_freqs`, `n_filter`). + """ + # Adopted from Librosa + # calculate the difference between each filter mid point and each stft freq point in hertz + f_diff = f_pts[1:] - f_pts[:-1] # (n_filter + 1) + slopes = f_pts.unsqueeze(0) - all_freqs.unsqueeze(1) # (n_freqs, n_filter + 2) + # create overlapping triangles + zero = torch.zeros(1) + down_slopes = (-1.0 * slopes[:, :-2]) / f_diff[:-1] # (n_freqs, n_filter) + up_slopes = slopes[:, 2:] / f_diff[1:] # (n_freqs, n_filter) + fb = torch.max(zero, torch.min(down_slopes, up_slopes)) + + return fb + + +# https://github.com/pytorch/audio/blob/d9942bae249329bd8c8bf5c92f0f108595fcb84f/torchaudio/prototype/functional/functional.py#L6 + + +def _hz_to_bark(freqs: float, bark_scale: str = "traunmuller") -> float: + r"""Convert Hz to Barks. + + Args: + freqs (float): Frequencies in Hz + bark_scale (str, optional): Scale to use: ``traunmuller``, ``schroeder`` or ``wang``. (Default: ``traunmuller``) + + Returns: + barks (float): Frequency in Barks + """ + + if bark_scale not in ["schroeder", "traunmuller", "wang"]: + raise ValueError( + 'bark_scale should be one of "schroeder", "traunmuller" or "wang".' + ) + + if bark_scale == "wang": + return 6.0 * math.asinh(freqs / 600.0) + elif bark_scale == "schroeder": + return 7.0 * math.asinh(freqs / 650.0) + # Traunmuller Bark scale + barks = ((26.81 * freqs) / (1960.0 + freqs)) - 0.53 + # Bark value correction + if barks < 2: + barks += 0.15 * (2 - barks) + elif barks > 20.1: + barks += 0.22 * (barks - 20.1) + + return barks + + +def _bark_to_hz(barks: torch.Tensor, bark_scale: str = "traunmuller") -> torch.Tensor: + """Convert bark bin numbers to frequencies. + + Args: + barks (torch.Tensor): Bark frequencies + bark_scale (str, optional): Scale to use: ``traunmuller``,``schroeder`` or ``wang``. (Default: ``traunmuller``) + + Returns: + freqs (torch.Tensor): Barks converted in Hz + """ + + if bark_scale not in ["schroeder", "traunmuller", "wang"]: + raise ValueError( + 'bark_scale should be one of "traunmuller", "schroeder" or "wang".' + ) + + if bark_scale == "wang": + return 600.0 * torch.sinh(barks / 6.0) + elif bark_scale == "schroeder": + return 650.0 * torch.sinh(barks / 7.0) + # Bark value correction + if any(barks < 2): + idx = barks < 2 + barks[idx] = (barks[idx] - 0.3) / 0.85 + elif any(barks > 20.1): + idx = barks > 20.1 + barks[idx] = (barks[idx] + 4.422) / 1.22 + + # Traunmuller Bark scale + freqs = 1960 * ((barks + 0.53) / (26.28 - barks)) + + return freqs + + +def _hz_to_octs(freqs, tuning=0.0, bins_per_octave=12): + a440 = 440.0 * 2.0 ** (tuning / bins_per_octave) + return torch.log2(freqs / (a440 / 16)) + + +def barkscale_fbanks( + n_freqs: int, + f_min: float, + f_max: float, + n_barks: int, + sample_rate: int, + bark_scale: str = "traunmuller", +) -> torch.Tensor: + r"""Create a frequency bin conversion matrix. + + .. devices:: CPU + + .. properties:: TorchScript + + .. image:: https://download.pytorch.org/torchaudio/doc-assets/bark_fbanks.png + :alt: Visualization of generated filter bank + + Args: + n_freqs (int): Number of frequencies to highlight/apply + f_min (float): Minimum frequency (Hz) + f_max (float): Maximum frequency (Hz) + n_barks (int): Number of mel filterbanks + sample_rate (int): Sample rate of the audio waveform + bark_scale (str, optional): Scale to use: ``traunmuller``,``schroeder`` or ``wang``. (Default: ``traunmuller``) + + Returns: + torch.Tensor: Triangular filter banks (fb matrix) of size (``n_freqs``, ``n_barks``) + meaning number of frequencies to highlight/apply to x the number of filterbanks. + Each column is a filterbank so that assuming there is a matrix A of + size (..., ``n_freqs``), the applied result would be + ``A * barkscale_fbanks(A.size(-1), ...)``. + + """ + + # freq bins + all_freqs = torch.linspace(0, sample_rate // 2, n_freqs) + + # calculate bark freq bins + m_min = _hz_to_bark(f_min, bark_scale=bark_scale) + m_max = _hz_to_bark(f_max, bark_scale=bark_scale) + + m_pts = torch.linspace(m_min, m_max, n_barks + 2) + f_pts = _bark_to_hz(m_pts, bark_scale=bark_scale) + + # create filterbank + fb = _create_triangular_filterbank(all_freqs, f_pts) + + if (fb.max(dim=0).values == 0.0).any(): + warnings.warn( + "At least one bark filterbank has all zero values. " + f"The value for `n_barks` ({n_barks}) may be set too high. " + f"Or, the value for `n_freqs` ({n_freqs}) may be set too low." + ) + + return fb + + +def compute_barkspectrum( + x: torch.Tensor, + fft_size: int = 32768, + n_bands: int = 24, + sample_rate: int = 44100, + f_min: float = 20.0, + f_max: float = 20000.0, + mode: str = "mid-side", + **kwargs, +): + """Compute bark-spectrogram. + + Args: + x: (bs, 2, seq_len) + fft_size: size of fft + n_bands: number of bark bins + sample_rate: sample rate of audio + f_min: minimum frequency + f_max: maximum frequency + mode: "mono", "stereo", or "mid-side" + + Returns: + X: (bs, n_bands) + + """ + # compute filterbank + fb = barkscale_fbanks((fft_size // 2) + 1, f_min, f_max, n_bands, sample_rate) + fb = fb.unsqueeze(0).type_as(x) + fb = fb.permute(0, 2, 1) + + if mode == "mono": + x = x.mean(dim=1) # average over channels + signals = [x] + elif mode == "stereo": + signals = [x[:, 0, :], x[:, 1, :]] + elif mode == "mid-side": + x_mid = x[:, 0, :] + x[:, 1, :] + x_side = x[:, 0, :] - x[:, 1, :] + signals = [x_mid, x_side] + else: + raise ValueError(f"Invalid mode {mode}") + + outputs = [] + for signal in signals: + X = torch.stft( + signal, + n_fft=fft_size, + hop_length=fft_size // 4, + return_complex=True, + ) # compute stft + X = torch.abs(X) # take magnitude + X = torch.mean(X, dim=-1, keepdim=True) # take mean over time + # X = X.permute(0, 2, 1) # swap time and freq dims + X = torch.matmul(fb, X) # apply filterbank + X = torch.log(X + 1e-8) + # X = torch.cat([X, X_log], dim=-1) + outputs.append(X) + + # stack into tensor + X = torch.cat(outputs, dim=-1) + # flatten to one dimension embeddings + X = X.view(X.shape[0], -1) + + # l2 normalize + X = torch.nn.functional.normalize(X, p=2, dim=-1) + + return X + + +def compute_rms_energy(x: torch.Tensor, **kwargs): + """Compute root mean square energy. + + Args: + x: (bs, chs, seq_len) + + Returns: + rms: (bs, ) + """ + rms = torch.sqrt(torch.mean(x**2, dim=-1).clamp(min=1e-8)) + return rms + + +def compute_crest_factor(x: torch.Tensor, **kwargs): + """Compute crest factor as ratio of peak to rms energy in dB. + + Args: + x: (bs, chs, seq_len) + + Returns: + cf: (bs, ) + + """ + # peak normalize + peak = torch.max(torch.abs(x), dim=1)[0] + x = x / peak[:, None].clamp(min=1e-8) + num = torch.max(torch.abs(x), dim=-1)[0] + den = compute_rms_energy(x).clamp(min=1e-8) + cf = 20 * torch.log10((num / den).clamp(min=1e-8)) + return cf + + +def compute_lufs( + x: torch.Tensor, + sample_rate: float, + **kwargs, +): + """Compute the loudness in dB LUFS of waveform. + + Args: + x: (bs, chs, seq_len) + sample_rate: sample rate of audio + + Returns: + lufs: (bs, ) + + """ + bs, chs, seq_len = x.shape + meter = pyln.Meter(sample_rate) + + # peak normalize + peak = torch.max(torch.abs(x), dim=1)[0] + x = x / peak[:, None].clamp(min=1e-8) + + # add stereo dim if needed + if x.shape[1] < 2: + x = x.repeat(1, 2, 1) + + # measure loudness in LUFS of each batch item + measurements = [] + for bidx in range(bs): + lufs = meter.integrated_loudness(x[bidx, ...].permute(1, 0).cpu().numpy()) + measurements.append(lufs) + + return torch.tensor(measurements).view(bs, 1).float().type_as(x) + + +def compute_spectral_centroid( + x: torch.Tensor, + sample_rate: float, + *args, + **kwargs, +): + """Compute the spectral centroid. + + Args: + x: (bs, chs, seq_len) + sample_rate: sample rate of audio + Returns: + sc: (bs, chs x 10) + """ + bs, chs, seq_len = x.shape + transform = torchaudio.transforms.SpectralCentroid( + sample_rate, n_fft=2048, win_length=2048, hop_length=1024 + ).to(x.device) + sc = transform(x) + + sc = torch.nan_to_num(sc, nan=0.0, posinf=0.0, neginf=0.0) + + # downsample + sc = torch.nn.functional.adaptive_avg_pool1d(sc, 10) + + # flatten to one dimension embeddings + sc = sc.view(sc.shape[0], -1) + + # normalize by the nyquist frequency + sc = sc / (sample_rate / 2) + + return sc diff --git a/utils/st_ito/methods/param.py b/utils/st_ito/methods/param.py new file mode 100644 index 0000000000000000000000000000000000000000..f55e334c31973169cf29e28ce7ffc6ed2a2f8455 --- /dev/null +++ b/utils/st_ito/methods/param.py @@ -0,0 +1,359 @@ +import torch +import itertools +import torchaudio +import pytorch_lightning as pl + + +from typing import List +from transformers import AutoFeatureExtractor, AutoModelForAudioClassification + + +def get_classifier_logits(extractor, model, audio: torch.Tensor, sr: int): + """Get the logits from a pretrained audio classifier (AST trained on AudioSet) for a given audio file.""" + bs, chs, seq_len = audio.shape + + if sr != 16000: + audio = torchaudio.functional.resample(audio, sr, 16000) + + audio = audio.mean(dim=1, keepdim=False) # must be mono + + # chunk into a list of mono tensors + audio_tensors = torch.chunk(audio, bs, dim=0) + + # convert to list of np arrays + audio_tensors = [audio.squeeze().cpu().numpy() for audio in audio_tensors] + + # Extract features (this has to be done on CPU) + outputs = extractor(audio_tensors, sampling_rate=16000, return_tensors="pt") + features = outputs.input_values + + # move to device + features = features.type_as(audio) + + with torch.no_grad(): + logits = model(features).logits + + # here is how you get the classes if you want that... + # predicted_class_ids = torch.argmax(logits, dim=-1).item() + # predicted_label = model.config.id2label[predicted_class_ids] + + return logits.squeeze() + + +class ParameterEstimator(pl.LightningModule): + def __init__( + self, + encoder: torch.nn.Module, + lr: float, + num_instances: int, + num_presets: int = 0, + num_adv_classes: int = 0, + adv_logits_type: str = "dataset", + adv_weight: float = 1.0, + weight_decay: float = 0.001, + max_epochs: int = 250, + embed_mode: str = "blind", + norm: str = None, + optimize_discriminator_only: bool = False, + ): + """ + + Args: + encoder (torch.nn.Module): Encoder model + lr (float): Learning rate + num_instances (int): Number of unique plugin instances for classification. + num_presets (int): Number of unique presets for classification. + num_adv_classes (int): Number of adversarial classes. Defaults to 0. + adv_logits_type (str, optional): Adversarial logits type. Choices are "classifier" or "dataset". Defaults to "dataset". + adv_weight (float, optional): Adversarial weight. Defaults to 1.0. + weight_decay (float): Weight decay + max_epochs (int, optional): Maximum number of epochs. Defaults to 500. + embed_mode (str, optional): Embedding mode. Choices are "blind", "diff", "concat". Defaults to "blind". + norm (str, optional): Normalization mode. Choices are "none" or "L2". Defaults to None. + optimize_discriminator_only (bool, optional): Optimize discriminator only. Defaults to False. + """ + super().__init__() + self.save_hyperparameters(ignore="encoder") + self.automatic_optimization = False + + # Base model f(.) + self.encoder = encoder + + if embed_mode == "blind": + input_dim = self.encoder.embed_dim * 2 + elif embed_mode == "diff": + input_dim = self.encoder.embed_dim * 2 + elif embed_mode == "concat": + input_dim = 2 * self.encoder.embed_dim * 2 + else: + raise ValueError(f"Invalid embed_mode {embed_mode}") + + self.instance_estimator = torch.nn.Sequential( + torch.nn.Linear(input_dim, 2 * input_dim), + torch.nn.ReLU(inplace=True), + torch.nn.Linear(2 * input_dim, num_instances), + ) + + print("num_presets", num_presets) + + if num_presets > 0: + self.preset_estimator = torch.nn.Sequential( + torch.nn.Linear(input_dim + num_instances, 2 * input_dim), + torch.nn.ReLU(inplace=True), + torch.nn.Linear(2 * input_dim, num_presets), + ) + + if num_adv_classes > 0: + self.discriminator = torch.nn.Sequential( + torch.nn.Linear(input_dim, 2 * input_dim), + torch.nn.ReLU(inplace=True), + torch.nn.Linear(2 * input_dim, num_adv_classes), + ) + + if adv_logits_type == "classifier": + self.extractor = AutoFeatureExtractor.from_pretrained( + "MIT/ast-finetuned-audioset-10-10-0.4593" + ) + self.model = AutoModelForAudioClassification.from_pretrained( + "MIT/ast-finetuned-audioset-10-10-0.4593" + ) + self.hparams.num_adv_classes = len(self.model.config.id2label) + + def configure_optimizers(self): + params_to_optimize = [ + {"params": self.encoder.parameters(), "lr": self.hparams.lr}, + {"params": self.instance_estimator.parameters(), "lr": self.hparams.lr}, + {"params": self.preset_estimator.parameters(), "lr": self.hparams.lr}, + ] + if self.hparams.num_adv_classes > 0: + opt_g = torch.optim.Adam( + params_to_optimize, + ) + opt_d = torch.optim.Adam( + self.discriminator.parameters(), + lr=self.hparams.lr, + ) + return [opt_g, opt_d], [] + else: + optimizer = torch.optim.Adam( + params_to_optimize, + lr=self.hparams.lr, + weight_decay=self.hparams.weight_decay, + ) + return optimizer + + def common_step(self, batch, mode="train"): + inputs, outputs, instance_index, preset_index, tar_index, content_logits = batch + + if mode == "train": + if self.hparams.num_adv_classes > 0: + optimizer_g, optimizer_d = self.optimizers() + else: + optimizer_g = self.optimizers() + + if not self.hparams.optimize_discriminator_only: + self.toggle_optimizer(optimizer_g) + + # encode outputs + output_mid_feats, output_side_feats = self.encoder(outputs) + + if self.hparams.norm == "L2": + outputs_mid_feats = torch.nn.functional.normalize( + output_mid_feats, p=2, dim=-1 + ) + outputs_side_feats = torch.nn.functional.normalize( + output_side_feats, p=2, dim=-1 + ) + elif self.hparams.norm is None: + pass + else: + raise ValueError(f"Invalid norm {self.hparams.norm}") + + # Encode all images + if self.hparams.embed_mode != "blind": + # encoder inputs + inputs_mid_feats, input_side_feats = self.encoder(inputs) + + if self.hparams.norm == "L2": + inputs_mid_feats = torch.nn.functional.normalize( + inputs_mid_feats, p=2, dim=-1 + ) + inputs_side_feats = torch.nn.functional.normalize( + input_side_feats, p=2, dim=-1 + ) + elif self.hparams.norm is None: + pass + else: + raise ValueError(f"Invalid norm {self.hparams.norm}") + + if self.hparams.embed_mode == "diff": + # take the difference between input and outputs + final_mid_feats = inputs_mid_feats - outputs_mid_feats + final_side_feats = input_side_feats - output_side_feats + elif self.hparams.embed_mode == "concat": + # concatenate input and outputs + final_feats = torch.cat( + ( + inputs_mid_feats, + outputs_mid_feats, + inputs_side_feats, + output_side_feats, + ), + dim=-1, + ) + else: + final_feats = torch.cat((outputs_mid_feats, outputs_side_feats), dim=-1) + + # compute loss on effect instance + instance_logits = self.instance_estimator(final_feats) + instance_loss = torch.nn.functional.cross_entropy( + instance_logits, instance_index + ) + loss = instance_loss + + self.log( + f"{mode}_instance_loss", + instance_loss, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + # make predictions for presets + if self.hparams.num_presets > 0: + # concatenate instance_pred and final features + concat_feats = torch.cat((instance_logits, final_feats), dim=-1) + + # predict preset + preset_pred = self.preset_estimator(concat_feats) + preset_loss = torch.nn.functional.cross_entropy(preset_pred, preset_index) + self.log( + f"{mode}_preset_loss", + preset_loss, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + loss += preset_loss + + # compute adversarial loss (if applicable) + if self.hparams.num_adv_classes > 0: + # compute discriminator loss + adv_logits = self.discriminator(final_feats) + + if self.hparams.adv_logits_type == "classifier": + adv_loss = -torch.nn.functional.cross_entropy( + adv_logits, content_logits + ) + elif self.hparams.adv_logits_type == "dataset": + adv_loss = -torch.nn.functional.cross_entropy(adv_logits, tar_index) + else: + raise ValueError( + f"Invalid adv_logits_type {self.hparams.adv_logits_type}" + ) + # loss is negated to perform gradient ascent + self.log( + f"{mode}_adv_loss", + adv_loss, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + loss += adv_loss + + if mode == "train" and not self.hparams.optimize_discriminator_only: + self.manual_backward(loss) + optimizer_g.step() + optimizer_g.zero_grad() + self.untoggle_optimizer(optimizer_g) + + # Logging loss + self.log( + mode + "_loss", + loss, + on_step=True, + on_epoch=True, + prog_bar=True, + sync_dist=True, + ) + + # compute instance prediction accuracy + instance_pred = torch.argmax(instance_logits, dim=-1) + instance_acc = torch.sum(instance_pred == instance_index).float() / len( + instance_index + ) + self.log( + f"{mode}_instance_acc", + instance_acc, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + # compute preset prediction accuracy + if self.hparams.num_presets > 0: + preset_pred = torch.argmax(preset_pred, dim=-1) + preset_acc = torch.sum(preset_pred == preset_index).float() / len( + preset_index + ) + self.log( + f"{mode}_preset_acc", + preset_acc, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + # compute discriminator loss + if self.hparams.num_adv_classes > 0: + # train the discriminator model + if mode == "train": + self.toggle_optimizer(optimizer_d) + + adv_logits = self.discriminator(final_feats.detach()) + if self.hparams.adv_logits_type == "classifier": + d_loss = torch.nn.functional.cross_entropy(adv_logits, content_logits) + elif self.hparams.adv_logits_type == "dataset": + d_loss = torch.nn.functional.cross_entropy(adv_logits, tar_index) + else: + raise ValueError( + f"Invalid adv_logits_type {self.hparams.adv_logits_type}" + ) + # scale the loss by the adversarial weight + d_loss *= self.hparams.adv_weight + + if mode == "train": + self.manual_backward(d_loss) + optimizer_d.step() + optimizer_d.zero_grad() + self.untoggle_optimizer(optimizer_d) + + self.log( + f"{mode}_d_loss", + d_loss, + on_step=True, + on_epoch=True, + sync_dist=True, + prog_bar=True, + ) + + # compute discriminator accuracy + adv_pred = torch.argmax(adv_logits, dim=-1) + adv_acc = torch.sum(adv_pred == tar_index).float() / len(tar_index) + self.log( + f"{mode}_adv_acc", + adv_acc, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + return loss, instance_logits, instance_index + + def training_step(self, batch, batch_idx): + loss, instance_pred, instance_index = self.common_step(batch, mode="train") + return loss + + def validation_step(self, batch, batch_idx): + loss, instance_pred, instance_index = self.common_step(batch, mode="val") + return instance_pred, instance_index diff --git a/utils/st_ito/methods/style.py b/utils/st_ito/methods/style.py new file mode 100644 index 0000000000000000000000000000000000000000..59e176e45691b7cacddf0e2e7f1f400efc56c225 --- /dev/null +++ b/utils/st_ito/methods/style.py @@ -0,0 +1,894 @@ +import json +import torch +import auraloss +import pedalboard +import torchaudio +import numpy as np +import dasp_pytorch +import pytorch_lightning as pl + +from typing import Any, List + + +def denormalize(p: torch.Tensor, min_val: float, max_val: float): + """When p is on (0,1) restore the original range of the parameter values. + + Args: + p (torch.Tensor): [bs, num_params] + min_val (float): minimum value of the parameter + max_val (float): maximum value of the parameter + + Returns: + torch.Tensor: [bs, num_params] + """ + return p * (max_val - min_val) + min_val + + +def apply_gain(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 1, "Number of parameters must be 1" + + gain_db = denormalize(params[:, 0], -48, 48) + + audio = dasp_pytorch.gain(audio, sample_rate, gain_db) + + return audio + + +def apply_distortion(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 1, "Number of parameters must be 1" + + gain_db = denormalize(params[:, 0], -48, 48) + + if chs == 2: + gain_db = gain_db.unsqueeze(1).repeat(1, 2) + + audio = dasp_pytorch.distortion(audio, sample_rate, gain_db) + + return audio + + +def apply_reverb(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 25, "Number of parameters must be 25" + + band0_gain = params[:, 0] + band1_gain = params[:, 1] + band2_gain = params[:, 2] + band3_gain = params[:, 3] + band4_gain = params[:, 4] + band5_gain = params[:, 5] + band6_gain = params[:, 6] + band7_gain = params[:, 7] + band8_gain = params[:, 8] + band9_gain = params[:, 9] + band10_gain = params[:, 10] + band11_gain = params[:, 11] + band0_decay = params[:, 12] + band1_decay = params[:, 13] + band2_decay = params[:, 14] + band3_decay = params[:, 15] + band4_decay = params[:, 16] + band5_decay = params[:, 17] + band6_decay = params[:, 18] + band7_decay = params[:, 19] + band8_decay = params[:, 20] + band9_decay = params[:, 21] + band10_decay = params[:, 22] + band11_decay = params[:, 23] + mix = params[:, 24] + + audio = dasp_pytorch.noise_shaped_reverberation( + audio, + sample_rate, + band0_gain, + band1_gain, + band2_gain, + band3_gain, + band4_gain, + band5_gain, + band6_gain, + band7_gain, + band8_gain, + band9_gain, + band10_gain, + band11_gain, + band0_decay, + band1_decay, + band2_decay, + band3_decay, + band4_decay, + band5_decay, + band6_decay, + band7_decay, + band8_decay, + band9_decay, + band10_decay, + band11_decay, + mix, + ) + + return audio + + +def apply_compressor(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert num_params == 6, "Number of parameters must be 6" + + threshold_db = denormalize(params[:, 0], -60, 0) + ratio = denormalize(params[:, 1], 1, 20) + attack_ms = denormalize(params[:, 2], 0.1, 250) + release_ms = denormalize(params[:, 3], 10, 2000) + knee_db = denormalize(params[:, 4], 1.0, 24.0) + makeup_gain_db = denormalize(params[:, 5], 0, 24.0) + + audio = dasp_pytorch.compressor( + audio, + sample_rate, + threshold_db, + ratio, + attack_ms, + release_ms, + knee_db, + makeup_gain_db, + lookahead_samples=512, + ) + + return audio + + +def apply_parametric_eq(audio: torch.Tensor, params: torch.Tensor, sample_rate: int): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + assert sample_rate >= 44100, "Sample rate must be at least 44100 Hz" + assert num_params == 15, "Number of parameters must be 15" + + # convert params to correct range + low_shelf_gain_db = denormalize(params[:, 0], -18, 18) + low_shelf_cutoff_freq = denormalize(params[:, 1], 20, 20000) + low_shelf_q_factor = denormalize(params[:, 2], 0.1, 10) + + band0_gain_db = denormalize(params[:, 3], -18, 18) + band0_center_freq = denormalize(params[:, 4], 20, 20000) + band0_q_factor = denormalize(params[:, 5], 0.1, 10) + + band1_gain_db = denormalize(params[:, 6], -18, 18) + band1_center_freq = denormalize(params[:, 7], 20, 20000) + band1_q_factor = denormalize(params[:, 8], 0.1, 10) + + band2_gain_db = denormalize(params[:, 9], -18, 18) + band2_center_freq = denormalize(params[:, 10], 20, 20000) + band2_q_factor = denormalize(params[:, 11], 0.1, 10) + + band3_gain_db = denormalize(params[:, 12], -18, 18) + band3_center_freq = denormalize(params[:, 13], 20, 20000) + band3_q_factor = denormalize(params[:, 14], 0.1, 10) + + high_shelf_gain_db = denormalize(params[:, 12], -18, 18) + high_shelf_cutoff_freq = denormalize(params[:, 13], 20, 20000) + high_shelf_q_factor = denormalize(params[:, 14], 0.1, 10) + + audio = dasp_pytorch.parametric_eq( + audio, + sample_rate, + low_shelf_gain_db, + low_shelf_cutoff_freq, + low_shelf_q_factor, + band0_gain_db, + band0_center_freq, + band0_q_factor, + band1_gain_db, + band1_center_freq, + band1_q_factor, + band2_gain_db, + band2_center_freq, + band2_q_factor, + band3_gain_db, + band3_center_freq, + band3_q_factor, + high_shelf_gain_db, + high_shelf_cutoff_freq, + high_shelf_q_factor, + ) + + return audio + + +def apply_simple_autodiff_processor( + audio: torch.Tensor, params: torch.Tensor, sample_rate: int, *args +): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + num_parametric_eq_params = 15 + num_compressor_params = 6 + + assert num_params == num_parametric_eq_params + num_compressor_params + + parametric_eq_params = params[:, :num_parametric_eq_params] + compressor_params = params[:, num_parametric_eq_params:] + + audio = apply_parametric_eq(audio, parametric_eq_params, sample_rate) + audio = apply_compressor(audio, compressor_params, sample_rate) + + return audio + + +def apply_complex_autodiff_processor( + audio: torch.Tensor, params: torch.Tensor, sample_rate: int, *args +): + bs, chs, seq_len = audio.shape + bs, num_params = params.shape + + num_parametric_eq_params = 15 + num_compressor_params = 6 + num_distortion_params = 1 + num_reverb_params = 25 + num_gain_params = 1 + + assert ( + num_params + == num_parametric_eq_params + + num_compressor_params + + num_distortion_params + + num_reverb_params + + num_gain_params + ) + + eq_param_start = 0 + eq_param_end = num_parametric_eq_params + parametric_eq_params = params[:, :num_parametric_eq_params] + + compressor_param_start = eq_param_end + compressor_param_end = compressor_param_start + num_compressor_params + compressor_params = params[:, compressor_param_start:compressor_param_end] + + distortion_param_start = compressor_param_end + distortion_param_end = distortion_param_start + num_distortion_params + distortion_params = params[:, distortion_param_start:distortion_param_end] + + reverb_param_start = distortion_param_end + reverb_param_end = reverb_param_start + num_reverb_params + reverb_params = params[:, reverb_param_start:reverb_param_end] + + gain_param_start = reverb_param_end + gain_param_end = gain_param_start + num_gain_params + gain_params = params[:, gain_param_start:gain_param_end] + + audio = apply_parametric_eq(audio, parametric_eq_params, sample_rate) + audio = apply_compressor(audio, compressor_params, sample_rate) + audio = apply_distortion(audio, distortion_params, sample_rate) + audio = apply_reverb(audio, reverb_params, sample_rate) + audio = apply_gain(audio, gain_params, sample_rate) + + return audio + + +def parameter_tensor_to_dict(params: torch.Tensor): + pass + + +def rademacher(size): + """Generates random samples from a Rademacher distribution +-1 + + Args: + size (int): Number of samples. + + Returns: + torch.Tensor: Tensor of shape (size,). + + """ + m = torch.distributions.binomial.Binomial(1, 0.5) + x = m.sample(size) + x[x == 0] = -1 + return x + + +class SPSAProcessor(torch.autograd.Function): + @staticmethod + def forward( + ctx, + audio: torch.Tensor, + params: torch.Tensor, + sample_rate: int, + plugins: List[dict], + epsilon: float = 0.01, + ): + ctx.save_for_backward(audio.cpu(), params.cpu()) + ctx.sample_rate = sample_rate + ctx.plugins = plugins + ctx.epsilon = epsilon + + # process audio + output = process_audio( + audio.detach().cpu(), + params.detach().cpu(), + sample_rate, + plugins, + ) + + # precompute the random perturbation + delta_k = rademacher(params.shape) + ctx.delta_k = delta_k + + # run the processor with the positive perturbation + params_pos = (params.cpu() + epsilon * delta_k).clamp(0, 1) + output_pos = process_audio( + audio.detach().cpu(), params_pos, sample_rate, plugins + ) + ctx.output_pos = output_pos + + # run the processor with the negative perturbation + params_neg = (params.cpu() - epsilon * delta_k).clamp(0, 1) + output_neg = process_audio( + audio.detach().cpu(), params_neg, sample_rate, plugins + ) + ctx.output_neg = output_neg + + return output.type_as(audio) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + audio, params = ctx.saved_tensors + plugins = ctx.plugins + sample_rate = ctx.sample_rate + epsilon = ctx.epsilon + output_pos = ctx.output_pos + output_neg = ctx.output_neg + delta_k = ctx.delta_k + + grads_input = None + grads_params = None + + needs_audio_grad = ctx.needs_input_grad[0] + needs_param_grad = ctx.needs_input_grad[1] + + if needs_audio_grad: + raise NotImplementedError("Input gradient not implemented.") + + # only estimate grads for parameters + if needs_param_grad: + grad_param = (output_pos - output_neg).mean(dim=-1) + + params_list = [] + # compute the gradient by iterating over each parameter + for pidx in range(params.shape[1]): + grad_p = grad_param / (2 * epsilon * delta_k[:, pidx : pidx + 1]) + grad_p_sum = ( + (grad_output * grad_p.type_as(grad_output).unsqueeze(-1)) + .sum(dim=-1) + .sum(dim=-1) + ) + params_list.append(grad_p_sum) + + grads_params = torch.stack(params_list, dim=-1) + + return grads_input, grads_params, None, None + + +def process_audio( + input_audio: torch.Tensor, + params: torch.Tensor, + sample_rate: int, + plugins: dict, +): + """Process audio with plugins and provided parameters on [0, 1]. + + Args: + input_audio (torch.Tensor): Audio tensor of shape (batch_size, 1, num_samples) + params (torch.Tensor): Parameter tensor of shape (batch_size, num_params) + sample_rate (int): Sample rate + plugins (dict): Dictionary of plugins + + Returns: + torch.Tensor: Processed audio tensor of shape (batch_size, 1, num_samples) + + """ + bs, chs, seq_len = input_audio.shape + + assert chs == 1, "Only mono audio is supported." + + input_audio = input_audio[:, 0:1, :] # convert to mono + + # move to cpu + input_audio_cpu = input_audio.detach().cpu().numpy() + params_cpu = params.detach().cpu() + + # allocate output + output_audio_numpy = np.zeros((bs, seq_len)) + + # iterate over batch + for bidx in range(bs): + pidx = 0 # reset parameter index + x = input_audio_cpu[bidx, :] # get the audio for this batch + # set the plugin parameters + for plugin_name, plugin in plugins.items(): + for name, parameter in plugin["instance"].parameters.items(): + if name in plugin["fixed_parameters"]: + parameter.raw_value = plugin["fixed_parameters"][name] + pidx += 1 + else: + parameter.raw_value = params_cpu[bidx, pidx].item() + pidx += 1 + + # double if stereo + if plugin["num_channels"] == 2: + x = np.concatenate((x, x), axis=0) + + # process audio + x = plugin["instance"].process(x, sample_rate=sample_rate) + + # convert to mono + if plugin["num_channels"] == 2: + x = x[0:1, :] + + output_audio_numpy[bidx, :] = x[0, :] # store the output + + output_audio_cpu = torch.from_numpy(output_audio_numpy) # convert back to tensor + output_audio = output_audio_cpu.type_as(input_audio) # move back to device + output_audio = output_audio.unsqueeze(1) # add channel dim back + + return output_audio + + +class ParameterRegressor(torch.nn.Module): + def __init__(self, input_dim: int, num_params: int) -> None: + super().__init__() + self.input_dim = input_dim + self.num_params = num_params + self.net = torch.nn.Sequential( + torch.nn.Linear(input_dim, 2 * input_dim), + torch.nn.ReLU(inplace=True), + torch.nn.Linear(2 * input_dim, num_params), + torch.nn.Sigmoid(), + ) + + def forward(self, embed: torch.Tensor) -> None: + return self.net(embed) + + +class ParameterClassifier(torch.nn.Module): + def __init__( + self, + input_dim: int, + num_params: int, + hidden_dim: int = 256, + num_bins: int = 64, + ) -> None: + super().__init__() + self.input_dim = input_dim + self.num_params = num_params + self.hidden_dim = hidden_dim + self.num_bins = num_bins + self.nets = torch.nn.ModuleList() + param_vals = torch.linspace(0, 1, num_bins) + self.register_buffer("param_vals", param_vals) + for _ in range(num_params): + self.nets.append( + torch.nn.Sequential( + torch.nn.Linear(input_dim, hidden_dim), + torch.nn.ReLU(inplace=True), + torch.nn.Linear(hidden_dim, num_bins), + ) + ) + + def params_to_onehot(self, params: torch.Tensor) -> torch.Tensor: + params_onehot = torch.zeros( + params.shape[0], self.num_params, self.num_bins + ).type_as(params) + for pidx in range(self.num_params): + param = params[:, pidx] + bin_idx = torch.searchsorted(self.param_vals, param) + params_onehot[:, pidx, bin_idx] = 1.0 + return params_onehot + + def params_to_index(self, params: torch.Tensor) -> torch.Tensor: + params_index = torch.zeros(params.shape[0], self.num_params).type_as(params) + for pidx in range(self.num_params): + param = params[:, pidx] + bin_idx = torch.searchsorted(self.param_vals, param) + params_index[:, pidx] = bin_idx + return params_index.long() + + def index_to_params(self, params_index: torch.Tensor) -> torch.Tensor: + params = torch.zeros(params_index.shape[0], self.num_params) + for pidx in range(self.num_params): + bin_idx = params_index[:, pidx] + param_val = torch.tensor([self.param_vals[bidx] for bidx in bin_idx]) + params[:, pidx] = param_val + return params + + def logits_to_params(self, param_logits: torch.Tensor) -> torch.Tensor: + """Convert logits to parameters. + + Args: + param_logits (torch.Tensor): Tensor of shape (batch_size, num_params, num_bins) + + Returns: + torch.Tensor: Tensor of shape (batch_size, num_params) + """ + params = torch.zeros(param_logits.shape[0], self.num_params) + for pidx in range(self.num_params): + bin_idx = torch.argmax(param_logits[:, pidx, :], dim=-1) + param_val = torch.tensor([self.param_vals[bidx] for bidx in bin_idx]) + params[:, pidx] = param_val + return params + + def forward(self, embed: torch.Tensor) -> None: + params = torch.zeros(embed.shape[0], self.num_params) + params_logits = torch.zeros( + embed.shape[0], self.num_params, self.num_bins + ).type_as(embed) + for pidx, net in enumerate(self.nets): + param_logits = net(embed) # make prediction + params_logits[:, pidx, :] = param_logits + + # convert logits to parameters + params = self.logits_to_params(params_logits) + + return params, params_logits + + +class StyleTransferSystem(pl.LightningModule): + def __init__( + self, + vst_json: str, + encoder: torch.nn.Module, + lr: float, + analysis_length: int = 131072, + weight_decay: float = 0.001, + max_epochs: int = 250, + loss_type: str = "parameter-regression", + grad_type: str = "autodiff", + autodiff_processor: str = "simple", + on_the_fly: bool = False, + split_section: bool = False, + num_bins: int = 64, + ): + """ + + Args: + vst_json (str): JSON file specifying the VST effect chain. + encoder (torch.nn.Module): Encoder model + lr (float): Learning rate + analysis_length (int, optional): Length of audio. Defaults to 131072. + weight_decay (float): Weight decay + max_epochs (int, optional): Maximum number of epochs. Defaults to 500. + loss_type (str, optional): Loss type. Choices are "parameter-regression", "parameter-classification", or "audio". Defaults to "parameter". + grad_type (str, optional): Gradient type. Choices are "spsa" or "autodiff". Defaults to "autodiff". + autodiff_processor (str, optional): Autodiff processor. Choices are "simple" or "complex". Defaults to "simple". + on_the_fly (bool, optional): Whether to use on-the-fly processing. Defaults to False. + split_section (bool, optional): Whether to split the audio into different sections. Defaults to False. + """ + super().__init__() + self.save_hyperparameters(ignore="encoder") + + # load the json file specifying the VST effect chain + with open(vst_json, "r") as fp: + plugins = json.load(fp) + + if grad_type == "autodiff" and loss_type == "audio": + print( + "Skipping loading of VSTs for audio loss when using autodiff effects." + ) + if self.hparams.autodiff_processor == "simple": + total_num_params = 21 + elif self.hparams.autodiff_processor == "complex": + total_num_params = 48 + else: + raise ValueError( + f"Invalid autodiff_processor {self.hparams.autodiff_processor}" + ) + else: + total_num_params = 0 + for plugin_name, plugin in plugins.items(): + plugin_instance = pedalboard.load_plugin(plugin["vst_filepath"]) + num_params = 0 + for name, parameter in plugin_instance.parameters.items(): + num_params += 1 + print(f"{plugin_name}: {name} = {parameter.raw_value}") + print() + + plugin["num_params"] = num_params + plugin["instance"] = plugin_instance + total_num_params += num_params + + self.plugins = plugins + self.total_num_params = total_num_params + self.sample_rate = 48000 + + # use multiresolution stft loss + self.mrstft_loss = auraloss.freq.MultiResolutionSTFTLoss() + + # Base model f(.) + self.encoder = encoder + input_dim = self.encoder.embed_dim * 4 + + if self.hparams.loss_type == "parameter-regression": + self.parameter_estimator = ParameterRegressor(input_dim, total_num_params) + self.process_fn = process_audio + elif self.hparams.loss_type == "parameter-classification": + self.parameter_estimator = ParameterClassifier( + input_dim, total_num_params, num_bins=self.hparams.num_bins + ) + self.process_fn = process_audio + elif self.hparams.loss_type == "audio" and self.hparams.grad_type == "spsa": + self.parameter_estimator = ParameterRegressor(input_dim, total_num_params) + self.process_fn = SPSAProcessor.apply + elif self.hparams.loss_type == "audio" and self.hparams.grad_type == "autodiff": + self.parameter_estimator = ParameterRegressor(input_dim, total_num_params) + if self.hparams.autodiff_processor == "simple": + self.process_fn = apply_simple_autodiff_processor + elif self.hparams.autodiff_processor == "complex": + self.process_fn = apply_complex_autodiff_processor + else: + raise ValueError( + f"Invalid autodiff_processor {self.hparams.autodiff_processor}" + ) + else: + raise ValueError(f"Invalid loss_type {self.hparams.loss_type}") + + def configure_optimizers(self): + optimizer = torch.optim.AdamW( + self.parameters(), + lr=self.hparams.lr, + weight_decay=self.hparams.weight_decay, + ) + ms1 = int(self.hparams.max_epochs * 0.8) + ms2 = int(self.hparams.max_epochs * 0.95) + print( + "Learning rate schedule:", + f"0 {self.hparams.lr:0.2e} -> ", + f"{ms1} {self.hparams.lr*0.1:0.2e} -> ", + f"{ms2} {self.hparams.lr*0.01:0.2e}", + ) + lr_scheduler = torch.optim.lr_scheduler.MultiStepLR( + optimizer, + milestones=[ms1, ms2], + gamma=0.1, + ) + return [optimizer], [lr_scheduler] + + def predict_params(self, input_audio: torch.Tensor, target_audio: torch.Tensor): + bs, chs, seq_len = target_audio.shape + assert chs == 2, "Target audio must be stereo." + bs, chs, seq_len = input_audio.shape + assert chs == 2, "Input audio must be stereo." + + start_idx = (seq_len - self.hparams.analysis_length) // 2 + end_idx = start_idx + self.hparams.analysis_length + + # crop audio to proper length + if input_audio.shape[-1] > self.hparams.analysis_length: + input_audio = input_audio[:, :, start_idx:end_idx] + elif target_audio.shape[-1] > self.hparams.analysis_length: + target_audio = target_audio[:, :, start_idx:end_idx] + + input_mid_embeds, input_side_embeds = self.encoder(input_audio) + target_mid_embeds, target_side_embeds = self.encoder(target_audio) + + # concatenate input and target embeddings + final_feats = torch.cat( + ( + input_mid_embeds, + input_side_embeds, + target_mid_embeds, + target_side_embeds, + ), + dim=-1, + ) + + # predict parameters + if self.hparams.loss_type == "parameter-regression": + params = self.parameter_estimator(final_feats) + params_logits = None + elif self.hparams.loss_type == "parameter-classification": + params, params_logits = self.parameter_estimator(final_feats) + elif self.hparams.loss_type == "audio": + params = self.parameter_estimator(final_feats) + params_logits = None + + return params, params_logits + + def forward( + self, + input_audio: torch.Tensor, + target_audio: torch.Tensor, + sample_rate: int = 48000, + render_audio: bool = False, + ): + bs, chs, seq_len = target_audio.size() + assert chs == 2, "Target audio must be stereo." + bs, chs, seq_len = input_audio.size() + assert chs == 2, "Input audio must be stereo." + + params, params_logits = self.predict_params(input_audio.clone(), target_audio) + + if render_audio: + output_audio = self.process_fn( + input_audio, params, sample_rate, self.plugins + ) + else: + output_audio = torch.zeros_like(input_audio) + + return output_audio, params, params_logits + + def common_step(self, batch, batch_idx: int, mode="train"): + input_audio, target_audio, target_params = batch + bs, chs, seq_len = input_audio.shape + + # peak normalize input + input_audio /= input_audio.abs().max().clamp(1e-8) + + # apply random gain to input + input_gain_db = -(torch.rand(bs) * 12).type_as(input_audio) + input_gain_lin = 10 ** (input_gain_db / 20) + input_audio *= input_gain_lin.view(bs, 1, 1) + + if self.hparams.on_the_fly: + with torch.no_grad(): + target_params = torch.rand(bs, self.total_num_params).type_as( + input_audio + ) + # hard code the gain parameter (the last one) to 0.5 (0 dB) + target_params[:, -1] = 0.5 + + # sometimes randomly bypass the reverb by setting mix to 0 + reverb_mix_params = torch.where(torch.rand(bs) > 0.5, 0.0, 1.0).type_as( + target_params + ) + target_params[:, -2] *= reverb_mix_params + + # apply random gain to input audio + input_audio /= input_audio.abs().max().clamp(1e-8) + gain_db = -(torch.rand(bs) * 24).type_as(input_audio) + gain_lin = 10 ** (gain_db / 20) + input_audio *= gain_lin.view(bs, 1, 1) + + target_audio = self.process_fn( + input_audio, target_params, self.sample_rate, self.plugins + ) + + # peak normalize within -48 dB + peaks = torch.max(torch.abs(target_audio), dim=-1).values + gain_lin = 1 / peaks + gain_db = 20 * torch.log10(gain_lin) + gain_db = gain_db.clamp(-48, 48) + gain_lin = 10 ** (gain_db / 20) + gain_lin = gain_lin.unsqueeze(-1) + target_audio *= gain_lin + + if input_audio.shape[1] == 1: + input_audio = torch.cat((input_audio, input_audio), dim=1) + + if target_audio.shape[1] == 1: + target_audio = torch.cat((target_audio, target_audio), dim=1) + + target_gain_db = -(torch.rand(bs) * 12).type_as(target_audio) + target_gain_lin = 10 ** (target_gain_db / 20) + target_audio *= target_gain_lin.view(bs, 1, 1) + + # section splitting augmentation + if self.hparams.split_section: + # split into A and B sections + input_audio_A = input_audio[:, :, : seq_len // 2] + input_audio_B = input_audio[:, :, seq_len // 2 :] + + target_audio_A = target_audio[:, :, : seq_len // 2] + target_audio_B = target_audio[:, :, seq_len // 2 :] + else: + input_audio_A = input_audio + input_audio_B = input_audio + + target_audio_A = target_audio + target_audio_B = target_audio + + # only render audio if needed + if self.hparams.loss_type == "audio": + render_audio = True + elif ( + self.hparams.loss_type == "parameter-regression" + or self.hparams.loss_type == "parameter-classification" + ) and mode == "val": + render_audio = True + else: + render_audio = False + + # run the model and get predicted parameters + output_audio_A, params, params_logits = self( + input_audio_A, + target_audio_B, + self.sample_rate, + render_audio=render_audio, + ) + + loss = 0 + + if self.hparams.loss_type == "audio": + audio_loss = self.mrstft_loss(output_audio_A, target_audio_A) + loss += audio_loss + + self.log( + f"{mode}_audio_loss", + audio_loss, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + if self.hparams.grad_type == "spsa": + param_loss = torch.nn.functional.mse_loss(params, target_params) + else: + param_loss = 0 + + elif self.hparams.loss_type == "parameter-regression": + param_loss = torch.nn.functional.mse_loss(params, target_params) + loss += param_loss + elif self.hparams.loss_type == "parameter-classification": + # convert target params to one-hot + target_params_index = self.parameter_estimator.params_to_index( + target_params + ) + # cross entropy loss expects shape: bs, num_bins, num_params + params_logits = params_logits.permute(0, 2, 1) + param_loss = torch.nn.functional.cross_entropy( + params_logits, target_params_index + ) + loss += param_loss + + if self.hparams.loss_type != "audio" and mode == "val": + # compute audio loss + audio_loss = self.mrstft_loss(output_audio_A, target_audio_A) + self.log( + f"{mode}_audio_loss", + audio_loss, + on_step=False, + on_epoch=True, + sync_dist=True, + ) + + self.log( + f"{mode}_param_loss", + param_loss, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + self.log( + f"{mode}_loss", + loss, + on_step=True, + on_epoch=True, + sync_dist=True, + ) + + # store in dict and move to cpu + data_dict = { + "input_audio_A": input_audio_A.detach().cpu(), + "input_audio_B": input_audio_B.detach().cpu(), + "target_audio_A": target_audio_A.detach().cpu(), + "target_audio_B": target_audio_B.detach().cpu(), + "output_audio_A": output_audio_A.detach().cpu(), + "params": params.detach().cpu(), + } + + return loss, data_dict + + def training_step(self, batch, batch_idx): + loss, _ = self.common_step(batch, batch_idx, mode="train") + return loss + + def validation_step(self, batch, batch_idx): + loss, data_dict = self.common_step(batch, batch_idx, mode="val") + return data_dict diff --git a/utils/st_ito/models/beats/Tokenizers.py b/utils/st_ito/models/beats/Tokenizers.py new file mode 100644 index 0000000000000000000000000000000000000000..67bf8b62a084969d4fa2444d7e81c249ab7ee1c5 --- /dev/null +++ b/utils/st_ito/models/beats/Tokenizers.py @@ -0,0 +1,199 @@ +# -------------------------------------------------------- +# BEATs: Audio Pre-Training with Acoustic Tokenizers (https://arxiv.org/abs/2212.09058) +# Github source: https://github.com/microsoft/unilm/tree/master/beats +# Copyright (c) 2022 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Based on fairseq code bases +# https://github.com/pytorch/fairseq +# -------------------------------------------------------- + + +import torch +import torch.nn as nn +from torch.nn import LayerNorm +import torchaudio.compliance.kaldi as ta_kaldi + +from .backbone import ( + TransformerEncoder, +) +from .quantizer import ( + NormEMAVectorQuantizer, +) + +import logging +from typing import Optional + +logger = logging.getLogger(__name__) + + +class TokenizersConfig: + def __init__(self, cfg=None): + self.input_patch_size: int = -1 # path size of patch embedding + self.embed_dim: int = 512 # patch embedding dimension + self.conv_bias: bool = False # include bias in conv encoder + + self.encoder_layers: int = 12 # num encoder layers in the transformer + self.encoder_embed_dim: int = 768 # encoder embedding dimension + self.encoder_ffn_embed_dim: int = 3072 # encoder embedding dimension for FFN + self.encoder_attention_heads: int = 12 # num encoder attention heads + self.activation_fn: str = "gelu" # activation function to use + + self.layer_norm_first: bool = False # apply layernorm first in the transformer + self.deep_norm: bool = False # apply deep_norm first in the transformer + + # dropouts + self.dropout: float = 0.1 # dropout probability for the transformer + self.attention_dropout: float = 0.1 # dropout probability for attention weights + self.activation_dropout: float = ( + 0.0 # dropout probability after activation in FFN + ) + self.encoder_layerdrop: float = ( + 0.0 # probability of dropping a tarnsformer layer + ) + self.dropout_input: float = ( + 0.0 # dropout to apply to the input (after feat extr) + ) + + # positional embeddings + self.conv_pos: int = ( + 128 # number of filters for convolutional positional embeddings + ) + self.conv_pos_groups: int = ( + 16 # number of groups for convolutional positional embedding + ) + + # relative position embedding + self.relative_position_embedding: bool = ( + False # apply relative position embedding + ) + self.num_buckets: int = 320 # number of buckets for relative position embedding + self.max_distance: int = ( + 1280 # maximum distance for relative position embedding + ) + self.gru_rel_pos: bool = False # apply gated relative position embedding + + # quantizer + self.quant_n: int = 1024 # codebook number in quantizer + self.quant_dim: int = 256 # codebook dimension in quantizer + + if cfg is not None: + self.update(cfg) + + def update(self, cfg: dict): + self.__dict__.update(cfg) + + +class Tokenizers(nn.Module): + def __init__( + self, + cfg: TokenizersConfig, + ) -> None: + super().__init__() + logger.info(f"Tokenizers Config: {cfg.__dict__}") + + self.cfg = cfg + + self.embed = cfg.embed_dim + self.post_extract_proj = ( + nn.Linear(self.embed, cfg.encoder_embed_dim) + if self.embed != cfg.encoder_embed_dim + else None + ) + + self.input_patch_size = cfg.input_patch_size + self.patch_embedding = nn.Conv2d( + 1, + self.embed, + kernel_size=self.input_patch_size, + stride=self.input_patch_size, + bias=cfg.conv_bias, + ) + + self.dropout_input = nn.Dropout(cfg.dropout_input) + + assert not cfg.deep_norm or not cfg.layer_norm_first + self.encoder = TransformerEncoder(cfg) + self.layer_norm = LayerNorm(self.embed) + + self.quantize = NormEMAVectorQuantizer( + n_embed=cfg.quant_n, + embedding_dim=cfg.quant_dim, + beta=1.0, + kmeans_init=True, + decay=0.99, + ) + self.quant_n = cfg.quant_n + self.quantize_layer = nn.Sequential( + nn.Linear(cfg.encoder_embed_dim, cfg.encoder_embed_dim), + nn.Tanh(), + nn.Linear(cfg.encoder_embed_dim, cfg.quant_dim), # for quantize + ) + + def forward_padding_mask( + self, + features: torch.Tensor, + padding_mask: torch.Tensor, + ) -> torch.Tensor: + extra = padding_mask.size(1) % features.size(1) + if extra > 0: + padding_mask = padding_mask[:, :-extra] + padding_mask = padding_mask.view(padding_mask.size(0), features.size(1), -1) + padding_mask = padding_mask.all(-1) + return padding_mask + + def preprocess( + self, + source: torch.Tensor, + fbank_mean: float = 15.41663, + fbank_std: float = 6.55582, + ) -> torch.Tensor: + fbanks = [] + for waveform in source: + waveform = waveform.unsqueeze(0) * 2**15 + fbank = ta_kaldi.fbank( + waveform, + num_mel_bins=128, + sample_frequency=16000, + frame_length=25, + frame_shift=10, + ) + fbanks.append(fbank) + fbank = torch.stack(fbanks, dim=0) + fbank = (fbank - fbank_mean) / (2 * fbank_std) + return fbank + + def extract_labels( + self, + source: torch.Tensor, + padding_mask: Optional[torch.Tensor] = None, + fbank_mean: float = 15.41663, + fbank_std: float = 6.55582, + ): + fbank = self.preprocess(source, fbank_mean=fbank_mean, fbank_std=fbank_std) + + if padding_mask is not None: + padding_mask = self.forward_padding_mask(fbank, padding_mask) + + fbank = fbank.unsqueeze(1) + features = self.patch_embedding(fbank) + features = features.reshape(features.shape[0], features.shape[1], -1) + features = features.transpose(1, 2) + features = self.layer_norm(features) + + if padding_mask is not None: + padding_mask = self.forward_padding_mask(features, padding_mask) + + if self.post_extract_proj is not None: + features = self.post_extract_proj(features) + + x = self.dropout_input(features) + + x, layer_results = self.encoder( + x, + padding_mask=padding_mask, + ) + + quantize_input = self.quantize_layer(x) + quantize_feature, embed_loss, embed_ind = self.quantize(quantize_input) + + return embed_ind diff --git a/utils/st_ito/models/beats/backbone.py b/utils/st_ito/models/beats/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..92ece77b6c2ed959c2f7f83dd66893040200f954 --- /dev/null +++ b/utils/st_ito/models/beats/backbone.py @@ -0,0 +1,813 @@ +# -------------------------------------------------------- +# BEATs: Audio Pre-Training with Acoustic Tokenizers (https://arxiv.org/abs/2212.09058) +# Github source: https://github.com/microsoft/unilm/tree/master/beats +# Copyright (c) 2022 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Based on fairseq code bases +# https://github.com/pytorch/fairseq +# -------------------------------------------------------- + +import math +import numpy as np +from typing import Dict, Optional, Tuple +import torch +from torch import Tensor, nn +import torch.nn.functional as F +from torch.nn import LayerNorm, Parameter +from .modules import ( + GradMultiply, + SamePad, + get_activation_fn, + GLU_Linear, + quant_noise, +) + + +class TransformerEncoder(nn.Module): + def __init__(self, args): + super().__init__() + + self.dropout = args.dropout + self.embedding_dim = args.encoder_embed_dim + + self.pos_conv = nn.Conv1d( + self.embedding_dim, + self.embedding_dim, + kernel_size=args.conv_pos, + padding=args.conv_pos // 2, + groups=args.conv_pos_groups, + ) + dropout = 0 + std = math.sqrt((4 * (1.0 - dropout)) / (args.conv_pos * self.embedding_dim)) + nn.init.normal_(self.pos_conv.weight, mean=0, std=std) + nn.init.constant_(self.pos_conv.bias, 0) + + self.pos_conv = nn.utils.weight_norm(self.pos_conv, name="weight", dim=2) + self.pos_conv = nn.Sequential(self.pos_conv, SamePad(args.conv_pos), nn.GELU()) + + if hasattr(args, "relative_position_embedding"): + self.relative_position_embedding = args.relative_position_embedding + self.num_buckets = args.num_buckets + self.max_distance = args.max_distance + else: + self.relative_position_embedding = False + self.num_buckets = 0 + self.max_distance = 0 + + self.layers = nn.ModuleList( + [ + TransformerSentenceEncoderLayer( + embedding_dim=self.embedding_dim, + ffn_embedding_dim=args.encoder_ffn_embed_dim, + num_attention_heads=args.encoder_attention_heads, + dropout=self.dropout, + attention_dropout=args.attention_dropout, + activation_dropout=args.activation_dropout, + activation_fn=args.activation_fn, + layer_norm_first=args.layer_norm_first, + deep_norm=args.deep_norm, + has_relative_attention_bias=self.relative_position_embedding, + num_buckets=self.num_buckets, + max_distance=self.max_distance, + gru_rel_pos=args.gru_rel_pos, + encoder_layers=args.encoder_layers, + ) + for i in range(args.encoder_layers) + ] + ) + if self.relative_position_embedding: + for i in range(1, args.encoder_layers): + del self.layers[i].self_attn.relative_attention_bias + self.layers[i].self_attn.relative_attention_bias = self.layers[ + 0 + ].self_attn.relative_attention_bias + + self.layer_norm_first = args.layer_norm_first + self.layer_norm = LayerNorm(self.embedding_dim) + self.layerdrop = args.encoder_layerdrop + + self.apply(init_bert_params) + + if args.deep_norm: + deep_norm_beta = math.pow(8 * args.encoder_layers, -1 / 4) + for i in range(args.encoder_layers): + nn.init.xavier_normal_(self.layers[i].self_attn.k_proj.weight, gain=1) + nn.init.xavier_normal_( + self.layers[i].self_attn.v_proj.weight, gain=deep_norm_beta + ) + nn.init.xavier_normal_(self.layers[i].self_attn.q_proj.weight, gain=1) + nn.init.xavier_normal_( + self.layers[i].self_attn.out_proj.weight, gain=deep_norm_beta + ) + nn.init.xavier_normal_(self.layers[i].fc1.weight, gain=deep_norm_beta) + nn.init.xavier_normal_(self.layers[i].fc2.weight, gain=deep_norm_beta) + + self.layer_wise_gradient_decay_ratio = getattr( + args, "layer_wise_gradient_decay_ratio", 1 + ) + + def forward(self, x, padding_mask=None, layer=None): + x, layer_results = self.extract_features(x, padding_mask, layer) + + if self.layer_norm_first and layer is None: + x = self.layer_norm(x) + + return x, layer_results + + def extract_features(self, x, padding_mask=None, tgt_layer=None): + + if padding_mask is not None: + x[padding_mask] = 0 + + x_conv = self.pos_conv(x.transpose(1, 2)) + x_conv = x_conv.transpose(1, 2) + x = x + x_conv + + if not self.layer_norm_first: + x = self.layer_norm(x) + + x = F.dropout(x, p=self.dropout, training=self.training) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + layer_results = [] + z = None + if tgt_layer is not None: + layer_results.append((x, z)) + r = None + pos_bias = None + for i, layer in enumerate(self.layers): + if self.layer_wise_gradient_decay_ratio != 1.0: + x = GradMultiply.apply(x, self.layer_wise_gradient_decay_ratio) + dropout_probability = np.random.random() + if not self.training or (dropout_probability > self.layerdrop): + x, z, pos_bias = layer( + x, + self_attn_padding_mask=padding_mask, + need_weights=False, + pos_bias=pos_bias, + ) + if tgt_layer is not None: + layer_results.append((x, z)) + if i == tgt_layer: + r = x + break + + if r is not None: + x = r + + # T x B x C -> B x T x C + x = x.transpose(0, 1) + + return x, layer_results + + +class TransformerSentenceEncoderLayer(nn.Module): + def __init__( + self, + embedding_dim: float = 768, + ffn_embedding_dim: float = 3072, + num_attention_heads: float = 8, + dropout: float = 0.1, + attention_dropout: float = 0.1, + activation_dropout: float = 0.1, + activation_fn: str = "relu", + layer_norm_first: bool = False, + deep_norm: bool = False, + has_relative_attention_bias: bool = False, + num_buckets: int = 0, + max_distance: int = 0, + rescale_init: bool = False, + gru_rel_pos: bool = False, + encoder_layers: int = 0, + ) -> None: + + super().__init__() + self.embedding_dim = embedding_dim + self.dropout = dropout + self.activation_dropout = activation_dropout + + self.activation_name = activation_fn + self.activation_fn = get_activation_fn(activation_fn) + self.self_attn = MultiheadAttention( + self.embedding_dim, + num_attention_heads, + dropout=attention_dropout, + self_attention=True, + has_relative_attention_bias=has_relative_attention_bias, + num_buckets=num_buckets, + max_distance=max_distance, + rescale_init=rescale_init, + gru_rel_pos=gru_rel_pos, + ) + + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(self.activation_dropout) + self.dropout3 = nn.Dropout(dropout) + + self.layer_norm_first = layer_norm_first + + self.self_attn_layer_norm = LayerNorm(self.embedding_dim) + + if self.activation_name == "glu": + self.fc1 = GLU_Linear(self.embedding_dim, ffn_embedding_dim, "swish") + else: + self.fc1 = nn.Linear(self.embedding_dim, ffn_embedding_dim) + self.fc2 = nn.Linear(ffn_embedding_dim, self.embedding_dim) + + self.final_layer_norm = LayerNorm(self.embedding_dim) + + self.deep_norm = deep_norm + if self.deep_norm: + self.deep_norm_alpha = math.pow(2 * encoder_layers, 1 / 4) + else: + self.deep_norm_alpha = 1 + + def forward( + self, + x: torch.Tensor, + self_attn_mask: torch.Tensor = None, + self_attn_padding_mask: torch.Tensor = None, + need_weights: bool = False, + pos_bias=None, + ): + residual = x + + if self.layer_norm_first: + x = self.self_attn_layer_norm(x) + x, attn, pos_bias = self.self_attn( + query=x, + key=x, + value=x, + key_padding_mask=self_attn_padding_mask, + need_weights=False, + attn_mask=self_attn_mask, + position_bias=pos_bias, + ) + x = self.dropout1(x) + x = residual + x + + residual = x + x = self.final_layer_norm(x) + if self.activation_name == "glu": + x = self.fc1(x) + else: + x = self.activation_fn(self.fc1(x)) + x = self.dropout2(x) + x = self.fc2(x) + x = self.dropout3(x) + x = residual + x + else: + x, attn, pos_bias = self.self_attn( + query=x, + key=x, + value=x, + key_padding_mask=self_attn_padding_mask, + need_weights=need_weights, + attn_mask=self_attn_mask, + position_bias=pos_bias, + ) + + x = self.dropout1(x) + x = residual * self.deep_norm_alpha + x + + x = self.self_attn_layer_norm(x) + + residual = x + if self.activation_name == "glu": + x = self.fc1(x) + else: + x = self.activation_fn(self.fc1(x)) + x = self.dropout2(x) + x = self.fc2(x) + x = self.dropout3(x) + x = residual * self.deep_norm_alpha + x + x = self.final_layer_norm(x) + + return x, attn, pos_bias + + +class MultiheadAttention(nn.Module): + """Multi-headed attention. + + See "Attention Is All You Need" for more details. + """ + + def __init__( + self, + embed_dim, + num_heads, + kdim=None, + vdim=None, + dropout=0.0, + bias=True, + add_bias_kv=False, + add_zero_attn=False, + self_attention=False, + encoder_decoder_attention=False, + q_noise=0.0, + qn_block_size=8, + has_relative_attention_bias=False, + num_buckets=32, + max_distance=128, + gru_rel_pos=False, + rescale_init=False, + ): + super().__init__() + self.embed_dim = embed_dim + self.kdim = kdim if kdim is not None else embed_dim + self.vdim = vdim if vdim is not None else embed_dim + self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim + + self.num_heads = num_heads + self.dropout_module = nn.Dropout(dropout) + + self.has_relative_attention_bias = has_relative_attention_bias + self.num_buckets = num_buckets + self.max_distance = max_distance + if self.has_relative_attention_bias: + self.relative_attention_bias = nn.Embedding(num_buckets, num_heads) + + self.head_dim = embed_dim // num_heads + self.q_head_dim = self.head_dim + self.k_head_dim = self.head_dim + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" + self.scaling = self.head_dim**-0.5 + + self.self_attention = self_attention + self.encoder_decoder_attention = encoder_decoder_attention + + assert not self.self_attention or self.qkv_same_dim, ( + "Self-attention requires query, key and " "value to be of the same size" + ) + + k_bias = True + if rescale_init: + k_bias = False + + k_embed_dim = embed_dim + q_embed_dim = embed_dim + + self.k_proj = quant_noise( + nn.Linear(self.kdim, k_embed_dim, bias=k_bias), q_noise, qn_block_size + ) + self.v_proj = quant_noise( + nn.Linear(self.vdim, embed_dim, bias=bias), q_noise, qn_block_size + ) + self.q_proj = quant_noise( + nn.Linear(embed_dim, q_embed_dim, bias=bias), q_noise, qn_block_size + ) + + self.out_proj = quant_noise( + nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size + ) + + if add_bias_kv: + self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim)) + self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim)) + else: + self.bias_k = self.bias_v = None + + self.add_zero_attn = add_zero_attn + + self.gru_rel_pos = gru_rel_pos + if self.gru_rel_pos: + self.grep_linear = nn.Linear(self.q_head_dim, 8) + self.grep_a = nn.Parameter(torch.ones(1, num_heads, 1, 1)) + + self.reset_parameters() + + def reset_parameters(self): + if self.qkv_same_dim: + # Empirically observed the convergence to be much better with + # the scaled initialization + nn.init.xavier_uniform_(self.k_proj.weight, gain=1 / math.sqrt(2)) + nn.init.xavier_uniform_(self.v_proj.weight, gain=1 / math.sqrt(2)) + nn.init.xavier_uniform_(self.q_proj.weight, gain=1 / math.sqrt(2)) + else: + nn.init.xavier_uniform_(self.k_proj.weight) + nn.init.xavier_uniform_(self.v_proj.weight) + nn.init.xavier_uniform_(self.q_proj.weight) + + nn.init.xavier_uniform_(self.out_proj.weight) + if self.out_proj.bias is not None: + nn.init.constant_(self.out_proj.bias, 0.0) + if self.bias_k is not None: + nn.init.xavier_normal_(self.bias_k) + if self.bias_v is not None: + nn.init.xavier_normal_(self.bias_v) + if self.has_relative_attention_bias: + nn.init.xavier_normal_(self.relative_attention_bias.weight) + + def _relative_positions_bucket(self, relative_positions, bidirectional=True): + num_buckets = self.num_buckets + max_distance = self.max_distance + relative_buckets = 0 + + if bidirectional: + num_buckets = num_buckets // 2 + relative_buckets += (relative_positions > 0).to(torch.long) * num_buckets + relative_positions = torch.abs(relative_positions) + else: + relative_positions = -torch.min( + relative_positions, torch.zeros_like(relative_positions) + ) + + max_exact = num_buckets // 2 + is_small = relative_positions < max_exact + + relative_postion_if_large = max_exact + ( + torch.log(relative_positions.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(torch.long) + relative_postion_if_large = torch.min( + relative_postion_if_large, + torch.full_like(relative_postion_if_large, num_buckets - 1), + ) + + relative_buckets += torch.where( + is_small, relative_positions, relative_postion_if_large + ) + return relative_buckets + + def compute_bias(self, query_length, key_length): + context_position = torch.arange(query_length, dtype=torch.long)[:, None] + memory_position = torch.arange(key_length, dtype=torch.long)[None, :] + relative_position = memory_position - context_position + relative_position_bucket = self._relative_positions_bucket( + relative_position, bidirectional=True + ) + relative_position_bucket = relative_position_bucket.to( + self.relative_attention_bias.weight.device + ) + values = self.relative_attention_bias(relative_position_bucket) + values = values.permute([2, 0, 1]) + return values + + def forward( + self, + query, + key: Optional[Tensor], + value: Optional[Tensor], + key_padding_mask: Optional[Tensor] = None, + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, + need_weights: bool = True, + static_kv: bool = False, + attn_mask: Optional[Tensor] = None, + before_softmax: bool = False, + need_head_weights: bool = False, + position_bias: Optional[Tensor] = None, + ) -> Tuple[Tensor, Optional[Tensor], Optional[Tensor]]: + """Input shape: Time x Batch x Channel + + Args: + key_padding_mask (ByteTensor, optional): mask to exclude + keys that are pads, of shape `(batch, src_len)`, where + padding elements are indicated by 1s. + need_weights (bool, optional): return the attention weights, + averaged over heads (default: False). + attn_mask (ByteTensor, optional): typically used to + implement causal attention, where the mask prevents the + attention from looking forward in time (default: None). + before_softmax (bool, optional): return the raw attention + weights and values before the attention softmax. + need_head_weights (bool, optional): return the attention + weights for each head. Implies *need_weights*. Default: + return the average attention weights over all heads. + """ + if need_head_weights: + need_weights = True + + is_tpu = query.device.type == "xla" + + tgt_len, bsz, embed_dim = query.size() + src_len = tgt_len + assert embed_dim == self.embed_dim + assert list(query.size()) == [tgt_len, bsz, embed_dim] + if key is not None: + src_len, key_bsz, _ = key.size() + if not torch.jit.is_scripting(): + assert key_bsz == bsz + assert value is not None + assert src_len, bsz == value.shape[:2] + + if self.has_relative_attention_bias and position_bias is None: + position_bias = self.compute_bias(tgt_len, src_len) + position_bias = ( + position_bias.unsqueeze(0) + .repeat(bsz, 1, 1, 1) + .view(bsz * self.num_heads, tgt_len, src_len) + ) + + if incremental_state is not None: + saved_state = self._get_input_buffer(incremental_state) + if saved_state is not None and "prev_key" in saved_state: + # previous time steps are cached - no need to recompute + # key and value if they are static + if static_kv: + assert self.encoder_decoder_attention and not self.self_attention + key = value = None + else: + saved_state = None + + if self.self_attention: + q = self.q_proj(query) + k = self.k_proj(query) + v = self.v_proj(query) + elif self.encoder_decoder_attention: + # encoder-decoder attention + q = self.q_proj(query) + if key is None: + assert value is None + k = v = None + else: + k = self.k_proj(key) + v = self.v_proj(key) + + else: + assert key is not None and value is not None + q = self.q_proj(query) + k = self.k_proj(key) + v = self.v_proj(value) + q *= self.scaling + alpha = 32 + q *= 1 / alpha + + if self.bias_k is not None: + assert self.bias_v is not None + k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)]) + v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)]) + if attn_mask is not None: + attn_mask = torch.cat( + [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1 + ) + if key_padding_mask is not None: + key_padding_mask = torch.cat( + [ + key_padding_mask, + key_padding_mask.new_zeros(key_padding_mask.size(0), 1), + ], + dim=1, + ) + + q = ( + q.contiguous() + .view(tgt_len, bsz * self.num_heads, self.q_head_dim) + .transpose(0, 1) + ) + if k is not None: + k = ( + k.contiguous() + .view(-1, bsz * self.num_heads, self.k_head_dim) + .transpose(0, 1) + ) + if v is not None: + v = ( + v.contiguous() + .view(-1, bsz * self.num_heads, self.head_dim) + .transpose(0, 1) + ) + + if saved_state is not None: + # saved states are stored with shape (bsz, num_heads, seq_len, head_dim) + if "prev_key" in saved_state: + _prev_key = saved_state["prev_key"] + assert _prev_key is not None + prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim) + if static_kv: + k = prev_key + else: + assert k is not None + k = torch.cat([prev_key, k], dim=1) + src_len = k.size(1) + if "prev_value" in saved_state: + _prev_value = saved_state["prev_value"] + assert _prev_value is not None + prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim) + if static_kv: + v = prev_value + else: + assert v is not None + v = torch.cat([prev_value, v], dim=1) + prev_key_padding_mask: Optional[Tensor] = None + if "prev_key_padding_mask" in saved_state: + prev_key_padding_mask = saved_state["prev_key_padding_mask"] + assert k is not None and v is not None + key_padding_mask = MultiheadAttention._append_prev_key_padding_mask( + key_padding_mask=key_padding_mask, + prev_key_padding_mask=prev_key_padding_mask, + batch_size=bsz, + src_len=k.size(1), + static_kv=static_kv, + ) + + saved_state["prev_key"] = k.view(bsz, self.num_heads, -1, self.head_dim) + saved_state["prev_value"] = v.view(bsz, self.num_heads, -1, self.head_dim) + saved_state["prev_key_padding_mask"] = key_padding_mask + # In this branch incremental_state is never None + assert incremental_state is not None + incremental_state = self._set_input_buffer(incremental_state, saved_state) + assert k is not None + assert k.size(1) == src_len + + # This is part of a workaround to get around fork/join parallelism + # not supporting Optional types. + if key_padding_mask is not None and key_padding_mask.dim() == 0: + key_padding_mask = None + + if key_padding_mask is not None: + assert key_padding_mask.size(0) == bsz + assert key_padding_mask.size(1) == src_len + + if self.add_zero_attn: + assert v is not None + src_len += 1 + k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], dim=1) + v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], dim=1) + if attn_mask is not None: + attn_mask = torch.cat( + [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1 + ) + if key_padding_mask is not None: + key_padding_mask = torch.cat( + [ + key_padding_mask, + torch.zeros(key_padding_mask.size(0), 1).type_as( + key_padding_mask + ), + ], + dim=1, + ) + + attn_weights = torch.bmm(q, k.transpose(1, 2)) + attn_weights = ( + attn_weights - attn_weights.max(dim=-1, keepdim=True)[0] + ) * alpha + attn_weights = self.apply_sparse_mask(attn_weights, tgt_len, src_len, bsz) + + assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len] + + if attn_mask is not None: + attn_mask = attn_mask.unsqueeze(0) + attn_weights += attn_mask + + if key_padding_mask is not None: + # don't attend to padding symbols + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + if not is_tpu: + attn_weights = attn_weights.masked_fill( + key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool), + float("-inf"), + ) + else: + attn_weights = attn_weights.transpose(0, 2) + attn_weights = attn_weights.masked_fill(key_padding_mask, float("-inf")) + attn_weights = attn_weights.transpose(0, 2) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if before_softmax: + return attn_weights, v, position_bias + + if position_bias is not None: + attn_mask_rel_pos = position_bias + if self.gru_rel_pos == 1: + query_layer = ( + q.view(bsz, self.num_heads, tgt_len, self.q_head_dim) + * alpha + / self.scaling + ) + _B, _H, _L, __ = query_layer.size() + gate_a, gate_b = torch.sigmoid( + self.grep_linear(query_layer) + .view(_B, _H, _L, 2, 4) + .sum(-1, keepdim=False) + ).chunk(2, dim=-1) + gate_a_1 = gate_a * (gate_b * self.grep_a - 1.0) + 2.0 + attn_mask_rel_pos = ( + gate_a_1.view(bsz * self.num_heads, tgt_len, 1) * position_bias + ) + + attn_mask_rel_pos = attn_mask_rel_pos.view(attn_weights.size()) + + attn_weights = attn_weights + attn_mask_rel_pos + + attn_weights_float = F.softmax(attn_weights, dim=-1) + attn_weights = attn_weights_float.type_as(attn_weights) + attn_probs = self.dropout_module(attn_weights) + + assert v is not None + attn = torch.bmm(attn_probs, v) + assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim] + attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) + attn = self.out_proj(attn) + attn_weights: Optional[Tensor] = None + if need_weights: + attn_weights = attn_weights_float.view( + bsz, self.num_heads, tgt_len, src_len + ).transpose(1, 0) + if not need_head_weights: + # average attention weights over heads + attn_weights = attn_weights.mean(dim=0) + + return attn, attn_weights, position_bias + + @staticmethod + def _append_prev_key_padding_mask( + key_padding_mask: Optional[Tensor], + prev_key_padding_mask: Optional[Tensor], + batch_size: int, + src_len: int, + static_kv: bool, + ) -> Optional[Tensor]: + # saved key padding masks have shape (bsz, seq_len) + if prev_key_padding_mask is not None and static_kv: + new_key_padding_mask = prev_key_padding_mask + elif prev_key_padding_mask is not None and key_padding_mask is not None: + new_key_padding_mask = torch.cat( + [prev_key_padding_mask.float(), key_padding_mask.float()], dim=1 + ) + # During incremental decoding, as the padding token enters and + # leaves the frame, there will be a time when prev or current + # is None + elif prev_key_padding_mask is not None: + if src_len > prev_key_padding_mask.size(1): + filler = torch.zeros( + (batch_size, src_len - prev_key_padding_mask.size(1)), + device=prev_key_padding_mask.device, + ) + new_key_padding_mask = torch.cat( + [prev_key_padding_mask.float(), filler.float()], dim=1 + ) + else: + new_key_padding_mask = prev_key_padding_mask.float() + elif key_padding_mask is not None: + if src_len > key_padding_mask.size(1): + filler = torch.zeros( + (batch_size, src_len - key_padding_mask.size(1)), + device=key_padding_mask.device, + ) + new_key_padding_mask = torch.cat( + [filler.float(), key_padding_mask.float()], dim=1 + ) + else: + new_key_padding_mask = key_padding_mask.float() + else: + new_key_padding_mask = prev_key_padding_mask + return new_key_padding_mask + + def _get_input_buffer( + self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] + ) -> Dict[str, Optional[Tensor]]: + result = self.get_incremental_state(incremental_state, "attn_state") + if result is not None: + return result + else: + empty_result: Dict[str, Optional[Tensor]] = {} + return empty_result + + def _set_input_buffer( + self, + incremental_state: Dict[str, Dict[str, Optional[Tensor]]], + buffer: Dict[str, Optional[Tensor]], + ): + return self.set_incremental_state(incremental_state, "attn_state", buffer) + + def apply_sparse_mask(self, attn_weights, tgt_len: int, src_len: int, bsz: int): + return attn_weights + + +def init_bert_params(module): + """ + Initialize the weights specific to the BERT Model. + This overrides the default initializations depending on the specified arguments. + 1. If normal_init_linear_weights is set then weights of linear + layer will be initialized using the normal distribution and + bais will be set to the specified value. + 2. If normal_init_embed_weights is set then weights of embedding + layer will be initialized using the normal distribution. + 3. If normal_init_proj_weights is set then weights of + in_project_weight for MultiHeadAttention initialized using + the normal distribution (to be validated). + """ + + def normal_(data): + # with FSDP, module params will be on CUDA, so we cast them back to CPU + # so that the RNG is consistent with and without FSDP + data.copy_(data.cpu().normal_(mean=0.0, std=0.02).to(data.device)) + + if isinstance(module, nn.Linear): + normal_(module.weight.data) + if module.bias is not None: + module.bias.data.zero_() + if isinstance(module, nn.Embedding): + normal_(module.weight.data) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + if isinstance(module, MultiheadAttention): + normal_(module.q_proj.weight.data) + normal_(module.k_proj.weight.data) + normal_(module.v_proj.weight.data) diff --git a/utils/st_ito/models/beats/beats.py b/utils/st_ito/models/beats/beats.py new file mode 100644 index 0000000000000000000000000000000000000000..ff4950ab07a6d796e2df413ad68100e0c9b2de23 --- /dev/null +++ b/utils/st_ito/models/beats/beats.py @@ -0,0 +1,206 @@ +# -------------------------------------------------------- +# BEATs: Audio Pre-Training with Acoustic Tokenizers (https://arxiv.org/abs/2212.09058) +# Github source: https://github.com/microsoft/unilm/tree/master/beats +# Copyright (c) 2022 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Based on fairseq code bases +# https://github.com/pytorch/fairseq +# -------------------------------------------------------- + + +import torch +import torch.nn as nn +from torch.nn import LayerNorm +import torchaudio.compliance.kaldi as ta_kaldi + +from .backbone import ( + TransformerEncoder, +) + +import logging +from typing import Optional + +logger = logging.getLogger(__name__) + + +class BEATsConfig: + def __init__(self, cfg=None): + self.input_patch_size: int = -1 # path size of patch embedding + self.embed_dim: int = 512 # patch embedding dimension + self.conv_bias: bool = False # include bias in conv encoder + + self.encoder_layers: int = 12 # num encoder layers in the transformer + self.encoder_embed_dim: int = 768 # encoder embedding dimension + self.encoder_ffn_embed_dim: int = 3072 # encoder embedding dimension for FFN + self.encoder_attention_heads: int = 12 # num encoder attention heads + self.activation_fn: str = "gelu" # activation function to use + + self.layer_wise_gradient_decay_ratio: float = ( + 1.0 # ratio for layer-wise gradient decay + ) + self.layer_norm_first: bool = False # apply layernorm first in the transformer + self.deep_norm: bool = False # apply deep_norm first in the transformer + + # dropouts + self.dropout: float = 0.1 # dropout probability for the transformer + self.attention_dropout: float = 0.1 # dropout probability for attention weights + self.activation_dropout: float = ( + 0.0 # dropout probability after activation in FFN + ) + self.encoder_layerdrop: float = ( + 0.0 # probability of dropping a tarnsformer layer + ) + self.dropout_input: float = ( + 0.0 # dropout to apply to the input (after feat extr) + ) + + # positional embeddings + self.conv_pos: int = ( + 128 # number of filters for convolutional positional embeddings + ) + self.conv_pos_groups: int = ( + 16 # number of groups for convolutional positional embedding + ) + + # relative position embedding + self.relative_position_embedding: bool = ( + False # apply relative position embedding + ) + self.num_buckets: int = 320 # number of buckets for relative position embedding + self.max_distance: int = ( + 1280 # maximum distance for relative position embedding + ) + self.gru_rel_pos: bool = False # apply gated relative position embedding + + # label predictor + self.finetuned_model: bool = False # whether the model is a fine-tuned model. + self.predictor_dropout: float = 0.1 # dropout probability for the predictor + self.predictor_class: int = 527 # target class number for the predictor + + if cfg is not None: + self.update(cfg) + + def update(self, cfg: dict): + self.__dict__.update(cfg) + + +class BEATs(nn.Module): + def __init__( + self, + cfg: BEATsConfig, + ) -> None: + super().__init__() + logger.info(f"BEATs Config: {cfg.__dict__}") + + self.cfg = cfg + + self.embed = cfg.embed_dim + self.post_extract_proj = ( + nn.Linear(self.embed, cfg.encoder_embed_dim) + if self.embed != cfg.encoder_embed_dim + else None + ) + + self.input_patch_size = cfg.input_patch_size + self.patch_embedding = nn.Conv2d( + 1, + self.embed, + kernel_size=self.input_patch_size, + stride=self.input_patch_size, + bias=cfg.conv_bias, + ) + + self.dropout_input = nn.Dropout(cfg.dropout_input) + + assert not cfg.deep_norm or not cfg.layer_norm_first + self.encoder = TransformerEncoder(cfg) + self.layer_norm = LayerNorm(self.embed) + + if cfg.finetuned_model: + self.predictor_dropout = nn.Dropout(cfg.predictor_dropout) + self.predictor = nn.Linear(cfg.encoder_embed_dim, cfg.predictor_class) + else: + self.predictor = None + + def forward_padding_mask( + self, + features: torch.Tensor, + padding_mask: torch.Tensor, + ) -> torch.Tensor: + extra = padding_mask.size(1) % features.size(1) + if extra > 0: + padding_mask = padding_mask[:, :-extra] + padding_mask = padding_mask.view(padding_mask.size(0), features.size(1), -1) + padding_mask = padding_mask.all(-1) + return padding_mask + + def preprocess( + self, + source: torch.Tensor, + fbank_mean: float = 15.41663, + fbank_std: float = 6.55582, + ) -> torch.Tensor: + fbanks = [] + for waveform in source: + waveform = waveform.unsqueeze(0) * 2**15 + fbank = ta_kaldi.fbank( + waveform, + num_mel_bins=128, + sample_frequency=16000, + frame_length=25, + frame_shift=10, + ) + fbanks.append(fbank) + fbank = torch.stack(fbanks, dim=0) + fbank = (fbank - fbank_mean) / (2 * fbank_std) + return fbank + + def extract_features( + self, + source: torch.Tensor, + padding_mask: Optional[torch.Tensor] = None, + fbank_mean: float = 15.41663, + fbank_std: float = 6.55582, + ): + fbank = self.preprocess(source, fbank_mean=fbank_mean, fbank_std=fbank_std) + + if padding_mask is not None: + padding_mask = self.forward_padding_mask(fbank, padding_mask) + + fbank = fbank.unsqueeze(1) + features = self.patch_embedding(fbank) + features = features.reshape(features.shape[0], features.shape[1], -1) + features = features.transpose(1, 2) + features = self.layer_norm(features) + + if padding_mask is not None: + padding_mask = self.forward_padding_mask(features, padding_mask) + + if self.post_extract_proj is not None: + features = self.post_extract_proj(features) + + x = self.dropout_input(features) + + x, layer_results = self.encoder( + x, + padding_mask=padding_mask, + ) + + if self.predictor is not None: + x = self.predictor_dropout(x) + logits = self.predictor(x) + + if padding_mask is not None and padding_mask.any(): + logits[padding_mask] = 0 + logits = logits.sum(dim=1) + logits = logits / (~padding_mask).sum(dim=1).unsqueeze(-1).expand_as( + logits + ) + else: + logits = logits.mean(dim=1) + + lprobs = torch.sigmoid(logits) + + return lprobs, padding_mask + else: + return x, padding_mask diff --git a/utils/st_ito/models/beats/modules.py b/utils/st_ito/models/beats/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..2de74299cc7dd5480feb94d178c4dcf7f9535328 --- /dev/null +++ b/utils/st_ito/models/beats/modules.py @@ -0,0 +1,221 @@ +# -------------------------------------------------------- +# BEATs: Audio Pre-Training with Acoustic Tokenizers (https://arxiv.org/abs/2212.09058) +# Github source: https://github.com/microsoft/unilm/tree/master/beats +# Copyright (c) 2022 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Based on fairseq code bases +# https://github.com/pytorch/fairseq +# -------------------------------------------------------- + +import math +import warnings +import torch +from torch import Tensor, nn +import torch.nn.functional as F + + +class GradMultiply(torch.autograd.Function): + @staticmethod + def forward(ctx, x, scale): + ctx.scale = scale + res = x.new(x) + return res + + @staticmethod + def backward(ctx, grad): + return grad * ctx.scale, None + + +class SamePad(nn.Module): + def __init__(self, kernel_size, causal=False): + super().__init__() + if causal: + self.remove = kernel_size - 1 + else: + self.remove = 1 if kernel_size % 2 == 0 else 0 + + def forward(self, x): + if self.remove > 0: + x = x[:, :, : -self.remove] + return x + + +class Swish(nn.Module): + def __init__(self): + super(Swish, self).__init__() + self.act = torch.nn.Sigmoid() + + def forward(self, x): + return x * self.act(x) + + +class GLU_Linear(nn.Module): + def __init__(self, input_dim, output_dim, glu_type="sigmoid", bias_in_glu=True): + super(GLU_Linear, self).__init__() + + self.glu_type = glu_type + self.output_dim = output_dim + + if glu_type == "sigmoid": + self.glu_act = torch.nn.Sigmoid() + elif glu_type == "swish": + self.glu_act = Swish() + elif glu_type == "relu": + self.glu_act = torch.nn.ReLU() + elif glu_type == "gelu": + self.glu_act = torch.nn.GELU() + + if bias_in_glu: + self.linear = nn.Linear(input_dim, output_dim * 2, True) + else: + self.linear = nn.Linear(input_dim, output_dim * 2, False) + + def forward(self, x): + # to be consistent with GLU_Linear, we assume the input always has the #channel (#dim) in the last dimension of the tensor, so need to switch the dimension first for 1D-Conv case + x = self.linear(x) + + if self.glu_type == "bilinear": + x = ( + x[:, :, 0 : self.output_dim] + * x[:, :, self.output_dim : self.output_dim * 2] + ) + else: + x = x[:, :, 0 : self.output_dim] * self.glu_act( + x[:, :, self.output_dim : self.output_dim * 2] + ) + + return x + + +def gelu_accurate(x): + if not hasattr(gelu_accurate, "_a"): + gelu_accurate._a = math.sqrt(2 / math.pi) + return ( + 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3)))) + ) + + +def gelu(x: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.gelu(x.float()).type_as(x) + + +def get_activation_fn(activation: str): + """Returns the activation function corresponding to `activation`""" + + if activation == "relu": + return F.relu + elif activation == "gelu": + return gelu + elif activation == "gelu_fast": + warnings.warn("--activation-fn=gelu_fast has been renamed to gelu_accurate") + return gelu_accurate + elif activation == "gelu_accurate": + return gelu_accurate + elif activation == "tanh": + return torch.tanh + elif activation == "linear": + return lambda x: x + elif activation == "glu": + return lambda x: x + else: + raise RuntimeError("--activation-fn {} not supported".format(activation)) + + +def quant_noise(module, p, block_size): + """ + Wraps modules and applies quantization noise to the weights for + subsequent quantization with Iterative Product Quantization as + described in "Training with Quantization Noise for Extreme Model Compression" + + Args: + - module: nn.Module + - p: amount of Quantization Noise + - block_size: size of the blocks for subsequent quantization with iPQ + + Remarks: + - Module weights must have the right sizes wrt the block size + - Only Linear, Embedding and Conv2d modules are supported for the moment + - For more detail on how to quantize by blocks with convolutional weights, + see "And the Bit Goes Down: Revisiting the Quantization of Neural Networks" + - We implement the simplest form of noise here as stated in the paper + which consists in randomly dropping blocks + """ + + # if no quantization noise, don't register hook + if p <= 0: + return module + + # supported modules + assert isinstance(module, (nn.Linear, nn.Embedding, nn.Conv2d)) + + # test whether module.weight has the right sizes wrt block_size + is_conv = module.weight.ndim == 4 + + # 2D matrix + if not is_conv: + assert ( + module.weight.size(1) % block_size == 0 + ), "Input features must be a multiple of block sizes" + + # 4D matrix + else: + # 1x1 convolutions + if module.kernel_size == (1, 1): + assert ( + module.in_channels % block_size == 0 + ), "Input channels must be a multiple of block sizes" + # regular convolutions + else: + k = module.kernel_size[0] * module.kernel_size[1] + assert k % block_size == 0, "Kernel size must be a multiple of block size" + + def _forward_pre_hook(mod, input): + # no noise for evaluation + if mod.training: + if not is_conv: + # gather weight and sizes + weight = mod.weight + in_features = weight.size(1) + out_features = weight.size(0) + + # split weight matrix into blocks and randomly drop selected blocks + mask = torch.zeros( + in_features // block_size * out_features, device=weight.device + ) + mask.bernoulli_(p) + mask = mask.repeat_interleave(block_size, -1).view(-1, in_features) + + else: + # gather weight and sizes + weight = mod.weight + in_channels = mod.in_channels + out_channels = mod.out_channels + + # split weight matrix into blocks and randomly drop selected blocks + if mod.kernel_size == (1, 1): + mask = torch.zeros( + int(in_channels // block_size * out_channels), + device=weight.device, + ) + mask.bernoulli_(p) + mask = mask.repeat_interleave(block_size, -1).view(-1, in_channels) + else: + mask = torch.zeros( + weight.size(0), weight.size(1), device=weight.device + ) + mask.bernoulli_(p) + mask = ( + mask.unsqueeze(2) + .unsqueeze(3) + .repeat(1, 1, mod.kernel_size[0], mod.kernel_size[1]) + ) + + # scale weights and apply mask + mask = mask.to( + torch.bool + ) # x.bool() is not currently supported in TorchScript + s = 1 / (1 - p) + mod.weight.data = s * weight.masked_fill(mask, 0) + + module.register_forward_pre_hook(_forward_pre_hook) + return module diff --git a/utils/st_ito/models/beats/quantizer.py b/utils/st_ito/models/beats/quantizer.py new file mode 100644 index 0000000000000000000000000000000000000000..dd020b61bf6fc9a98263c61e8fbf2d503abf361c --- /dev/null +++ b/utils/st_ito/models/beats/quantizer.py @@ -0,0 +1,247 @@ +# -------------------------------------------------------- +# BEATs: Audio Pre-Training with Acoustic Tokenizers (https://arxiv.org/abs/2212.09058) +# Github source: https://github.com/microsoft/unilm/tree/master/beats +# Copyright (c) 2022 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Based on VQGAN code bases +# https://github.com/CompVis/taming-transformers +# --------------------------------------------------------' + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as distributed + +try: + from einops import rearrange, repeat +except ImportError: + pass + + +def l2norm(t): + return F.normalize(t, p=2, dim=-1) + + +def ema_inplace(moving_avg, new, decay): + moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay)) + + +def sample_vectors(samples, num): + num_samples, device = samples.shape[0], samples.device + + if num_samples >= num: + indices = torch.randperm(num_samples, device=device)[:num] + else: + indices = torch.randint(0, num_samples, (num,), device=device) + + return samples[indices] + + +def kmeans(samples, num_clusters, num_iters=10, use_cosine_sim=False): + dim, dtype, device = samples.shape[-1], samples.dtype, samples.device + + means = sample_vectors(samples, num_clusters) + + for _ in range(num_iters): + if use_cosine_sim: + dists = samples @ means.t() + else: + diffs = rearrange(samples, "n d -> n () d") - rearrange( + means, "c d -> () c d" + ) + dists = -(diffs**2).sum(dim=-1) + + buckets = dists.max(dim=-1).indices + bins = torch.bincount(buckets, minlength=num_clusters) + zero_mask = bins == 0 + bins_min_clamped = bins.masked_fill(zero_mask, 1) + + new_means = buckets.new_zeros(num_clusters, dim, dtype=dtype) + new_means.scatter_add_(0, repeat(buckets, "n -> n d", d=dim), samples) + new_means = new_means / bins_min_clamped[..., None] + + if use_cosine_sim: + new_means = l2norm(new_means) + + means = torch.where(zero_mask[..., None], means, new_means) + + return means, bins + + +class EmbeddingEMA(nn.Module): + def __init__( + self, + num_tokens, + codebook_dim, + decay=0.99, + eps=1e-5, + kmeans_init=True, + codebook_init_path="", + ): + super().__init__() + self.num_tokens = num_tokens + self.codebook_dim = codebook_dim + self.decay = decay + self.eps = eps + if codebook_init_path == "": + if not kmeans_init: + weight = torch.randn(num_tokens, codebook_dim) + weight = l2norm(weight) + else: + weight = torch.zeros(num_tokens, codebook_dim) + self.register_buffer("initted", torch.Tensor([not kmeans_init])) + else: + print(f"load init codebook weight from {codebook_init_path}") + codebook_ckpt_weight = torch.load(codebook_init_path, map_location="cpu") + weight = codebook_ckpt_weight.clone() + self.register_buffer("initted", torch.Tensor([True])) + + self.weight = nn.Parameter(weight, requires_grad=False) + self.cluster_size = nn.Parameter(torch.zeros(num_tokens), requires_grad=False) + self.embed_avg = nn.Parameter(weight.clone(), requires_grad=False) + # self.register_buffer('initted', torch.Tensor([not kmeans_init])) + self.update = True + + @torch.jit.ignore + def init_embed_(self, data): + if self.initted: + return + print("Performing Kemans init for codebook") + embed, cluster_size = kmeans(data, self.num_tokens, 10, use_cosine_sim=True) + self.weight.data.copy_(embed) + self.cluster_size.data.copy_(cluster_size) + self.initted.data.copy_(torch.Tensor([True])) + + def forward(self, embed_id): + return F.embedding(embed_id, self.weight) + + def cluster_size_ema_update(self, new_cluster_size): + self.cluster_size.data.mul_(self.decay).add_( + new_cluster_size, alpha=1 - self.decay + ) + + def embed_avg_ema_update(self, new_embed_avg): + self.embed_avg.data.mul_(self.decay).add_(new_embed_avg, alpha=1 - self.decay) + + def weight_update(self, num_tokens): + n = self.cluster_size.sum() + smoothed_cluster_size = ( + (self.cluster_size + self.eps) / (n + num_tokens * self.eps) * n + ) + # normalize embedding average with smoothed cluster size + embed_normalized = self.embed_avg / smoothed_cluster_size.unsqueeze(1) + # embed_normalized = l2norm(self.embed_avg / smoothed_cluster_size.unsqueeze(1)) + self.weight.data.copy_(embed_normalized) + + +def norm_ema_inplace(moving_avg, new, decay): + moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay)) + moving_avg.data.copy_(l2norm(moving_avg.data)) + + +class NormEMAVectorQuantizer(nn.Module): + def __init__( + self, + n_embed, + embedding_dim, + beta, + decay=0.99, + eps=1e-5, + statistic_code_usage=True, + kmeans_init=False, + codebook_init_path="", + ): + super().__init__() + self.codebook_dim = embedding_dim + self.num_tokens = n_embed + self.beta = beta + self.decay = decay + + # learnable = True if orthogonal_reg_weight > 0 else False + self.embedding = EmbeddingEMA( + self.num_tokens, + self.codebook_dim, + decay, + eps, + kmeans_init, + codebook_init_path, + ) + + self.statistic_code_usage = statistic_code_usage + if statistic_code_usage: + self.register_buffer("cluster_size", torch.zeros(n_embed)) + if distributed.is_available() and distributed.is_initialized(): + print( + "ddp is enable, so use ddp_reduce to sync the statistic_code_usage for each gpu!" + ) + self.all_reduce_fn = distributed.all_reduce + else: + self.all_reduce_fn = nn.Identity() + + def reset_cluster_size(self, device): + if self.statistic_code_usage: + self.register_buffer("cluster_size", torch.zeros(self.num_tokens)) + self.cluster_size = self.cluster_size.to(device) + + def forward(self, z): + # reshape z -> (batch, height, width, channel) and flatten + # z, 'b c h w -> b h w c' + # z = rearrange(z, 'b c h w -> b h w c') + # z = z.transpose(1, 2) + z = l2norm(z) + z_flattened = z.reshape(-1, self.codebook_dim) + + self.embedding.init_embed_(z_flattened) + + d = ( + z_flattened.pow(2).sum(dim=1, keepdim=True) + + self.embedding.weight.pow(2).sum(dim=1) + - 2 * torch.einsum("bd,nd->bn", z_flattened, self.embedding.weight) + ) # 'n d -> d n' + + encoding_indices = torch.argmin(d, dim=1) + + z_q = self.embedding(encoding_indices).view(z.shape) + + encodings = F.one_hot(encoding_indices, self.num_tokens).type(z.dtype) + + if not self.training: + with torch.no_grad(): + cluster_size = encodings.sum(0) + self.all_reduce_fn(cluster_size) + ema_inplace(self.cluster_size, cluster_size, self.decay) + + if self.training and self.embedding.update: + # EMA cluster size + + bins = encodings.sum(0) + self.all_reduce_fn(bins) + + # self.embedding.cluster_size_ema_update(bins) + ema_inplace(self.cluster_size, bins, self.decay) + + zero_mask = bins == 0 + bins = bins.masked_fill(zero_mask, 1.0) + + embed_sum = z_flattened.t() @ encodings + self.all_reduce_fn(embed_sum) + + embed_normalized = (embed_sum / bins.unsqueeze(0)).t() + embed_normalized = l2norm(embed_normalized) + + embed_normalized = torch.where( + zero_mask[..., None], self.embedding.weight, embed_normalized + ) + norm_ema_inplace(self.embedding.weight, embed_normalized, self.decay) + + # compute loss for embedding + loss = self.beta * F.mse_loss(z_q.detach(), z) + + # preserve gradients + z_q = z + (z_q - z).detach() + + # reshape back to match original input shape + # z_q, 'b h w c -> b c h w' + # z_q = rearrange(z_q, 'b h w c -> b c h w') + # z_q = z_q.transpose(1, 2) + return z_q, loss, encoding_indices diff --git a/utils/st_ito/models/clap.py b/utils/st_ito/models/clap.py new file mode 100644 index 0000000000000000000000000000000000000000..1fda64386272a5b196cf9c4bc95df343d3de0871 --- /dev/null +++ b/utils/st_ito/models/clap.py @@ -0,0 +1,42 @@ +import torch +import laion_clap +import torchaudio + + +class CLAP(torch.nn.Module): + def __init__(self, pretrained: bool = False, frozen: bool = False) -> None: + super().__init__() + self.pretrained = pretrained + self.frozen = frozen + + # load the model + self.encoder = laion_clap.CLAP_Module(enable_fusion=False) + if self.pretrained: + self.encoder.load_ckpt() # download the default pretrained checkpoint. + + self.embed_dim = 512 + + def forward(self, x: torch.Tensor): + """ """ + bs, chs, seq_len = x.size() + + x_mid = (x[:, 0, :] + x[:, 1, :]) / 2 + x_side = (x[:, 0, :] - x[:, 1, :]) / 2 + + if self.frozen: + with torch.no_grad(): + embed_mid = self.encoder.get_audio_embedding_from_data( + x=x_mid, use_tensor=True + ) + embed_side = self.encoder.get_audio_embedding_from_data( + x=x_side, use_tensor=True + ) + else: + embed_mid = self.encoder.get_audio_embedding_from_data( + x=x_mid, use_tensor=True + ) + embed_side = self.encoder.get_audio_embedding_from_data( + x=x_side, use_tensor=True + ) + + return embed_mid, embed_side diff --git a/utils/st_ito/models/dstcn.py b/utils/st_ito/models/dstcn.py new file mode 100644 index 0000000000000000000000000000000000000000..4ce6b101747dd0d1bac4aff30894540777d1565c --- /dev/null +++ b/utils/st_ito/models/dstcn.py @@ -0,0 +1,152 @@ +import torch +import pytorch_lightning as pl + + +def get_activation(act_type, ch=None): + """Helper function to construct activation functions by a string. + + Args: + act_type (str): One of 'ReLU', 'PReLU', 'SELU', 'ELU'. + ch (int, optional): Number of channels to use for PReLU. + + Returns: + torch.nn.Module activation function. + """ + + if act_type == "PReLU": + return torch.nn.PReLU(ch) + elif act_type == "ReLU": + return torch.nn.ReLU() + elif act_type == "SELU": + return torch.nn.SELU() + elif act_type == "ELU": + return torch.nn.ELU() + + +class dsTCNBlock(torch.nn.Module): + def __init__( + self, + in_ch, + out_ch, + kernel_size, + stride=1, + dilation=1, + norm_type=None, + act_type="PReLU", + ): + super(dsTCNBlock, self).__init__() + self.in_ch = in_ch + self.out_ch = out_ch + self.kernel_size = kernel_size + self.stride = stride + self.norm_type = norm_type + + pad_value = ((kernel_size - 1) * dilation) // 2 + + self.conv1 = torch.nn.Conv1d( + in_ch, + out_ch, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + padding=pad_value, + ) + self.act1 = get_activation(act_type, out_ch) + + if norm_type == "BatchNorm": + self.norm1 = torch.nn.BatchNorm1d(out_ch) + # self.norm2 = torch.nn.BatchNorm1d(out_ch) + self.res_norm = torch.nn.BatchNorm1d(out_ch) + else: + self.norm1 = torch.nn.Identity() + self.res_norm = torch.nn.Identity() + + self.res_conv = torch.nn.Conv1d(in_ch, out_ch, kernel_size=1, stride=stride) + + def forward(self, x): + x_res = x # store input for later + + # -- first section -- + x = self.conv1(x) + x = self.norm1(x) + x = self.act1(x) + + # -- residual connection -- + x_res = self.res_conv(x_res) + x_res = self.res_norm(x_res) + + return x + x_res + + +class dsTCNModel(torch.nn.Module): + """Downsampling Temporal convolutional network. + + Args: + ninputs (int): Number of input channels (mono = 1, stereo 2). Default: 1 + nblocks (int): Number of total TCN blocks. Default: 8 + kernel_size (int): Width of the convolutional kernels. Default: 15 + stride (int): Stide size when applying convolutional filter. Default: 2 + dialation_growth (int): Compute the dilation factor at each block as dilation_growth ** (n % stack_size). Default: 8 + channel_growth (int): Compute the output channels at each black as in_ch * channel_growth. Default: 1 + channel_width (int): When channel_growth = 1 all blocks use convolutions with this many channels. Default: 32 + stack_size (int): Number of blocks that constitute a single stack of blocks. Default: 10 + grouped (bool): Use grouped convolutions to reduce the total number of parameters. Default: False + causal (bool): Causal TCN configuration does not consider future input values. Default: False + skip_connections (bool): Skip connections from each block to the output. Default: False + norm_type (str): Type of normalization layer to use 'BatchNorm', 'LayerNorm', 'InstanceNorm'. Default: None + """ + + def __init__( + self, + embed_dim: int, + ninputs: int = 1, + nblocks: int = 8, + kernel_size: int = 13, + stride: int = 4, + dilation_growth: int = 8, + channel_growth: int = 2, + channel_width: int = 32, + stack_size: int = 4, + norm_type: str = None, + act_type: str = "PReLU", + *args, + **kwargs, + ): + super().__init__() + self.embed_dim = embed_dim + self.nblocks = nblocks + self.kernel_size = kernel_size + self.stack_size = stack_size + self.stride = stride + + self.blocks = torch.nn.ModuleList() + for n in range(nblocks): + in_ch = ninputs if n == 0 else out_ch + out_ch = channel_width if n == 0 else in_ch * channel_growth + dilation = dilation_growth ** (n % stack_size) + self.blocks.append( + dsTCNBlock( + in_ch, + out_ch, + kernel_size, + stride, + dilation, + norm_type, + act_type, + ) + ) + + self.output = torch.nn.Conv1d(out_ch, 1, 1) + self.fc = torch.nn.Linear(out_ch, embed_dim) + + def forward(self, x): + for block in self.blocks: + x = block(x) + + (x1, _) = torch.max(x, dim=2) + x2 = torch.mean(x, dim=2) + x = x1 + x2 + x = torch.nn.functional.dropout(x, p=0.5, training=self.training) + x = self.fc(x) + + return x diff --git a/utils/st_ito/models/fx_encoder.py b/utils/st_ito/models/fx_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..9a504d603fd890f84df231367ee37aa74795157c --- /dev/null +++ b/utils/st_ito/models/fx_encoder.py @@ -0,0 +1,363 @@ +# adapted from https://github.com/jhtonyKoo/music_mixing_style_transfer +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.init as init +import torch +import torchaudio + +from typing import List + + +# Convoluaionl Block +# consists of multiple (number of layer_num) convolutional layers +# only the final convoluational layer outputs the desired 'out_channels' +class ConvBlock(nn.Module): + def __init__( + self, + dimension, + layer_num, + in_channels, + out_channels, + kernel_size, + stride=1, + padding="SAME", + dilation=1, + bias=True, + norm="batch", + activation="relu", + last_activation="relu", + mode="conv", + ): + super(ConvBlock, self).__init__() + + conv_block = [] + if dimension == 1: + for i in range(layer_num - 1): + conv_block.append( + Conv1d_layer( + in_channels, + in_channels, + kernel_size, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=activation, + ) + ) + conv_block.append( + Conv1d_layer( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=last_activation, + mode=mode, + ) + ) + elif dimension == 2: + for i in range(layer_num - 1): + conv_block.append( + Conv2d_layer( + in_channels, + in_channels, + kernel_size, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=activation, + ) + ) + conv_block.append( + Conv2d_layer( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=last_activation, + mode=mode, + ) + ) + self.conv_block = nn.Sequential(*conv_block) + + def forward(self, input): + return self.conv_block(input) + + +# 1-dimensional convolutional layer +# in the order of conv -> norm -> activation +class Conv1d_layer(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding="SAME", + dilation=1, + bias=True, + norm="batch", + activation="relu", + mode="conv", + ): + super(Conv1d_layer, self).__init__() + + self.conv1d = nn.Sequential() + + """ padding """ + if mode == "deconv": + padding = int(dilation * (kernel_size - 1) / 2) + out_padding = 0 if stride == 1 else 1 + elif mode == "conv" or "alias_free" in mode: + if padding == "SAME": + pad = int((kernel_size - 1) * dilation) + l_pad = int(pad // 2) + r_pad = pad - l_pad + padding_area = (l_pad, r_pad) + elif padding == "VALID": + padding_area = (0, 0) + else: + pass + + """ convolutional layer """ + if mode == "deconv": + self.conv1d.add_module( + "deconv1d", + nn.ConvTranspose1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + output_padding=out_padding, + dilation=dilation, + bias=bias, + ), + ) + elif mode == "conv": + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module( + f"{mode}1d", + nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=0, + dilation=dilation, + bias=bias, + ), + ) + elif "alias_free" in mode: + if "up" in mode: + up_factor = stride * 2 + down_factor = 2 + elif "down" in mode: + up_factor = 2 + down_factor = stride * 2 + else: + raise ValueError("choose alias-free method : 'up' or 'down'") + # procedure : conv -> upsample -> lrelu -> low-pass filter -> downsample + # the torchaudio.transforms.Resample's default resampling_method is 'sinc_interpolation' which performs low-pass filter during the process + # details at https://pytorch.org/audio/stable/transforms.html + self.conv1d.add_module(f"{mode}1d_pad", nn.ReflectionPad1d(padding_area)) + self.conv1d.add_module( + f"{mode}1d", + nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=dilation, + bias=bias, + ), + ) + self.conv1d.add_module( + f"{mode}upsample", + torchaudio.transforms.Resample(orig_freq=1, new_freq=up_factor), + ) + self.conv1d.add_module(f"{mode}lrelu", nn.LeakyReLU()) + self.conv1d.add_module( + f"{mode}downsample", + torchaudio.transforms.Resample(orig_freq=down_factor, new_freq=1), + ) + + """ normalization """ + if norm == "batch": + self.conv1d.add_module("batch_norm", nn.BatchNorm1d(out_channels)) + # self.conv1d.add_module("batch_norm", nn.SyncBatchNorm(out_channels)) + + """ activation """ + if "alias_free" not in mode: + if activation == "relu": + self.conv1d.add_module("relu", nn.ReLU()) + elif activation == "lrelu": + self.conv1d.add_module("lrelu", nn.LeakyReLU()) + + def forward(self, input): + # input shape should be : batch x channel x height x width + output = self.conv1d(input) + return output + + +# Residual Block +# the input is added after the first convolutional layer, retaining its original channel size +# therefore, the second convolutional layer's output channel may differ +class Res_ConvBlock(nn.Module): + def __init__( + self, + dimension, + in_channels, + out_channels, + kernel_size, + stride=1, + padding="SAME", + dilation=1, + bias=True, + norm="batch", + activation="relu", + last_activation="relu", + mode="conv", + ): + super(Res_ConvBlock, self).__init__() + + if dimension == 1: + self.conv1 = Conv1d_layer( + in_channels, + in_channels, + kernel_size, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=activation, + ) + self.conv2 = Conv1d_layer( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=last_activation, + mode=mode, + ) + elif dimension == 2: + self.conv1 = Conv2d_layer( + in_channels, + in_channels, + kernel_size, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=activation, + ) + self.conv2 = Conv2d_layer( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + norm=norm, + activation=last_activation, + mode=mode, + ) + + def forward(self, input): + c1_out = self.conv1(input) + input + c2_out = self.conv2(c1_out) + return c2_out + + +# FXencoder that extracts audio effects from music recordings trained with a contrastive objective +class FXencoder(torch.nn.Module): + def __init__( + self, + channels: List[int] = [ + 16, + 32, + 64, + 128, + 256, + 256, + 512, + 512, + 1024, + 1024, + 2048, + 2048, + ], + kernels: List[int] = [25, 25, 15, 15, 10, 10, 10, 10, 5, 5, 5, 5], + strides: List[int] = [4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1], + dilation: List[int] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + bias: bool = True, + norm: str = "batch", + conv_block: str = "res", + activation: str = "relu", + ): + super(FXencoder, self).__init__() + # input is stereo channeled audio + channels.insert(0, 2) + + # encoder layers + encoder = [] + for i in range(len(kernels)): + if conv_block == "res": + encoder.append( + Res_ConvBlock( + dimension=1, + in_channels=channels[i], + out_channels=channels[i + 1], + kernel_size=kernels[i], + stride=strides[i], + padding="SAME", + dilation=dilation[i], + norm=norm, + activation=activation, + last_activation=activation, + ) + ) + elif conv_block == "conv": + encoder.append( + ConvBlock( + dimension=1, + layer_num=1, + in_channels=channels[i], + out_channels=channels[i + 1], + kernel_size=kernels[i], + stride=strides[i], + padding="VALID", + dilation=dilation[i], + norm=norm, + activation=activation, + last_activation=activation, + mode="conv", + ) + ) + self.encoder = nn.Sequential(*encoder) + + # pooling method + self.glob_pool = nn.AdaptiveAvgPool1d(1) + + # network forward operation + def forward(self, input): + enc_output = self.encoder(input) + glob_pooled = self.glob_pool(enc_output).squeeze(-1) + + # outputs c feature + return glob_pooled diff --git a/utils/st_ito/models/gcn.py b/utils/st_ito/models/gcn.py new file mode 100644 index 0000000000000000000000000000000000000000..cab8065597326d76e91ad1aa543964da9e1a7921 --- /dev/null +++ b/utils/st_ito/models/gcn.py @@ -0,0 +1,960 @@ +import math +import torch +import numpy as np +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Sequential as Seq + +from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from timm.models.helpers import load_pretrained +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ +from timm.models.registry import register_model +from torchlibrosa.stft import Spectrogram, LogmelFilterBank +from torchlibrosa.augmentation import SpecAugmentation + +# 2022.06.17-Changed for building ViG model +# Huawei Technologies Co., Ltd. +# modified from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# -------------------------------------------------------- +# Position embedding utils +# -------------------------------------------------------- + + +# -------------------------------------------------------- +# relative position embedding +# References: https://arxiv.org/abs/2009.13658 +# -------------------------------------------------------- +def get_2d_relative_pos_embed(embed_dim, grid_size): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, grid_size*grid_size] + """ + pos_embed = get_2d_sincos_pos_embed(embed_dim, grid_size) + relative_pos = 2 * np.matmul(pos_embed, pos_embed.transpose()) / pos_embed.shape[1] + return relative_pos + + +# -------------------------------------------------------- +# 2D sine-cosine position embedding +# References: +# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py +# MoCo v3: https://github.com/facebookresearch/moco-v3 +# -------------------------------------------------------- +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token: + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +# 2022.06.17-Changed for building ViG model +# Huawei Technologies Co., Ltd. +import math +import torch +from torch import nn +import torch.nn.functional as F + + +def pairwise_distance(x): + """ + Compute pairwise distance of a point cloud. + Args: + x: tensor (batch_size, num_points, num_dims) + Returns: + pairwise distance: (batch_size, num_points, num_points) + """ + with torch.no_grad(): + x_inner = -2 * torch.matmul(x, x.transpose(2, 1)) + x_square = torch.sum(torch.mul(x, x), dim=-1, keepdim=True) + return x_square + x_inner + x_square.transpose(2, 1) + + +def part_pairwise_distance(x, start_idx=0, end_idx=1): + """ + Compute pairwise distance of a point cloud. + Args: + x: tensor (batch_size, num_points, num_dims) + Returns: + pairwise distance: (batch_size, num_points, num_points) + """ + with torch.no_grad(): + x_part = x[:, start_idx:end_idx] + x_square_part = torch.sum(torch.mul(x_part, x_part), dim=-1, keepdim=True) + x_inner = -2 * torch.matmul(x_part, x.transpose(2, 1)) + x_square = torch.sum(torch.mul(x, x), dim=-1, keepdim=True) + return x_square_part + x_inner + x_square.transpose(2, 1) + + +def xy_pairwise_distance(x, y): + """ + Compute pairwise distance of a point cloud. + Args: + x: tensor (batch_size, num_points, num_dims) + Returns: + pairwise distance: (batch_size, num_points, num_points) + """ + with torch.no_grad(): + xy_inner = -2 * torch.matmul(x, y.transpose(2, 1)) + x_square = torch.sum(torch.mul(x, x), dim=-1, keepdim=True) + y_square = torch.sum(torch.mul(y, y), dim=-1, keepdim=True) + return x_square + xy_inner + y_square.transpose(2, 1) + + +def dense_knn_matrix(x, k=16, relative_pos=None): + """Get KNN based on the pairwise distance. + Args: + x: (batch_size, num_dims, num_points, 1) + k: int + Returns: + nearest neighbors: (batch_size, num_points, k) (batch_size, num_points, k) + """ + with torch.no_grad(): + x = x.transpose(2, 1).squeeze(-1) + batch_size, n_points, n_dims = x.shape + ### memory efficient implementation ### + n_part = 10000 + if n_points > n_part: + nn_idx_list = [] + groups = math.ceil(n_points / n_part) + for i in range(groups): + start_idx = n_part * i + end_idx = min(n_points, n_part * (i + 1)) + dist = part_pairwise_distance(x.detach(), start_idx, end_idx) + if relative_pos is not None: + dist += relative_pos[:, start_idx:end_idx] + _, nn_idx_part = torch.topk(-dist, k=k) + nn_idx_list += [nn_idx_part] + nn_idx = torch.cat(nn_idx_list, dim=1) + else: + dist = pairwise_distance(x.detach()) + if relative_pos is not None: + dist += relative_pos + _, nn_idx = torch.topk(-dist, k=k) # b, n, k + ###### + center_idx = ( + torch.arange(0, n_points, device=x.device) + .repeat(batch_size, k, 1) + .transpose(2, 1) + ) + return torch.stack((nn_idx, center_idx), dim=0) + + +def xy_dense_knn_matrix(x, y, k=16, relative_pos=None): + """Get KNN based on the pairwise distance. + Args: + x: (batch_size, num_dims, num_points, 1) + k: int + Returns: + nearest neighbors: (batch_size, num_points, k) (batch_size, num_points, k) + """ + relative_pos = None + with torch.no_grad(): + x = x.transpose(2, 1).squeeze(-1) + y = y.transpose(2, 1).squeeze(-1) + batch_size, n_points, n_dims = x.shape + dist = xy_pairwise_distance(x.detach(), y.detach()) + if relative_pos is not None: + dist += relative_pos + + _, nn_idx = torch.topk(-dist, k=k) + center_idx = ( + torch.arange(0, n_points, device=x.device) + .repeat(batch_size, k, 1) + .transpose(2, 1) + ) + return torch.stack((nn_idx, center_idx), dim=0) + + +class DenseDilated(nn.Module): + """ + Find dilated neighbor from neighbor list + + edge_index: (2, batch_size, num_points, k) + """ + + def __init__(self, k=9, dilation=1, stochastic=False, epsilon=0.0): + super(DenseDilated, self).__init__() + self.dilation = dilation + self.stochastic = stochastic + self.epsilon = epsilon + self.k = k + + def forward(self, edge_index): + if self.stochastic: + if torch.rand(1) < self.epsilon and self.training: + num = self.k * self.dilation + randnum = torch.randperm(num)[: self.k] + edge_index = edge_index[:, :, :, randnum] + else: + edge_index = edge_index[:, :, :, :: self.dilation] + else: + edge_index = edge_index[:, :, :, :: self.dilation] + return edge_index + + +class DenseDilatedKnnGraph(nn.Module): + """ + Find the neighbors' indices based on dilated knn + """ + + def __init__(self, k=9, dilation=1, stochastic=False, epsilon=0.0): + super(DenseDilatedKnnGraph, self).__init__() + self.dilation = dilation + self.stochastic = stochastic + self.epsilon = epsilon + self.k = k + self._dilated = DenseDilated(k, dilation, stochastic, epsilon) + + def forward(self, x, y=None, relative_pos=None): + if y is not None: + #### normalize + x = F.normalize(x, p=2.0, dim=1) + y = F.normalize(y, p=2.0, dim=1) + #### + edge_index = xy_dense_knn_matrix(x, y, self.k * self.dilation, relative_pos) + else: + #### normalize + x = F.normalize(x, p=2.0, dim=1) + #### + edge_index = dense_knn_matrix(x, self.k * self.dilation, relative_pos) + return self._dilated(edge_index) + + +# 2022.06.17-Changed for building ViG model +# Huawei Technologies Co., Ltd. +import torch +from torch import nn +from torch.nn import Sequential as Seq, Linear as Lin, Conv2d + + +############################## +# Basic layers +############################## +def act_layer(act, inplace=False, neg_slope=0.2, n_prelu=1): + # activation layer + + act = act.lower() + if act == "relu": + layer = nn.ReLU(inplace) + elif act == "leakyrelu": + layer = nn.LeakyReLU(neg_slope, inplace) + elif act == "prelu": + layer = nn.PReLU(num_parameters=n_prelu, init=neg_slope) + elif act == "gelu": + layer = nn.GELU() + elif act == "hswish": + layer = nn.Hardswish(inplace) + else: + raise NotImplementedError("activation layer [%s] is not found" % act) + return layer + + +def norm_layer(norm, nc): + # normalization layer 2d + norm = norm.lower() + if norm == "batch": + layer = nn.BatchNorm2d(nc, affine=True) + elif norm == "instance": + layer = nn.InstanceNorm2d(nc, affine=False) + else: + raise NotImplementedError("normalization layer [%s] is not found" % norm) + return layer + + +class MLP(Seq): + def __init__(self, channels, act="relu", norm=None, bias=True): + m = [] + for i in range(1, len(channels)): + m.append(Lin(channels[i - 1], channels[i], bias)) + if act is not None and act.lower() != "none": + m.append(act_layer(act)) + if norm is not None and norm.lower() != "none": + m.append(norm_layer(norm, channels[-1])) + super(MLP, self).__init__(*m) + + +class BasicConv(Seq): + def __init__(self, channels, act="relu", norm=None, bias=True, drop=0.0): + m = [] + for i in range(1, len(channels)): + m.append(Conv2d(channels[i - 1], channels[i], 1, bias=bias, groups=4)) + if norm is not None and norm.lower() != "none": + m.append(norm_layer(norm, channels[-1])) + if act is not None and act.lower() != "none": + m.append(act_layer(act)) + if drop > 0: + m.append(nn.Dropout2d(drop)) + + super(BasicConv, self).__init__(*m) + + self.reset_parameters() + + def reset_parameters(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.InstanceNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + +def batched_index_select(x, idx): + r"""fetches neighbors features from a given neighbor idx + + Args: + x (Tensor): input feature Tensor + :math:`\mathbf{X} \in \mathbb{R}^{B \times C \times N \times 1}`. + idx (Tensor): edge_idx + :math:`\mathbf{X} \in \mathbb{R}^{B \times N \times l}`. + Returns: + Tensor: output neighbors features + :math:`\mathbf{X} \in \mathbb{R}^{B \times C \times N \times k}`. + """ + batch_size, num_dims, num_vertices_reduced = x.shape[:3] + _, num_vertices, k = idx.shape + idx_base = ( + torch.arange(0, batch_size, device=idx.device).view(-1, 1, 1) + * num_vertices_reduced + ) + idx = idx + idx_base + idx = idx.contiguous().view(-1) + + x = x.transpose(2, 1) + feature = x.contiguous().view(batch_size * num_vertices_reduced, -1)[idx, :] + feature = ( + feature.view(batch_size, num_vertices, k, num_dims) + .permute(0, 3, 1, 2) + .contiguous() + ) + return feature + + +# 2022.06.17-Changed for building ViG model +# Huawei Technologies Co., Ltd. + + +class MRConv2d(nn.Module): + """ + Max-Relative Graph Convolution (Paper: https://arxiv.org/abs/1904.03751) for dense data type + """ + + def __init__(self, in_channels, out_channels, act="relu", norm=None, bias=True): + super(MRConv2d, self).__init__() + self.nn = BasicConv([in_channels * 2, out_channels], act, norm, bias) + + def forward(self, x, edge_index, y=None): + x_i = batched_index_select(x, edge_index[1]) + if y is not None: + x_j = batched_index_select(y, edge_index[0]) + else: + x_j = batched_index_select(x, edge_index[0]) + x_j, _ = torch.max(x_j - x_i, -1, keepdim=True) + b, c, n, _ = x.shape + x = torch.cat([x.unsqueeze(2), x_j.unsqueeze(2)], dim=2).reshape(b, 2 * c, n, _) + return self.nn(x) + + +class EdgeConv2d(nn.Module): + """ + Edge convolution layer (with activation, batch normalization) for dense data type + """ + + def __init__(self, in_channels, out_channels, act="relu", norm=None, bias=True): + super(EdgeConv2d, self).__init__() + self.nn = BasicConv([in_channels * 2, out_channels], act, norm, bias) + + def forward(self, x, edge_index, y=None): + x_i = batched_index_select(x, edge_index[1]) + if y is not None: + x_j = batched_index_select(y, edge_index[0]) + else: + x_j = batched_index_select(x, edge_index[0]) + max_value, _ = torch.max( + self.nn(torch.cat([x_i, x_j - x_i], dim=1)), -1, keepdim=True + ) + return max_value + + +class GraphSAGE(nn.Module): + """ + GraphSAGE Graph Convolution (Paper: https://arxiv.org/abs/1706.02216) for dense data type + """ + + def __init__(self, in_channels, out_channels, act="relu", norm=None, bias=True): + super(GraphSAGE, self).__init__() + self.nn1 = BasicConv([in_channels, in_channels], act, norm, bias) + self.nn2 = BasicConv([in_channels * 2, out_channels], act, norm, bias) + + def forward(self, x, edge_index, y=None): + if y is not None: + x_j = batched_index_select(y, edge_index[0]) + else: + x_j = batched_index_select(x, edge_index[0]) + x_j, _ = torch.max(self.nn1(x_j), -1, keepdim=True) + return self.nn2(torch.cat([x, x_j], dim=1)) + + +class GINConv2d(nn.Module): + """ + GIN Graph Convolution (Paper: https://arxiv.org/abs/1810.00826) for dense data type + """ + + def __init__(self, in_channels, out_channels, act="relu", norm=None, bias=True): + super(GINConv2d, self).__init__() + self.nn = BasicConv([in_channels, out_channels], act, norm, bias) + eps_init = 0.0 + self.eps = nn.Parameter(torch.Tensor([eps_init])) + + def forward(self, x, edge_index, y=None): + if y is not None: + x_j = batched_index_select(y, edge_index[0]) + else: + x_j = batched_index_select(x, edge_index[0]) + x_j = torch.sum(x_j, -1, keepdim=True) + return self.nn((1 + self.eps) * x + x_j) + + +class GraphConv2d(nn.Module): + """ + Static graph convolution layer + """ + + def __init__( + self, in_channels, out_channels, conv="edge", act="relu", norm=None, bias=True + ): + super(GraphConv2d, self).__init__() + if conv == "edge": + self.gconv = EdgeConv2d(in_channels, out_channels, act, norm, bias) + elif conv == "mr": + self.gconv = MRConv2d(in_channels, out_channels, act, norm, bias) + elif conv == "sage": + self.gconv = GraphSAGE(in_channels, out_channels, act, norm, bias) + elif conv == "gin": + self.gconv = GINConv2d(in_channels, out_channels, act, norm, bias) + else: + raise NotImplementedError("conv:{} is not supported".format(conv)) + + def forward(self, x, edge_index, y=None): + return self.gconv(x, edge_index, y) + + +class DyGraphConv2d(GraphConv2d): + """ + Dynamic graph convolution layer + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size=9, + dilation=1, + conv="edge", + act="relu", + norm=None, + bias=True, + stochastic=False, + epsilon=0.0, + r=1, + ): + super(DyGraphConv2d, self).__init__( + in_channels, out_channels, conv, act, norm, bias + ) + self.k = kernel_size + self.d = dilation + self.r = r + self.dilated_knn_graph = DenseDilatedKnnGraph( + kernel_size, dilation, stochastic, epsilon + ) + + def forward(self, x, relative_pos=None): + B, C, H, W = x.shape + y = None + if self.r > 1: + y = F.avg_pool2d(x, self.r, self.r) + y = y.reshape(B, C, -1, 1).contiguous() + x = x.reshape(B, C, -1, 1).contiguous() + edge_index = self.dilated_knn_graph(x, y, relative_pos) + x = super(DyGraphConv2d, self).forward(x, edge_index, y) + return x.reshape(B, -1, H, W).contiguous() + + +class Grapher(nn.Module): + """ + Grapher module with graph convolution and fc layers + """ + + def __init__( + self, + in_channels, + kernel_size=9, + dilation=1, + conv="edge", + act="relu", + norm=None, + bias=True, + stochastic=False, + epsilon=0.0, + r=1, + n=196, + drop_path=0.0, + relative_pos=False, + ): + super(Grapher, self).__init__() + self.channels = in_channels + self.n = n + self.r = r + self.fc1 = nn.Sequential( + nn.Conv2d(in_channels, in_channels, 1, stride=1, padding=0), + nn.BatchNorm2d(in_channels), + ) + self.graph_conv = DyGraphConv2d( + in_channels, + in_channels * 2, + kernel_size, + dilation, + conv, + act, + norm, + bias, + stochastic, + epsilon, + r, + ) + self.fc2 = nn.Sequential( + nn.Conv2d(in_channels * 2, in_channels, 1, stride=1, padding=0), + nn.BatchNorm2d(in_channels), + ) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.relative_pos = False + if relative_pos: + relative_pos_tensor = ( + torch.from_numpy( + np.float32(get_2d_relative_pos_embed(in_channels, int(n**0.5))) + ) + .unsqueeze(0) + .unsqueeze(1) + ) + relative_pos_tensor = F.interpolate( + relative_pos_tensor, + size=(n, n // (r * r)), + mode="bicubic", + align_corners=False, + ) + self.relative_pos = nn.Parameter( + -relative_pos_tensor.squeeze(1), requires_grad=False + ) + + def _get_relative_pos(self, relative_pos, H, W): + if relative_pos is None or H * W == self.n: + return relative_pos + else: + N = H * W + N_reduced = N // (self.r * self.r) + return F.interpolate( + relative_pos.unsqueeze(0), size=(N, N_reduced), mode="bicubic" + ).squeeze(0) + + def forward(self, x): + _tmp = x + x = self.fc1(x) + B, C, H, W = x.shape + relative_pos = self._get_relative_pos(self.relative_pos, H, W) + x = self.graph_conv(x, relative_pos) + x = self.fc2(x) + x = self.drop_path(x) + _tmp + return x + + +class FFN(nn.Module): + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act="relu", + drop_path=0.0, + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Sequential( + nn.Conv2d(in_features, hidden_features, 1, stride=1, padding=0), + nn.BatchNorm2d(hidden_features), + ) + self.act = act_layer(act) + self.fc2 = nn.Sequential( + nn.Conv2d(hidden_features, out_features, 1, stride=1, padding=0), + nn.BatchNorm2d(out_features), + ) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + def forward(self, x): + shortcut = x + x = self.fc1(x) + x = self.act(x) + x = self.fc2(x) + x = self.drop_path(x) + shortcut + return x # .reshape(B, C, N, 1) + + +class Stem(nn.Module): + """Image to Visual Embedding + Overlap: https://arxiv.org/pdf/2106.13797.pdf + """ + + def __init__(self, img_size=224, in_dim=1, out_dim=768, act="relu"): + super().__init__() + self.convs = nn.Sequential( + nn.Conv2d(in_dim, out_dim // 2, 3, stride=2, padding=1), + nn.BatchNorm2d(out_dim // 2), + act_layer(act), + nn.Conv2d(out_dim // 2, out_dim, 3, stride=2, padding=1), + nn.BatchNorm2d(out_dim), + act_layer(act), + nn.Conv2d(out_dim, out_dim, 3, stride=1, padding=1), + nn.BatchNorm2d(out_dim), + ) + + def forward(self, x): + x = self.convs(x) + return x + + +class Downsample(nn.Module): + """Convolution-based downsample""" + + def __init__(self, in_dim=3, out_dim=768): + super().__init__() + self.conv = nn.Sequential( + nn.Conv2d(in_dim, out_dim, 3, stride=2, padding=1), + nn.BatchNorm2d(out_dim), + ) + + def forward(self, x): + x = self.conv(x) + return x + + +class DeepGCN(torch.nn.Module): + """Graph Model. + Args: + mel_bins: (int) Number of mel bins + classes_num: (int) Number of classes + in_channels:(int) Number of input channels + k: (int) Number of nearest neighbors + conv: (str) Graph conv type + act: (str) Activation function + bias: (bool) Use bias + dropout: (float) Dropout rate + use_dilation: (bool) Use dilation in graph conv + epsilon: (float) epsilon value for stochastic depth + use_stochastic: (bool) Use stochastic depth + drop_path: (float) Drop path rate + embed_dim: (int) Embedding dimension + model_size: (str) Graph model size (s,m,b) + norm: (str) Normalization type + use_stdnorm:(bool) Use standard normalization + use_batchnorm: (bool) Use batch normalization + num_frames:(int) Number of time frames in spectrogram + """ + + def __init__( + self, + mel_bins: int, + classes_num: int, + in_channels: int, + k: int, + conv: str, + act: str, + bias: bool, + dropout: float, + use_dilation: bool, + epsilon: float, + use_stochastic: bool, + drop_path: float, + embed_dim: int, + model_size: str, + norm: str, + use_stdnorm: bool, + use_batchnorm: bool, + num_frames: int, + ): + super().__init__() + + window = "hann" + center = True + pad_mode = "reflect" + ref = 1.0 + amin = 1e-10 + top_db = None + window_size = 2048 + hop_size = 512 + sample_rate = 48000 + mel_bins = 128 + fmin = 20 + fmax = 4000.0 + + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram( + n_fft=window_size, + hop_length=hop_size, + win_length=window_size, + window=window, + center=center, + pad_mode=pad_mode, + freeze_parameters=True, + ) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank( + sr=sample_rate, + n_fft=window_size, + n_mels=mel_bins, + fmin=fmin, + fmax=fmax, + ref=ref, + amin=amin, + top_db=top_db, + freeze_parameters=True, + ) + + if model_size == "t": + blocks = [2, 2, 6, 2] + channels = [48, 96, 240, 384] + + elif model_size == "s": + print("Using small model") + blocks = [2, 2, 6, 2] + channels = [80, 160, 400, 640] + + elif model_size == "m": + print("Using Medium model") + blocks = [2, 2, 16, 2] + channels = [96, 192, 384, 768] + + elif model_size == "b": + print("Using big model") + blocks = [2, 2, 18, 2] + channels = [128, 256, 512, 1024] + + self.mel_bins = mel_bins + self.classes_num = classes_num + self.in_channels = in_channels + self.k = k + self.conv = conv + self.act = act + self.dropout = dropout + self.use_dilation = use_dilation + self.epsilon = epsilon + self.stochastic = use_stochastic + self.drop_path = drop_path + self.embed_dim = embed_dim + self.model_size = model_size + self.num_frames = num_frames + self.norm = norm + self.use_stdnorm = use_stdnorm + self.use_batchnorm = use_batchnorm + + # Spec augmenter + self.spec_augmenter = SpecAugmentation( + time_drop_width=64, + time_stripes_num=2, + freq_drop_width=8, + freq_stripes_num=2, + ) + + self.bn0 = nn.BatchNorm2d(self.mel_bins) + self.n_blocks = sum(blocks) + reduce_ratios = [4, 2, 1, 1] + dpr = [ + x.item() for x in torch.linspace(0, drop_path, self.n_blocks) + ] # stochastic depth decay rule + num_knn = [ + int(x.item()) for x in torch.linspace(k, k, self.n_blocks) + ] # number of knn's k + + max_dilation = (mel_bins // 16 * num_frames // 16) // max(num_knn) + + self.stem = Stem(out_dim=channels[0], act=act) + self.pos_embed = nn.Parameter( + torch.zeros(1, channels[0], self.mel_bins // 4, self.num_frames // 4) + ) + + self.HW = self.mel_bins // 4 * self.num_frames // 4 + self.backbone = nn.ModuleList([]) + idx = 0 + for i in range(len(blocks)): + if i > 0: + self.backbone.append(Downsample(channels[i - 1], channels[i])) + self.HW = self.HW // 4 + for j in range(blocks[i]): + self.backbone += [ + Seq( + Grapher( + channels[i], + num_knn[idx], + min(idx // 4 + 1, max_dilation), + conv, + act, + self.norm, + bias, + self.stochastic, + epsilon, + reduce_ratios[i], + n=self.HW, + drop_path=dpr[idx], + relative_pos=True, + ), + FFN(channels[i], channels[i] * 4, act=act, drop_path=dpr[idx]), + ) + ] + idx += 1 + self.backbone = Seq(*self.backbone) + + self.prediction = Seq( + nn.Conv2d(channels[-1], 1024, 1, bias=True), + act_layer(act), + nn.Dropout(self.dropout), + nn.Conv2d(1024, self.classes_num, 1, bias=True), + ) + self.model_init() + + def reset_parameters(self): + stdv = 1.0 / math.sqrt(self.adj.size(1)) + self.adj.data.uniform_(-stdv, stdv) + self.adj.data.fill_diagonal_(0) + + def model_init(self): + for m in self.modules(): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + m.weight.requires_grad = True + if m.bias is not None: + m.bias.data.zero_() + m.bias.requires_grad = True + + def forward(self, x: torch.Tensor): + B, C, N = x.shape + x = x.reshape(B, -1) + x = self.spectrogram_extractor(x) + x = self.logmel_extractor(x) + batch, channel, frames, mel_bins = x.shape + + if self.use_stdnorm: + std = x.std(dim=(-1, -2), keepdim=True) + mu = x.mean(dim=(-1, -2), keepdim=True) + x = (x - mu) / std + + if self.use_batchnorm: + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + + if self.training: + x = self.spec_augmenter(x) + + x = x.transpose(2, 3) # (batch_size, 1, mel_bins, time_steps) + x = self.stem(x) + + if x.shape[3] != self.num_frames // 4: + # self.pos_embed = F.interpolate(self.pos_embed, size=(x.shape[0],x.shape[1],x.shape[2], x.shape[3]), mode='bicubic', align_corners=False) + self.pos_embed = nn.Parameter( + F.interpolate( + self.pos_embed, + size=(x.shape[2], x.shape[3]), + mode="bicubic", + align_corners=False, + ) + ) + + x += self.pos_embed + + for i in range(len(self.backbone)): + x = self.backbone[i](x) + + x = F.adaptive_avg_pool2d(x, 1) + + x = self.prediction(x) + return x.squeeze(-1).squeeze(-1) + + +def __main__(): + graph_model = DeepGCN( + mel_bins=mel_bins, + classes_num=128, + in_channels=1, + k=9, + conv="mr", + act="gelu", + bias=True, + dropout=0.0, + use_dilation=True, + epsilon=0.2, + use_stochastic=False, + drop_path=0.1, + embed_dim=96, + model_size="m", + norm="batch", + use_stdnorm=True, + use_batchnorm=True, + num_frames=frames, + ) + + y = graph_model(x) + print(y.shape) + + +if __name__ == "__main__": + __main__() diff --git a/utils/st_ito/models/htast.py b/utils/st_ito/models/htast.py new file mode 100644 index 0000000000000000000000000000000000000000..3f2f48ccb587706d0d882eb2817c2839a0f4e1d7 --- /dev/null +++ b/utils/st_ito/models/htast.py @@ -0,0 +1,1005 @@ +import math +import torch +import random +import warnings +import torch.nn as nn +import collections.abc +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint + +from torch import Tensor +from itertools import repeat +from torchlibrosa.stft import Spectrogram, LogmelFilterBank +from torchlibrosa.augmentation import SpecAugmentation + + +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +# The `to_2tuple` function is an instance of the `_ntuple` function with a parameter of 2. +# This function will convert its input into a tuple of 2 elements. +to_2tuple = _ntuple(2) + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +class PatchEmbed(nn.Module): + """2D Image to Patch Embedding""" + + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + patch_stride=16, + ): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patch_stride = to_2tuple(patch_stride) + self.img_size = img_size + self.patch_size = patch_size + self.patch_stride = patch_stride + self.grid_size = ( + img_size[0] // patch_stride[0], + img_size[1] // patch_stride[1], + ) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + self.in_chans = in_chans + self.embed_dim = embed_dim + + padding = ( + (patch_size[0] - patch_stride[0]) // 2, + (patch_size[1] - patch_stride[1]) // 2, + ) + + self.proj = nn.Conv2d( + in_chans, + embed_dim, + kernel_size=patch_size, + stride=patch_stride, + padding=padding, + ) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + B, C, H, W = x.shape + assert ( + H == self.img_size[0] and W == self.img_size[1] + ), f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +class PatchMerging(nn.Module): + r"""Patch Merging Layer. + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." + + x = x.view(B, H, W, C) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + def extra_repr(self): + return f"input_resolution={self.input_resolution}, dim={self.dim}" + + +class WindowAttention(nn.Module): + r"""Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) + ) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = ( + coords_flatten[:, :, None] - coords_flatten[:, None, :] + ) # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0 + ).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=0.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B_, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = ( + qkv[0], + qkv[1], + qkv[2], + ) # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1, + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze( + 1 + ).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x, attn + + def extra_repr(self): + return f"dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}" + + +class Mlp(nn.Module): + """MLP as used in Vision Transformer, MLP-Mixer and related networks""" + + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.0, + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = ( + x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + ) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view( + B, H // window_size, W // window_size, window_size, window_size, -1 + ) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * ( + x.ndim - 1 + ) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + norm_before_mlp="ln", + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) + else drop_path, + norm_layer=norm_layer, + norm_before_mlp=norm_before_mlp, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample( + input_resolution, dim=dim, norm_layer=norm_layer + ) + else: + self.downsample = None + + def forward(self, x): + attns = [] + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x, attn = blk(x) + if not self.training: + attns.append(attn.unsqueeze(0)) + if self.downsample is not None: + x = self.downsample(x) + if not self.training: + attn = torch.cat(attns, dim=0) + attn = torch.mean(attn, dim=0) + return x, attn + + def extra_repr(self): + return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}" + + +class SwinTransformerBlock(nn.Module): + r"""Swin Transformer Block. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__( + self, + dim, + input_resolution, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + norm_before_mlp="ln", + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.norm_before_mlp = norm_before_mlp + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert ( + 0 <= self.shift_size < self.window_size + ), "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + if self.norm_before_mlp == "ln": + self.norm2 = nn.LayerNorm(dim) + elif self.norm_before_mlp == "bn": + self.norm2 = lambda x: nn.BatchNorm1d(dim)(x.transpose(1, 2)).transpose( + 1, 2 + ) + else: + raise NotImplementedError + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) + + if self.shift_size > 0: + # calculate attention mask for SW-MSA + H, W = self.input_resolution + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill( + attn_mask != 0, float(-100.0) + ).masked_fill(attn_mask == 0, float(0.0)) + else: + attn_mask = None + + self.register_buffer("attn_mask", attn_mask) + + def forward(self, x): + # pdb.set_trace() + H, W = self.input_resolution + # print("H: ", H) + # print("W: ", W) + # pdb.set_trace() + B, L, C = x.shape + # assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2) + ) + else: + shifted_x = x + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows, attn = self.attn( + x_windows, mask=self.attn_mask + ) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2) + ) + else: + x = shifted_x + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x, attn + + def extra_repr(self): + return ( + f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " + f"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}" + ) + + +class HTSAT(nn.Module): + r"""HTSAT model + Args: + spec_size (int): Spectrogram size. + sample_rate (float): Sample rate. + window_size (int): Window size for spectrogram in samples. + hop_size (int): Hop size for spectrogram in samples. + mel_bins (int): Number of mel bins. + fmin (float): Minimum frequency for mel bins. + fmax (float): Maximum frequency for mel bins. + classes_num (int): Number of classes. + patch_size (int): Patch size for images. + patch_stride (tuple[int]): Patch stride for images. + embed_dim (int): Embedding dimension. + in_channels (int): Number of input channels. + patch_window_size (int): Window size for patching. + depths (list[int]): Depth of each Swin Transformer stage. + num_heads (list[int]): Number of attention heads in different stages. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + drop_rate (float): Dropout rate. Default: 0.0 + attn_drop_rate (float): Attention dropout rate. Default: 0.0 + drop_path_rate (float): Stochastic depth rate. Default: 0.1 + patch_norm (bool): If True, add a normalization layer after patch embedding. Default: True + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.0 + norm_before_mlp (str): Type of normalization before MLP. Default: 'ln' + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + spec_size: int, + sample_rate: float, + window_size: int, + hop_size: int, + mel_bins: int, + fmin: float, + fmax: float, + embed_dim: int, + patch_size: int, + patch_stride: tuple, + internal_embed_dim: int, + in_channels: int, + patch_window_size: int, + depths: list, + num_heads: list, + qkv_bias: bool, + drop_rate: float, + attn_drop_rate: float, + drop_path_rate: float, + patch_norm: bool, + mlp_ratio: float, + norm_before_mlp: str, + use_checkpoint: bool, + use_batchnorm: bool = False, + use_stdnorm: bool = False, + norm_layer: str = "ln", + ): + super().__init__() + self.spec_size = spec_size + self.sample_rate = sample_rate + self.window_size = window_size + self.hop_size = hop_size + self.mel_bins = mel_bins + self.fmin = fmin + self.fmax = fmax + self.embed_dim = embed_dim + self.patch_size = patch_size + self.patch_stride = patch_stride + self.internal_embed_dim = internal_embed_dim + self.in_chans = in_channels + self.patch_window_size = patch_window_size + self.depths = depths + self.num_layers = len(self.depths) + self.num_heads = num_heads + self.num_features = int(self.internal_embed_dim * 2 ** (self.num_layers - 1)) + + self.drop_rate = drop_rate + self.attn_drop_rate = attn_drop_rate + self.drop_path_rate = drop_path_rate + self.use_stdnorm = use_stdnorm + self.use_batchnorm = use_batchnorm + + self.qkv_bias = qkv_bias + self.qk_scale = None + self.patch_norm = patch_norm + self.norm_layer = norm_layer if self.patch_norm else None + self.norm_before_mlp = norm_before_mlp + self.mlp_ratio = mlp_ratio + + if self.norm_layer == "ln": + self.norm_layer = nn.LayerNorm + else: + raise ValueError(f"Unrecognized norm layer: {self.norm_layer}") + + self.use_checkpoint = use_checkpoint + self.interpolate_ratio = 32 + + window = "hann" + center = True + pad_mode = "reflect" + ref = 1.0 + amin = 1e-10 + top_db = None + + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram( + n_fft=window_size, + hop_length=hop_size, + win_length=window_size, + window=window, + center=center, + pad_mode=pad_mode, + freeze_parameters=True, + ) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank( + sr=sample_rate, + n_fft=window_size, + n_mels=mel_bins, + fmin=fmin, + fmax=fmax, + ref=ref, + amin=amin, + top_db=top_db, + freeze_parameters=True, + ) + + # Spec augmenter + self.spec_augmenter = SpecAugmentation( + time_drop_width=64, + time_stripes_num=2, + freq_drop_width=8, + freq_stripes_num=2, + ) + + self.bn0 = nn.BatchNorm2d(self.mel_bins) + self.freq_ratio = self.spec_size // self.mel_bins + self.patch_embed = PatchEmbed( + img_size=self.spec_size, + patch_size=self.patch_size, + in_chans=self.in_chans, + embed_dim=self.internal_embed_dim, + norm_layer=self.norm_layer, + patch_stride=patch_stride, + ) + num_patches = self.patch_embed.num_patches + patches_resolution = self.patch_embed.grid_size + self.patches_resolution = patches_resolution + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, num_patches, self.internal_embed_dim) + ) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=self.drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, self.drop_path_rate, sum(self.depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(self.internal_embed_dim * 2**i_layer), + input_resolution=( + patches_resolution[0] // (2**i_layer), + patches_resolution[1] // (2**i_layer), + ), + depth=self.depths[i_layer], + num_heads=self.num_heads[i_layer], + window_size=self.window_size, + mlp_ratio=self.mlp_ratio, + qkv_bias=self.qkv_bias, + qk_scale=self.qk_scale, + drop=self.drop_rate, + attn_drop=self.attn_drop_rate, + drop_path=dpr[ + sum(self.depths[:i_layer]) : sum(self.depths[: i_layer + 1]) + ], + norm_layer=self.norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + norm_before_mlp=self.norm_before_mlp, + ) + self.layers.append(layer) + + self.norm = self.norm_layer(self.num_features) + self.avgpool = nn.AdaptiveAvgPool1d(1) + self.maxpool = nn.AdaptiveMaxPool1d(1) + + self.head = ( + nn.Linear(self.num_features, embed_dim) if embed_dim > 0 else nn.Identity() + ) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def reshape_wav2img(self, x): + target_T = int(self.spec_size * self.freq_ratio) + target_F = self.spec_size // self.freq_ratio + B, C, T, F = x.shape + + if T > target_T: + x = x[:, :, :target_T, :] + + B, C, T, F = x.shape + + assert ( + T <= target_T and F <= target_F + ), "the wav size should less than or equal to the swin input size" + # to avoid bicubic zero error + if T < target_T: + x = nn.functional.interpolate( + x, (target_T, x.shape[3]), mode="bicubic", align_corners=True + ) + if F < target_F: + x = nn.functional.interpolate( + x, (x.shape[2], target_F), mode="bicubic", align_corners=True + ) + x = x.permute(0, 1, 3, 2).contiguous() + x = x.reshape( + x.shape[0], + x.shape[1], + x.shape[2], + self.freq_ratio, + x.shape[3] // self.freq_ratio, + ) + # print(x.shape) + x = x.permute(0, 1, 3, 2, 4).contiguous() + x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3], x.shape[4]) + return x + + # Repeat the wavform to a img size, if you want to use the pretrained swin transformer model + def repeat_wat2img(self, x, cur_pos): + B, C, T, F = x.shape + target_T = int(self.spec_size * self.freq_ratio) + target_F = self.spec_size // self.freq_ratio + + assert ( + T <= target_T and F <= target_F + ), "the wav size should less than or equal to the swin input size" + # to avoid bicubic zero error + if T <= target_T: + x = nn.functional.interpolate( + x, (target_T, x.shape[3]), mode="bicubic", align_corners=True + ) + if F <= target_F: + x = nn.functional.interpolate( + x, (x.shape[2], target_F), mode="bicubic", align_corners=True + ) + x = x.permute(0, 1, 3, 2).contiguous() # B C F T + x = x[:, :, :, cur_pos : cur_pos + self.spec_size] + x = x.repeat(repeats=(1, 1, 4, 1)) + return x + + def forward(self, x: torch.Tensor): + batch_size, chs, seq_len = x.size() + x = x.view(batch_size, -1) + + x = self.spectrogram_extractor(x) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + + if self.use_stdnorm: + std = x.std(dim=(-1, -2), keepdim=True) + mu = x.mean(dim=(-1, -2), keepdim=True) + x = (x - mu) / std + + if self.use_batchnorm: + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + + if self.training: + x = self.spec_augmenter(x) + + frames_num = x.shape[2] + cur_pos = random.randint(0, (self.freq_ratio - 1) * self.spec_size - 1) + x = self.reshape_wav2img(x) + + x = self.patch_embed(x) + x = x + self.absolute_pos_embed + x = self.pos_drop(x) + + for i, layer in enumerate(self.layers): + x, attn = layer(x) + x = self.norm(x) + x = self.avgpool(x.transpose(1, 2)) + x = torch.flatten(x, 1) + x = self.head(x) + return x + + +def __main__(): + htast = HTSAT( + spec_size=256, + sample_rate=48000, + window_size=2048, + hop_size=512, + mel_bins=128, + fmin=20.0, + fmax=4000.0, + classes_num=128, + patch_size=16, + patch_stride=(4, 4), + internal_embed_dim=96, + in_channels=1, + patch_window_size=8, + depths=[2, 2, 6, 2], + num_heads=[4, 8, 16, 32], + qkv_bias=True, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.1, + patch_norm=True, + mlp_ratio=4.0, + norm_before_mlp="ln", + use_checkpoint=False, + use_batchnorm=True, + use_stdnorm=True, + ) + x = torch.rand(1, 2, 262144) + y = htast(x) # 1, 128 + print(y.shape) + + +if __name__ == "__main__": + __main__() diff --git a/utils/st_ito/models/panns.py b/utils/st_ito/models/panns.py new file mode 100644 index 0000000000000000000000000000000000000000..f032cdc64719758ef00c7153a22d1dee0a1e7774 --- /dev/null +++ b/utils/st_ito/models/panns.py @@ -0,0 +1,281 @@ +# Adapted from https://github.com/qiuqiangkong/audioset_tagging_cnn/blob/master/pytorch/models.py +# Under MIT License +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchlibrosa.stft import Spectrogram, LogmelFilterBank +from torchlibrosa.augmentation import SpecAugmentation + + +def init_layer(layer): + """Initialize a Linear or Convolutional layer.""" + nn.init.xavier_uniform_(layer.weight) + + if hasattr(layer, "bias"): + if layer.bias is not None: + layer.bias.data.fill_(0.0) + + +def init_bn(bn): + """Initialize a Batchnorm layer.""" + bn.bias.data.fill_(0.0) + bn.weight.data.fill_(1.0) + + +class ConvBlock(nn.Module): + def __init__(self, in_channels, out_channels, use_batchnorm: bool = True): + super(ConvBlock, self).__init__() + self.use_batchnorm = use_batchnorm + + self.conv1 = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=(3, 3), + stride=(1, 1), + padding=(1, 1), + bias=False, + ) + + self.conv2 = nn.Conv2d( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=(3, 3), + stride=(1, 1), + padding=(1, 1), + bias=False, + ) + + if self.use_batchnorm: + self.bn1 = nn.BatchNorm2d(out_channels) + self.bn2 = nn.BatchNorm2d(out_channels) + else: + self.bn1 = nn.Identity() + self.bn2 = nn.Identity() + + self.init_weight() + + def init_weight(self): + init_layer(self.conv1) + init_layer(self.conv2) + + if self.use_batchnorm: + init_bn(self.bn1) + init_bn(self.bn2) + + def forward(self, input, pool_size=(2, 2), pool_type="avg"): + x = input + x = F.relu_(self.bn1(self.conv1(x))) + x = F.relu_(self.bn2(self.conv2(x))) + if pool_type == "max": + x = F.max_pool2d(x, kernel_size=pool_size) + elif pool_type == "avg": + x = F.avg_pool2d(x, kernel_size=pool_size) + elif pool_type == "avg+max": + x1 = F.avg_pool2d(x, kernel_size=pool_size) + x2 = F.max_pool2d(x, kernel_size=pool_size) + x = x1 + x2 + else: + raise Exception("Incorrect argument!") + + return x + + +class ConvBlock5x5(nn.Module): + def __init__(self, in_channels, out_channels): + super(ConvBlock5x5, self).__init__() + + self.conv1 = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=(5, 5), + stride=(1, 1), + padding=(2, 2), + bias=False, + ) + + self.bn1 = nn.BatchNorm2d(out_channels) + + self.init_weight() + + def init_weight(self): + init_layer(self.conv1) + init_bn(self.bn1) + + def forward(self, input, pool_size=(2, 2), pool_type="avg"): + x = input + x = F.relu_(self.bn1(self.conv1(x))) + if pool_type == "max": + x = F.max_pool2d(x, kernel_size=pool_size) + elif pool_type == "avg": + x = F.avg_pool2d(x, kernel_size=pool_size) + elif pool_type == "avg+max": + x1 = F.avg_pool2d(x, kernel_size=pool_size) + x2 = F.max_pool2d(x, kernel_size=pool_size) + x = x1 + x2 + else: + raise Exception("Incorrect argument!") + + return x + + +class Cnn14(nn.Module): + def __init__( + self, + embed_dim: int, + sample_rate: float, + window_size: int, + hop_size: int, + mel_bins: int, + fmin: float, + fmax: float, + use_batchnorm: bool = False, + input_norm: str = "batchnorm", + ): + super(Cnn14, self).__init__() + self.embed_dim = embed_dim + self.use_batchnorm = use_batchnorm + self.input_norm = input_norm + + window = "hann" + center = True + pad_mode = "reflect" + ref = 1.0 + amin = 1e-10 + top_db = None + + # Spectrogram extractor + self.spectrogram_extractor = Spectrogram( + n_fft=window_size, + hop_length=hop_size, + win_length=window_size, + window=window, + center=center, + pad_mode=pad_mode, + freeze_parameters=True, + ) + + # Logmel feature extractor + self.logmel_extractor = LogmelFilterBank( + sr=sample_rate, + n_fft=window_size, + n_mels=mel_bins, + fmin=fmin, + fmax=fmax, + ref=ref, + amin=amin, + top_db=top_db, + freeze_parameters=True, + ) + + # Spec augmenter + self.spec_augmenter = SpecAugmentation( + time_drop_width=64, + time_stripes_num=2, + freq_drop_width=8, + freq_stripes_num=2, + ) + + self.bn0 = nn.BatchNorm2d(mel_bins) + + self.conv_block1 = ConvBlock( + in_channels=1, out_channels=64, use_batchnorm=use_batchnorm + ) + self.conv_block2 = ConvBlock( + in_channels=64, out_channels=128, use_batchnorm=use_batchnorm + ) + self.conv_block3 = ConvBlock( + in_channels=128, out_channels=256, use_batchnorm=use_batchnorm + ) + self.conv_block4 = ConvBlock( + in_channels=256, out_channels=512, use_batchnorm=use_batchnorm + ) + self.conv_block5 = ConvBlock( + in_channels=512, out_channels=1024, use_batchnorm=use_batchnorm + ) + self.conv_block6 = ConvBlock( + in_channels=1024, out_channels=2048, use_batchnorm=use_batchnorm + ) + + self.fc_mid = nn.Linear(2048, embed_dim, bias=True) + self.fc_side = nn.Linear(2048, embed_dim, bias=True) + + self.init_weight() + + def init_weight(self): + init_bn(self.bn0) + init_layer(self.fc_mid) + init_layer(self.fc_side) + + def forward(self, x: torch.Tensor): + """ + input (torch.Tensor): Waveform tensor with shape (batch_size, chs, seq_len) + """ + batch_size, chs, seq_len = x.size() + + # compute mid and side signals + if chs == 1: + pass + elif chs == 2: + x_mid = (x[:, 0, :] + x[:, 1, :]) / 2 + x_side = (x[:, 0, :] - x[:, 1, :]) / 2 + # stack along batch dim + x = torch.stack([x_mid, x_side], dim=1) + else: + raise ValueError(f"Invalid number of channels: {chs}") + + # move to batch dim + x = x.view(batch_size * chs, seq_len) + + # extract logmel features + x = self.spectrogram_extractor(x) # (batch_size, 1, time_steps, freq_bins) + x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins) + + if self.input_norm == "batchnorm": + # this normalizes over mel bins which is problematic for equalization + x = x.transpose(1, 3) + x = self.bn0(x) + x = x.transpose(1, 3) + elif self.input_norm == "minmax": + x = x.clamp(-80, 40.0) # clamp the logmels between -80 and 40 + x = (x + 80) / 120 # normalize the logmels between 0 and 1 + x = (x * 2) - 1 # normalize the logmels between -1 and 1 + elif self.input_norm == "none": + pass + else: + raise ValueError(f"Invalid input_norm: {self.input_norm}") + + if self.training: + x = self.spec_augmenter(x) + + x = self.conv_block1(x, pool_size=(2, 2), pool_type="avg") + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block2(x, pool_size=(2, 2), pool_type="avg") + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block3(x, pool_size=(2, 2), pool_type="avg") + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block4(x, pool_size=(2, 2), pool_type="avg") + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block5(x, pool_size=(2, 2), pool_type="avg") + x = F.dropout(x, p=0.2, training=self.training) + x = self.conv_block6(x, pool_size=(1, 1), pool_type="avg") + x = F.dropout(x, p=0.2, training=self.training) + x = torch.mean(x, dim=3) + + (x1, _) = torch.max(x, dim=2) + x2 = torch.mean(x, dim=2) + x = x1 + x2 + + # move mid and side back to channel dim + x = x.view(batch_size, chs, -1) + + if chs == 1: + x_mid = x[:, 0, :] + mid_embed = self.fc_mid(x_mid) + side_embed = mid_embed + elif chs == 2: + x_mid = x[:, 0, :] + x_side = x[:, 1, :] + mid_embed = self.fc_mid(x_mid) + side_embed = self.fc_side(x_side) + + return mid_embed, side_embed diff --git a/utils/st_ito/models/pretrained.py b/utils/st_ito/models/pretrained.py new file mode 100644 index 0000000000000000000000000000000000000000..1cc30c187468eaa27d2d940ef3881b4ec6d74040 --- /dev/null +++ b/utils/st_ito/models/pretrained.py @@ -0,0 +1,199 @@ +import torch +import laion_clap +import torch +import sys + +# sys.path.append("VGGish") +# from VGGish.network.vggish import VGGish +# from VGGish.vggish.vggish_input import waveform_to_examples +# import VGGish.vggish.vggish_params as vggish_params +import resampy +import torchaudio +import numpy as np +import wav2clip +from panns_inference import AudioTagging +from transformers import Wav2Vec2Model + + +class CLAP(torch.nn.Module): + def __init__(self): + super().__init__() + self.model = laion_clap.CLAP_Module(enable_fusion=False) + self.model.load_ckpt() # download the default pretrained checkpoint. + + def forward(self, x): + # Remove channel dim + x = x.squeeze(1, -1) + embeddings = self.model.get_audio_embedding_from_data(x=x, use_tensor=True) + return embeddings + + +class HEAR(torch.nn.Module): + def __init__(self, model_choice: str): + super().__init__() + if model_choice == "hearbaseline": + from hearbaseline import load_model, get_scene_embeddings + + raise NotImplementedError("Model not implemented") + self.model = load_model() + self.get_scene_embeddings = get_scene_embeddings + + def forward(self, x): + # Remove channel dim + x = x.squeeze(1) + embeddings = self.get_scene_embeddings(x, self.model) + return embeddings + + +class Wav2Vec2(torch.nn.Module): + def __init__(self): + super().__init__() + self.model = Wav2Vec2Model.from_pretrained( + "facebook/wav2vec2-large-960h-lv60-self" + ) + + def forward(self, x): + x = x.squeeze(1) + output = self.model(x, output_hidden_states=True) + out = output.hidden_states[0] + for i in range(1, len(output.hidden_states)): + out = out + output.hidden_states[i] + return out / len(output.hidden_states) + + +class Wav2Clip(torch.nn.Module): + def __init__(self): + super().__init__() + self.model = wav2clip.get_model() + + def forward(self, x): + # Remove channel dim + x = x.squeeze(1) + embeddings = self.model(x) + return embeddings + + +class PANNS(torch.nn.Module): + def __init__(self, device: str = "cpu"): + super().__init__() + self.model = AudioTagging(checkpoint_path=None, device=device) + self.model.model.eval() + + def forward(self, x): + # Remove channel dim + x = x.squeeze(1) + output_dict = self.model.model(x, None) + embedding = output_dict["embedding"] + return embedding + + +class VGGISH(torch.nn.Module): + def __init__(self, sr: int = 16000): + super().__init__() + self.sr = sr + self.model = VGGish() + self.model.load_state_dict(torch.load("VGGish/checkpoints/vggish_model.pt")) + # Remove the last layer + layers = list(self.model.children()) + layers[-1] = layers[-1][:-1] + self.model = torch.nn.Sequential(*layers) + self.linear = torch.nn.Linear(131072, 128) + n_fft = 2 ** int( + np.ceil( + np.log( + int( + vggish_params.STFT_WINDOW_LENGTH_SECONDS + * vggish_params.SAMPLE_RATE + ) + ) + / np.log(2.0) + ) + ) + self.melspec = torchaudio.transforms.MelSpectrogram( + n_fft=n_fft, + sample_rate=vggish_params.SAMPLE_RATE, + win_length=int( + vggish_params.STFT_WINDOW_LENGTH_SECONDS * vggish_params.SAMPLE_RATE + ), + hop_length=int( + vggish_params.STFT_HOP_LENGTH_SECONDS * vggish_params.SAMPLE_RATE + ), + n_mels=vggish_params.NUM_MEL_BINS, + f_min=vggish_params.MEL_MIN_HZ, + f_max=vggish_params.MEL_MAX_HZ, + ) + + def forward(self, x): + bs = x.shape[0] + # # Remove channel dim + x = x.view(bs, -1) + embeddings = [] + for i in range(bs): + # Original code + # input_batch = waveform_to_examples(x[i].detach().numpy(), self.sr) + input_batch = self.waveform_to_examples(x[i]) + input_batch = input_batch.unsqueeze(1) + embedding_batch_torch = self.model(input_batch) + embedding_batch_torch = torch.mean(embedding_batch_torch.float(), axis=0) + embeddings.append(embedding_batch_torch) + embeddings = torch.stack(embeddings) + return embeddings + + # Following functions are modified from VGGish to allow for gradiens to pass through + def waveform_to_examples(self, data): + """Converts audio waveform into an array of examples for VGGish. + Args: + data: np.array of either one dimension (mono) or two dimensions + (multi-channel, with the outer dimension representing channels). + Each sample is generally expected to lie in the range [-1.0, +1.0], + although this is not required. + sample_rate: Sample rate of data. + Returns: + 3-D np.array of shape [num_examples, num_frames, num_bands] which represents + a sequence of examples, each of which contains a patch of log mel + spectrogram, covering num_frames frames of audio and num_bands mel frequency + bands, where the frame length is vggish_params.STFT_HOP_LENGTH_SECONDS. + """ + # Resample to the rate assumed by VGGish. + if self.sr != vggish_params.SAMPLE_RATE: + data = resampy.resample(data, self.sr, vggish_params.SAMPLE_RATE) + + # Compute mel spectrogram features. + mel = self.melspec(data) + # Convert to log scale. + log_mel = torch.log(mel + vggish_params.LOG_OFFSET).T + # Frame features into examples. + features_sample_rate = 1.0 / vggish_params.STFT_HOP_LENGTH_SECONDS + example_window_length = int( + round(vggish_params.EXAMPLE_WINDOW_SECONDS * features_sample_rate) + ) + example_hop_length = int( + round(vggish_params.EXAMPLE_HOP_SECONDS * features_sample_rate) + ) + log_mel_examples = frame( + log_mel, window_length=example_window_length, hop_length=example_hop_length + ) + return log_mel_examples + + +def frame(data, window_length, hop_length): + """Convert array into a sequence of successive possibly overlapping frames. + An n-dimensional array of shape (num_samples, ...) is converted into an + (n+1)-D array of shape (num_frames, window_length, ...), where each frame + starts hop_length points after the preceding one. + This is accomplished using stride_tricks, so the original data is not + copied. However, there is no zero-padding, so any incomplete frames at the + end are not included. + Args: + data: np.array of dimension N >= 1. + window_length: Number of samples in each frame. + hop_length: Advance (in samples) between each window. + Returns: + (N+1)-D np.array with as many rows as there are complete frames that can be + extracted. + """ + num_samples = data.shape[0] + num_frames = 1 + int(np.floor((num_samples - window_length) / hop_length)) + shape = (num_frames, window_length) + data.shape[1:] + strides = (data.stride()[0] * hop_length,) + data.stride() + return torch.as_strided(data, size=shape, stride=strides) diff --git a/utils/st_ito/style_transfer.py b/utils/st_ito/style_transfer.py new file mode 100644 index 0000000000000000000000000000000000000000..37e6f0734de90c2efe70e17c34fdea219ef416f1 --- /dev/null +++ b/utils/st_ito/style_transfer.py @@ -0,0 +1,692 @@ +# inference-time optimization methods +import os +import cma +import torch +import torchaudio +import pedalboard +import numpy as np +import scipy.signal +import pyloudnorm as pyln +import multiprocessing as mp + +from typing import List + +# ------- audio processing methods ------- + + +def load_plugins(plugins: dict): + total_num_params = 0 + init_params = [] + for plugin_name, plugin in plugins.items(): + if "vst_filepath" in plugin: + plugin_instance = pedalboard.load_plugin(plugin["vst_filepath"]) + elif "class_path" in plugin: + plugin_instance = plugin["class_path"]() + else: + raise ValueError(f"Plugin must contain 'vst_filepath' or 'class_path'.") + + plugin["parameter_names"] = ["our_bypass"] + init_params.append(0.0) + num_params = 1 + for name, parameter in plugin_instance.parameters.items(): + num_params += 1 + print(f"{plugin_name}: {name} = {parameter.raw_value}") + init_params.append(parameter.raw_value) + plugin["parameter_names"].append(name) + print() + + plugin["num_params"] = num_params + plugin["instance"] = plugin_instance + total_num_params += num_params + + return plugins, total_num_params, init_params + + +def process_audio( + x: np.ndarray, + w: np.ndarray, + sr: int, + plugins: List[dict], + normalize_stages: bool = False, +): + """Process audio with plugins and provided parameters on [0, 1]. + + Args: + x (np.ndarray): Audio vector of shape (1, num_samples) + w (np.ndarray): Parameter vector of shape (num_params,) + sr (int): Sample rate + plugins (List[dict]): List of plugin dicts + normalize_stages (bool): Normalize the output of each stage + """ + + # reflect pad to avoid transient artifacts + # x = np.pad(x, ((0, 0), (2048, 0)), mode="reflect") + + widx = 0 + for plugin_name, plugin in plugins.items(): + if "instance" not in plugin: + if "vst_filepath" in plugin: + plugin_instance = pedalboard.load_plugin(plugin["vst_filepath"]) + elif "class_path" in plugin: + plugin_instance = plugin["class_path"]() + else: + raise ValueError(f"Plugin must contain 'vst_filepath' or 'class_path'.") + + plugin["instance"] = plugin_instance + for name in plugin["parameter_names"]: + if not name == "our_bypass": + parameter = plugin["instance"].parameters[name] + if name in plugin["fixed_parameters"]: + if "vst_filepath" in plugin: + parameter.raw_value = plugin["fixed_parameters"][name] + else: + parameter.set_value(plugin["fixed_parameters"][name]) + widx += 1 + else: + parameter.raw_value = w[widx] + widx += 1 + else: + if w[widx] > 0.5: + widx += 1 + continue # don't run this plugin + widx += 1 + + if plugin["num_channels"] == 2 and x.shape[0] == 1: + x = np.concatenate((x, x), axis=0) + + # process left and right channels separately + if plugin["num_channels"] == 1 and x.shape[0] == 2: + # process audio + x_l = plugin["instance"].process(x[0:1, :], sample_rate=sr) + x_r = plugin["instance"].process(x[1:2, :], sample_rate=sr) + x = np.concatenate((x_l, x_r), axis=0) + else: + x = plugin["instance"].process(x, sample_rate=sr) + + if normalize_stages: + x /= np.clip(np.max(np.abs(x)), a_min=1e-8, a_max=None) + + # crop early transient + # x = x[:, 2048:] + + # peak normalize + x /= np.clip(np.max(np.abs(x)), a_min=1e-8, a_max=None) + + return x + + +# ------ main methods ------- + + +def run_input( + input_audio: torch.Tensor, + target_audio: torch.Tensor, + sample_rate: int, + plugins: List[dict], + model: torch.nn.Module, + *args, + **kwargs, +): + bs, chs, seq_len = input_audio.shape + result = { + "output_audio": input_audio, + } + + return result + + +def run_random( + input_audio: torch.Tensor, + target_audio: torch.Tensor, + sample_rate: int, + plugins: List[dict], + model: torch.nn.Module, + *args, + **kwargs, +): + bs, chs, seq_len = input_audio.shape + total_num_params = sum([plugin["num_params"] for plugin in plugins.values()]) + w = torch.rand(total_num_params) + output_audio = process_audio( + input_audio.squeeze(0).numpy(), w.numpy(), sample_rate, plugins + ) + output_audio = torch.from_numpy(output_audio).unsqueeze(0) + + result = { + "output_audio": output_audio, + "param_dict": parameters_to_dict(w.numpy(), plugins), + } + + return result + + +def smooth_spectrum(H: np.ndarray): + # apply Savgol filter for smoothed target curve + return scipy.signal.savgol_filter(H, 1025, 2) + + +def get_average_spectrum(x: torch.Tensor, n_fft: int = 16384): + print(x.shape) + # if stereo sum to mono + if x.shape[0] == 2: + x = x.mean(dim=0, keepdim=True) + # x = x[:, : self.n_fft] + X = torch.stft(x, n_fft, return_complex=True, normalized=True) + # fft_size = self.next_power_of_2(x.shape[-1]) + # X = torch.fft.rfft(x, n=fft_size) + + X = X.abs() # convert to magnitude + X = X.mean(dim=-1).view(-1) # average across frames + + return X + + +def run_rule_based( + input_audio: torch.Tensor, + target_audio: torch.Tensor, + sample_rate: int, + plugins: List[dict], + model: torch.nn.Module, + n_fft: int = 16384, + n_taps: int = 2048, + **kwargs, +): + """Rule-based style transfer baseline. + + This approach performs style transfer by designing a matched EQ curve based on + the target audio and performs a simple iterative hill climbing optimization to + find the best value of the threshold parameter for the dynamics processor. + + Args: + input_audio (torch.Tensor): Input audio tensor of shape (bs, chs, seq_len) + target_audio (torch.Tensor): Target audio tensor of shape (bs, chs, seq_len) + sample_rate (int): Sample rate + plugins (List[dict]): List of plugin dicts (Not used in this method) + model (torch.nn.Module): Model (Not used in this method) + + """ + bs, chs, seq_len = input_audio.shape + bs, chs, seq_len = target_audio.shape + + audio_outputs = [] + for bidx in range(bs): + # get batch audio + input_audio_elem = input_audio[bidx, :, :] + target_audio_elem = target_audio[bidx, :, :] + + # peak normalize to -12 dBFS + input_audio_elem /= torch.max(torch.abs(input_audio_elem)).clamp(min=1e-8) + target_audio_elem /= torch.max(torch.abs(target_audio_elem)).clamp(min=1e-8) + input_audio_elem *= 10 ** (-12 / 20) + target_audio_elem *= 10 ** (-12 / 20) + + # ------------ design the matched EQ ------------ + in_spec = get_average_spectrum(input_audio_elem, n_fft=n_fft) + ref_spec = get_average_spectrum(target_audio_elem, n_fft=n_fft) + sm_in_spec = smooth_spectrum(in_spec) + sm_ref_spec = smooth_spectrum(ref_spec) + + # design inverse FIR filter to match target EQ + freqs = np.linspace(0, 1.0, num=(n_fft // 2) + 1) + response = sm_ref_spec / sm_in_spec + response[-1] = 0.0 # zero gain at nyquist + + b = scipy.signal.firwin2( + n_taps, + freqs * (sample_rate / 2), + response, + fs=sample_rate, + ) + + # apply the filter + x_filt = scipy.signal.lfilter(b, [1.0], input_audio_elem.numpy()) + input_audio_elem = torch.tensor(x_filt.astype("float32")) + + # peak normalize again + input_audio_elem /= torch.max(torch.abs(input_audio_elem)).clamp(min=1e-8) + input_audio_elem *= 10 ** (-12 / 20) + + # ------------ apply dynamics processor ------------ + meter = pyln.Meter(sample_rate) + input_lufs = meter.integrated_loudness(input_audio_elem.permute(1, 0).numpy()) + target_lufs = meter.integrated_loudness(target_audio_elem.permute(1, 0).numpy()) + + delta_lufs = target_lufs - input_lufs + + threshold_db = 0.0 + x_comp = input_audio_elem.numpy() + x_comp_new = input_audio_elem.numpy() + while delta_lufs > 0.25 and threshold_db > -80.0: + x_comp = x_comp_new # use the last setting + x_comp_new = pedalboard.Compressor( + threshold_db=threshold_db, ratio=3.0, attack_ms=1.0, release_ms=100.0 + ).process(x_comp, sample_rate=sample_rate) + x_comp_new /= np.max(np.abs(x_comp_new)) + x_comp_new *= 10 ** (-12 / 20) + x_lufs = meter.integrated_loudness(x_comp_new.T) + delta_lufs = target_lufs - x_lufs + threshold_db -= 0.5 # decrease threshold + + # convert to torch + x_comp_new = torch.tensor(x_comp_new.astype("float32")) + audio_outputs.append(x_comp_new) + + result = { + "output_audio": torch.stack(audio_outputs, dim=0), + } + + return result + + +def run_deepafx_st( + input_audio: torch.Tensor, + target_audio: torch.Tensor, + sample_rate: int, + plugins: List[dict], + model: torch.nn.Module, + *args, + **kwargs, +): + bs, chs, seq_len = input_audio.size() + bs, chs, seq_len = target_audio.size() + + with torch.no_grad(): + # peak normalize + # input_audio /= torch.max(torch.abs(input_audio)).clamp(min=1e-8) + # target_audio /= torch.max(torch.abs(target_audio)).clamp(min=1e-8) + + output_audio, params, params_logits = model( + input_audio, + target_audio, + sample_rate=sample_rate, + render_audio=True, + ) + + if output_audio.shape[1] == 1: + output_audio = output_audio.repeat(1, chs, 1) + + # store parameters in dict + param_dict = {} + for idx in range(params.shape[-1]): + param_dict[f"{idx}"] = params[..., idx].item() + + result = { + "output_audio": output_audio, + "params": param_dict, + } + + return result + + +# ----------- Evolutionary Strategies ------------ + + +def parameters_to_dict(w: np.ndarray, plugins: List[dict]): + """Convert parameter vector to dictionary. + + Args: + w (np.ndarray): Parameter vector + plugins (List[dict]): List of plugin dicts + """ + widx = 0 + w_dict = {} + for plugin_name, plugin in plugins.items(): + if plugin_name not in w_dict: + w_dict[plugin_name] = {} + + for name in plugin["parameter_names"]: + if name == "our_bypass": + w_dict[plugin_name][name] = w[widx] + widx += 1 + continue + else: + parameter = plugin["instance"].parameters[name] + if name in plugin["fixed_parameters"]: + if "vst_filepath" in plugin: + parameter.raw_value = plugin["fixed_parameters"][name] + else: + parameter.set_value(plugin["fixed_parameters"][name]) + widx += 1 + else: + parameter.raw_value = w[widx] + widx += 1 + + if hasattr(parameter, "get_value"): + w_dict[plugin_name][name] = parameter.get_value() + else: + w_dict[plugin_name][name] = parameter.raw_value + + return w_dict + + +def savepop_to_disk( + iteration: str, + fvals: List[float], + output_embeds: List[torch.Tensor], + output_audios: List[torch.Tensor], + run_dir: str, + sample_rate: int, +): + # create directory for the current iteration + pop_dir = os.path.join(run_dir, f"pop_{iteration}") + os.makedirs(pop_dir, exist_ok=True) + # save the current population + pop_members = [] + + for idx, (fval, output_audio, output_embed) in enumerate( + zip(fvals, output_audios, output_embeds) + ): + pop_members.append((fval, output_audio, output_embed)) + + # sort by fval + pop_members = sorted(pop_members, key=lambda x: x[0]) + + # save to disk + for idx, (fval, output_audio, _) in enumerate(pop_members): + output_audio_filepath = os.path.join( + pop_dir, + f"output_audio_pop_{idx}_fval_{fval:0.4e}.wav", + ) + output_audio /= torch.max(torch.abs(output_audio)).clamp(min=1e-8) + torchaudio.save( + output_audio_filepath, + output_audio.squeeze(0), + sample_rate, + backend="soundfile", + ) + + +def run_es( + input_audio: torch.Tensor, + target_audio: torch.Tensor, + sample_rate: int, + plugins: List[dict], + model: torch.nn.Module, + embed_func: callable, + content_model: torch.nn.Module = None, + content_embed_func: callable = None, + max_iters: int = 100, + w0: torch.Tensor = None, + find_w0: bool = True, + sigma0: float = 0.1, + distance: str = "cosine", + random_crop: bool = False, + popsize: int = 32, + parallel: bool = False, + dropout: float = 0.0, + savepop: bool = False, + run_dir: str = ".", + *args, + **kwargs, +): + """Run CMA-ES optimization to find the best parameters. + + Args: + input_audio (torch.Tensor): Input audio tensor of shape (bs, chs, seq_len) + target_audio (torch.Tensor): Target audio tensor of shape (bs, chs, seq_len) + sample_rate (int): Sample rate + plugins (List[dict]): List of plugin dicts + model (torch.nn.Module): Model + embed_func (callable): Embedding function + normalization (str): Normalization type + max_iters (int): Max iterations + w0 (torch.Tensor): Initial parameters + find_w0 (bool): Evaluate random initial parameters selecting the best + sigma0 (float): Initial sigma + distance (str): Distance type + random_crop (bool): Apply random crop to the input audio + popsize (int): Population size + parallel (bool): Use parallel processing + savepop (bool): Save population to disk + + """ + # count total number of parameters + total_num_params = sum([plugin["num_params"] for plugin in plugins.values()]) + + bs, chs, seq_len = input_audio.shape + + # input_audio = input_audio.unsqueeze(0) + # target_audio = target_audio.unsqueeze(0) + + # peak normalize + input_audio /= torch.max(torch.abs(input_audio)).clamp(min=1e-8) + target_audio /= torch.max(torch.abs(target_audio)).clamp(min=1e-8) + + # compute target embedding (only once) + target_embed = embed_func( + target_audio, + model, + sample_rate, + ) + + print(content_model) + + if content_model is not None: + target_content_embeds = content_embed_func( + target_audio, + content_model, + sample_rate, + ) + else: + target_content_embeds = None + + # define evaluation function + def evaluate( + W: torch.Tensor, + x: torch.Tensor, + sample_rate: int, + plugins: List[dict], + target_embeds: torch.Tensor, + target_content_embeds: torch.Tensor = None, + parallel: bool = False, + dropout: float = 0.0, + ): + """Evaluate the current solution. + + Args: + W (np.ndarray): List of plugin parameters + x (np.ndarray): input audio + sample_rate (int): Sample rate + plugins (List[dict]): List of plugin dicts + target_embeds (torch.Tensor): Target embedding + parallel (bool): Use parallel processing + dropout (float): Dropout rate + + Returns: + dict : Dictionary containing the output audio, parameters, and distance + """ + + if parallel: + with mp.Pool(processes=16) as pool: + args = [(x.squeeze(0).numpy(), w, sample_rate, plugins) for w in W] + output_audios = pool.starmap(process_audio, args) + else: + output_audios = [] + crop_len = 262144 + if ( + random_crop and (x.shape[-1] - crop_len) > 16384 + ): # apply random crop (same crop for all in population) + start_idx = np.random.randint(16384, x.shape[-1] - crop_len) + else: + start_idx = 0 + for w in W: + if x.shape[-1] > crop_len: + x_crop = ( + x[:, :, start_idx : start_idx + crop_len] if random_crop else x + ) + else: + x_crop = torch.nn.functional.pad(x, (0, 262144 - x.shape[-1])) + output_audios.append( + process_audio(x_crop.squeeze(0).numpy(), w, sample_rate, plugins) + ) + + output_audios = [ + torch.from_numpy(output_audio) for output_audio in output_audios + ] + + # cat along batch dim + output_audios = torch.stack(output_audios, dim=0) + + # compute embedding + output_embeds = embed_func( + output_audios, + model, + sample_rate, + ) + + if content_model is not None: + output_content_embeds = content_embed_func( + output_audios, + content_model, + sample_rate, + ) + + dists = [] + for output_idx, (embed_name, output_embed) in enumerate(output_embeds.items()): + + target_embed = target_embeds[embed_name] + + # apply dropout to the embeddings + if dropout > 0.0: + output_embed = torch.nn.functional.dropout(output_embed, p=dropout) + + # compute distance + with torch.no_grad(): + dist = torch.cosine_similarity(output_embed, target_embed, dim=-1) + dist = -dist + + # aggregate distances + dists.append(dist) + + # now add the content distance + if target_content_embeds is not None: + for embed_name, output_content_embed in output_content_embeds.items(): + dist = torch.cosine_similarity( + output_content_embed, target_content_embeds[embed_name], dim=-1 + ) + dist = -dist + dists.append(2 * dist) + + dists = torch.stack(dists, dim=0) + dist = dists.mean(dim=0) + + return dist.tolist(), output_embeds, output_audios + + # setup CMA-ES + if find_w0: + print("Finding the best w0...") + tmp_w0s = [] + for n in range(popsize): + tmp_w0 = np.random.rand(total_num_params) + tmp_w0s.append(tmp_w0) + fvals, output_embeds, output_audios = evaluate( + tmp_w0s, + input_audio, + sample_rate, + plugins, + target_embed, + target_content_embeds=target_content_embeds, + parallel=parallel, + dropout=dropout, + ) + # find the best w0 + print(fvals) + w0 = tmp_w0s[np.argmin(fvals)] + if savepop: + savepop_to_disk( + -1, + fvals, + output_embeds, + output_audios, + run_dir, + sample_rate, + ) + else: + if w0 is None: + w0 = np.ones(total_num_params) * 0.5 + else: + w0 = w0.numpy() + + # convert parameters to dictionary + init_param_dict = parameters_to_dict(w0, plugins) + print(init_param_dict) + + es = cma.CMAEvolutionStrategy(w0, sigma0, {"bounds": [0, 1], "popsize": popsize}) + + # tracking history + fval_history = [] + wopt_history = [] + iters_without_improvement = 0 + + # run CMA-ES + for iteration in range(max_iters): + x = input_audio.clone() + W = es.ask() + fvals, output_embeds, output_audios = evaluate( + W, + x, + sample_rate, + plugins, + target_embed, + target_content_embeds=target_content_embeds, + parallel=parallel, + dropout=( + dropout if (iteration + 1) < max_iters else 0.0 + ), # on the last iteration, do not apply dropout + ) + + # save best + wopt_history.append(es.result[0]) + fval_history.append(es.result[1]) + + if savepop: + savepop_to_disk( + iteration, + fvals, + output_embeds, + output_audios, + run_dir, + sample_rate, + ) + es.tell(W, fvals) + es.disp() + + # check if the the best fval has improved + if iteration > 0: + fval_delta = min(fvals) - min(fval_history) + else: + fval_delta = -0.02 + + if fval_delta > -0.01: + iters_without_improvement += 1 + print( + f"Solution has not improved for {iters_without_improvement} iterations." + ) + else: + iters_without_improvement = 0 + + if iters_without_improvement > 10: + print("Stopping early due to no improvement.") + break + + wopt = es.result[0] + fopt = es.result[1] + + # save the current solution + output_audio = torch.from_numpy( + process_audio(x.squeeze(0).numpy(), wopt, sample_rate, plugins) + ) + + # convert parameters to dictionary + param_dict = parameters_to_dict(wopt, plugins) + + result = { + "output_audio": output_audio, + "params": param_dict, + "fopt": fopt, + "wopt": wopt, + "fval_history": fval_history, + "wopt_history": wopt_history, + } + + return result diff --git a/utils/st_ito/utils.py b/utils/st_ito/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b7874557589e7b4f4b8434dd19a4169b462b5811 --- /dev/null +++ b/utils/st_ito/utils.py @@ -0,0 +1,608 @@ +import os +import yaml +import torch +import torchaudio +import pyloudnorm as pyln + +from collections import OrderedDict +from importlib import import_module + +# external models +import wav2clip +import laion_clap +from transformers import Wav2Vec2Model +from st_ito.models.beats.beats import BEATs, BEATsConfig + +from st_ito.models.fx_encoder import FXencoder +from st_ito.methods.style import StyleTransferSystem + +# features +from st_ito.features import ( + compute_lufs, + compute_rms_energy, + compute_barkspectrum, + compute_crest_factor, + compute_spectral_centroid, +) + +# ------------------ Normalization functions ------------------ + + +def apply_fade_in(x: torch.Tensor, num_samples: int = 16384): + """Apply fade in to the first num_samples of the audio signal. + + Args: + x (torch.Tensor): Input audio tensor + num_samples (int, optional): Number of samples to apply fade in. Defaults to 16384. + + Returns: + torch.Tensor: Audio tensor with fade in applied + """ + fade = torch.linspace(0, 1, num_samples, device=x.device) + x[..., :num_samples] = x[..., :num_samples] * fade + return x + + +def batch_peak_normalize(x: torch.Tensor): + peak = torch.max(torch.abs(x), dim=1)[0] + x = x / peak[:, None].clamp(min=1e-8) + return x + + +def batch_loudness_normalize(x: torch.Tensor, meter: pyln.Meter, target_lufs: float): + for batch_idx in range(x.shape[0]): + lufs = meter.integrated_loudness( + x[batch_idx : batch_idx + 1, ...].permute(1, 0).cpu().numpy() + ) + gain_db = target_lufs - lufs + gain_lin = 10 ** (gain_db / 20) + x[batch_idx, :] = gain_lin * x[batch_idx, :] + return x + + +# ------------------- MIR feature extract---------------- # + + +def load_mir_feature_extractor(use_gpu: bool = False): + class Model: + def __init__(self) -> None: + self.embed_dim = 49 + + model = Model() + + return model + + +def get_mir_feature_embeds( + x: torch.Tensor, model: torch.nn.Module, sample_rate: float, **kwargs +): + + lufs_feat = compute_lufs(x, sample_rate) + rms_feat = compute_rms_energy(x) + crest_feat = compute_crest_factor(x) + barkspectrum_feat = compute_barkspectrum(x, sample_rate, mode="mono") + spectral_centroid_feat = compute_spectral_centroid(x, sample_rate) + + # combine features into tensor + features = { + "lufs": lufs_feat, + "rms": rms_feat, + "crest": crest_feat, + "barkspectrum": barkspectrum_feat, + "spectral_centroid": spectral_centroid_feat, + } + return features + + +# ------------------- audio feature (MFCC) extractor ---------------- # + + +def load_mfcc_feature_extractor(use_gpu: bool = False): + transform = torchaudio.transforms.MFCC( + sample_rate=48000, + n_mfcc=25, + melkwargs={ + "n_fft": 2048, + "hop_length": 1024, + "n_mels": 128, + "center": False, + }, + ) + transform.embed_dim = 25 * 3 + if torch.cuda.is_available() and use_gpu: + transform = transform.cuda() + transform.device = "cuda" + return transform + + +def get_mfcc_feature_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + midside: bool = False, + **kwargs, +): + bs, chs, seq_len = x.shape + + # in_device = x.device + # x = x.to(model.device) + + if sample_rate != 48000: + x = torchaudio.functional.resample(x, sample_rate, 48000) + + if chs == 2 and midside: # process as mid-side + x_mid = x[:, 0, :] + x[:, 1, :] + x_side = x[:, 0, :] - x[:, 1, :] + # stack back + x = torch.stack([x_mid, x_side], dim=1) + else: + x = x.mean(dim=1, keepdim=True) + + # Get embeddings + with torch.no_grad(): + embeddings = model(x) + embeddings_mean = embeddings.mean(dim=-1) + embeddings_std = embeddings.std(dim=-1) + embeddings_max = embeddings.max(dim=-1)[0] + embeddings = torch.cat( + [embeddings_mean, embeddings_std, embeddings_max], dim=-1 + ) + embeddings = embeddings.view(bs, -1) + + # l2 normalize + embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=-1) + + embeddings = { + "mono": embeddings, + } + + return embeddings + + +# ------------------- deepafx-st (style transfer) ---------------- # +def load_deepafx_st_model( + ckpt_path: str, use_gpu: bool = False, encoder_only: bool = False +): + config_path = os.path.join(os.path.dirname(ckpt_path), "config.yaml") + + with open(config_path) as f: + config = yaml.safe_load(f) + + encoder_configs = config["model"]["init_args"]["encoder"] + + module_path, class_name = encoder_configs["class_path"].rsplit(".", 1) + module = import_module(module_path) + encoder_model = getattr(module, class_name)(**encoder_configs["init_args"]) + + model = StyleTransferSystem.load_from_checkpoint( + ckpt_path, + vst_json=config["model"]["init_args"]["vst_json"], + encoder=encoder_model, + map_location="cpu", + ) + model.eval() + + if encoder_only: + model = model.encoder + + if use_gpu: + model = model.cuda() + + return model + + +def get_deepafx_st_embeds( + x: torch.Tensor, model: torch.nn.Module, sample_rate: float, **kwargs +): + bs, chs, seq_len = x.shape + + x_device = x + + # move audio to model device + x = x.type_as(next(model.parameters())) + + if sample_rate != 48000: + x = torchaudio.functional.resample(x, sample_rate, 48000) + + # if stereo -> mono + if x.shape[1] == 2: + x = torch.mean(x, dim=1, keepdim=True) + + with torch.no_grad(): + mid_embeddings, side_embeddings = model(x) + + embeddings = { + "mid": mid_embeddings.type_as(x_device), + "side": side_embeddings.type_as(x_device), + } + + return embeddings + + +# ------------------- BEATs ---------------- # + + +def load_beats_model(use_gpu: bool = False): + # load the pre-trained checkpoints + ckpt_path = "checkpoints/BEATs_iter3_plus_AS2M.pt" + + if not os.path.isfile(ckpt_path): + os.system( + f"""wget -P checkpoints/ "https://valle.blob.core.windows.net/share/BEATs/BEATs_iter3_plus_AS2M.pt?sv=2020-08-04&st=2023-03-01T07%3A51%3A05Z&se=2033-03-02T07%3A51%3A00Z&sr=c&sp=rl&sig=QJXmSJG9DbMKf48UDIU1MfzIro8HQOf3sqlNXiflY1I%3D" """ + ) + os.rename( + "BEATs_iter3_plus_AS2M.pt?sv=2020-08-04&st=2023-03-01T07%3A51%3A05Z&se=2033-03-02T07%3A51%3A00Z&sr=c&sp=rl&sig=QJXmSJG9DbMKf48UDIU1MfzIro8HQOf3sqlNXiflY1I%3D", + "BEATs_iter3_plus_AS2M.pt", + ) + + checkpoint = torch.load(ckpt_path, map_location="cpu") + + cfg = BEATsConfig(checkpoint["cfg"]) + BEATs_model = BEATs(cfg) + BEATs_model.load_state_dict(checkpoint["model"]) + BEATs_model.eval() + + if use_gpu: + BEATs_model.cuda() + + return BEATs_model + + +def get_beats_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + **kwargs, +): + bs, chs, seq_len = x.shape + + x = x.mean(dim=1, keepdim=False) + + # extract the the audio representation + padding_mask = torch.zeros(1, seq_len).bool().type_as(x) + model.embed_dim = 768 + + with torch.no_grad(): + embed = model.extract_features(x, padding_mask=padding_mask)[0] + + # take mean across time (2nd dim) + embed = embed.mean(dim=1) + + embeddings = { + "mono": embed, + } + + return embeddings + + +# ------------------- pretrained wav2vec ---------------- # +def load_wav2vec2_model(use_gpu: bool = False): + model = Wav2Vec2Model.from_pretrained("facebook/wav2vec2-large-960h-lv60-self") + if torch.cuda.is_available() and use_gpu: + model = model.cuda() + model.embed_dim = 1024 + model.eval() + + return model + + +def get_wav2vec2_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + no_grad: bool = False, + **kwargs, +): + bs, chs, seq_len = x.shape + + x = x.mean(dim=1, keepdim=True) + + if sample_rate != 16000: + x = torchaudio.functional.resample(x, sample_rate, 16000) + + with torch.no_grad(): + x = x.squeeze(1) + output = model(x, output_hidden_states=True) + out = output.hidden_states[0] + for i in range(1, len(output.hidden_states)): + out = out + output.hidden_states[i] + out = out / len(output.hidden_states) + + embeddings = { + "mono": out.mean(dim=1), + } + return embeddings + + +# ------------------- pretrained wav2clip ---------------- # + + +def load_wav2clip_model(use_gpu: bool = False): + model = wav2clip.get_model() + if torch.cuda.is_available() and use_gpu: + model = model.cuda() + model.eval() + model.embed_dim = 512 + return model + + +def get_wav2clip_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + no_grad: bool = False, + **kwargs, +): + bs, chs, seq_len = x.shape + x = x.mean(dim=1, keepdim=True) + + if sample_rate != 16000: + x = torchaudio.functional.resample(x, sample_rate, 16000) + with torch.no_grad(): + embeddings = model.forward(x.view(bs, -1).detach()) + + embeddings = { + "mono": embeddings, + } + return embeddings + + +# ------------------- pretrained vggish ---------------- # + + +def load_vggish_model(use_gpu: bool = False): + model = torch.hub.load("harritaylor/torchvggish", "vggish") + if torch.cuda.is_available() and use_gpu: + model = model.cuda() + model.eval() + model.embed_dim = 128 + return model + + +def get_vggish_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + no_grad: bool = False, + **kwargs, +): + bs, chs, seq_len = x.shape + x = torch.mean(x, dim=1) + + embeddings = [] + + for batch_idx in range(bs): + batch_audio = x[batch_idx, :].detach().cpu().numpy() + embedding = model.forward(batch_audio, fs=sample_rate) + embedding = embedding.sum(dim=0, keepdim=True) / embedding.shape[0] + embeddings.append(embedding) + + embeddings = torch.cat(embeddings, dim=0) + embeddings = {"mono": embeddings} + + return embeddings + + +# ------------------- pretrained CLAP model ---------------- # + + +def load_clap_model(use_gpu: bool = False, midside: bool = False): + model = laion_clap.CLAP_Module(enable_fusion=False) + model.load_ckpt() # download the default pretrained checkpoint. + model.midside = midside + if torch.cuda.is_available() and use_gpu: + model = model.cuda() + model.device = "cuda" + model.eval() + model.embed_dim = 512 + return model + + +def get_clap_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + midside: bool = False, + **kwargs, +): + bs, chs, seq_len = x.shape + + if sample_rate != 48000: + x = torchaudio.functional.resample(x, sample_rate, 48000) + + # if stereo -> mono + if x.shape[1] == 2: + if midside: + x_mid = x[:, 0, :] + x[:, 1, :] + x_side = x[:, 0, :] - x[:, 1, :] + with torch.no_grad(): + embeddings_mid = model.get_audio_embedding_from_data( + x=x_mid, use_tensor=True + ) + embeddings_side = model.get_audio_embedding_from_data( + x=x_side, use_tensor=True + ) + + embeddings = {"mid": embeddings_mid, "side": embeddings_side} + + else: # mono + x = torch.mean(x, dim=1, keepdim=False) + with torch.no_grad(): + embeddings = model.get_audio_embedding_from_data(x=x, use_tensor=True) + embeddings = {"mono": embeddings} + else: + with torch.no_grad(): + embeddings = model.get_audio_embedding_from_data(x=x, use_tensor=True) + embeddings = {"mono": embeddings} + + return embeddings + + +# -------- self-supervised parameter estimation model -------- # + + +def get_param_embeds( + x: torch.Tensor, + model: torch.nn.Module, + sample_rate: float, + requires_grad: bool = False, + peak_normalize: bool = False, + dropout: float = 0.0, +): + bs, chs, seq_len = x.shape + + x_device = x + + # move audio to model device + x = x.type_as(next(model.parameters())) + + # if peak_normalize: + # x = batch_peak_normalize(x) + + if sample_rate != 48000: + x = torchaudio.functional.resample(x, sample_rate, 48000) + + seq_len = x.shape[-1] # update seq_len after resampling + # if longer than 262144 crop, else repeat pad to 262144 + # if seq_len > 262144: + # x = x[:, :, :262144] + # else: + # x = torch.nn.functional.pad(x, (0, 262144 - seq_len), "replicate") + + # peak normalize each batch item + for batch_idx in range(bs): + x[batch_idx, ...] /= x[batch_idx, ...].abs().max().clamp(1e-8) + + if not requires_grad: + with torch.no_grad(): + mid_embeddings, side_embeddings = model(x) + else: + mid_embeddings, side_embeddings = model(x) + + # add dropout + if dropout > 0.0: + mid_embeddings = torch.nn.functional.dropout( + mid_embeddings, p=dropout, training=True + ) + side_embeddings = torch.nn.functional.dropout( + side_embeddings, p=dropout, training=True + ) + + # check for nan + if torch.isnan(mid_embeddings).any(): + print("Warning: NaNs found in mid_embeddings") + mid_embeddings = torch.nan_to_num(mid_embeddings) + elif torch.isnan(side_embeddings).any(): + print("Warning: NaNs found in side_embeddings") + side_embeddings = torch.nan_to_num(side_embeddings) + + # l2 normalize + mid_embeddings = torch.nn.functional.normalize(mid_embeddings, p=2, dim=-1) + side_embeddings = torch.nn.functional.normalize(side_embeddings, p=2, dim=-1) + + embeddings = { + "mid": mid_embeddings.type_as(x_device), + "side": side_embeddings.type_as(x_device), + } + + return embeddings + + +def load_param_model(ckpt_path: str = None, use_gpu: bool = False): + + if ckpt_path is None: # look in tmp direcory + ckpt_path = os.path.join(os.getcwd(), "tmp", "afx-rep.ckpt") + os.makedirs("tmp", exist_ok=True) + if not os.path.isfile(ckpt_path): + # download from huggingfacehub + os.system( + "wget -O tmp/afx-rep.ckpt https://huggingface.co/csteinmetz1/afx-rep/resolve/main/afx-rep.ckpt" + ) + os.system( + "wget -O tmp/config.yaml https://huggingface.co/csteinmetz1/afx-rep/resolve/main/config.yaml" + ) + + config_path = os.path.join(os.path.dirname(ckpt_path), "config.yaml") + + with open(config_path) as f: + config = yaml.safe_load(f) + + encoder_configs = config["model"]["init_args"]["encoder"] + + module_path, class_name = encoder_configs["class_path"].rsplit(".", 1) + module_path = module_path.replace("lcap", "st_ito") + module = import_module(module_path) + model = getattr(module, class_name)(**encoder_configs["init_args"]) + + checkpoint = torch.load(ckpt_path, map_location="cpu") + + # load state dicts + state_dict = {} + for k, v in checkpoint["state_dict"].items(): + if k.startswith("encoder"): + state_dict[k.replace("encoder.", "", 1)] = v + + model.load_state_dict(state_dict) + model.eval() + + if use_gpu: + model.cuda() + + return model + + +def get_fx_encoder_embeds( + x: torch.Tensor, model: torch.nn.Module, sample_rate: float, **kwargs +): + bs, chs, seq_len = x.shape + + x_in = x + x = x.type_as(next(model.parameters())) + + # resample and ensure stereo + x = torchaudio.functional.resample(x, sample_rate, 44100) + + # peak normalize + x /= x.abs().max().clamp(1e-8) + + if chs == 1: + x = x.repeat(1, 2, 1) + + # signal shape should be : [channel, signal duration] + with torch.no_grad(): + embed = model(x) + + embeddings = { + "stereo": embed.type_as(x_in), + } + + return embeddings + + +def load_fx_encoder_model(use_gpu: bool = False): + ckpt_path = "checkpoints/FXencoder_ps.pt" + # look if checkpoint exists + if not os.path.exists(ckpt_path): + print( + f"Checkpoint file not found at {ckpt_path}. Download from https://drive.google.com/file/d/1BFABsJRUVgJS5UE5iuM03dbfBjmI9LT5/view" + ) + + ddp = True + model = FXencoder() # default configs + checkpoint = torch.load(ckpt_path, map_location="cpu") + + new_state_dict = OrderedDict() + for k, v in checkpoint["model"].items(): + # remove `module.` if the model was trained with DDP + name = k[7:] if ddp else k + new_state_dict[name] = v + + # load params + model.load_state_dict(new_state_dict) + model.embed_dim = 2048 + model.eval() + + if use_gpu: + model.cuda() + + return model diff --git a/utils/torch_utils/training_stats.py b/utils/torch_utils/training_stats.py new file mode 100644 index 0000000000000000000000000000000000000000..b30ef1548416257a25090141c6ac764d5fc67c03 --- /dev/null +++ b/utils/torch_utils/training_stats.py @@ -0,0 +1,270 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# This work is licensed under a Creative Commons +# Attribution-NonCommercial-ShareAlike 4.0 International License. +# You should have received a copy of the license along with this +# work. If not, see http://creativecommons.org/licenses/by-nc-sa/4.0/ + +"""Facilities for reporting and collecting training statistics across +multiple processes and devices. The interface is designed to minimize +synchronization overhead as well as the amount of boilerplate in user +code.""" + +import re +import numpy as np +import torch +import utils.dnnlib as dnnlib + + +#---------------------------------------------------------------------------- + +_num_moments = 3 # [num_scalars, sum_of_scalars, sum_of_squares] +_reduce_dtype = torch.float32 # Data type to use for initial per-tensor reduction. +_counter_dtype = torch.float64 # Data type to use for the internal counters. +_rank = 0 # Rank of the current process. +_sync_device = None # Device to use for multiprocess communication. None = single-process. +_sync_called = False # Has _sync() been called yet? +_counters = dict() # Running counters on each device, updated by report(): name => device => torch.Tensor +_cumulative = dict() # Cumulative counters on the CPU, updated by _sync(): name => torch.Tensor + +#---------------------------------------------------------------------------- + +def init_multiprocessing(rank, sync_device): + r"""Initializes `torch_utils.training_stats` for collecting statistics + across multiple processes. + + This function must be called after + `torch.distributed.init_process_group()` and before `Collector.update()`. + The call is not necessary if multi-process collection is not needed. + + Args: + rank: Rank of the current process. + sync_device: PyTorch device to use for inter-process + communication, or None to disable multi-process + collection. Typically `torch.device('cuda', rank)`. + """ + global _rank, _sync_device + assert not _sync_called + _rank = rank + _sync_device = sync_device + +#---------------------------------------------------------------------------- + +def report(name, value): + r"""Broadcasts the given set of scalars to all interested instances of + `Collector`, across device and process boundaries. + + This function is expected to be extremely cheap and can be safely + called from anywhere in the training loop, loss function, or inside a + `torch.nn.Module`. + + Warning: The current implementation expects the set of unique names to + be consistent across processes. Please make sure that `report()` is + called at least once for each unique name by each process, and in the + same order. If a given process has no scalars to broadcast, it can do + `report(name, [])` (empty list). + + Args: + name: Arbitrary string specifying the name of the statistic. + Averages are accumulated separately for each unique name. + value: Arbitrary set of scalars. Can be a list, tuple, + NumPy array, PyTorch tensor, or Python scalar. + + Returns: + The same `value` that was passed in. + """ + if name not in _counters: + _counters[name] = dict() + + elems = torch.as_tensor(value) + if elems.numel() == 0: + return value + + elems = elems.detach().flatten().to(_reduce_dtype) + moments = torch.stack([ + torch.ones_like(elems).sum(), + elems.sum(), + elems.square().sum(), + ]) + assert moments.ndim == 1 and moments.shape[0] == _num_moments + moments = moments.to(_counter_dtype) + + device = moments.device + if device not in _counters[name]: + _counters[name][device] = torch.zeros_like(moments) + _counters[name][device].add_(moments) + return value + +#---------------------------------------------------------------------------- + +def report0(name, value): + r"""Broadcasts the given set of scalars by the first process (`rank = 0`), + but ignores any scalars provided by the other processes. + See `report()` for further details. + """ + report(name, value if _rank == 0 else []) + return value + +#---------------------------------------------------------------------------- + +class Collector: + r"""Collects the scalars broadcasted by `report()` and `report0()` and + computes their long-term averages (mean and standard deviation) over + user-defined periods of time. + + The averages are first collected into internal counters that are not + directly visible to the user. They are then copied to the user-visible + state as a result of calling `update()` and can then be queried using + `mean()`, `std()`, `as_dict()`, etc. Calling `update()` also resets the + internal counters for the next round, so that the user-visible state + effectively reflects averages collected between the last two calls to + `update()`. + + Args: + regex: Regular expression defining which statistics to + collect. The default is to collect everything. + keep_previous: Whether to retain the previous averages if no + scalars were collected on a given round + (default: True). + """ + def __init__(self, regex='.*', keep_previous=True): + self._regex = re.compile(regex) + self._keep_previous = keep_previous + self._cumulative = dict() + self._moments = dict() + self.update() + self._moments.clear() + + def names(self): + r"""Returns the names of all statistics broadcasted so far that + match the regular expression specified at construction time. + """ + return [name for name in _counters if self._regex.fullmatch(name)] + + def update(self): + r"""Copies current values of the internal counters to the + user-visible state and resets them for the next round. + + If `keep_previous=True` was specified at construction time, the + operation is skipped for statistics that have received no scalars + since the last update, retaining their previous averages. + + This method performs a number of GPU-to-CPU transfers and one + `torch.distributed.all_reduce()`. It is intended to be called + periodically in the main training loop, typically once every + N training steps. + """ + if not self._keep_previous: + self._moments.clear() + for name, cumulative in _sync(self.names()): + if name not in self._cumulative: + self._cumulative[name] = torch.zeros([_num_moments], dtype=_counter_dtype) + delta = cumulative - self._cumulative[name] + self._cumulative[name].copy_(cumulative) + if float(delta[0]) != 0: + self._moments[name] = delta + + def _get_delta(self, name): + r"""Returns the raw moments that were accumulated for the given + statistic between the last two calls to `update()`, or zero if + no scalars were collected. + """ + assert self._regex.fullmatch(name) + if name not in self._moments: + self._moments[name] = torch.zeros([_num_moments], dtype=_counter_dtype) + return self._moments[name] + + def num(self, name): + r"""Returns the number of scalars that were accumulated for the given + statistic between the last two calls to `update()`, or zero if + no scalars were collected. + """ + delta = self._get_delta(name) + return int(delta[0]) + + def mean(self, name): + r"""Returns the mean of the scalars that were accumulated for the + given statistic between the last two calls to `update()`, or NaN if + no scalars were collected. + """ + delta = self._get_delta(name) + if int(delta[0]) == 0: + return float('nan') + return float(delta[1] / delta[0]) + + def std(self, name): + r"""Returns the standard deviation of the scalars that were + accumulated for the given statistic between the last two calls to + `update()`, or NaN if no scalars were collected. + """ + delta = self._get_delta(name) + if int(delta[0]) == 0 or not np.isfinite(float(delta[1])): + return float('nan') + if int(delta[0]) == 1: + return float(0) + mean = float(delta[1] / delta[0]) + raw_var = float(delta[2] / delta[0]) + return np.sqrt(max(raw_var - np.square(mean), 0)) + + def as_dict(self): + r"""Returns the averages accumulated between the last two calls to + `update()` as an `dnnlib.EasyDict`. The contents are as follows: + + dnnlib.EasyDict( + NAME = dnnlib.EasyDict(num=FLOAT, mean=FLOAT, std=FLOAT), + ... + ) + """ + stats = dnnlib.EasyDict() + for name in self.names(): + stats[name] = dnnlib.EasyDict(num=self.num(name), mean=self.mean(name), std=self.std(name)) + return stats + + def __getitem__(self, name): + r"""Convenience getter. + `collector[name]` is a synonym for `collector.mean(name)`. + """ + return self.mean(name) + +#---------------------------------------------------------------------------- + +def _sync(names): + r"""Synchronize the global cumulative counters across devices and + processes. Called internally by `Collector.update()`. + """ + if len(names) == 0: + return [] + global _sync_called + _sync_called = True + + # Collect deltas within current rank. + deltas = [] + device = _sync_device if _sync_device is not None else torch.device('cpu') + for name in names: + delta = torch.zeros([_num_moments], dtype=_counter_dtype, device=device) + for counter in _counters[name].values(): + delta.add_(counter.to(device)) + counter.copy_(torch.zeros_like(counter)) + deltas.append(delta) + deltas = torch.stack(deltas) + + # Sum deltas across ranks. + if _sync_device is not None: + torch.distributed.all_reduce(deltas) + + # Update cumulative values. + deltas = deltas.cpu() + for idx, name in enumerate(names): + if name not in _cumulative: + _cumulative[name] = torch.zeros([_num_moments], dtype=_counter_dtype) + _cumulative[name].add_(deltas[idx]) + + # Return name-value pairs. + return [(name, _cumulative[name]) for name in names] + +#---------------------------------------------------------------------------- +# Convenience. + +default_collector = Collector() + +#---------------------------------------------------------------------------- \ No newline at end of file diff --git a/utils/training_utils.py b/utils/training_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..efc79c2aa7feb2d66c0dc0002164bb22c7a89ffc --- /dev/null +++ b/utils/training_utils.py @@ -0,0 +1,719 @@ + +import math +import torch +import torchaudio +import numpy as np +import scipy.signal +class EMAWarmup: + """Implements an EMA warmup using an inverse decay schedule. + If inv_gamma=1 and power=1, implements a simple average. inv_gamma=1, power=2/3 are + good values for models you plan to train for a million or more steps (reaches decay + factor 0.999 at 31.6K steps, 0.9999 at 1M steps), inv_gamma=1, power=3/4 for models + you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 at + 215.4k steps). + Args: + inv_gamma (float): Inverse multiplicative factor of EMA warmup. Default: 1. + power (float): Exponential factor of EMA warmup. Default: 1. + min_value (float): The minimum EMA decay rate. Default: 0. + max_value (float): The maximum EMA decay rate. Default: 1. + start_at (int): The epoch to start averaging at. Default: 0. + last_epoch (int): The index of last epoch. Default: 0. + """ + + def __init__(self, inv_gamma=1., power=1., min_value=0., max_value=1., start_at=0, + last_epoch=0): + self.inv_gamma = inv_gamma + self.power = power + self.min_value = min_value + self.max_value = max_value + self.start_at = start_at + self.last_epoch = last_epoch + + def state_dict(self): + """Returns the state of the class as a :class:`dict`.""" + return dict(self.__dict__.items()) + + def load_state_dict(self, state_dict): + """Loads the class's state. + Args: + state_dict (dict): scaler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + self.__dict__.update(state_dict) + + def get_value(self): + """Gets the current EMA decay rate.""" + epoch = max(0, self.last_epoch - self.start_at) + value = 1 - (1 + epoch / self.inv_gamma) ** -self.power + return 0. if epoch < 0 else min(self.max_value, max(self.min_value, value)) + + def step(self): + """Updates the step count.""" + self.last_epoch += 1 + + +def resample_batch(audio, fs, fs_target, length_target=None): + + device=audio.device + dtype=audio.dtype + B=audio.shape[0] + #if possible resampe in a batched way + #check if all the fs are the same and equal to 44100 + #print(fs_target) + if fs_target==22050: + if (fs==44100).all(): + audio=torchaudio.functional.resample(audio, 2,1) + return audio[:, 0:length_target] #trow away the last samples + elif (fs==48000).all(): + #approcimate resamppleint + audio=torchaudio.functional.resample(audio, 160*2,147) + return audio[:, 0:length_target] + else: + #if revious is unsuccesful bccause we have examples at 441000 and 48000 in the same batch,, just iterate over the batch + proc_batch=torch.zeros((B,length_target), device=device) + for i, (a, f_s) in enumerate(zip(audio, fs)): #I hope this shit wll not slow down everythingh + if f_s==44100: + #resample by 2 + a=torchaudio.functional.resample(a, 2,1) + elif f_s==48000: + a=torchaudio.functional.resample(a, 160*2,147) + elif f_s==22050: + pass + else: + print("WARNING, strange fs", f_s) + + proc_batch[i]=a[0:length_target] + return proc_batch + elif fs_target==44100: + if (fs==44100).all(): + return audio[:, 0:length_target] #trow away the last samples + elif (fs==48000).all(): + #approcimate resamppleint + audio=torchaudio.functional.resample(audio, 160,147) + return audio[:, 0:length_target] + else: + #if revious is unsuccesful bccause we have examples at 441000 and 48000 in the same batch,, just iterate over the batch + #B,C,L=audio.shape + #proc_batch=torch.zeros((B,C,L), device=device) + proc_batch=torch.zeros((B,length_target), device=device) + #print("debigging resample batch") + #print(audio.shape,fs.shape) + #for i, (a, f_s) in enumerate(zip(audio, fs.tolist())): #I hope this shit wll not slow down everythingh + for i, (a, f_s) in enumerate(zip(audio, fs)): #I hope this shit wll not slow down everythingh + #print(i,a.shape,f_s) + if f_s==44100: + #resample by 2 + pass + elif f_s==22050: + a=torchaudio.functional.resample(a, 1,2) + elif f_s==48000: + a=torchaudio.functional.resample(a, 160,147) + elif f_s==96000: + a=torchaudio.functional.resample(a, 320, 147) + else: + print("WARNING, strange fs", f_s) + + + proc_batch[i]=a[...,0:length_target] + return proc_batch + else: + if (fs==44100).all(): + audio=torchaudio.functional.resample(audio, 44100, fs_target) + return audio[...,0:length_target] #trow away the last samples + elif (fs==48000).all(): + print("resampling 48000 to 16000", length_target, audio.shape) + #approcimate resamppleint + audio=torchaudio.functional.resample(audio, 48000,fs_target) + print(audio.shape) + return audio[..., 0:length_target] + else: + #if revious is unsuccesful bccause we have examples at 441000 and 48000 in the same batch,, just iterate over the batch + proc_batch=torch.zeros((B,length_target), device=device) + for i, (a, f_s) in enumerate(zip(audio, fs)): #I hope this shit wll not slow down everythingh + if f_s==44100: + #resample by 2 + a=torchaudio.functional.resample(a, 44100,fs_target) + elif f_s==48000: + a=torchaudio.functional.resample(a, 48000,fs_target) + else: + print("WARNING, strange fs", f_s) + + proc_batch[i]=a[...,0:length_target] + return proc_batch + +def load_state_dict( state_dict, network=None, ema=None, optimizer=None, log=True): + ''' + utility for loading state dicts for different models. This function sequentially tries different strategies + args: + state_dict: the state dict to load + returns: + True if the state dict was loaded, False otherwise + Assuming the operations are don in_place, this function will not create a copy of the network and optimizer (I hope) + ''' + #print(state_dict) + if log: print("Loading state dict") + if log: + print(state_dict.keys()) + #if there + try: + if log: print("Attempt 1: trying with strict=True") + if network is not None: + network.load_state_dict(state_dict['network']) + if optimizer is not None: + optimizer.load_state_dict(state_dict['optimizer']) + if ema is not None: + ema.load_state_dict(state_dict['ema']) + return True + except Exception as e: + if log: + print("Could not load state dict") + print(e) + try: + print("assuming the network was saved from a DDP model, and I forgot to add .module to the keys") + #verify that the keys are the same but with .module removed + if network is not None: + network_state_dict = network.state_dict() + for key in list(state_dict['network'].keys()): + print("checking", key) + if key.startswith('module.'): + new_key = key.replace('module.', '') + state_dict['network'][new_key] = state_dict['network'].pop(key) + + network.load_state_dict(state_dict['network']) + + if optimizer is not None: + optimizer.load_state_dict(state_dict['optimizer']) + if ema is not None: + ema.load_state_dict(state_dict['ema']) + + print("loaded state dict with .module removed from the keys") + + return True + + except Exception as e: + if log: + print("Could not load state dict") + print(e) + + try: + if log: print("Attempt 2: trying with strict=False") + if network is not None: + network.load_state_dict(state_dict['network'], strict=False) + #we cannot load the optimizer in this setting + #self.optimizer.load_state_dict(state_dict['optimizer'], strict=False) + if ema is not None: + ema.load_state_dict(state_dict['ema'], strict=False) + return True + except Exception as e: + if log: + print("Could not load state dict") + print(e) + print("training from scratch") + try: + if log: print("Attempt 3: trying with strict=False,but making sure that the shapes are fine") + if ema is not None: + ema_state_dict = ema.state_dict() + if network is not None: + network_state_dict = network.state_dict() + i=0 + if network is not None: + for name, param in state_dict['network'].items(): + if log: print("checking",name) + if name in network_state_dict.keys(): + if network_state_dict[name].shape==param.shape: + network_state_dict[name]=param + if log: + print("assigning",name) + i+=1 + network.load_state_dict(network_state_dict) + if ema is not None: + for name, param in state_dict['ema'].items(): + if log: print("checking",name) + if name in ema_state_dict.keys(): + if ema_state_dict[name].shape==param.shape: + ema_state_dict[name]=param + if log: + print("assigning",name) + i+=1 + + ema.load_state_dict(ema_state_dict) + + if i==0: + if log: print("WARNING, no parameters were loaded") + raise Exception("No parameters were loaded") + elif i>0: + if log: print("loaded", i, "parameters") + return True + + except Exception as e: + print(e) + print("the second strict=False failed") + + + try: + if log: print("Attempt 4: Assuming the naming is different, with the network and ema called 'state_dict'") + if network is not None: + network.load_state_dict(state_dict['state_dict']) + if ema is not None: + ema.load_state_dict(state_dict['state_dict']) + except Exception as e: + if log: + print("Could not load state dict") + print(e) + print("training from scratch") + print("It failed 3 times!! but not giving up") + #print the names of the parameters in self.network + + try: + if log: print("Attempt 5: trying to load with different names, now model='model' and ema='ema_weights'") + if ema is not None: + dic_ema = {} + for (key, tensor) in zip(state_dict['model'].keys(), state_dict['ema_weights']): + dic_ema[key] = tensor + ema.load_state_dict(dic_ema) + return True + except Exception as e: + if log: + print(e) + + try: + if log: print("Attempt 6: If there is something wrong with the name of the ema parameters, we can try to load them using the names of the parameters in the model") + if ema is not None: + dic_ema = {} + i=0 + for (key, tensor) in zip(state_dict['model'].keys(), state_dict['model'].values()): + if tensor.requires_grad: + dic_ema[key]=state_dict['ema_weights'][i] + i=i+1 + else: + dic_ema[key]=tensor + ema.load_state_dict(dic_ema) + return True + except Exception as e: + if log: + print(e) + + + try: + #assign the parameters in state_dict to self.network using a for loop + print("Attempt 7: Trying to load the parameters one by one. This is for the dance diffusion model, looking for parameters starting with 'diffusion.' or 'diffusion_ema.'") + if ema is not None: + ema_state_dict = ema.state_dict() + if network is not None: + network_state_dict = ema.state_dict() + i=0 + if network is not None: + for name, param in state_dict['state_dict'].items(): + print("checking",name) + if name.startswith("diffusion."): + i+=1 + name=name.replace("diffusion.","") + if network_state_dict[name].shape==param.shape: + #print(param.shape, network.state_dict()[name].shape) + network_state_dict[name]=param + #print("assigning",name) + + network.load_state_dict(network_state_dict, strict=False) + + if ema is not None: + for name, param in state_dict['state_dict'].items(): + if name.startswith("diffusion_ema."): + i+=1 + name=name.replace("diffusion_ema.","") + if ema_state_dict[name].shape==param.shape: + if log: + print(param.shape, ema.state_dict()[name].shape) + ema_state_dict[name]=param + + ema.load_state_dict(ema_state_dict, strict=False) + + if i==0: + print("WARNING, no parameters were loaded") + raise Exception("No parameters were loaded") + elif i>0: + print("loaded", i, "parameters") + return True + except Exception as e: + if log: + print(e) + #try: + # this is for the dmae1d mddel, assuming there is only one network + if network is not None: + network.load_state_dict(state_dict, strict=True) + if ema is not None: + ema.load_state_dict(state_dict, strict=True) + return True + + #except Exception as e: + # if log: + # print(e) + + return False + + +def unnormalize(x,stds, args): + #unnormalize the STN separated audio + new_std=args.exp.normalization.target_std + if new_std=="sigma_data": + new_std=args.diff_params.sigma_data + x=stds*x/(new_std+1e-8) + return x +def normalize( xS, xT, xN, args, return_std=False): + #normalize the STN separated audio + if args.exp.normalization.mode=="None": + pass + elif args.exp.normalization.mode=="residual_noise": + #normalize the residual noise + + std=xN.std(dim=-1, keepdim=True).mean(dim=1, keepdim=True) + new_std=args.exp.normalization.target_std + + if new_std=="sigma_data": + new_std=args.diff_params.sigma_data + + #print(std, new_std) + + xN=new_std*xN/(std+1e-8) + #print(xN.std(dim=-1, keepdim=True)) + xS=new_std*xS/(std+1e-8) + xT=new_std*xT/(std+1e-8) + elif args.exp.normalization.mode=="residual_noise_batch": + #normalize the residual noise per batch + #get the std of the entire batch + std=xN.std(dim=(0,1,2),unbiased=True, keepdim=False) + + new_std=args.exp.normalization.target_std + + if new_std=="sigma_data": + new_std=args.diff_params.sigma_data + + #print(std, new_std) + + xN=new_std*xN/(std+1e-8) + #print(xN.std(dim=-1, keepdim=True).mean(dim=1, keepdim=True)) + xS=new_std*xS/(std+1e-8) + xT=new_std*xT/(std+1e-8) + + elif args.exp.normalization.mode=="all": + std=(xN+xS+xT).std(dim=-1, keepdim=True).mean(dim=1, keepdim=True) + new_std=args.exp.normalization.target_std + if new_std=="sigma_data": + new_std=args.diff_params.sigma_data + xN=new_std*xN/(std+1e-8) + xS=new_std*xS/(std+1e-8) + xT=new_std*xT/(std+1e-8) + #print("std",xN.std(dim=-1, keepdim=True).mean(dim=1, keepdim=True)) + else: + print("normalization mode not recognized") + pass + + try: + if return_std: + return xS, xT, xN, std + except Exception as e: + print(e) + print("warning!, std cannot be returned") + pass + + return xS, xT, xN + + +#def find_time_offset(x: torch.Tensor, y: torch.Tensor, sign_shift=False): + + +#def find_time_offset(y: torch.Tensor, x: torch.Tensor, margin=3000, check_sign_flip=False): +# x = x.double() +# y = y.double() +# N = x.size(-1) +# M = y.size(-1) +# #print("x",x.shape) +# #print("y",y.shape) +# #print(N,M) + +# X = torch.fft.rfft(x, n=N + M - 1) + +# Y = torch.fft.rfft(y, n=N + M - 1) +# print("X",X.shape, "Y",Y.shape, "x",x.shape, "y",y.shape) + +# corr = torch.fft.irfft(X.conj() * Y) +# print("corr",corr.shape) + +# shifts = torch.argmax(corr, dim=-1) - x.shape[-1] + +# return shifts + + +def align_batch(y, x, sample_rate): + + x = x.double() + y = y.double() + N = x.size(-1) + M = y.size(-1) + y_device=y.device + x_device=x.device + y_dtype=y.dtype + x_dtype=x.dtype + + X = torch.fft.rfft(x, n=N + M - 1) + X_flipped=torch.fft.rfft(x*-1, n=N + M - 1) + + Y = torch.fft.rfft(y, n=N + M - 1) + + corr = torch.fft.irfft(X.conj() * Y) + corr_flipped = torch.fft.irfft(X_flipped.conj() * Y) + + corr=corr.sum(dim=1) + corr_flipped=corr_flipped.sum(dim=1) + + #print("corr",corr.shape) + shifts = torch.argmax(corr, dim=-1) + + shifts_flipped = torch.argmax(corr_flipped, dim=-1) +# corr_values_flipped= corr_flipped[...,shifts_flipped] + #shifts= torch.where(shifts >= N, shifts - N - M + 1, shifts) + #shifts_flipped= torch.where(shifts_flipped >= N, shifts_flipped - N - M + 1, shifts_flipped) + + shifts=shifts.to(torch.int64) + + result=[] + for i in range(len(shifts)): + corr_value=corr[i, shifts[i]] + corr_value_flipped=corr_flipped[i, shifts_flipped[i]] + if corr_value < corr_value_flipped: + shift=shifts_flipped[i].item() + if shift >=N: + shift = shift - N - M + 1 + result.append(torch.roll(x[i]*-1, shifts=shift, dims=-1) ) + else: + shift=shifts[i].item() + if shift >=N: + shift = shift - N - M + 1 + result.append(torch.roll(x[i], shifts=shift, dims=-1) ) + + + x= torch.stack(result) + #x = torch.stack([torch.roll(x[i], shifts=shifts[i].item(), dims=-1) for i in range(x.shape[0])]) + + return y.to(y_device).to(y_dtype), x.to(x_device).to(x_dtype) + +def get_pink_noise_magnitude(freqs, device='cpu'): + a=torch.ones_like(freqs, device=device) + return a / torch.sqrt(torch.clamp(freqs, min=1e-6)) # Avoid division by zero + +def generate_pink_noise(shape, device='cpu'): + """ + Generate pink noise with 1/f frequency scaling for a batch of stereo signals. + + Args: + shape (tuple): Shape of the noise signal (B, 2, T). + device (str): Device for tensor computation ('cpu' or 'cuda'). + + Returns: + torch.Tensor: Pink noise signal with shape (B, 2, T). + """ + B, C, T = shape + + # Generate white noise + white_noise = torch.randn(B, C, T, device=device) + + # Perform FFT to move to frequency domain + fft = torch.fft.rfft(white_noise, dim=-1) + + # Generate frequency bins + freqs = torch.fft.rfftfreq(T, d=1.0).to(device) + + # Scale the amplitude by 1/sqrt(frequency) to approximate pink noise + # Avoid division by zero by clamping frequencies to a minimum value + H= get_pink_noise_magnitude(freqs, device=device) + fft*=H.unsqueeze(0).unsqueeze(0) + + # Perform inverse FFT to return to time domain + pink_noise = torch.fft.irfft(fft, n=T, dim=-1) + + return pink_noise + +def add_pink_noise(signal, snr_db): + """ + Add pink noise to a signal based on the desired SNR. + + Args: + signal (torch.Tensor): Original signal with shape (B, 2, T). + snr_db (float): Desired Signal-to-Noise Ratio in decibels. + + Returns: + torch.Tensor: Noisy signal with the specified SNR. + """ + # Calculate signal power + signal_power = torch.mean(signal ** 2, dim=(-1, -2), keepdim=True) + + + # Calculate noise power based on desired SNR + snr_linear = 10 ** (snr_db / 10) + noise_power = signal_power / snr_linear.view(-1, 1, 1) # Adjust shape for broadcasting + + # Generate pink noise + pink_noise = generate_pink_noise(signal.shape, device=signal.device) + + # Scale pink noise to achieve the desired noise power + noise_scaling_factor = torch.sqrt(noise_power / torch.mean(pink_noise ** 2, dim=-1, keepdim=True)) + + scaled_noise = pink_noise * noise_scaling_factor + + # Add scaled noise to the original signal + noisy_signal = signal + scaled_noise + + return noisy_signal + + +def Gauss_smooth(X,f,Noct=1, smooth_filter=None): + """ + based on https://github.com/IoSR-Surrey/MatlabToolbox/blob/4bff1bb2da7c95de0ce2713e7c710a0afa70c705/%2Biosr/%2Bdsp/smoothSpectrum.m + Smooths the magnitude spectrum X using a Gaussian filter. + + Args: + X (torch.Tensor): Input spectrum to be smoothed, shape (B, N,). + f (torch.Tensor): Frequency bins corresponding to the spectrum, shape (N,). + Noct (int, optional): Number of octaves for smoothing. Default is 1 + Returns: + torch.Tensor: Smoothed spectrum, same shape as X. + """ + + def gauss_f(f_x,F,Noct): + sigma = (F/Noct)/np.pi + g = torch.exp(-(((f_x-F)**2)/(2*(sigma**2)))) + g = g/torch.sum(g) + return g + + shape=X.shape + + x_oct = X.clone().view(-1, shape[-1]) # Initialize smoothed output + if Noct>0: + for i in range(1,len(f)): + g = gauss_f(f,f[i],Noct) + g=g.to(X.device).view(1, -1) + x_oct[...,i] = torch.sum(g*X) + + + x_oct = x_oct.clamp(min=0) # Ensure non-negative values + + return x_oct.view(shape) # Reshape back to original dimensions + +def prepare_smooth_filter(f, Noct=1): + f_matrix = f.unsqueeze(0) # Shape: [1, N] + F_matrix = f.unsqueeze(1) # Shape: [N, 1] + + # Calculate sigma for each center frequency + sigma = (F_matrix / Noct) / np.pi # Shape: [N, 1] + + # Calculate Gaussian weights for all frequency pairs at once + g = torch.exp(-((f_matrix - F_matrix)**2) / (2 * (sigma**2))) # Shape: [N, N] + + # Normalize each row to sum to 1 + g = g / g.sum(dim=1, keepdim=True) # Shape: [N, N] + + # Move to the same device as X + return g + +def Gauss_smooth_vectorized(X, f, Noct=1, smooth_filter=None): + """ + based on https://github.com/IoSR-Surrey/MatlabToolbox/blob/4bff1bb2da7c95de0ce2713e7c710a0afa70c705/%2Biosr/%2Bdsp/smoothSpectrum.m + Smooths the magnitude spectrum X using a Gaussian filter. + + Args: + X (torch.Tensor): Input spectrum to be smoothed, shape (B, N,). + f (torch.Tensor): Frequency bins corresponding to the spectrum, shape (N,). + Noct (int, optional): Number of octaves for smoothing. Default is 1 + Returns: + torch.Tensor: Smoothed spectrum, same shape as X. + """ + shape = X.shape + x_oct = X.clone().view(-1, shape[-1]) # Initialize smoothed output + + if Noct > 0: + # Vectorized implementation + # Create a matrix of all frequency pairs + if smooth_filter is None: + f_matrix = f.unsqueeze(0) # Shape: [1, N] + F_matrix = f.unsqueeze(1) # Shape: [N, 1] + + # Calculate sigma for each center frequency + sigma = (F_matrix / Noct) / np.pi # Shape: [N, 1] + + # Calculate Gaussian weights for all frequency pairs at once + g = torch.exp(-((f_matrix - F_matrix)**2) / (2 * (sigma**2))) # Shape: [N, N] + + # Normalize each row to sum to 1 + g = g / g.sum(dim=1, keepdim=True) # Shape: [N, N] + + # Move to the same device as X + else: + g=smooth_filter + + g = g.to(X.device) + + # Apply the filter to each batch element + # Skip the first bin (i=0) as in the original loop + x_oct_new = torch.matmul(x_oct, g[1:].T) # Shape: [B, N-1] + + # Keep the first bin unchanged and update the rest + x_oct[:, 1:] = x_oct_new + + # Ensure non-negative values + x_oct = x_oct.clamp(min=0) + + return x_oct.view(shape) # Reshape back to original dimensions + +def create_music_mean_spectrum_curve(freqs_Hz, device='cpu'): + """ + Create a target curve approximating the mean spectrum of music. + + Args: + freqs_Hz (torch.Tensor or np.ndarray): Frequency bins (Hz). + device (str): Device for tensor computation ('cpu' or 'cuda'). + + Returns: + torch.Tensor: Target curve for equalization. + """ + import torch + import numpy as np + + # Convert to numpy for easier manipulation if needed + if isinstance(freqs_Hz, torch.Tensor): + freqs_np = freqs_Hz.cpu().numpy() + else: + freqs_np = freqs_Hz + + # Create the curve in segments + target = np.ones_like(freqs_np) + + # Define the segments + f1 = 100 # First transition point + f2 = 1000 # Second transition point + f3 = 5000 # Third transition point + + # Calculate the curve for each segment + for i, f in enumerate(freqs_np): + if f < f1: + # Below 100Hz: -3dB/octave roll-off + target[i] = np.sqrt(f / f1) + elif f <= f2: + # 100Hz to 1kHz: Flat + target[i] = 1.0 + elif f <= f3: + # 1kHz to 10kHz: -3dB/octave roll-off + target[i] = np.sqrt(f2 / f) + else: + # Above 10kHz: -6dB/octave roll-off + # Calculate what the value would be at 10kHz using the -3dB/octave formula + val_at_f3 = np.sqrt(f2 / f3) + # Continue from that value with a -6dB/octave slope + target[i] = val_at_f3 * (f3 / f) + + # Convert back to tensor + target_curve = torch.tensor(target, device=device) + target_curve[0]=target_curve[1] # Ensure the first value is not zero to avoid division issues + + #decrease 20dB + target_curve = target_curve * 0.1 # Scale down to -20dB + + return target_curve \ No newline at end of file