import pandas as pd import numpy as np import torch import os import re import torchaudio from typing import NamedTuple, Tuple, List, Optional, Union from tqdm import tqdm from meta_clap_consistency import CLAPTextConsistencyMetric from lpaps import LPAPS class CombinedRes(NamedTuple): ours: pd.DataFrame sdedit: pd.DataFrame orig: pd.DataFrame musicgen: pd.DataFrame musicgen_large: pd.DataFrame ddim: pd.DataFrame class ScorePair(NamedTuple): lpaps: dict clap: dict class Scores(NamedTuple): ours: ScorePair sdedit: ScorePair orig: ScorePair musicgen: ScorePair ddim: ScorePair def compute_lpaps_with_windows(aud1: torch.Tensor, aud1_sr: int, aud2: torch.Tensor, aud2_sr: int, model: LPAPS, windows_size1: Optional[int] = None, windows_size2: Optional[int] = None, overlap: float = 0.1, method: str = 'mean', device: str = 'cuda:0') -> float: """Calculate the LPAPS score for the given audio files, windowed. If windows_size1 or windows_size2 is None, it will default to 10 seconds. :param torch.Tensor aud1: The first audio file to compute LPAPS for :param int aud1_sr: The sample rate of the first audio file :param torch.Tensor aud2: The second audio file to compute LPAPS for :param int aud2_sr: The sample rate of the second audio file :param LPAPS model: The LPAPS model to use :param Optional[int] windows_size1: Window size in seconds for the first audio file. Defaults to 10 seconds (None) :param Optional[int] windows_size2: Window size in seconds for the second audio file. Defaults to 10 seconds (None) :param float overlap: The overlap factor of the windows, defaults to 0.1 :param str method: method to use to combine scores, defaults to 'mean', choices=['mean', 'median', 'max', 'min'] :param _type_ device: Torch device to use, defaults to 'cuda:0' :raises ValueError: Using an unknown method :return float: The combined LPAPS score """ if windows_size1 is None: windows_size1 = int(aud1_sr * 10) if windows_size2 is None: windows_size2 = int(aud2_sr * 10) scores = [] for i, j in zip(range(0, aud1.shape[-1], int(windows_size1 * (1 - overlap))), range(0, aud2.shape[-1], int(windows_size2 * (1 - overlap)))): window1 = aud1[:, i:i + windows_size1] window2 = aud2[:, j:j + windows_size2] scores.append(model(window1.unsqueeze(0).to(device), window2.unsqueeze(0).to(device), torch.tensor([aud1_sr], device=device), torch.tensor([aud2_sr], device=device)).item()) if method == 'mean': func = np.mean elif method == 'median': func = np.median elif method == 'max': func = np.max elif method == 'min': func = np.min else: raise ValueError(f'Unknown method: {method}') return func(scores) def compute_clap_with_windows(aud: torch.Tensor, aud_sr: int, prompt: str, model: CLAPTextConsistencyMetric, windows_size: Optional[int] = None, overlap: float = 0.1, method: str = 'mean', device: torch.device = 'cuda:0') -> float: """Calculate the CLAP score for the given audio file and prompt, windowed. If windows_size is None, it will default to 10 seconds. :param torch.Tensor aud: The audio file to compute CLAP for :param int aud_sr: The sample rate of the audio file :param str prompt: The prompt to compute CLAP relative to :param CLAPTextConsistencyMetric model: The CLAP model to use :param Optional[int] windows_size: Window size in seconds. Defaults to 10 seconds (None) :param float overlap: The overlap factor of the windows, defaults to 0.1 :param str method: method to use to combine scores, defaults to 'mean', choices=['mean', 'median', 'max', 'min'] :param _type_ device: Torch device to use, defaults to 'cuda:0' :raises ValueError: Using an unknown method :return float: The combined CLAP score """ if windows_size is None: windows_size = int(aud_sr * 10) scores = [] for i in range(0, aud.shape[-1], int(windows_size * (1 - overlap))): window = aud[:, i:i + windows_size] model.update(window.unsqueeze(0).to(device), [prompt], torch.tensor([aud_sr], device=device)) scores.append(model.compute()) model.reset() if method == 'mean': func = np.mean elif method == 'median': func = np.median elif method == 'max': func = np.max elif method == 'min': func = np.min else: raise ValueError(f'Unknown method: {method}') return func(scores) def calc_scores(clap_ckpt_name: str, device, ours_dirs: Union[str, List[str]], sdedit_dirs: Union[str, List[str]], musicgen_dir: str, ddim_dirs: Union[str, List[str]], inputs_orig: str, prev_pt: Optional[str] = None, win_length: Optional[int] = None, overlap: float = 0.1, method: str = 'mean', clap_checkpoint_path: str = 'clap/pretrained') -> Scores: """Calculate the CLAP and LPAPS scores for the given directories Note: This function is built for our used directory structure, it's here as an example. :param str clap_ckpt_name: the name of the CLAP checkpoint to use :param _type_ device: torch device to use :param Union[str, List[str]] ours_dirs: our results directories :param Union[str, List[str]] sdedit_dirs: sdedit results directories :param str musicgen_dir: musicgen results directory :param Union[str, List[str]] ddim_dirs: ddim results directories :param str inputs_orig: input directory :param Optional[str] prev_pt: path to checkpoint of calculated scores, defaults to None :param Optional[int] win_length: window length in seconds to use, defaults to None :param float overlap: fraction of overlap between windows, defaults to 0.1 :param str method: method to use to combine scores, defaults to 'mean', choices=['mean', 'median', 'max', 'min'] :param str clap_checkpoint_path: the path where CLAP's weights are stored, defaults to 'clap/pretrained' :return Scores: """ clap_model = CLAPTextConsistencyMetric(model_path=os.path.join(clap_checkpoint_path, clap_ckpt_name), model_arch='HTSAT-base' if 'fusion' not in clap_ckpt_name else 'HTSAT-tiny', enable_fusion='fusion' in clap_ckpt_name ).to(device) clap_model.eval() lpaps_model = LPAPS(net='clap', device=device, net_kwargs={'model_arch': 'HTSAT-base' if 'fusion' not in clap_ckpt_name else 'HTSAT-tiny', 'chkpt': clap_ckpt_name, 'enable_fusion': 'fusion' in clap_ckpt_name}, checkpoint_path=clap_checkpoint_path) clap_score_orig = {} all_clap_ours = {} all_clap_sdedit = {} all_clap_musicgen = {} all_clap_ddim = {} all_lpaps_orig = {} all_lpaps_ours = {} all_lpaps_sdedit = {} all_lpaps_musicgen = {} all_lpaps_ddim = {} if prev_pt is not None and os.path.exists(prev_pt): prev_pt: Scores = torch.load(prev_pt)['scores'] all_clap_ours = prev_pt.ours.clap all_lpaps_ours = prev_pt.ours.lpaps all_clap_sdedit = prev_pt.sdedit.clap all_lpaps_sdedit = prev_pt.sdedit.lpaps clap_score_orig = prev_pt.orig.clap all_lpaps_orig = prev_pt.orig.lpaps all_clap_musicgen = prev_pt.musicgen.clap all_lpaps_musicgen = prev_pt.musicgen.lpaps all_clap_ddim = prev_pt.ddim.clap all_lpaps_ddim = prev_pt.ddim.lpaps if ours_dirs is str: ours_dirs = [ours_dirs] sdedit_dirs = [sdedit_dirs] ddim_dirs = [ddim_dirs] audio_inputs = os.listdir(ours_dirs[0]) with torch.no_grad(): for audio_input in tqdm(audio_inputs): audio_dir_musicgen = os.path.join(musicgen_dir, audio_input) # Initialize the dictionaries for the audio input for score_dict in [all_clap_ours, clap_score_orig, all_clap_sdedit, all_clap_musicgen, all_clap_ddim, all_lpaps_ours, all_lpaps_orig, all_lpaps_sdedit, all_lpaps_musicgen, all_lpaps_ddim]: if audio_input not in score_dict: score_dict[audio_input] = {} orig_aud_path = os.path.join(inputs_orig, audio_input + ".wav") orig_aud, orig_aud_sr = torchaudio.load(orig_aud_path) # LPAPS relative to itself if all_lpaps_orig[audio_input] is not float: all_lpaps_orig[audio_input] = calc_lpaps_win(lpaps_model, orig_aud, orig_aud, orig_aud_sr, orig_aud_sr, win_length, method, overlap, device) # Get MusicGen files try: musicgen_target_prompts = {x[7:-4]: x for x in os.listdir(audio_dir_musicgen)} except FileNotFoundError: # We didn't run this in musicgen somehow print(f'Missing MUSICGEN {audio_input}') continue for ours_dir, sdedit_dir, ddim_dir in zip(ours_dirs, sdedit_dirs, ddim_dirs): audio_dir_ours = os.path.join(ours_dir, audio_input) audio_dir_sdedit = os.path.join(sdedit_dir, audio_input) audio_dir_ddim = os.path.join(ddim_dir, audio_input) # Get SDedit files try: sdedit_target_prompts = {x[4:-7].replace('_', ' '): x for x in os.listdir(audio_dir_sdedit)} except FileNotFoundError: print(f'Missing SDEDIT {audio_input}') # We didn't get to that input in sdedit yet continue # Get DDIM files try: {x[4:-7].replace('_', ' '): x for x in os.listdir(audio_dir_ddim)} except FileNotFoundError: # We didn't run this in ddim somehow print(f'Missing DDIM {audio_input}') continue for src_prompt_dir in os.listdir(audio_dir_ours): src_prompt = src_prompt_dir[4:].replace('_', ' ') # Initialize the dictionaries for the source prompt for score_dict in [all_lpaps_ours, all_lpaps_ddim, all_clap_ours, all_clap_ddim]: if score_dict[audio_input].get(src_prompt) is None: score_dict[audio_input][src_prompt] = {} src_prompt_ddim_dir = os.path.join(audio_dir_ddim, src_prompt_dir) src_prompt_dir = os.path.join(audio_dir_ours, src_prompt_dir) for target_prompt in os.listdir(src_prompt_dir): inner_dir = os.path.join(src_prompt_dir, target_prompt) inner_ddim_dir = os.path.join(src_prompt_ddim_dir, target_prompt) target_prompt = target_prompt[4:-7].replace('_', ' ') if src_prompt == target_prompt: print(f'Skipping {audio_input}, {src_prompt}, {target_prompt}') continue # Calc orig CLAP score for CLAP accuracy if clap_score_orig[audio_input].get(target_prompt) is None: clap_score_orig[audio_input][target_prompt] = calc_clap_win( clap_model, orig_aud, orig_aud_sr, target_prompt, win_length, method, overlap, device) # Initialize the dictionaries for the target prompt for score_dict in [all_clap_ours, all_clap_ddim, all_lpaps_ours, all_lpaps_ddim]: if score_dict[audio_input][src_prompt].get(target_prompt) is None: score_dict[audio_input][src_prompt][target_prompt] = {} # CALC for OURS ours_skips = set([int(re.findall(r'_skip_(\d+)_', x)[0]) for x in os.listdir(inner_dir) if x.endswith('.wav') and not x.startswith('orig')]) for skip in ours_skips: for score_dict in [all_clap_ours, all_lpaps_ours]: if score_dict[audio_input][src_prompt][target_prompt].get(skip) is None: score_dict[audio_input][src_prompt][target_prompt][skip] = {} ours_tarcfgs = set([int(re.findall(r'_cfg_d_(\d+).0_', x)[0]) for x in os.listdir(inner_dir) if x.endswith('.wav') and not x.startswith('orig') and f'_skip_{skip}_' in x]) for tarcfg in ours_tarcfgs: for score_dict in [all_clap_ours, all_lpaps_ours]: if score_dict[audio_input][src_prompt][target_prompt][skip].get(tarcfg) is None: score_dict[audio_input][src_prompt][target_prompt][skip][tarcfg] = {} ours_auds = {float(re.findall(r'cfg_e_(\d+\.\d+)_', x)[0]): os.path.join(inner_dir, x) for x in os.listdir(inner_dir) if x.endswith('.wav') and not x.startswith('orig') and f'_skip_{skip}_' in x and f'_cfg_d_{tarcfg}.0_' in x} for srccfg in ours_auds: if all_clap_ours[audio_input][src_prompt][target_prompt][skip][tarcfg].get(srccfg) is None or \ all_lpaps_ours[audio_input][src_prompt][target_prompt][skip][tarcfg].get(srccfg) is None: ours_aud, ours_aud_sr = torchaudio.load(ours_auds[srccfg]) all_clap_ours[audio_input][src_prompt][target_prompt][skip][tarcfg][srccfg] = calc_clap_win( clap_model, ours_aud, ours_aud_sr, target_prompt, win_length, method, overlap, device) all_lpaps_ours[audio_input][src_prompt][target_prompt][skip][tarcfg][srccfg] = calc_lpaps_win( lpaps_model, ours_aud, orig_aud, ours_aud_sr, orig_aud_sr, win_length, method, overlap, device) # CALC for DDIM try: ddim_skips = [int(re.findall(r'_skip_(\d+)_', x)[0]) for x in os.listdir(inner_ddim_dir) if x.endswith('.wav') and not x.startswith('orig') and 'skip' in x] if len([x for x in os.listdir(inner_ddim_dir) if x.endswith('.wav') and not x.startswith('orig') if '200timesteps' in x]): ddim_skips += [0] ddim_skips = set(ddim_skips) for skip in ddim_skips: for score_dict in [all_clap_ddim, all_lpaps_ddim]: if score_dict[audio_input][src_prompt][target_prompt].get(skip) is None: score_dict[audio_input][src_prompt][target_prompt][skip] = {} ddim_tarcfgs = set([int(re.findall(r'_cfg_d_(\d+).0_', x)[0]) for x in os.listdir(inner_ddim_dir) if x.endswith('.wav') and not x.startswith('orig') and (('skip' in x and f'_skip_{skip}_' in x) or ('200timesteps' in x and skip == 0))]) for tarcfg in ddim_tarcfgs: for score_dict in [all_clap_ddim, all_lpaps_ddim]: if score_dict[audio_input][src_prompt][target_prompt][skip].get(tarcfg) is None: score_dict[audio_input][src_prompt][target_prompt][skip][tarcfg] = {} ddim_auds = {float(re.findall(r'cfg_e_(\d+\.\d+)_', x)[0]): os.path.join(inner_ddim_dir, x) for x in os.listdir(inner_ddim_dir) if x.endswith('.wav') and not x.startswith('orig') and (('skip' in x and f'_skip_{skip}_' in x) or ('200timesteps' in x and skip == 0)) and f'_cfg_d_{tarcfg}.0_' in x} for srccfg in ddim_auds: if all_lpaps_ddim[audio_input][src_prompt][target_prompt][skip][tarcfg].get(srccfg) is None or \ all_clap_ddim[audio_input][src_prompt][target_prompt][skip][tarcfg].get(srccfg) is None: ddim_aud, ddim_aud_sr = torchaudio.load(ddim_auds[srccfg]) all_clap_ddim[audio_input][src_prompt][target_prompt][skip][tarcfg][srccfg] = calc_clap_win( clap_model, ddim_aud, ddim_aud_sr, target_prompt, win_length, method, overlap, device) all_lpaps_ddim[audio_input][src_prompt][target_prompt][skip][tarcfg][srccfg] = calc_lpaps_win( lpaps_model, ddim_aud, orig_aud, ddim_aud_sr, orig_aud_sr, win_length, method, overlap, device) except (KeyError, FileNotFoundError, IndexError): print(f'Missing DDIM {audio_input}, {src_prompt}, {target_prompt}, {skip}, {tarcfg}, {srccfg}') # CALC for MUSICGEN try: musicgen_tmp = musicgen_target_prompts[target_prompt] except KeyError: # We didn't get to that prompt in musicgen yet print(f'Missing MUSICGEN {audio_input}, {src_prompt}, {target_prompt}') continue for score_dict in [all_clap_musicgen, all_lpaps_musicgen]: if score_dict[audio_input].get(target_prompt) is None: score_dict[audio_input][target_prompt] = [] # TODO WHY DOES THIS HAPPEN if not len([x for x in all_clap_musicgen[audio_input][target_prompt]]) or \ not len([x for x in all_lpaps_musicgen[audio_input][target_prompt]]): musicgen_aud, musicgen_aud_sr = torchaudio.load(os.path.join(audio_dir_musicgen, musicgen_tmp)) all_clap_musicgen[audio_input][target_prompt].append( {'score': calc_clap_win(clap_model, musicgen_aud, musicgen_aud_sr, target_prompt, win_length, method, overlap, device)}) all_lpaps_musicgen[audio_input][target_prompt].append( {'score': calc_lpaps_win(lpaps_model, musicgen_aud, orig_aud, musicgen_aud_sr, orig_aud_sr, win_length, method, overlap, device)}) # CALC for SDEDIT try: sdedit_tmp = sdedit_target_prompts[target_prompt] except KeyError: # We didn't get to that prompt in sdedit yet print(f'Missing SDEDIT {audio_input}, {src_prompt}, {target_prompt}') continue for score_dict in [all_clap_sdedit, all_lpaps_sdedit]: if score_dict[audio_input].get(target_prompt) is None: score_dict[audio_input][target_prompt] = {} sdedit_skips = set([int(re.findall(r'_skip(\d+)', x)[0]) for x in os.listdir(os.path.join(audio_dir_sdedit, sdedit_tmp)) if x.endswith('.wav') and not x.startswith('orig')]) for skip in sdedit_skips: for score_dict in [all_clap_sdedit, all_lpaps_sdedit]: if score_dict[audio_input][target_prompt].get(skip) is None: score_dict[audio_input][target_prompt][skip] = {} sdedits_auds = {int(re.findall(r'cfg(\d+).0', x)[0]): os.path.join(audio_dir_sdedit, sdedit_tmp, x) for x in os.listdir(os.path.join(audio_dir_sdedit, sdedit_tmp)) if x.endswith('.wav') and not x.startswith('orig') and f'_skip{skip}_' in x} if not len(sdedits_auds): # If we didn't print the skip in the filename it's 12 sdedits_auds = {12: os.path.join(audio_dir_sdedit, sdedit_tmp, [x for x in os.listdir(os.path.join(audio_dir_sdedit, sdedit_tmp)) if x.endswith('.wav') and not x.startswith('orig') and f'_skip{skip}' in x][0])} for tarcfg in sdedits_auds: if all_clap_sdedit[audio_input][target_prompt][skip].get(tarcfg) is None or \ all_lpaps_sdedit[audio_input][target_prompt][skip].get(tarcfg) is None: sdedits_aud, sdedits_aud_sr = torchaudio.load(sdedits_auds[tarcfg]) all_clap_sdedit[audio_input][target_prompt][skip][tarcfg] = calc_clap_win( clap_model, sdedits_aud, sdedits_aud_sr, target_prompt, win_length, method, overlap, device) all_lpaps_sdedit[audio_input][target_prompt][skip][tarcfg] = calc_lpaps_win( lpaps_model, sdedits_aud, orig_aud, sdedits_aud_sr, orig_aud_sr, win_length, method, overlap, device) return Scores(ours=ScorePair(clap=all_clap_ours, lpaps=all_lpaps_ours), sdedit=ScorePair(clap=all_clap_sdedit, lpaps=all_lpaps_sdedit), orig=ScorePair(clap=clap_score_orig, lpaps=all_lpaps_orig), musicgen=ScorePair(clap=all_clap_musicgen, lpaps=all_lpaps_musicgen), ddim=ScorePair(clap=all_clap_ddim, lpaps=all_lpaps_ddim)) def calc_clap_win(clap_model: CLAPTextConsistencyMetric, aud: torch.Tensor, sr: int, target_prompt: str, win_length: int, method: str, overlap: float, device: torch.device) -> float: """Calculate the CLAP score between an audio file and a prompt, with optional windowing :param CLAPTextConsistencyMetric clap_model: An initialized CLAP model to use :param torch.Tensor aud: The audio file to compute CLAP for :param int sr: The sample rate of the audio file :param str target_prompt: The prompt to compute CLAP relative to :param int win_length: The length of the window in seconds :param str method: The method to use to combine the scores, between 'mean', 'median', 'max', 'min' :param float overlap: The overlap fraction between windows in the range [0, 1] :param torch.device device: torch device to use :return float: The CLAP score """ if win_length is None: clap_model.update(aud.unsqueeze(0).to(device), [target_prompt], torch.tensor([sr], device=device)) tmp = clap_model.compute() clap_model.reset() return tmp else: return compute_clap_with_windows( aud, sr, target_prompt, clap_model, device=device, windows_size=win_length * sr, overlap=overlap, method=method) def calc_lpaps_win(lpaps_model: LPAPS, aud1: torch.Tensor, aud2: torch.Tensor, sr1: int, sr2: int, win_length: int, method: str, overlap: float, device: torch.device) -> float: """Calculate the LPAPS score between two audio files, with optional windowing :param LPAPS lpaps_model: An initialized LPAPS model to use :param torch.Tensor aud1: First audio file :param torch.Tensor aud2: Second audio file :param int sr1: Sample rate of the first audio file :param int sr2: Sample rate of the second audio file :param int win_length: The length of the window in seconds :param str method: The method to use to combine the scores, between 'mean', 'median', 'max', 'min' :param float overlap: The overlap fraction between windows in the range [0, 1] :param torch.device device: torch device to use :return float: The LPAPS score """ if win_length is None: return lpaps_model(aud1.unsqueeze(0).to(device), aud2.unsqueeze(0).to(device), torch.tensor([sr1], device=device), torch.tensor([sr2], device=device)).item() else: return compute_lpaps_with_windows(aud1, sr1, aud2, sr2, lpaps_model, windows_size1=win_length * sr1, windows_size2=win_length * sr2, overlap=overlap, method=method, device=device) def combine_scores(scores: Scores) -> Tuple[CombinedRes, CombinedRes]: """Combine the scores into easier-to-handel DataFrames :param Scores scores: The scores to combine :return Tuple[CombinedRes, CombinedRes]: The combined scores """ all_clap_ours = scores.ours.clap all_lpaps_ours = scores.ours.lpaps all_clap_sdedit = scores.sdedit.clap all_lpaps_sdedit = scores.sdedit.lpaps clap_score_orig = scores.orig.clap all_lpaps_orig = scores.orig.lpaps all_clap_musicgen = scores.musicgen.clap all_lpaps_musicgen = scores.musicgen.lpaps all_clap_ddim = scores.ddim.clap all_lpaps_ddim = scores.ddim.lpaps all_lpaps_orig_df = pd.DataFrame.from_dict(all_lpaps_orig, orient='index').reset_index() all_lpaps_orig_df = all_lpaps_orig_df.melt(id_vars='index', var_name='target_prompt', value_name='lpaps_score') all_lpaps_orig_df = all_lpaps_orig_df[~all_lpaps_orig_df['lpaps_score'].isna()] all_lpaps_orig_df.rename(columns={'index': 'input_name'}, inplace=True) all_lpaps_orig_df.sort_values(by=['input_name', 'target_prompt'], inplace=True) df_all_lpaps_musicgen = pd.DataFrame([(input_name, prompt, item['score']) for input_name, prompts_dict in all_lpaps_musicgen.items() for prompt, items in prompts_dict.items() for item in items], columns=['input_name', 'target_prompt', 'lpaps_score']) # Flatten the nested dictionary flattened_data = [] for input_name, source_prompts in all_lpaps_ours.items(): for source_prompt, target_prompts in source_prompts.items(): for target_prompt, skips in target_prompts.items(): for skip, tarcfgs in skips.items(): for tarcfg, srccfgs in tarcfgs.items(): for srccfg, lpaps_score in srccfgs.items(): flattened_data.append({ 'input_name': input_name, 'source_prompt': source_prompt, 'target_prompt': target_prompt, 'skip': skip, 'tarcfg': tarcfg, 'srccfg': srccfg, 'lpaps_score': lpaps_score }) # Create the dataframe df_all_lpaps_ours = pd.DataFrame(flattened_data) # Flatten the nested dictionary flattened_data = [] for input_name, source_prompts in all_lpaps_ddim.items(): for source_prompt, target_prompts in source_prompts.items(): for target_prompt, skips in target_prompts.items(): for skip, tarcfgs in skips.items(): for tarcfg, srccfgs in tarcfgs.items(): for srccfg, lpaps_score in srccfgs.items(): flattened_data.append({ 'input_name': input_name, 'source_prompt': source_prompt, 'target_prompt': target_prompt, 'lpaps_score': lpaps_score, 'skip': skip, 'tarcfg': tarcfg, 'srccfg': srccfg }) # Create the dataframe df_all_lpaps_ddim = pd.DataFrame(flattened_data) # Flatten the nested dictionary flattened_data = [] for input_name, target_prompts in all_lpaps_sdedit.items(): for target_prompt, skips in target_prompts.items(): for skip, tarcfgs in skips.items(): for tarcfg, lpaps_score in tarcfgs.items(): flattened_data.append({ 'input_name': input_name, 'target_prompt': target_prompt, 'skip': skip, 'tarcfg': tarcfg, 'lpaps_score': lpaps_score }) # Create the dataframe df_all_lpaps_sdedit = pd.DataFrame(flattened_data) clap_score_orig_df = pd.DataFrame.from_dict(clap_score_orig, orient='index').reset_index() clap_score_orig_df = clap_score_orig_df.melt(id_vars='index', var_name='target_prompt', value_name='clap_score') clap_score_orig_df = clap_score_orig_df[~clap_score_orig_df['clap_score'].isna()] clap_score_orig_df.rename(columns={'index': 'input_name'}, inplace=True) clap_score_orig_df.sort_values(by=['input_name', 'target_prompt'], inplace=True) df_all_clap_musicgen = pd.DataFrame([(input_name, prompt, item['score']) for input_name, prompts_dict in all_clap_musicgen.items() for prompt, items in prompts_dict.items() for item in items], columns=['input_name', 'target_prompt', 'clap_score']) # Flatten the nested dictionary flattened_data = [] for input_name, source_prompts in all_clap_ours.items(): for source_prompt, target_prompts in source_prompts.items(): for target_prompt, skips in target_prompts.items(): for skip, tarcfgs in skips.items(): for tarcfg, srccfgs in tarcfgs.items(): for srcfg, clap_score in srccfgs.items(): flattened_data.append({ 'input_name': input_name, 'source_prompt': source_prompt, 'target_prompt': target_prompt, 'skip': skip, 'tarcfg': tarcfg, 'srccfg': srcfg, 'clap_score': clap_score }) # Create the dataframe df_all_clap_ours = pd.DataFrame(flattened_data) # Flatten the nested dictionary flattened_data = [] for input_name, source_prompts in all_clap_ddim.items(): for source_prompt, target_prompts in source_prompts.items(): for target_prompt, skips in target_prompts.items(): for skip, tarcfgs in skips.items(): for tarcfg, srccfgs in tarcfgs.items(): for srccfg, clap_score in srccfgs.items(): flattened_data.append({ 'input_name': input_name, 'source_prompt': source_prompt, 'target_prompt': target_prompt, 'clap_score': clap_score, 'skip': skip, 'tarcfg': tarcfg, 'srccfg': srccfg }) # Create the dataframe df_all_clap_ddim = pd.DataFrame(flattened_data) # Flatten the nested dictionary flattened_data = [] for input_name, target_prompts in all_clap_sdedit.items(): for target_prompt, skips in target_prompts.items(): for skip, tarcfgs in skips.items(): for tarcfg, clap_score in tarcfgs.items(): flattened_data.append({ 'input_name': input_name, 'target_prompt': target_prompt, 'skip': skip, 'tarcfg': tarcfg, 'clap_score': clap_score }) # Create the dataframe df_all_clap_sdedit = pd.DataFrame(flattened_data) return CombinedRes(ours=df_all_clap_ours, sdedit=df_all_clap_sdedit, orig=clap_score_orig_df, musicgen=df_all_clap_musicgen, musicgen_large=None, ddim=df_all_clap_ddim), CombinedRes(ours=df_all_lpaps_ours, sdedit=df_all_lpaps_sdedit, orig=all_lpaps_orig_df, musicgen=df_all_lpaps_musicgen, musicgen_large=None, ddim=df_all_lpaps_ddim)