Spaces:
Running on Zero
Running on Zero
| import matplotlib | |
| import soundfile as sf | |
| import torch | |
| from matplotlib import cm | |
| from matplotlib import pyplot as plt | |
| import matplotlib.pyplot as plt | |
| #matplotlib.use("tkAgg") #original but makes error | |
| matplotlib.use("Agg") | |
| from sklearn.manifold import TSNE | |
| from sklearn.decomposition import PCA | |
| import numpy | |
| from tqdm import tqdm | |
| from Preprocessing.ProsodicConditionExtractor import ProsodicConditionExtractor | |
| class Visualizer: | |
| def __init__(self, sr=48000, device="cpu"): | |
| """ | |
| Args: | |
| sr: The sampling rate of the audios you want to visualize. | |
| """ | |
| self.tsne = TSNE(n_jobs=-1, n_iter_without_progress=4000, n_iter=20000) | |
| self.pca = PCA(n_components=2) | |
| self.pros_cond_ext = ProsodicConditionExtractor(sr=sr, device=device) | |
| self.sr = sr | |
| def visualize_speaker_embeddings(self, label_to_filepaths, title_of_plot, save_file_path=None, include_pca=True, legend=True, colors=None): | |
| label_list = list() | |
| embedding_list = list() | |
| ordered_labels = sorted(list(label_to_filepaths.keys())) | |
| for label in tqdm(ordered_labels): | |
| for filepath in tqdm(label_to_filepaths[label]): | |
| wave, sr = sf.read(filepath) | |
| if len(wave) / sr < 1: | |
| continue | |
| if self.sr != sr: | |
| print("One of the Audios you included doesn't match the sampling rate of this visualizer object, " | |
| "creating a new condition extractor. Results will be correct, but if there are too many cases " | |
| "of changing samplingrate, this will run very slowly.") | |
| self.pros_cond_ext = ProsodicConditionExtractor(sr=sr) | |
| self.sr = sr | |
| embedding_list.append(self.pros_cond_ext.extract_condition_from_reference_wave(wave).squeeze().numpy()) | |
| label_list.append(label) | |
| embeddings_as_array = numpy.array(embedding_list) | |
| dimensionality_reduced_embeddings_tsne = self.tsne.fit_transform(embeddings_as_array) | |
| self._plot_embeddings(projected_data=dimensionality_reduced_embeddings_tsne, | |
| labels=label_list, | |
| title=title_of_plot + " t-SNE" if include_pca else title_of_plot, | |
| save_file_path=save_file_path, | |
| legend=legend, | |
| colors=colors) | |
| print(save_file_path) | |
| if include_pca: | |
| dimensionality_reduced_embeddings_pca = self.pca.fit_transform(embeddings_as_array) | |
| self._plot_embeddings(projected_data=dimensionality_reduced_embeddings_pca, | |
| labels=label_list, | |
| title=title_of_plot + " PCA", | |
| save_file_path=save_file_path, | |
| legend=legend, | |
| colors=colors) | |
| def _plot_embeddings(self, projected_data, labels, title, save_file_path, legend, colors): | |
| if colors is None: | |
| colors = cm.gist_rainbow(numpy.linspace(0, 1, len(set(labels)))) | |
| label_to_color = dict() | |
| for index, label in enumerate(sorted(list(set(labels)))): | |
| label_to_color[label] = colors[index] | |
| labels_to_points_x = dict() | |
| labels_to_points_y = dict() | |
| for label in labels: | |
| labels_to_points_x[label] = list() | |
| labels_to_points_y[label] = list() | |
| for index, label in enumerate(labels): | |
| labels_to_points_x[label].append(projected_data[index][0]) | |
| labels_to_points_y[label].append(projected_data[index][1]) | |
| fig, ax = plt.subplots() | |
| for label in sorted(list(set(labels))): | |
| x = numpy.array(labels_to_points_x[label]) | |
| y = numpy.array(labels_to_points_y[label]) | |
| ax.scatter(x=x, | |
| y=y, | |
| c=label_to_color[label], | |
| label=label, | |
| alpha=0.9) | |
| if legend: | |
| ax.legend() | |
| fig.tight_layout() | |
| ax.axis('off') | |
| fig.subplots_adjust(top=0.9, bottom=0.0, right=1.0, left=0.0) | |
| ax.set_title(title) | |
| if save_file_path is not None: | |
| plt.savefig(save_file_path) | |
| print(save_file_path) | |
| else: | |
| plt.show() | |
| plt.close() | |
| def calculate_spk_sim(self, reference_path, comparisons): | |
| embedding_list = list() | |
| for filepath in tqdm(comparisons): | |
| wave, sr = sf.read(filepath) | |
| if len(wave) / sr < 1: | |
| continue | |
| if self.sr != sr: | |
| print("One of the Audios you included doesn't match the sampling rate of this visualizer object, " | |
| "creating a new condition extractor. Results will be correct, but if there are too many cases " | |
| "of changing samplingrate, this will run very slowly.") | |
| self.pros_cond_ext = ProsodicConditionExtractor(sr=sr) | |
| self.sr = sr | |
| embedding_list.append(self.pros_cond_ext.extract_condition_from_reference_wave(wave).squeeze()) | |
| wave, sr = sf.read(reference_path) | |
| if self.sr != sr: | |
| self.pros_cond_ext = ProsodicConditionExtractor(sr=sr) | |
| self.sr = sr | |
| reference_embedding = self.pros_cond_ext.extract_condition_from_reference_wave(wave).squeeze() | |
| sims = list() | |
| for comp_emb in embedding_list: | |
| sims.append(torch.cosine_similarity(reference_embedding, comp_emb, dim=0)) | |
| return (sum(sims) / len(sims)).item(), numpy.std(sims) | |