Spaces:
Running on Zero
Running on Zero
File size: 5,700 Bytes
cc348e2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | import matplotlib
import soundfile as sf
import torch
from matplotlib import cm
from matplotlib import pyplot as plt
import matplotlib.pyplot as plt
#matplotlib.use("tkAgg") #original but makes error
matplotlib.use("Agg")
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA
import numpy
from tqdm import tqdm
from Preprocessing.ProsodicConditionExtractor import ProsodicConditionExtractor
class Visualizer:
def __init__(self, sr=48000, device="cpu"):
"""
Args:
sr: The sampling rate of the audios you want to visualize.
"""
self.tsne = TSNE(n_jobs=-1, n_iter_without_progress=4000, n_iter=20000)
self.pca = PCA(n_components=2)
self.pros_cond_ext = ProsodicConditionExtractor(sr=sr, device=device)
self.sr = sr
def visualize_speaker_embeddings(self, label_to_filepaths, title_of_plot, save_file_path=None, include_pca=True, legend=True, colors=None):
label_list = list()
embedding_list = list()
ordered_labels = sorted(list(label_to_filepaths.keys()))
for label in tqdm(ordered_labels):
for filepath in tqdm(label_to_filepaths[label]):
wave, sr = sf.read(filepath)
if len(wave) / sr < 1:
continue
if self.sr != sr:
print("One of the Audios you included doesn't match the sampling rate of this visualizer object, "
"creating a new condition extractor. Results will be correct, but if there are too many cases "
"of changing samplingrate, this will run very slowly.")
self.pros_cond_ext = ProsodicConditionExtractor(sr=sr)
self.sr = sr
embedding_list.append(self.pros_cond_ext.extract_condition_from_reference_wave(wave).squeeze().numpy())
label_list.append(label)
embeddings_as_array = numpy.array(embedding_list)
dimensionality_reduced_embeddings_tsne = self.tsne.fit_transform(embeddings_as_array)
self._plot_embeddings(projected_data=dimensionality_reduced_embeddings_tsne,
labels=label_list,
title=title_of_plot + " t-SNE" if include_pca else title_of_plot,
save_file_path=save_file_path,
legend=legend,
colors=colors)
print(save_file_path)
if include_pca:
dimensionality_reduced_embeddings_pca = self.pca.fit_transform(embeddings_as_array)
self._plot_embeddings(projected_data=dimensionality_reduced_embeddings_pca,
labels=label_list,
title=title_of_plot + " PCA",
save_file_path=save_file_path,
legend=legend,
colors=colors)
def _plot_embeddings(self, projected_data, labels, title, save_file_path, legend, colors):
if colors is None:
colors = cm.gist_rainbow(numpy.linspace(0, 1, len(set(labels))))
label_to_color = dict()
for index, label in enumerate(sorted(list(set(labels)))):
label_to_color[label] = colors[index]
labels_to_points_x = dict()
labels_to_points_y = dict()
for label in labels:
labels_to_points_x[label] = list()
labels_to_points_y[label] = list()
for index, label in enumerate(labels):
labels_to_points_x[label].append(projected_data[index][0])
labels_to_points_y[label].append(projected_data[index][1])
fig, ax = plt.subplots()
for label in sorted(list(set(labels))):
x = numpy.array(labels_to_points_x[label])
y = numpy.array(labels_to_points_y[label])
ax.scatter(x=x,
y=y,
c=label_to_color[label],
label=label,
alpha=0.9)
if legend:
ax.legend()
fig.tight_layout()
ax.axis('off')
fig.subplots_adjust(top=0.9, bottom=0.0, right=1.0, left=0.0)
ax.set_title(title)
if save_file_path is not None:
plt.savefig(save_file_path)
print(save_file_path)
else:
plt.show()
plt.close()
def calculate_spk_sim(self, reference_path, comparisons):
embedding_list = list()
for filepath in tqdm(comparisons):
wave, sr = sf.read(filepath)
if len(wave) / sr < 1:
continue
if self.sr != sr:
print("One of the Audios you included doesn't match the sampling rate of this visualizer object, "
"creating a new condition extractor. Results will be correct, but if there are too many cases "
"of changing samplingrate, this will run very slowly.")
self.pros_cond_ext = ProsodicConditionExtractor(sr=sr)
self.sr = sr
embedding_list.append(self.pros_cond_ext.extract_condition_from_reference_wave(wave).squeeze())
wave, sr = sf.read(reference_path)
if self.sr != sr:
self.pros_cond_ext = ProsodicConditionExtractor(sr=sr)
self.sr = sr
reference_embedding = self.pros_cond_ext.extract_condition_from_reference_wave(wave).squeeze()
sims = list()
for comp_emb in embedding_list:
sims.append(torch.cosine_similarity(reference_embedding, comp_emb, dim=0))
return (sum(sims) / len(sims)).item(), numpy.std(sims)
|