Download preprocessor/preprocessor.py from lushy/fastspeech2-ljspeech: direct link, hf CLI and curl.
- Browser
- Download file 9.73 kB
-
https://huggingface.co/lushy/fastspeech2-ljspeech/resolve/main/preprocessor/preprocessor.py
- Command line
-
hf download hf://lushy/fastspeech2-ljspeech/preprocessor/preprocessor.py
-
curl -L -o preprocessor.py https://huggingface.co/lushy/fastspeech2-ljspeech/resolve/main/preprocessor/preprocessor.py
9.73 kB
| import os | |
| import random | |
| import json | |
| from pathlib import Path | |
| import tgt | |
| import librosa | |
| import numpy as np | |
| import pyworld as pw | |
| from scipy.interpolate import interp1d | |
| from sklearn.preprocessing import StandardScaler | |
| from tqdm import tqdm | |
| import audio as Audio | |
| class Preprocessor: | |
| def __init__(self, config): | |
| self.config = config | |
| self.in_dir = Path(config["path"]["raw_path"]) | |
| self.out_dir = Path(config["path"]["preprocessed_path"]) | |
| self.val_size = config["preprocessing"]["val_size"] | |
| self.sampling_rate = config["preprocessing"]["audio"]["sampling_rate"] | |
| self.hop_length = config["preprocessing"]["stft"]["hop_length"] | |
| assert config["preprocessing"]["pitch"]["feature"] in ["phoneme_level", "frame_level"] | |
| assert config["preprocessing"]["energy"]["feature"] in ["phoneme_level", "frame_level"] | |
| self.pitch_phoneme_averaging = config["preprocessing"]["pitch"]["feature"] == "phoneme_level" | |
| self.energy_phoneme_averaging = config["preprocessing"]["energy"]["feature"] == "phoneme_level" | |
| self.pitch_normalization = config["preprocessing"]["pitch"]["normalization"] | |
| self.energy_normalization = config["preprocessing"]["energy"]["normalization"] | |
| self.STFT = Audio.stft.TacotronSTFT( | |
| config["preprocessing"]["stft"]["filter_length"], | |
| config["preprocessing"]["stft"]["hop_length"], | |
| config["preprocessing"]["stft"]["win_length"], | |
| config["preprocessing"]["mel"]["n_mel_channels"], | |
| config["preprocessing"]["audio"]["sampling_rate"], | |
| config["preprocessing"]["mel"]["mel_fmin"], | |
| config["preprocessing"]["mel"]["mel_fmax"], | |
| ) | |
| def build_from_path(self): | |
| (self.out_dir / "mel").mkdir(parents=True, exist_ok=True) | |
| (self.out_dir / "pitch").mkdir(parents=True, exist_ok=True) | |
| (self.out_dir / "energy").mkdir(parents=True, exist_ok=True) | |
| (self.out_dir / "duration").mkdir(parents=True, exist_ok=True) | |
| print("Processing Data ...") | |
| out = list() | |
| n_frames = 0 | |
| pitch_scaler = StandardScaler() | |
| energy_scaler = StandardScaler() | |
| speakers = {} | |
| search_dirs = [p for p in self.in_dir.iterdir() if p.is_dir()] | |
| for i, speaker_dir in enumerate(tqdm(search_dirs)): | |
| speaker = speaker_dir.name | |
| speakers[speaker] = i | |
| for wav_path in speaker_dir.iterdir(): | |
| if wav_path.suffix != ".wav": | |
| continue | |
| basename = wav_path.stem | |
| tg_path = self.out_dir / "TextGrid" / speaker / f"{basename}.TextGrid" | |
| if tg_path.exists(): | |
| ret = self.process_utterance(speaker, basename) | |
| if ret is None: | |
| continue | |
| info, pitch, energy, n = ret | |
| out.append(info) | |
| if len(pitch) > 0: | |
| pitch_scaler.partial_fit(pitch.reshape((-1, 1))) | |
| if len(energy) > 0: | |
| energy_scaler.partial_fit(energy.reshape((-1, 1))) | |
| n_frames += n | |
| print("Computing statistic quantities ...") | |
| if hasattr(pitch_scaler, "mean_") and pitch_scaler.mean_ is not None and n_frames > 0: | |
| pitch_mean = pitch_scaler.mean_[0] if self.pitch_normalization else 0 | |
| pitch_std = pitch_scaler.scale_[0] if self.pitch_normalization else 1 | |
| energy_mean = energy_scaler.mean_[0] if self.energy_normalization else 0 | |
| energy_std = energy_scaler.scale_[0] if self.energy_normalization else 1 | |
| pitch_min, pitch_max = self.normalize(self.out_dir / "pitch", pitch_mean, pitch_std) | |
| energy_min, energy_max = self.normalize(self.out_dir / "energy", energy_mean, energy_std) | |
| else: | |
| print("\n[!] Warning: No audio samples matched with a corresponding TextGrid file.") | |
| pitch_min, pitch_max, pitch_mean, pitch_std = 0, 1, 0, 1 | |
| energy_min, energy_max, energy_mean, energy_std = 0, 1, 0, 1 | |
| with open(self.out_dir / "speakers.json", "w") as f: | |
| f.write(json.dumps(speakers)) | |
| with open(self.out_dir / "stats.json", "w") as f: | |
| stats = { | |
| "pitch": [float(pitch_min), float(pitch_max), float(pitch_mean), float(pitch_std)], | |
| "energy": [float(energy_min), float(energy_max), float(energy_mean), float(energy_std)], | |
| } | |
| f.write(json.dumps(stats)) | |
| print(f"Total time: {n_frames * self.hop_length / self.sampling_rate / 3600:.2f} hours") | |
| random.shuffle(out) | |
| out = [r for r in out if r is not None] | |
| with open(self.out_dir / "train.txt", "w", encoding="utf-8") as f: | |
| for m in out[self.val_size :]: | |
| f.write(f"{m}\n") | |
| with open(self.out_dir / "val.txt", "w", encoding="utf-8") as f: | |
| for m in out[: self.val_size]: | |
| f.write(f"{m}\n") | |
| return out | |
| def process_utterance(self, speaker, basename): | |
| wav_path = self.in_dir / speaker / f"{basename}.wav" | |
| text_path = self.in_dir / speaker / f"{basename}.lab" | |
| tg_path = self.out_dir / "TextGrid" / speaker / f"{basename}.TextGrid" | |
| textgrid = tgt.io.read_textgrid(str(tg_path)) | |
| phone, duration, start, end = self.get_alignment(textgrid.get_tier_by_name("phones")) | |
| text = "{" + " ".join(phone) + "}" | |
| if start >= end: | |
| return None | |
| wav, _ = librosa.load(str(wav_path)) | |
| wav = wav[int(self.sampling_rate * start) : int(self.sampling_rate * end)].astype(np.float32) | |
| with open(text_path, "r", encoding="utf-8") as f: | |
| raw_text = f.readline().strip("\n") | |
| pitch, t = pw.dio( | |
| wav.astype(np.float64), | |
| self.sampling_rate, | |
| frame_period=self.hop_length / self.sampling_rate * 1000, | |
| ) | |
| pitch = pw.stonemask(wav.astype(np.float64), pitch, t, self.sampling_rate) | |
| pitch = pitch[: sum(duration)] | |
| if np.sum(pitch != 0) <= 1: | |
| return None | |
| mel_spectrogram, energy = Audio.tools.get_mel_from_wav(wav, self.STFT) | |
| mel_spectrogram = mel_spectrogram[:, : sum(duration)] | |
| energy = energy[: sum(duration)] | |
| if self.pitch_phoneme_averaging: | |
| nonzero_ids = np.where(pitch != 0)[0] | |
| interp_fn = interp1d( | |
| nonzero_ids, | |
| pitch[nonzero_ids], | |
| fill_value=(pitch[nonzero_ids[0]], pitch[nonzero_ids[-1]]), | |
| bounds_error=False, | |
| ) | |
| pitch = interp_fn(np.arange(0, len(pitch))) | |
| pos = 0 | |
| for i, d in enumerate(duration): | |
| pitch[i] = np.mean(pitch[pos : pos + d]) if d > 0 else 0 | |
| pos += d | |
| pitch = pitch[: len(duration)] | |
| if self.energy_phoneme_averaging: | |
| pos = 0 | |
| for i, d in enumerate(duration): | |
| energy[i] = np.mean(energy[pos : pos + d]) if d > 0 else 0 | |
| pos += d | |
| energy = energy[: len(duration)] | |
| np.save(self.out_dir / "duration" / f"{speaker}-duration-{basename}.npy", duration) | |
| np.save(self.out_dir / "pitch" / f"{speaker}-pitch-{basename}.npy", pitch) | |
| np.save(self.out_dir / "energy" / f"{speaker}-energy-{basename}.npy", energy) | |
| np.save(self.out_dir / "mel" / f"{speaker}-mel-{basename}.npy", mel_spectrogram.T) | |
| return ( | |
| "|".join([basename, speaker, text, raw_text]), | |
| self.remove_outlier(pitch), | |
| self.remove_outlier(energy), | |
| mel_spectrogram.shape[1], | |
| ) | |
| def get_alignment(self, tier): | |
| sil_phones = ["sil", "sp", "spn"] | |
| phones, durations = [], [] | |
| start_time, end_time, end_idx = 0, 0, 0 | |
| for t in tier._objects: | |
| s, e, p = t.start_time, t.end_time, t.text | |
| if phones == []: | |
| if p in sil_phones: | |
| continue | |
| else: | |
| start_time = s | |
| if p not in sil_phones: | |
| phones.append(p) | |
| end_time = e | |
| end_idx = len(phones) | |
| else: | |
| phones.append(p) | |
| durations.append( | |
| int(np.round(e * self.sampling_rate / self.hop_length) - np.round(s * self.sampling_rate / self.hop_length)) | |
| ) | |
| return phones[:end_idx], durations[:end_idx], start_time, end_time | |
| def remove_outlier[T](self, values: T) -> T: | |
| values = np.array(values) | |
| p25 = np.percentile(values, 25) | |
| p75 = np.percentile(values, 75) | |
| lower = p25 - 1.5 * (p75 - p25) | |
| upper = p75 + 1.5 * (p75 - p25) | |
| normal_indices = np.logical_and(values > lower, values < upper) | |
| return values[normal_indices] | |
| def normalize(self, in_dir: Path, mean: float, std: float): | |
| max_value = np.finfo(np.float64).min | |
| min_value = np.finfo(np.float64).max | |
| for file_path in in_dir.iterdir(): | |
| if file_path.is_file(): | |
| values = (np.load(file_path) - mean) / std | |
| np.save(file_path, values) | |
| max_value = max(max_value, max(values)) | |
| min_value = min(min_value, min(values)) | |
| return min_value, max_value |