Spaces:
Sleeping
Sleeping
Download inference_cli.py from Spider-8251/UniFlow-Audio: direct link, hf CLI and curl.
- Browser
- Download file 13.3 kB
-
https://huggingface.co/spaces/Spider-8251/UniFlow-Audio/resolve/main/inference_cli.py
- Command line
-
hf download hf://spaces/Spider-8251/UniFlow-Audio/inference_cli.py
-
curl -L -o inference_cli.py https://huggingface.co/spaces/Spider-8251/UniFlow-Audio/resolve/main/inference_cli.py
13.3 kB
| #!/usr/bin/env python3 | |
| import json | |
| import os | |
| from pathlib import Path | |
| from typing import Any, Callable | |
| import fire | |
| import torch | |
| import torchaudio | |
| import soundfile as sf | |
| import numpy as np | |
| from modeling_uniflow_audio import UniFlowAudioModel | |
| from constants import TIME_ALIGNED_TASKS, NON_TIME_ALIGNED_TASKS | |
| from utils.phonemize import sentence_to_phones | |
| AUDIO_SUPER_RESOLUTION_OUTPUT_SAMPLE_RATE = 44_100 | |
| class InferenceCLI: | |
| def __init__(self): | |
| self.model_name_or_path = None | |
| self.device = torch.device( | |
| "cuda" if torch.cuda.is_available() else "cpu" | |
| ) | |
| self.g2p = None | |
| self.word2phone = None | |
| self.speaker_model = None | |
| self.svs_processor = None | |
| self.singer_mapping = None | |
| self.video_preprocessor = None | |
| self.video_size = (256, 256) | |
| self.video_fps = 10 | |
| def init_model(self, model_name_or_path): | |
| self.model_name_or_path = model_name_or_path | |
| if Path(self.model_name_or_path).exists(): | |
| self.model = UniFlowAudioModel(model_name_or_path) | |
| else: | |
| self.model = UniFlowAudioModel(f"wsntxxn/{model_name_or_path}") | |
| self.model.to(self.device) | |
| self.sample_rate = self.model.config["sample_rate"] | |
| def init_svs_processor(self, ): | |
| from utils.diffsinger_utilities import SVSInputConverter, TokenTextEncoder | |
| if self.svs_processor is None: | |
| phoneme_list = json.load(open(self.model.svs_phone_set_path, "r")) | |
| self.svs_processor = { | |
| "converter": | |
| SVSInputConverter( | |
| self.model.svs_singer_mapping, self.model.svs_pinyin2ph | |
| ), | |
| "tokenizer": | |
| TokenTextEncoder( | |
| None, vocab_list=phoneme_list, replace_oov=',' | |
| ) | |
| } | |
| def init_video_preprocessor(self, ): | |
| if self.video_preprocessor is None: | |
| from third_party.mmaudio.feature_extractor import init_feature_extractor | |
| self.video_preprocessor = init_feature_extractor() | |
| def on_inference_start(self, model_name_or_path): | |
| if self.model_name_or_path is None or model_name_or_path != self.model_name_or_path: | |
| self.init_model(model_name_or_path) | |
| def add_prehook(func: Callable, ): | |
| def wrapper(self, *args, **kwargs): | |
| model_name_or_path = kwargs.get( | |
| "model_name_or_path", "UniFlow-Audio-v1.1-Large" | |
| ) | |
| self.on_inference_start(model_name_or_path) | |
| return func(self, *args, **kwargs) | |
| return wrapper | |
| def t2a( | |
| self, | |
| caption: str, | |
| model_name_or_path: str = "UniFlow-Audio-v1.1-Large", | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| ): | |
| self._run_inference( | |
| content=caption, | |
| task="text_to_audio", | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| model_name_or_path=model_name_or_path, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path | |
| ) | |
| def t2m( | |
| self, | |
| caption: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| ): | |
| self._run_inference( | |
| content=caption, | |
| task="text_to_music", | |
| model_name_or_path=model_name_or_path, | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path, | |
| ) | |
| def tts( | |
| self, | |
| transcript: str, | |
| ref_speaker_speech: str, | |
| model_name_or_path: str = "UniFlow-Audio-v1.1-Large", | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| ): | |
| from g2p_en import G2p | |
| import nltk | |
| if not self.g2p: | |
| if not os.path.exists( | |
| os.path.expanduser( | |
| "~/nltk_data/taggers/averaged_perceptron_tagger_eng" | |
| ) | |
| ): | |
| nltk.download("averaged_perceptron_tagger_eng") | |
| self.g2p = G2p() | |
| if not self.word2phone: | |
| self.word2phone = json.load( | |
| open( | |
| self.model.tts_word2phone_dict_path, "r", encoding="utf-8" | |
| ) | |
| ) | |
| # OOV word will use a g2p model to predict phoneme | |
| phonemes, OOV_list = sentence_to_phones( | |
| transcript, self.word2phone, self.g2p | |
| ) | |
| print(phonemes) | |
| phone_indices = [ | |
| self.model.tts_phone2id.get( | |
| p, self.model.tts_phone2id.get("spn", 0) | |
| ) for p in phonemes | |
| ] | |
| prompt_waveform, orig_sr = torchaudio.load(ref_speaker_speech) | |
| prompt_waveform = prompt_waveform.mean(0) | |
| prompt_waveform = torchaudio.functional.resample( | |
| prompt_waveform, | |
| orig_freq=orig_sr, | |
| new_freq=self.model.tts_prompt_sr | |
| ) | |
| content = { | |
| "phoneme": np.array(phone_indices, dtype=np.int64), | |
| "prompt_waveform": prompt_waveform, | |
| } | |
| self._run_inference( | |
| content=content, | |
| task="text_to_speech", | |
| model_name_or_path=model_name_or_path, | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path, | |
| ) | |
| def _audio_input_inference( | |
| self, | |
| input_audio: str, | |
| task: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| output_sample_rate: int | None = None, | |
| ): | |
| waveform, orig_sr = torchaudio.load(input_audio) | |
| waveform = waveform.mean(0) | |
| waveform = torchaudio.functional.resample( | |
| waveform, orig_freq=orig_sr, new_freq=self.sample_rate | |
| ) | |
| return self._run_inference( | |
| content=waveform, | |
| task=task, | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| model_name_or_path=model_name_or_path, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path, | |
| output_sample_rate=output_sample_rate, | |
| ) | |
| def se( | |
| self, | |
| noisy_speech: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 1.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| ): | |
| self._audio_input_inference( | |
| input_audio=noisy_speech, | |
| task="speech_enhancement", | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| model_name_or_path=model_name_or_path, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path | |
| ) | |
| def sr( | |
| self, | |
| low_sr_audio: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 1.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| output_sample_rate: int = AUDIO_SUPER_RESOLUTION_OUTPUT_SAMPLE_RATE, | |
| ): | |
| self._audio_input_inference( | |
| input_audio=low_sr_audio, | |
| task="audio_super_resolution", | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| model_name_or_path=model_name_or_path, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path, | |
| output_sample_rate=output_sample_rate, | |
| ) | |
| def v2a( | |
| self, | |
| video: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.mp4", | |
| ): | |
| from utils.video import merge_audio_video | |
| from third_party.mmaudio.feature_extractor import get_video_feature | |
| self.init_video_preprocessor() | |
| video_feature = get_video_feature(video, self.video_preprocessor) | |
| waveform = self._run_inference( | |
| content=video_feature, | |
| task="video_to_audio", | |
| model_name_or_path=model_name_or_path, | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path, | |
| ) | |
| merge_audio_video( | |
| waveform, video, output_path, audio_fps=self.sample_rate | |
| ) | |
| def svs( | |
| self, | |
| singer: str, | |
| music_score: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| ): | |
| self.init_svs_processor() | |
| text, note, note_dur = music_score.split('<sep>') | |
| if singer not in self.model.svs_singer_mapping: | |
| print(f"Unsupported singer {singer}, available singers: ") | |
| print(list(self.model.svs_singer_mapping.keys())) | |
| raise KeyError | |
| midi = self.svs_processor["converter"].preprocess_input({ | |
| "spk_name": singer, | |
| "text": text, | |
| "notes": note, | |
| "notes_duration": note_dur, | |
| }) | |
| midi["phoneme"] = self.svs_processor["tokenizer"].encode( | |
| midi["phoneme"] | |
| ) | |
| self._run_inference( | |
| content=midi, | |
| task="singing_voice_synthesis", | |
| model_name_or_path=model_name_or_path, | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| guidance_scale=guidance_scale, | |
| num_steps=num_steps, | |
| output_path=output_path, | |
| ) | |
| def _run_inference( | |
| self, | |
| content: Any, | |
| task: str, | |
| model_name_or_path: str, | |
| instruction: str | None = None, | |
| instruction_idx: int | None = None, | |
| guidance_scale: float = 5.0, | |
| num_steps: int = 25, | |
| output_path: str = "./output.wav", | |
| output_sample_rate: int | None = None, | |
| ): | |
| if self.model_name_or_path is None or model_name_or_path != self.model_name_or_path: | |
| self.init_model(model_name_or_path) | |
| time_aligned = torch.zeros(1, 2).bool() | |
| if task in TIME_ALIGNED_TASKS: | |
| time_aligned[0, 0] = True | |
| if task in NON_TIME_ALIGNED_TASKS: | |
| time_aligned[0, 1] = True | |
| if instruction: | |
| instruction = [instruction] | |
| if instruction_idx: | |
| instruction_idx = [instruction_idx] | |
| waveform = self.model.sample( | |
| content=[content], | |
| task=[task], | |
| time_aligned=time_aligned, | |
| instruction=instruction, | |
| instruction_idx=instruction_idx, | |
| num_steps=num_steps, | |
| guidance_scale=guidance_scale, | |
| disable_progress=False | |
| ) | |
| waveform = waveform[0, 0].cpu().numpy() | |
| # The model and codec run at their native sample rate (24 kHz for the | |
| # published checkpoints). For audio super-resolution, convert the | |
| # decoded waveform to the requested delivery rate before writing it. | |
| # Resampling the samples (instead of only changing the WAV header) | |
| # preserves the audio's duration and pitch. | |
| write_sample_rate = output_sample_rate or self.sample_rate | |
| if write_sample_rate != self.sample_rate: | |
| waveform = torchaudio.functional.resample( | |
| torch.from_numpy(waveform), | |
| orig_freq=self.sample_rate, | |
| new_freq=write_sample_rate, | |
| ).numpy() | |
| output_path = output_path.__str__() | |
| if not output_path.endswith(".mp4"): | |
| sf.write(output_path, waveform, write_sample_rate) | |
| return waveform | |
| if __name__ == "__main__": | |
| fire.Fire(InferenceCLI) | |