UniFlow-Audio / inference_cli.py
Spider-8251's picture
Upload 2 files
767a55c verified
Raw History Blame Contribute Delete
13.3 kB
#!/usr/bin/env python3
import json
import os
from pathlib import Path
from typing import Any, Callable
import fire
import torch
import torchaudio
import soundfile as sf
import numpy as np
from modeling_uniflow_audio import UniFlowAudioModel
from constants import TIME_ALIGNED_TASKS, NON_TIME_ALIGNED_TASKS
from utils.phonemize import sentence_to_phones
AUDIO_SUPER_RESOLUTION_OUTPUT_SAMPLE_RATE = 44_100
class InferenceCLI:
def __init__(self):
self.model_name_or_path = None
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu"
)
self.g2p = None
self.word2phone = None
self.speaker_model = None
self.svs_processor = None
self.singer_mapping = None
self.video_preprocessor = None
self.video_size = (256, 256)
self.video_fps = 10
def init_model(self, model_name_or_path):
self.model_name_or_path = model_name_or_path
if Path(self.model_name_or_path).exists():
self.model = UniFlowAudioModel(model_name_or_path)
else:
self.model = UniFlowAudioModel(f"wsntxxn/{model_name_or_path}")
self.model.to(self.device)
self.sample_rate = self.model.config["sample_rate"]
def init_svs_processor(self, ):
from utils.diffsinger_utilities import SVSInputConverter, TokenTextEncoder
if self.svs_processor is None:
phoneme_list = json.load(open(self.model.svs_phone_set_path, "r"))
self.svs_processor = {
"converter":
SVSInputConverter(
self.model.svs_singer_mapping, self.model.svs_pinyin2ph
),
"tokenizer":
TokenTextEncoder(
None, vocab_list=phoneme_list, replace_oov=','
)
}
def init_video_preprocessor(self, ):
if self.video_preprocessor is None:
from third_party.mmaudio.feature_extractor import init_feature_extractor
self.video_preprocessor = init_feature_extractor()
def on_inference_start(self, model_name_or_path):
if self.model_name_or_path is None or model_name_or_path != self.model_name_or_path:
self.init_model(model_name_or_path)
@staticmethod
def add_prehook(func: Callable, ):
def wrapper(self, *args, **kwargs):
model_name_or_path = kwargs.get(
"model_name_or_path", "UniFlow-Audio-v1.1-Large"
)
self.on_inference_start(model_name_or_path)
return func(self, *args, **kwargs)
return wrapper
@add_prehook
def t2a(
self,
caption: str,
model_name_or_path: str = "UniFlow-Audio-v1.1-Large",
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.wav",
):
self._run_inference(
content=caption,
task="text_to_audio",
instruction=instruction,
instruction_idx=instruction_idx,
model_name_or_path=model_name_or_path,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path
)
@add_prehook
def t2m(
self,
caption: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.wav",
):
self._run_inference(
content=caption,
task="text_to_music",
model_name_or_path=model_name_or_path,
instruction=instruction,
instruction_idx=instruction_idx,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path,
)
@add_prehook
def tts(
self,
transcript: str,
ref_speaker_speech: str,
model_name_or_path: str = "UniFlow-Audio-v1.1-Large",
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.wav",
):
from g2p_en import G2p
import nltk
if not self.g2p:
if not os.path.exists(
os.path.expanduser(
"~/nltk_data/taggers/averaged_perceptron_tagger_eng"
)
):
nltk.download("averaged_perceptron_tagger_eng")
self.g2p = G2p()
if not self.word2phone:
self.word2phone = json.load(
open(
self.model.tts_word2phone_dict_path, "r", encoding="utf-8"
)
)
# OOV word will use a g2p model to predict phoneme
phonemes, OOV_list = sentence_to_phones(
transcript, self.word2phone, self.g2p
)
print(phonemes)
phone_indices = [
self.model.tts_phone2id.get(
p, self.model.tts_phone2id.get("spn", 0)
) for p in phonemes
]
prompt_waveform, orig_sr = torchaudio.load(ref_speaker_speech)
prompt_waveform = prompt_waveform.mean(0)
prompt_waveform = torchaudio.functional.resample(
prompt_waveform,
orig_freq=orig_sr,
new_freq=self.model.tts_prompt_sr
)
content = {
"phoneme": np.array(phone_indices, dtype=np.int64),
"prompt_waveform": prompt_waveform,
}
self._run_inference(
content=content,
task="text_to_speech",
model_name_or_path=model_name_or_path,
instruction=instruction,
instruction_idx=instruction_idx,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path,
)
@add_prehook
def _audio_input_inference(
self,
input_audio: str,
task: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.wav",
output_sample_rate: int | None = None,
):
waveform, orig_sr = torchaudio.load(input_audio)
waveform = waveform.mean(0)
waveform = torchaudio.functional.resample(
waveform, orig_freq=orig_sr, new_freq=self.sample_rate
)
return self._run_inference(
content=waveform,
task=task,
instruction=instruction,
instruction_idx=instruction_idx,
model_name_or_path=model_name_or_path,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path,
output_sample_rate=output_sample_rate,
)
def se(
self,
noisy_speech: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 1.0,
num_steps: int = 25,
output_path: str = "./output.wav",
):
self._audio_input_inference(
input_audio=noisy_speech,
task="speech_enhancement",
instruction=instruction,
instruction_idx=instruction_idx,
model_name_or_path=model_name_or_path,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path
)
def sr(
self,
low_sr_audio: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 1.0,
num_steps: int = 25,
output_path: str = "./output.wav",
output_sample_rate: int = AUDIO_SUPER_RESOLUTION_OUTPUT_SAMPLE_RATE,
):
self._audio_input_inference(
input_audio=low_sr_audio,
task="audio_super_resolution",
instruction=instruction,
instruction_idx=instruction_idx,
model_name_or_path=model_name_or_path,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path,
output_sample_rate=output_sample_rate,
)
@add_prehook
def v2a(
self,
video: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.mp4",
):
from utils.video import merge_audio_video
from third_party.mmaudio.feature_extractor import get_video_feature
self.init_video_preprocessor()
video_feature = get_video_feature(video, self.video_preprocessor)
waveform = self._run_inference(
content=video_feature,
task="video_to_audio",
model_name_or_path=model_name_or_path,
instruction=instruction,
instruction_idx=instruction_idx,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path,
)
merge_audio_video(
waveform, video, output_path, audio_fps=self.sample_rate
)
@add_prehook
def svs(
self,
singer: str,
music_score: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.wav",
):
self.init_svs_processor()
text, note, note_dur = music_score.split('<sep>')
if singer not in self.model.svs_singer_mapping:
print(f"Unsupported singer {singer}, available singers: ")
print(list(self.model.svs_singer_mapping.keys()))
raise KeyError
midi = self.svs_processor["converter"].preprocess_input({
"spk_name": singer,
"text": text,
"notes": note,
"notes_duration": note_dur,
})
midi["phoneme"] = self.svs_processor["tokenizer"].encode(
midi["phoneme"]
)
self._run_inference(
content=midi,
task="singing_voice_synthesis",
model_name_or_path=model_name_or_path,
instruction=instruction,
instruction_idx=instruction_idx,
guidance_scale=guidance_scale,
num_steps=num_steps,
output_path=output_path,
)
def _run_inference(
self,
content: Any,
task: str,
model_name_or_path: str,
instruction: str | None = None,
instruction_idx: int | None = None,
guidance_scale: float = 5.0,
num_steps: int = 25,
output_path: str = "./output.wav",
output_sample_rate: int | None = None,
):
if self.model_name_or_path is None or model_name_or_path != self.model_name_or_path:
self.init_model(model_name_or_path)
time_aligned = torch.zeros(1, 2).bool()
if task in TIME_ALIGNED_TASKS:
time_aligned[0, 0] = True
if task in NON_TIME_ALIGNED_TASKS:
time_aligned[0, 1] = True
if instruction:
instruction = [instruction]
if instruction_idx:
instruction_idx = [instruction_idx]
waveform = self.model.sample(
content=[content],
task=[task],
time_aligned=time_aligned,
instruction=instruction,
instruction_idx=instruction_idx,
num_steps=num_steps,
guidance_scale=guidance_scale,
disable_progress=False
)
waveform = waveform[0, 0].cpu().numpy()
# The model and codec run at their native sample rate (24 kHz for the
# published checkpoints). For audio super-resolution, convert the
# decoded waveform to the requested delivery rate before writing it.
# Resampling the samples (instead of only changing the WAV header)
# preserves the audio's duration and pitch.
write_sample_rate = output_sample_rate or self.sample_rate
if write_sample_rate != self.sample_rate:
waveform = torchaudio.functional.resample(
torch.from_numpy(waveform),
orig_freq=self.sample_rate,
new_freq=write_sample_rate,
).numpy()
output_path = output_path.__str__()
if not output_path.endswith(".mp4"):
sf.write(output_path, waveform, write_sample_rate)
return waveform
if __name__ == "__main__":
fire.Fire(InferenceCLI)