# coding=utf-8 import os import numpy as np import torch import torchaudio import gradio as gr import spaces from funasr import AutoModel model = AutoModel( model="FunAudioLLM/SenseVoiceSmall", vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", vad_kwargs={"max_single_segment_time": 30000}, hub="hf", device="cuda", ) emo_dict = { "<|HAPPY|>": "๐Ÿ˜Š", "<|SAD|>": "๐Ÿ˜”", "<|ANGRY|>": "๐Ÿ˜ก", "<|NEUTRAL|>": "", "<|FEARFUL|>": "๐Ÿ˜ฐ", "<|DISGUSTED|>": "๐Ÿคข", "<|SURPRISED|>": "๐Ÿ˜ฎ", } event_dict = { "<|BGM|>": "๐ŸŽผ", "<|Speech|>": "", "<|Applause|>": "๐Ÿ‘", "<|Laughter|>": "๐Ÿ˜€", "<|Cry|>": "๐Ÿ˜ญ", "<|Sneeze|>": "๐Ÿคง", "<|Breath|>": "", "<|Cough|>": "๐Ÿ˜ท", } emoji_dict = { "<|nospeech|><|Event_UNK|>": "โ“", "<|zh|>": "", "<|en|>": "", "<|yue|>": "", "<|ja|>": "", "<|ko|>": "", "<|nospeech|>": "", "<|HAPPY|>": "๐Ÿ˜Š", "<|SAD|>": "๐Ÿ˜”", "<|ANGRY|>": "๐Ÿ˜ก", "<|NEUTRAL|>": "", "<|BGM|>": "๐ŸŽผ", "<|Speech|>": "", "<|Applause|>": "๐Ÿ‘", "<|Laughter|>": "๐Ÿ˜€", "<|FEARFUL|>": "๐Ÿ˜ฐ", "<|DISGUSTED|>": "๐Ÿคข", "<|SURPRISED|>": "๐Ÿ˜ฎ", "<|Cry|>": "๐Ÿ˜ญ", "<|EMO_UNKNOWN|>": "", "<|Sneeze|>": "๐Ÿคง", "<|Breath|>": "", "<|Cough|>": "๐Ÿ˜ท", "<|Sing|>": "", "<|Speech_Noise|>": "", "<|withitn|>": "", "<|woitn|>": "", "<|GBG|>": "", "<|Event_UNK|>": "", } lang_dict = { "<|zh|>": "<|lang|>", "<|en|>": "<|lang|>", "<|yue|>": "<|lang|>", "<|ja|>": "<|lang|>", "<|ko|>": "<|lang|>", "<|nospeech|>": "<|lang|>", } emo_set = {"๐Ÿ˜Š", "๐Ÿ˜”", "๐Ÿ˜ก", "๐Ÿ˜ฐ", "๐Ÿคข", "๐Ÿ˜ฎ"} event_set = {"๐ŸŽผ", "๐Ÿ‘", "๐Ÿ˜€", "๐Ÿ˜ญ", "๐Ÿคง", "๐Ÿ˜ท"} def format_str_v2(s): sptk_dict = {} for sptk in emoji_dict: sptk_dict[sptk] = s.count(sptk) s = s.replace(sptk, "") emo = "<|NEUTRAL|>" for e in emo_dict: if sptk_dict.get(e, 0) > sptk_dict.get(emo, 0): emo = e for e in event_dict: if sptk_dict.get(e, 0) > 0: s = event_dict[e] + s s = s + emo_dict[emo] for emoji in emo_set.union(event_set): s = s.replace(" " + emoji, emoji) s = s.replace(emoji + " ", emoji) return s.strip() def format_str_v3(s): def get_emo(s): return s[-1] if s and s[-1] in emo_set else None def get_event(s): return s[0] if s and s[0] in event_set else None s = s.replace("<|nospeech|><|Event_UNK|>", "โ“") for lang in lang_dict: s = s.replace(lang, "<|lang|>") s_list = [format_str_v2(s_i).strip(" ") for s_i in s.split("<|lang|>")] new_s = " " + s_list[0] cur_ent_event = get_event(new_s) for i in range(1, len(s_list)): if len(s_list[i]) == 0: continue if get_event(s_list[i]) == cur_ent_event and get_event(s_list[i]) is not None: s_list[i] = s_list[i][1:] cur_ent_event = get_event(s_list[i]) if get_emo(s_list[i]) is not None and get_emo(s_list[i]) == get_emo(new_s): new_s = new_s[:-1] new_s += s_list[i].strip().lstrip() new_s = new_s.replace("The.", " ") return new_s.strip() @spaces.GPU def model_inference(input_wav, language, fs=16000): language = "auto" if not language else language if isinstance(input_wav, tuple): fs, input_wav = input_wav input_wav = input_wav.astype(np.float32) / np.iinfo(np.int16).max if len(input_wav.shape) > 1: input_wav = input_wav.mean(-1) if fs != 16000: resampler = torchaudio.transforms.Resample(fs, 16000) input_wav_t = torch.from_numpy(input_wav).to(torch.float32) input_wav = resampler(input_wav_t[None, :])[0, :].numpy() text = model.generate( input=input_wav, cache={}, language=language, use_itn=True, batch_size_s=500, merge_vad=True, ) text = text[0]["text"] text = format_str_v3(text) return text audio_examples = [ ["example/zh.mp3", "auto"], ["example/en.mp3", "auto"], ["example/yue.mp3", "auto"], ["example/ja.mp3", "auto"], ["example/ko.mp3", "auto"], ["example/emo_1.wav", "auto"], ["example/emo_2.wav", "auto"], ["example/emo_3.wav", "auto"], ["example/rich_1.wav", "auto"], ["example/rich_2.wav", "auto"], ["example/longwav_1.wav", "auto"], ["example/longwav_2.wav", "auto"], ] description_html = """

๐ŸŽ™๏ธ SenseVoice

Speech Recognition + Emotion Detection + Audio Events โ€” All in One Model

5 languages (zh/en/yue/ja/ko) ยท 7x faster than Whisper-small ยท 17x faster than Whisper-large

โญ GitHub ยท ๐Ÿ› ๏ธ FunASR Toolkit ยท ๐Ÿ“„ Paper ยท ๐Ÿš€ Fun-ASR (31 Languages)

""" guide_html = """
How it works: Upload audio or record via microphone โ†’ SenseVoice transcribes speech and detects emotions (๐Ÿ˜Š๐Ÿ˜ก๐Ÿ˜”) and sound events (๐ŸŽผ๐Ÿ‘๐Ÿ˜€๐Ÿ˜ญ๐Ÿคง). Event labels appear at the front of text, emotions at the end.
""" def launch(): with gr.Blocks(theme=gr.themes.Soft(), title="SenseVoice - Speech Understanding") as demo: gr.HTML(description_html) gr.HTML(guide_html) with gr.Row(): with gr.Column(): audio_inputs = gr.Audio(label="Upload audio or use microphone") with gr.Accordion("Language (auto-detect by default)", open=False): language_inputs = gr.Dropdown( choices=["auto", "zh", "en", "yue", "ja", "ko"], value="auto", label="Language", ) fn_button = gr.Button("Recognize", variant="primary", size="lg") text_outputs = gr.Textbox(label="Result", lines=5, show_copy_button=True) gr.Examples( examples=audio_examples, inputs=[audio_inputs, language_inputs], examples_per_page=12, ) fn_button.click(model_inference, inputs=[audio_inputs, language_inputs], outputs=text_outputs) demo.launch() if __name__ == "__main__": launch()