app.py

# coding=utf-8

import os
import numpy as np
import torch
import torchaudio
import gradio as gr
import spaces

from funasr import AutoModel

model = AutoModel(
    model="FunAudioLLM/SenseVoiceSmall",
    vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
    vad_kwargs={"max_single_segment_time": 30000},
    hub="hf",
    device="cuda",
)

emo_dict = {
    "<|HAPPY|>": "😊", "<|SAD|>": "😔", "<|ANGRY|>": "😡",
    "<|NEUTRAL|>": "", "<|FEARFUL|>": "😰", "<|DISGUSTED|>": "🤢", "<|SURPRISED|>": "😮",
}

event_dict = {
    "<|BGM|>": "🎼", "<|Speech|>": "", "<|Applause|>": "👏",
    "<|Laughter|>": "😀", "<|Cry|>": "😭", "<|Sneeze|>": "🤧",
    "<|Breath|>": "", "<|Cough|>": "😷",
}

emoji_dict = {
    "<|nospeech|><|Event_UNK|>": "❓",
    "<|zh|>": "", "<|en|>": "", "<|yue|>": "", "<|ja|>": "", "<|ko|>": "",
    "<|nospeech|>": "",
    "<|HAPPY|>": "😊", "<|SAD|>": "😔", "<|ANGRY|>": "😡", "<|NEUTRAL|>": "",
    "<|BGM|>": "🎼", "<|Speech|>": "", "<|Applause|>": "👏", "<|Laughter|>": "😀",
    "<|FEARFUL|>": "😰", "<|DISGUSTED|>": "🤢", "<|SURPRISED|>": "😮",
    "<|Cry|>": "😭", "<|EMO_UNKNOWN|>": "", "<|Sneeze|>": "🤧",
    "<|Breath|>": "", "<|Cough|>": "😷", "<|Sing|>": "",
    "<|Speech_Noise|>": "", "<|withitn|>": "", "<|woitn|>": "",
    "<|GBG|>": "", "<|Event_UNK|>": "",
}

lang_dict = {
    "<|zh|>": "<|lang|>", "<|en|>": "<|lang|>", "<|yue|>": "<|lang|>",
    "<|ja|>": "<|lang|>", "<|ko|>": "<|lang|>", "<|nospeech|>": "<|lang|>",
}

emo_set = {"😊", "😔", "😡", "😰", "🤢", "😮"}
event_set = {"🎼", "👏", "😀", "😭", "🤧", "😷"}


def format_str_v2(s):
    sptk_dict = {}
    for sptk in emoji_dict:
        sptk_dict[sptk] = s.count(sptk)
        s = s.replace(sptk, "")
    emo = "<|NEUTRAL|>"
    for e in emo_dict:
        if sptk_dict.get(e, 0) > sptk_dict.get(emo, 0):
            emo = e
    for e in event_dict:
        if sptk_dict.get(e, 0) > 0:
            s = event_dict[e] + s
    s = s + emo_dict[emo]
    for emoji in emo_set.union(event_set):
        s = s.replace(" " + emoji, emoji)
        s = s.replace(emoji + " ", emoji)
    return s.strip()


def format_str_v3(s):
    def get_emo(s):
        return s[-1] if s and s[-1] in emo_set else None
    def get_event(s):
        return s[0] if s and s[0] in event_set else None

    s = s.replace("<|nospeech|><|Event_UNK|>", "❓")
    for lang in lang_dict:
        s = s.replace(lang, "<|lang|>")
    s_list = [format_str_v2(s_i).strip(" ") for s_i in s.split("<|lang|>")]
    new_s = " " + s_list[0]
    cur_ent_event = get_event(new_s)
    for i in range(1, len(s_list)):
        if len(s_list[i]) == 0:
            continue
        if get_event(s_list[i]) == cur_ent_event and get_event(s_list[i]) is not None:
            s_list[i] = s_list[i][1:]
        cur_ent_event = get_event(s_list[i])
        if get_emo(s_list[i]) is not None and get_emo(s_list[i]) == get_emo(new_s):
            new_s = new_s[:-1]
        new_s += s_list[i].strip().lstrip()
    new_s = new_s.replace("The.", " ")
    return new_s.strip()


@spaces.GPU
def model_inference(input_wav, language, fs=16000):
    language = "auto" if not language else language

    if isinstance(input_wav, tuple):
        fs, input_wav = input_wav
        input_wav = input_wav.astype(np.float32) / np.iinfo(np.int16).max
        if len(input_wav.shape) > 1:
            input_wav = input_wav.mean(-1)
        if fs != 16000:
            resampler = torchaudio.transforms.Resample(fs, 16000)
            input_wav_t = torch.from_numpy(input_wav).to(torch.float32)
            input_wav = resampler(input_wav_t[None, :])[0, :].numpy()

    text = model.generate(
        input=input_wav,
        cache={},
        language=language,
        use_itn=True,
        batch_size_s=500,
        merge_vad=True,
    )

    text = text[0]["text"]
    text = format_str_v3(text)
    return text


audio_examples = [
    ["example/zh.mp3", "auto"],
    ["example/en.mp3", "auto"],
    ["example/yue.mp3", "auto"],
    ["example/ja.mp3", "auto"],
    ["example/ko.mp3", "auto"],
    ["example/emo_1.wav", "auto"],
    ["example/emo_2.wav", "auto"],
    ["example/emo_3.wav", "auto"],
    ["example/rich_1.wav", "auto"],
    ["example/rich_2.wav", "auto"],
    ["example/longwav_1.wav", "auto"],
    ["example/longwav_2.wav", "auto"],
]


description_html = """
<div style="text-align: center; max-width: 800px; margin: 0 auto;">
    <h1 style="font-size: 2em; margin-bottom: 0.2em;">🎙️ SenseVoice</h1>
    <p style="font-size: 1.2em; color: #555; margin-bottom: 0.5em;">Speech Recognition + Emotion Detection + Audio Events — All in One Model</p>
    <p style="font-size: 1em; color: #777;">
        <strong>5 languages</strong> (zh/en/yue/ja/ko) · <strong>7x faster</strong> than Whisper-small · <strong>17x faster</strong> than Whisper-large
    </p>
    <p style="font-size: 0.9em; margin-top: 1em;">
        <a href="https://github.com/FunAudioLLM/SenseVoice" target="_blank">⭐ GitHub</a> ·
        <a href="https://github.com/modelscope/FunASR" target="_blank">🛠️ FunASR Toolkit</a> ·
        <a href="https://arxiv.org/abs/2407.04051" target="_blank">📄 Paper</a> ·
        <a href="https://github.com/FunAudioLLM/Fun-ASR" target="_blank">🚀 Fun-ASR (31 Languages)</a>
    </p>
</div>
"""

guide_html = """
<div style="background: #f8f9fa; border-radius: 8px; padding: 12px 16px; margin: 8px 0; font-size: 0.9em;">
    <strong>How it works:</strong> Upload audio or record via microphone → SenseVoice transcribes speech and detects emotions (😊😡😔) and sound events (🎼👏😀😭🤧).
    Event labels appear at the front of text, emotions at the end.
</div>
"""


def launch():
    with gr.Blocks(theme=gr.themes.Soft(), title="SenseVoice - Speech Understanding") as demo:
        gr.HTML(description_html)
        gr.HTML(guide_html)
        with gr.Row():
            with gr.Column():
                audio_inputs = gr.Audio(label="Upload audio or use microphone")
                with gr.Accordion("Language (auto-detect by default)", open=False):
                    language_inputs = gr.Dropdown(
                        choices=["auto", "zh", "en", "yue", "ja", "ko"],
                        value="auto",
                        label="Language",
                    )
                fn_button = gr.Button("Recognize", variant="primary", size="lg")
                text_outputs = gr.Textbox(label="Result", lines=5, show_copy_button=True)
            gr.Examples(
                examples=audio_examples,
                inputs=[audio_inputs, language_inputs],
                examples_per_page=12,
            )

        fn_button.click(model_inference, inputs=[audio_inputs, language_inputs], outputs=text_outputs)

    demo.launch()


if __name__ == "__main__":
    launch()