cosyvoice-api / app.py
WEICALVIN's picture
Update app.py
b09994e verified
Raw History Blame Contribute Delete
11 kB
import contextlib
import importlib
import os
import re
import shutil
import subprocess
import sys
import threading
from pathlib import Path
from typing import Optional
import gradio as gr
import spaces
import torch
import torchaudio
from huggingface_hub import snapshot_download
COSYVOICE_REPO = Path("/tmp/CosyVoice")
MODEL_DIR = Path("/tmp/CosyVoice2-0.5B")
PROMPT_WAV = Path("/tmp/p.wav")
OUTPUT_WAV = Path("/tmp/out.wav")
_state_lock = threading.Lock()
_inference_lock = threading.RLock()
_model = None
_model_error: Optional[str] = None
_model_status = "服務啟動中,準備下載模型…"
# 只對已知 module/package 名稱作補裝,避免把錯誤訊息任意當 pip 套件安裝。
MODULE_TO_PACKAGE = {
"hyperpyyaml": "HyperPyYAML",
"onnxruntime": "onnxruntime",
"pyworld": "pyworld",
"pypinyin": "pypinyin",
"cn2an": "cn2an",
"jieba": "jieba",
"langid": "langid",
"wetext": "wetext",
"conformer": "conformer",
"modelscope": "modelscope",
"whisper": "openai-whisper==20231117",
"webrtcvad": "webrtcvad-wheels",
}
def set_model_status(message: str) -> None:
global _model_status
with _state_lock:
_model_status = message
def get_model_status() -> str:
with _state_lock:
status = _model_status
error = _model_error
if error:
return f"模型載入失敗:{error}"
return status
def try_install_missing_module(error: ModuleNotFoundError) -> bool:
match = re.search(r"No module named ['\"]([^'\"]+)['\"]", str(error))
if not match:
return False
module_name = match.group(1).split(".", 1)[0]
package_name = MODULE_TO_PACKAGE.get(module_name)
if not package_name:
return False
set_model_status(f"缺少套件 {package_name},正在嘗試補裝…")
pip_command = [sys.executable, "-m", "pip", "install", "--no-cache-dir"]
if module_name == "whisper":
# Whisper 20231117 的 setup.py 匯入 pkg_resources;使用主環境中
# requirements.txt 固定的 setuptools<81,避免隔離環境挑到 setuptools 82+。
pip_command.append("--no-build-isolation")
pip_command.append(package_name)
result = subprocess.run(
pip_command,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=False,
)
if result.returncode != 0:
raise RuntimeError(
f"補裝 {package_name} 失敗:\n{(result.stdout or '')[-3000:]}"
)
importlib.invalidate_caches()
return True
def prepare_cosyvoice_source() -> None:
if not (COSYVOICE_REPO / ".git").exists():
if COSYVOICE_REPO.exists():
shutil.rmtree(COSYVOICE_REPO)
set_model_status("正在下載 CosyVoice 原始碼與子模組…")
subprocess.run(
[
"git",
"clone",
"--depth",
"1",
"--recursive",
"https://github.com/FunAudioLLM/CosyVoice.git",
str(COSYVOICE_REPO),
],
check=True,
)
for path in (
COSYVOICE_REPO,
COSYVOICE_REPO / "third_party" / "Matcha-TTS",
):
path_string = str(path)
if path_string not in sys.path:
sys.path.insert(0, path_string)
def load_cosyvoice_cpu():
prepare_cosyvoice_source()
attempts = 0
while attempts < 10:
try:
# 延遲 import,讓確定缺少的模組有機會補裝後重試。
from cosyvoice.cli.cosyvoice import CosyVoice2
# Spaces 背景初始化階段不應嘗試使用未配發的 CUDA 裝置。
original_is_available = torch.cuda.is_available
try:
torch.cuda.is_available = lambda: False
model = CosyVoice2(
model_dir=str(MODEL_DIR),
load_jit=False,
load_trt=False,
load_vllm=False,
fp16=False,
)
finally:
torch.cuda.is_available = original_is_available
return model
except ModuleNotFoundError as error:
attempts += 1
if not try_install_missing_module(error):
raise
raise RuntimeError("嘗試補裝依賴 10 次後仍無法載入 CosyVoice。")
def load_model_in_background() -> None:
global _model, _model_error
try:
set_model_status("正在從 Hugging Face 下載 CosyVoice2 模型…")
snapshot_download(
repo_id="FunAudioLLM/CosyVoice2-0.5B",
local_dir=str(MODEL_DIR),
)
set_model_status("模型下載完成,正在以 CPU 載入模型結構…")
loaded_model = load_cosyvoice_cpu()
with _state_lock:
_model = loaded_model
_model_error = None
_model_status = "模型載入完成,可以開始配音。"
except Exception as error:
message = f"{type(error).__name__}: {error}"
with _state_lock:
_model_error = message
_model_status = f"模型載入失敗:{message}"
print(f"[CosyVoice 初始化錯誤] {message}", flush=True)
def move_model_to(device: torch.device) -> None:
"""CosyVoice2 的 model 不是單一 nn.Module,需逐一移動三個子網路。"""
if _model is None:
raise RuntimeError("模型尚未載入完成。")
cosyvoice_model = _model.model
cosyvoice_model.llm.to(device)
cosyvoice_model.flow.to(device)
cosyvoice_model.hift.to(device)
# frontend 會以此裝置建立文字、聲學特徵及 speaker embedding 張量。
cosyvoice_model.device = device
_model.frontend.device = device
# CPU 預載時 llm_context 是 nullcontext;在 GPU 配發期間改用 CUDA stream。
if device.type == "cuda":
cosyvoice_model.llm_context = torch.cuda.stream(torch.cuda.Stream(device))
else:
cosyvoice_model.llm_context = contextlib.nullcontext()
def clone_tts(text: str, audio_path: str) -> str:
if not text or not text.strip():
raise gr.Error("請輸入要合成的文字。")
if not audio_path or not os.path.isfile(audio_path):
raise gr.Error("請先上傳參考音訊。")
if _model is None:
if _model_error:
raise gr.Error(f"模型載入失敗:{_model_error}")
raise gr.Error("模型仍在背景下載或載入中,請稍後再試。")
# 固定暫存路徑需序列化,避免並行請求互相覆蓋音檔。
with _inference_lock:
if not torch.cuda.is_available():
raise gr.Error("目前尚未取得 ZeroGPU 裝置,請稍後重試。")
move_attempted = False
try:
set_model_status("已取得 ZeroGPU,正在將 CosyVoice2 切換至 CUDA…")
move_attempted = True
move_model_to(torch.device("cuda"))
set_model_status("正在轉換參考音訊為 16 kHz 單聲道 WAV…")
conversion = subprocess.run(
[
"ffmpeg",
"-nostdin",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-i",
audio_path,
"-ac",
"1",
"-ar",
"16000",
"-c:a",
"pcm_s16le",
str(PROMPT_WAV),
],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
if conversion.returncode != 0 or not PROMPT_WAV.is_file():
detail = conversion.stderr[-2000:] if conversion.stderr else "沒有產生 WAV 檔案。"
raise RuntimeError(f"ffmpeg 轉檔失敗:{detail}")
set_model_status("正在進行跨語言語音克隆…")
speech_chunks = []
for result in _model.inference_cross_lingual(
text.strip(),
str(PROMPT_WAV),
stream=False,
):
speech = result.get("tts_speech")
if speech is not None and speech.numel() > 0:
speech_chunks.append(speech.detach().cpu())
if not speech_chunks:
raise RuntimeError("CosyVoice2 沒有產生音訊資料。")
speech = torch.cat(speech_chunks, dim=1)
torchaudio.save(
str(OUTPUT_WAV),
speech,
_model.sample_rate,
encoding="PCM_S",
bits_per_sample=16,
)
set_model_status("語音生成完成。")
return str(OUTPUT_WAV)
except gr.Error:
raise
except Exception as error:
set_model_status(f"推理失敗:{type(error).__name__}: {error}")
raise gr.Error(f"語音克隆失敗:{type(error).__name__}: {error}") from error
finally:
# 清理仍在 GPU 函式執行範圍內完成,避免函式外保留 CUDA 權重。
if move_attempted:
try:
move_model_to(torch.device("cpu"))
torch.cuda.empty_cache()
except Exception as cleanup_error:
print(f"[CUDA 清理警告] {cleanup_error}", flush=True)
# API 名稱會成為 /gradio_api/call/clone_tts。
clone_tts_gpu = spaces.GPU(duration=120)(clone_tts)
with gr.Blocks(title="聲影 AI|VoxCut 語音克隆") as demo:
gr.Markdown("# 聲影 AI|VoxCut 語音克隆")
gr.Markdown("上傳清晰的參考人聲,再輸入要合成的文字。")
status_box = gr.Textbox(
label="模型/服務狀態",
value=get_model_status,
interactive=False,
lines=2,
)
status_timer = gr.Timer(value=1.0, active=True)
status_timer.tick(
fn=get_model_status,
inputs=None,
outputs=status_box,
)
text_input = gr.Textbox(
label="目標文字",
placeholder="輸入要合成的文字",
lines=3,
)
audio_input = gr.Audio(
label="參考音訊",
sources=["upload", "microphone"],
type="filepath",
)
generate_button = gr.Button("AI 克隆原聲配音", variant="primary")
audio_output = gr.Audio(label="生成語音", type="filepath")
generate_button.click(
fn=clone_tts_gpu,
inputs=[text_input, audio_input],
outputs=audio_output,
api_name="clone_tts",
)
# 在 Gradio 啟動前立刻開始背景工作;下載/載入不阻塞 Web 服務啟動。
threading.Thread(
target=load_model_in_background,
name="cosyvoice-model-loader",
daemon=True,
).start()
demo.queue().launch(server_name="0.0.0.0", server_port=7860)