my-tiny-audio / app.py
aaron-hu26's picture
Initial release: Qwen3.5-0.8B + Granite Speech 5.0 ASR and GGUF models
373d740 verified
Raw History Blame Contribute Delete
10.8 kB
#!/usr/bin/env python3
"""
🎙️ Tiny-Audio 单文件绿色独立版 Web 界面 (Standalone Web UI)
特点:
- 零项目目录依赖: 模型架构代码已全部内置于本脚本中,不再依赖 tiny_audio 包。
- 跨平台兼容: 无论是在 Windows、Mac (Apple Silicon) 还是 Linux 上均可直接运行。
- 只需携带本文件和 outputs/ 权重目录即可在任意电脑上部署运行。
依赖安装:
pip install torch transformers peft gradio soundfile
"""
import os
import sys
import time
from pathlib import Path
import gradio as gr
import numpy as np
import soundfile as sf
import torch
from peft import PeftModel
from torch import nn
from transformers import (
AutoFeatureExtractor,
AutoModelForCausalLM,
AutoTokenizer,
GraniteSpeech5Encoder,
PreTrainedTokenizerBase,
)
from transformers.models.llama.modeling_llama import LlamaRMSNorm
# ==============================================================================
# 1. 内置投影层与基座加载逻辑 (完全自包含)
# ==============================================================================
def _frame_stack_length(length: int | torch.Tensor, k: int) -> int | torch.Tensor:
return (length - k) // k + 1
def _frame_stack(x: torch.Tensor, k: int) -> torch.Tensor:
batch, seq, dim = x.shape
out_len = _frame_stack_length(seq, k)
return x[:, : out_len * k, :].reshape(batch, out_len, dim * k)
class MLPAudioProjector(nn.Module):
def __init__(self, config: object) -> None:
super().__init__()
encoder_dim: int = getattr(config, "encoder_dim", 1024)
llm_dim: int = getattr(config, "llm_dim", 2048)
self.k: int = getattr(config, "projector_pool_stride", 1)
in_dim = encoder_dim * self.k
hidden_dim: int = getattr(config, "projector_hidden_dim", None) or llm_dim
self.input_norm = LlamaRMSNorm(in_dim, eps=1e-6)
self.linear_1 = nn.Linear(in_dim, hidden_dim)
self.act = nn.GELU()
self.linear_2 = nn.Linear(hidden_dim, llm_dim)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = _frame_stack(x, self.k)
x = self.input_norm(x)
x = self.linear_1(x)
x = self.act(x)
return self.linear_2(x)
class StandaloneASRModel(nn.Module):
def __init__(
self,
encoder_model_id: str = "ibm-granite/granite-speech-5.0-470m-turboctc-nc",
decoder_model_id: str = "Qwen/Qwen3.5-0.8B",
device: str = "cuda" if torch.cuda.is_available() else "cpu",
dtype: torch.dtype = torch.bfloat16,
):
super().__init__()
self.device = device
self.dtype = dtype
print("1/3 加载音频特征提取器与 Granite-Speech-5.0 编码器...")
self.feature_extractor = AutoFeatureExtractor.from_pretrained(encoder_model_id)
self.encoder = GraniteSpeech5Encoder.from_pretrained(
encoder_model_id,
torch_dtype=dtype,
attn_implementation="sdpa",
low_cpu_mem_usage=True,
).to(device=device, dtype=dtype)
self.encoder.eval().requires_grad_(False)
print("2/3 加载 Qwen 分词器与文本解码器...")
self.tokenizer = AutoTokenizer.from_pretrained(decoder_model_id, trust_remote_code=True)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
self.decoder = AutoModelForCausalLM.from_pretrained(
decoder_model_id,
dtype=dtype,
attn_implementation="sdpa",
low_cpu_mem_usage=True,
trust_remote_code=True,
).to(device=device, dtype=dtype)
self.decoder.eval().requires_grad_(False)
# 注册 <audio> 特殊标记
self.audio_token = "<audio>"
if self.audio_token not in self.tokenizer.get_vocab():
existing = getattr(self.tokenizer, "additional_special_tokens", None) or []
self.tokenizer.add_special_tokens(
{"additional_special_tokens": [*existing, self.audio_token]}
)
self.decoder.resize_token_embeddings(len(self.tokenizer), pad_to_multiple_of=128)
self.audio_token_id = self.tokenizer.convert_tokens_to_ids(self.audio_token)
print("3/3 组装 MLP 投影层...")
class ProjCfg:
encoder_dim = self.encoder.config.hidden_size
llm_dim = self.decoder.config.hidden_size
projector_pool_stride = 1
projector_hidden_dim = self.decoder.config.hidden_size
self.projector = MLPAudioProjector(ProjCfg()).to(device=device, dtype=dtype)
# ==============================================================================
# 2. 单例模型管理与转录执行
# ==============================================================================
_MODEL = None
_CKPT = None
def get_model(checkpoint_dir: str):
global _MODEL, _CKPT
resolved = Path(checkpoint_dir).resolve()
if _MODEL is not None and _CKPT == str(resolved):
return _MODEL
device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" and torch.cuda.is_bf16_supported() else torch.float32
print(f"\n📦 正在初始化基座并加载微调权重: {resolved}")
merged_decoder_dir = resolved / "decoder"
decoder_id = str(merged_decoder_dir) if (merged_decoder_dir / "model.safetensors").exists() else "Qwen/Qwen3.5-0.8B"
model = StandaloneASRModel(decoder_model_id=decoder_id, device=device, dtype=dtype)
# 1. 载入 Projector 权重
proj_pt = resolved / "projector.pt"
if proj_pt.exists():
print(f"🔗 载入 Projector 权重: {proj_pt}")
model.projector.load_state_dict(torch.load(proj_pt, map_location=device))
else:
print(f"⚠️ 警告: 未找到 {proj_pt}")
# 2. 载入 LoRA 适配层并融合 (若已是融合权重目录则自动跳过)
adapter_dir = resolved / "decoder_adapter"
if adapter_dir.exists():
print(f"🧠 载入 Decoder LoRA 权重: {adapter_dir}")
model.decoder = PeftModel.from_pretrained(model.decoder, str(adapter_dir))
if hasattr(model.decoder, "merge_and_unload"):
model.decoder = model.decoder.merge_and_unload()
print("🚀 LoRA 权重已成功融合至基座矩阵!")
elif (merged_decoder_dir / "model.safetensors").exists():
print("🚀 已直接加载完全融合的独立 Decoder 权重 (无需 LoRA 转换)!")
model.eval()
_MODEL = model
_CKPT = str(resolved)
print("✅ 模型就绪!\n")
return _MODEL
def transcribe_audio(audio_path: str | None, checkpoint_dir: str, max_new_tokens: int = 128):
if not audio_path:
return "⚠️ 请先录制或上传音频!", ""
start_t = time.time()
try:
model = get_model(checkpoint_dir)
device = model.device
# 1. 读取音频波形并重采样
speech, sr = sf.read(audio_path)
if speech.ndim > 1:
speech = speech.mean(axis=-1)
if sr != 16000:
import torchaudio.transforms as T
speech = T.Resample(sr, 16000)(torch.tensor(speech, dtype=torch.float32)).numpy()
sr = 16000
audio_len = len(speech) / sr
# 2. 提取特征
audio_in = model.feature_extractor(speech, sampling_rate=16000, return_tensors="pt")
feats = audio_in.input_features.to(device=device, dtype=model.dtype)
# 3. 前向提取与特征替换
with torch.no_grad():
hidden = model.encoder(input_features=feats).last_hidden_state
proj_dtype = model.projector.linear_1.weight.dtype
embeds = model.projector(hidden.to(dtype=proj_dtype))
num_tokens = embeds.shape[1]
prompt = (
f"<|im_start|>user\n{'<audio>' * num_tokens} Transcribe the speech to text.<|im_end|>\n"
f"<|im_start|>assistant\n"
)
input_ids = model.tokenizer.encode(prompt, return_tensors="pt").to(device)
inputs_embeds = model.decoder.get_input_embeddings()(input_ids)
is_audio = input_ids == model.audio_token_id
inputs_embeds = inputs_embeds.masked_scatter(is_audio.unsqueeze(-1), embeds.to(inputs_embeds.dtype))
# 4. 生成文本
out = model.decoder.generate(
inputs_embeds=inputs_embeds,
max_new_tokens=max_new_tokens,
do_sample=False,
eos_token_id=model.tokenizer.eos_token_id,
pad_token_id=model.tokenizer.pad_token_id,
)
text = model.tokenizer.decode(out[0], skip_special_tokens=True).strip()
elapsed = time.time() - start_t
rtf = elapsed / max(audio_len, 0.01)
metrics = f"⏱️ **音频时长**: `{audio_len:.2f}s` | ⚡ **推理耗时**: `{elapsed:.2f}s` | 🚀 **实时率 (RTF)**: `{rtf:.2f}x`"
return text, metrics
except Exception as e:
import traceback
return f"❌ 运行异常:\n{traceback.format_exc()}", ""
# ==============================================================================
# 3. Gradio Web 界面构建
# ==============================================================================
with gr.Blocks(title="Tiny-Audio Standalone ASR") as demo:
gr.HTML(
"""
<div style="text-align: center; margin-bottom: 1.5rem;">
<h1 style="font-size: 2rem; color: #2563eb;">🎙️ Tiny-Audio 语音转录系统 (独立部署版)</h1>
<p style="color: #64748b;">单文件自包含运行脚本 · 无缝跨平台兼容 · 支持麦克风与音频文件</p>
</div>
"""
)
with gr.Row():
with gr.Column():
audio_in = gr.Audio(sources=["microphone", "upload"], type="filepath", label="🎤 麦克风录音 / 上传音频文件")
with gr.Accordion("⚙️ 模型配置", open=False):
ckpt_in = gr.Textbox(value=".", label="检查点权重路径")
tokens_slider = gr.Slider(minimum=32, maximum=512, value=128, step=32, label="Max New Tokens")
with gr.Row():
clear_btn = gr.ClearButton([audio_in], value="清空")
submit_btn = gr.Button("开始识别 (Transcribe)", variant="primary", scale=2)
with gr.Column():
text_out = gr.Textbox(label="📝 识别结果", lines=7, placeholder="识别结果将在此显示...")
metrics_out = gr.Markdown()
submit_btn.click(
fn=transcribe_audio,
inputs=[audio_in, ckpt_in, tokens_slider],
outputs=[text_out, metrics_out],
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860, share=True)