File size: 9,739 Bytes
5b45479
f5e5f37
 
5b45479
fa22051
 
 
 
5b45479
 
 
95e8192
f5e5f37
 
95e8192
 
 
 
 
 
f5e5f37
5b45479
 
 
fa22051
38c14e1
fa22051
38c14e1
5b45479
 
 
 
 
 
 
f5e5f37
5b45479
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f5e5f37
5b45479
 
 
 
 
 
 
 
 
 
 
 
 
fa22051
5b45479
 
 
f5e5f37
5b45479
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fa22051
5b45479
 
 
 
 
 
 
 
 
 
 
 
fa22051
5b45479
 
f5e5f37
fa22051
5b45479
 
 
f5e5f37
5b45479
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f5e5f37
5b45479
fa22051
f5e5f37
 
 
 
 
 
 
 
5b45479
 
 
 
f5e5f37
5b45479
f5e5f37
 
 
5b45479
 
 
 
 
f5e5f37
5b45479
 
 
 
 
 
 
 
f5e5f37
fa22051
5b45479
 
 
 
 
 
 
 
 
fa22051
5b45479
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f5e5f37
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
# =====================================================================
#  Chép lời tiếng Việt — ZipFormer-150M fine-tuned (Tối ưu tốc độ)
#  Hugging Face Space (Gradio + ZeroGPU + FP16 + Inference Mode)
# =====================================================================
import os
import sys
import types
import subprocess
from pathlib import Path

import torch

# 1. Vô hiệu hóa lỗi torch.set_num_interop_threads khi train.py được import
_orig_set_interop = torch.set_num_interop_threads
def _safe_set_interop(n):
    try:
        _orig_set_interop(n)
    except RuntimeError:
        pass
torch.set_num_interop_threads = _safe_set_interop

import torch.nn.functional as F
import torchaudio
from torch.nn.utils.rnn import pad_sequence
import sentencepiece as spm
import gradio as gr
import spaces  # Thư viện ZeroGPU của Hugging Face

MODEL_REPO = "hynt/ZipFormer-150M-CR-CTC-RNNT-6000h"
FINETUNED_CKPT = "checkpoints/zipformer_best.pt"
FINETUNED_REPO = os.environ.get("FINETUNED_REPO", "").strip()
AUDIO_SCALE = 1.0
MAX_SEC = 90


# ---------- K2 stub ----------
def swoosh_l(x):
    return (F.softplus(x - 4.0) - 0.08 * x - 0.035).to(x.dtype)

def swoosh_r(x):
    return (F.softplus(x - 1.0) - 0.08 * x - 0.313261687).to(x.dtype)

def swoosh_l_forward_and_deriv(x):
    return swoosh_l(x), (torch.sigmoid(x - 4.0) - 0.08).to(x.dtype)

def swoosh_r_forward_and_deriv(x):
    return swoosh_r(x), (torch.sigmoid(x - 1.0) - 0.08).to(x.dtype)

def _need_real_k2(*a, **k):
    raise NotImplementedError("k2 thật chưa được cài: chỉ dùng CTC (PyTorch).")

k2_mock = types.ModuleType("k2")
k2_mock.__path__ = []
k2_mock.__file__ = "k2/__init__.py"
for _n, _f in {
    "swoosh_l": swoosh_l, "swoosh_l_forward": swoosh_l,
    "swoosh_r": swoosh_r, "swoosh_r_forward": swoosh_r,
    "swoosh_l_forward_and_deriv": swoosh_l_forward_and_deriv,
    "swoosh_r_forward_and_deriv": swoosh_r_forward_and_deriv,
    "ctc_loss": _need_real_k2, "rnnt_loss": _need_real_k2,
}.items():
    setattr(k2_mock, _n, _f)
for _n in ["RaggedTensor", "RaggedShape", "Fsa"]:
    setattr(k2_mock, _n, type(_n, (object,), {}))
k2_mock.SymbolTable = type("SymbolTable", (object,),
                            {"from_file": classmethod(lambda cls, f: cls())})
k2_version = types.ModuleType("k2.version")
k2_version.__file__ = "k2/version.py"
k2_version.__version__ = "1.0.0.stub"
k2_mock.version = k2_version
_k2 = types.ModuleType("_k2")
_k2.RaggedInt = type("RaggedInt", (object,), {})
_k2.simple_ragged_index_select = lambda *a, **k: None
sys.modules.update({
    "k2": k2_mock, "k2.version": k2_version,
    "k2.ragged": types.ModuleType("k2.ragged"),
    "k2.fsa": types.ModuleType("k2.fsa"),
    "_k2": _k2,
})


# ---------- Tải pretrained + Icefall ----------
from huggingface_hub import snapshot_download

repo_path = snapshot_download(repo_id=MODEL_REPO,
                              allow_patterns=["*.pt", "*.model", "*.txt", "*.yaml"])

if not os.path.exists("/tmp/icefall"):
    subprocess.run(["git", "clone", "-q", "--depth", "1",
                    "https://github.com/k2-fsa/icefall.git", "/tmp/icefall"], check=True)
for p in ["/tmp/icefall", "/tmp/icefall/egs/librispeech/ASR",
          "/tmp/icefall/egs/librispeech/ASR/zipformer"]:
    if p not in sys.path:
        sys.path.append(p)

import scaling  # noqa: F401
from train import get_model, get_parser, get_params


# ---------- Dựng model & nạp trọng số ----------
import train as _train
_train.get_env_info = lambda: {"k2-version": "stub"}

bpe_files = sorted(Path(repo_path).rglob("bpe.model"))
assert bpe_files, "Không tìm thấy bpe.model trong repo pretrained"
sp = spm.SentencePieceProcessor()
sp.load(str(bpe_files[0]))
VOCAB = sp.get_piece_size()
assert sp.id_to_piece(0) == "<blk>", "blank phải là id 0"

pts = sorted(Path(repo_path).rglob("*.pt"))
ckpt_path = next((p for p in pts if p.name == "pretrained.pt"), None) \
         or next((p for p in pts if p.name.startswith("epoch-")), None) or pts[0]
ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False)
state_dict = ckpt["model"] if isinstance(ckpt, dict) and "model" in ckpt else ckpt

ARCH = ("--num-encoder-layers 2,2,4,5,4,2 --feedforward-dim 512,768,1536,2048,1536,768 "
        "--encoder-dim 192,256,512,768,512,256 --encoder-unmasked-dim 192,192,256,320,256,192 "
        "--cnn-module-kernel 31,31,15,15,15,31 --num-heads 4,4,4,8,4,4 "
        "--downsampling-factor 1,2,4,8,4,2 --decoder-dim 512 --joiner-dim 512 --context-size 2 "
        "--query-head-dim 32 --value-head-dim 12 --pos-head-dim 4 --pos-dim 48 "
        "--causal 0 --use-transducer 1 --use-ctc 1").split()

parser = get_parser()
args, _ = parser.parse_known_args(ARCH)
params = get_params()
params.update(vars(args))
params.blank_id = 0
params.vocab_size = VOCAB
params.feature_dim = 80
params.use_transducer = True
params.use_ctc = True
params.use_attention_decoder = False

model = get_model(params)
res = model.load_state_dict(state_dict, strict=False)
assert len(res.missing_keys) == 0, f"Thiếu trọng số: {res.missing_keys[:5]}"

ft_path = None
if FINETUNED_REPO:
    try:
        ft_dir = snapshot_download(repo_id=FINETUNED_REPO, allow_patterns=["*.pt"])
        cand = sorted(Path(ft_dir).rglob("*.pt"))
        ft_path = str(cand[0]) if cand else None
    except Exception as e:
        print(f"Không tải được FINETUNED_REPO={FINETUNED_REPO}: {e}")
if ft_path is None and os.path.exists(FINETUNED_CKPT):
    ft_path = FINETUNED_CKPT

if ft_path:
    st = torch.load(ft_path, map_location="cpu", weights_only=False)
    st = st["model"] if isinstance(st, dict) and "model" in st else st
    r = model.load_state_dict(st, strict=False)
    print(f"Đã nạp checkpoint fine-tune | missing={len(r.missing_keys)} "
          f"unexpected={len(r.unexpected_keys)}")
    USING_FINETUNED = True
else:
    print("CHƯA có checkpoint fine-tune — đang dùng model gốc.")
    USING_FINETUNED = False

# Giữ mô hình trên CPU lúc khởi tạo
model.to("cpu").eval()
N_PARAMS = sum(p.numel() for p in model.parameters()) / 1e6


# ---------- Fbank + giải mã CTC (Tối ưu FP16 & Inference Mode) ----------
def load_fbank(path):
    wav, sr = torchaudio.load(path)
    if wav.size(0) > 1:
        wav = wav.mean(0, keepdim=True)
    if sr != 16000:
        wav = torchaudio.functional.resample(wav, sr, 16000)
    return torchaudio.compliance.kaldi.fbank(
        wav * AUDIO_SCALE, num_mel_bins=80, sample_frequency=16000,
        snip_edges=False, dither=0.0, low_freq=20.0, high_freq=-400.0, energy_floor=1e-10)


def encode(model, x, x_lens):
    if hasattr(model, "forward_encoder"):
        return model.forward_encoder(x, x_lens)
    from icefall.utils import make_pad_mask
    x, x_lens = model.encoder_embed(x, x_lens)
    out, out_lens = model.encoder(x.permute(1, 0, 2), x_lens, make_pad_mask(x_lens))
    return out.permute(1, 0, 2), out_lens


@torch.inference_mode()
def ctc_greedy_decode(model, paths, batch_size=4):
    device = "cuda" if torch.cuda.is_available() else "cpu"
    
    # Đẩy model sang GPU và ép kiểu dữ liệu FP16 để tăng tốc tính toán
    model.to(device)
    use_fp16 = (device == "cuda")
    if use_fp16:
        model.half()
    model.eval()

    hyps = []
    for i in range(0, len(paths), batch_size):
        feats = [load_fbank(p) for p in paths[i:i + batch_size]]
        lens = torch.tensor([f.size(0) for f in feats], dtype=torch.long)
        
        x = pad_sequence(feats, batch_first=True).to(device)
        if use_fp16:
            x = x.half()

        enc, enc_lens = encode(model, x, lens.to(device))
        best = model.ctc_output(enc).argmax(-1).cpu()
        for b in range(best.size(0)):
            ids = torch.unique_consecutive(best[b, : int(enc_lens[b])]).tolist()
            hyps.append(sp.decode([t for t in ids if t != 0]))
            
    return hyps


def sentence_case(t: str) -> str:
    t = (t or "").strip().lower()
    return t[0].upper() + t[1:] if t else t


# ---------- Giao diện web (Gradio + ZeroGPU) ----------
@spaces.GPU
def transcribe(audio_path):
    if not audio_path:
        return "", ""
    info = torchaudio.info(audio_path)
    duration = info.num_frames / info.sample_rate
    if duration < 0.3:
        raise gr.Error("Âm thanh quá ngắn (dưới 0,3 giây).")
    if duration > MAX_SEC:
        raise gr.Error(f"Âm thanh dài {duration:.0f} giây, vượt giới hạn {MAX_SEC} giây.")
    
    text = ctc_greedy_decode(model, [audio_path])[0]
    status = f"{duration:.1f} giây · " + (
        "mô hình đã fine-tune" if USING_FINETUNED else
        "⚠️ chưa tìm thấy checkpoint fine-tune, đang dùng model gốc")
    return sentence_case(text), status


CSS = """
.gradio-container { max-width: 720px !important; font-family: 'Be Vietnam Pro', system-ui, sans-serif; }
#out_text textarea { font-family: Georgia, 'Source Serif 4', serif !important; font-size: 1.35rem !important; line-height: 1.5 !important; }
"""

with gr.Blocks(css=CSS, title="Chép lời tiếng Việt") as demo:
    gr.Markdown(
        f"# Chép lời tiếng Việt\n"
        f"Tải lên hoặc ghi âm, mô hình ZipFormer-150M ({N_PARAMS:.1f}M tham số) "
        f"sẽ chuyển giọng nói thành văn bản. Tối đa {MAX_SEC} giây mỗi lần."
    )
    audio_in = gr.Audio(sources=["upload", "microphone"], type="filepath", label="Âm thanh")
    btn = gr.Button("Chép lời", variant="primary")
    out_text = gr.Textbox(label="Văn bản", elem_id="out_text", lines=4)
    out_status = gr.Markdown()
    btn.click(transcribe, inputs=audio_in, outputs=[out_text, out_status])

demo.queue(max_size=20).launch(show_api=False)