Text-to-Speech
MLX
ONNX
PyTorch
Portuguese
tts
pt-br
brazilian-portuguese
apple-silicon
pivo-ptbr / code /speak.py
itxtoledo's picture
pivo-ptbr: MLX, ONNX, PyTorch, código e avaliação
ee383fc verified
Raw History Blame Contribute Delete
6 kB
"""Texto -> voz com o pivo-ptbr, em MLX (Mac) ou ONNX Runtime (qualquer CPU).
python code/speak.py --backend mlx --voice F_s4 --text "Olá! Esta é a voz da SemFence." -o ola.wav
python code/speak.py --backend onnx --voice M_sm4 --text-file texto.txt --speed 1.1 -o ola.wav
Precisa do fonemizador: pip install git+https://github.com/SemFence/dizione-ptbr git+https://github.com/SemFence/tiny-voca-ptbr
"""
import argparse, json, math, sys
from pathlib import Path
import numpy as np
import soundfile as sf
from scipy.signal import lfilter
from tiny_voca.infer import finished, sentences, shorter # divisão em frases, igual à do tiny-voca-ptbr
from tiny_voca.text import PHONE_IDS, NothingToPronounce, phonemize
ROOT = Path(__file__).resolve().parent.parent
VOICES = {"F_s4": 0, "M_sm4": 1}
RATE, PEAK, MAX_FRAMES = 16000, 0.99, 3000
def phase_features(dur, max_token_frames=200):
d = np.asarray(dur, np.int64)
idx = np.repeat(np.arange(len(d)), d)
starts = np.cumsum(d) - d
counts = np.maximum(d[idx], 1).astype(np.float32)
frac = (np.arange(len(idx), dtype=np.float32) - starts[idx] + 0.5) / counts
return idx, np.stack([2 * frac - 1, np.sin(np.pi * frac), np.cos(np.pi * frac),
np.log1p(counts) / np.log1p(max_token_frames)], -1).astype(np.float32)
def durations(log_dur, ids, speed, max_token_frames=200):
d = np.rint(np.expm1(np.clip(log_dur, 0, math.log1p(max_token_frames))) / speed).astype(np.int64)
d = np.where(np.isin(ids, list(PHONE_IDS)), np.maximum(d, 1), d)
d = np.minimum(d, max_token_frames)
d[np.asarray(ids) == 0] = 0
if d.sum() > MAX_FRAMES:
raise ValueError("frase longa demais")
return d
class MLXBackend:
def __init__(self):
import mlx.core as mx
mx.set_default_device(mx.cpu) # o modelo é minúsculo: a CPU ganha da GPU nos passos do vocoder
sys.path.insert(0, str(Path(__file__).parent)); import pivo_mlx
self.m, self.voc, _ = pivo_mlx.load(ROOT / "mlx"); self.mod = pivo_mlx
def __call__(self, ph, voice, speed):
mask = np.array([i in PHONE_IDS for i in ph.ids])
feats, _ = self.mod.acoustic_features(self.m, list(ph.ids), list(ph.stress), voice, speed, mask)
return self.voc(feats)
class ONNXBackend:
def __init__(self):
import onnxruntime as ort
o = ort.SessionOptions(); o.intra_op_num_threads = 1
d = ROOT / "onnx"
self.s = {n: ort.InferenceSession(str(d / f"{n}.onnx"), o, providers=["CPUExecutionProvider"])
for n in ("encoder", "decoder", "conditioning", "vocoder_step")}
self.cfg = json.loads((d / "metadata.json").read_text())["config"]
def __call__(self, ph, voice, speed):
c, s, n = self.cfg, self.s, len(ph.ids)
i64 = lambda v: np.asarray(v, np.int64)
enc, log_dur = s["encoder"].run(None, {"tokens": i64([ph.ids]), "stress": i64([ph.stress]),
"voice": i64([voice]), "lengths": i64([n])})
dur = durations(log_dur[0], ph.ids, speed, c["max_token_frames"])
idx, phase = phase_features(dur, c["max_token_frames"])
expanded = np.concatenate([enc[0][idx], phase], -1)[None].astype(np.float32)
frames, hd = expanded.shape[1], c["width"] // c["heads"]
cache = {k: np.zeros((1, c["kv_heads"], 0, hd), np.float32) for k in ("key0", "value0", "key1", "value1")}
feats = []
for p in range(0, frames, 64):
f, k0, v0, k1, v1 = s["decoder"].run(None, {"expanded": expanded[:, p:p + 64], "position": np.array(p, np.int64), **cache})
feats.append(f[0]); cache = dict(key0=k0, value0=v0, key1=k1, value1=v1)
feats = np.concatenate(feats)
padded = np.concatenate([feats[:1], feats, feats[-1:]])
per = np.rint(256 / np.power(2.0, np.clip(padded[:, 18], -10, 10) + 1.5)).clip(32, 255).astype(np.int64)
cond = s["conditioning"].run(None, {"features": padded[None], "periods": per[None]})[0][0]
st = {k: np.zeros((1, w), np.float32) for k, w in zip(("memory", "gru1", "gru2", "gru3", "frame_state"), (256, 160, 128, 128, 164))}
out = []
for k in range(frames):
for q in range(4):
r = s["vocoder_step"].run(None, {"conditioning": cond[k:k + 1, q * 80:(q + 1) * 80], "period": per[k + 1:k + 2], **st})
st = dict(zip(st, r[1:])); out.append(r[0][0])
return lfilter([1.0], [1.0, -0.85], np.concatenate(out)).astype(np.float32)
def speak(backend, text, voice="F_s4", speed=1.0):
waves = []
def read(piece):
try:
ph = phonemize(piece)
except NothingToPronounce:
return
if len(ph.ids) <= 400:
try:
waves.append(backend(ph, VOICES[voice], speed)); return
except ValueError:
pass
parts = shorter(piece)
if not parts:
raise ValueError(f"palavra longa demais: {piece[:40]!r}")
for part in parts:
read(part)
for sentence in sentences(finished(text)):
read(sentence)
if not waves:
raise NothingToPronounce("nada para ler")
wave = np.concatenate(waves)
peak = float(np.abs(wave).max())
return wave * (PEAK / peak) if peak > PEAK else wave
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--backend", choices=["mlx", "onnx"], default="onnx")
ap.add_argument("--voice", choices=list(VOICES), default="F_s4")
ap.add_argument("--text"); ap.add_argument("--text-file")
ap.add_argument("--speed", type=float, default=1.0)
ap.add_argument("-o", "--output", default="saida.wav")
a = ap.parse_args()
text = Path(a.text_file).read_text() if a.text_file else a.text
y = speak(MLXBackend() if a.backend == "mlx" else ONNXBackend(), text, a.voice, a.speed)
sf.write(a.output, y, RATE); print(a.output, round(len(y) / RATE, 2), "s")