Text-to-Speech
MLX
ONNX
PyTorch
Portuguese
tts
pt-br
brazilian-portuguese
apple-silicon
File size: 6,003 Bytes
ee383fc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""Texto -> voz com o pivo-ptbr, em MLX (Mac) ou ONNX Runtime (qualquer CPU).

  python code/speak.py --backend mlx  --voice F_s4 --text "Olá! Esta é a voz da SemFence." -o ola.wav
  python code/speak.py --backend onnx --voice M_sm4 --text-file texto.txt --speed 1.1 -o ola.wav

Precisa do fonemizador: pip install git+https://github.com/SemFence/dizione-ptbr git+https://github.com/SemFence/tiny-voca-ptbr
"""
import argparse, json, math, sys
from pathlib import Path
import numpy as np
import soundfile as sf
from scipy.signal import lfilter

from tiny_voca.infer import finished, sentences, shorter          # divisão em frases, igual à do tiny-voca-ptbr
from tiny_voca.text import PHONE_IDS, NothingToPronounce, phonemize

ROOT = Path(__file__).resolve().parent.parent
VOICES = {"F_s4": 0, "M_sm4": 1}
RATE, PEAK, MAX_FRAMES = 16000, 0.99, 3000


def phase_features(dur, max_token_frames=200):
    d = np.asarray(dur, np.int64)
    idx = np.repeat(np.arange(len(d)), d)
    starts = np.cumsum(d) - d
    counts = np.maximum(d[idx], 1).astype(np.float32)
    frac = (np.arange(len(idx), dtype=np.float32) - starts[idx] + 0.5) / counts
    return idx, np.stack([2 * frac - 1, np.sin(np.pi * frac), np.cos(np.pi * frac),
                          np.log1p(counts) / np.log1p(max_token_frames)], -1).astype(np.float32)


def durations(log_dur, ids, speed, max_token_frames=200):
    d = np.rint(np.expm1(np.clip(log_dur, 0, math.log1p(max_token_frames))) / speed).astype(np.int64)
    d = np.where(np.isin(ids, list(PHONE_IDS)), np.maximum(d, 1), d)
    d = np.minimum(d, max_token_frames)
    d[np.asarray(ids) == 0] = 0
    if d.sum() > MAX_FRAMES:
        raise ValueError("frase longa demais")
    return d


class MLXBackend:
    def __init__(self):
        import mlx.core as mx
        mx.set_default_device(mx.cpu)             # o modelo é minúsculo: a CPU ganha da GPU nos passos do vocoder
        sys.path.insert(0, str(Path(__file__).parent)); import pivo_mlx
        self.m, self.voc, _ = pivo_mlx.load(ROOT / "mlx"); self.mod = pivo_mlx

    def __call__(self, ph, voice, speed):
        mask = np.array([i in PHONE_IDS for i in ph.ids])
        feats, _ = self.mod.acoustic_features(self.m, list(ph.ids), list(ph.stress), voice, speed, mask)
        return self.voc(feats)


class ONNXBackend:
    def __init__(self):
        import onnxruntime as ort
        o = ort.SessionOptions(); o.intra_op_num_threads = 1
        d = ROOT / "onnx"
        self.s = {n: ort.InferenceSession(str(d / f"{n}.onnx"), o, providers=["CPUExecutionProvider"])
                  for n in ("encoder", "decoder", "conditioning", "vocoder_step")}
        self.cfg = json.loads((d / "metadata.json").read_text())["config"]

    def __call__(self, ph, voice, speed):
        c, s, n = self.cfg, self.s, len(ph.ids)
        i64 = lambda v: np.asarray(v, np.int64)
        enc, log_dur = s["encoder"].run(None, {"tokens": i64([ph.ids]), "stress": i64([ph.stress]),
                                                "voice": i64([voice]), "lengths": i64([n])})
        dur = durations(log_dur[0], ph.ids, speed, c["max_token_frames"])
        idx, phase = phase_features(dur, c["max_token_frames"])
        expanded = np.concatenate([enc[0][idx], phase], -1)[None].astype(np.float32)
        frames, hd = expanded.shape[1], c["width"] // c["heads"]
        cache = {k: np.zeros((1, c["kv_heads"], 0, hd), np.float32) for k in ("key0", "value0", "key1", "value1")}
        feats = []
        for p in range(0, frames, 64):
            f, k0, v0, k1, v1 = s["decoder"].run(None, {"expanded": expanded[:, p:p + 64], "position": np.array(p, np.int64), **cache})
            feats.append(f[0]); cache = dict(key0=k0, value0=v0, key1=k1, value1=v1)
        feats = np.concatenate(feats)
        padded = np.concatenate([feats[:1], feats, feats[-1:]])
        per = np.rint(256 / np.power(2.0, np.clip(padded[:, 18], -10, 10) + 1.5)).clip(32, 255).astype(np.int64)
        cond = s["conditioning"].run(None, {"features": padded[None], "periods": per[None]})[0][0]
        st = {k: np.zeros((1, w), np.float32) for k, w in zip(("memory", "gru1", "gru2", "gru3", "frame_state"), (256, 160, 128, 128, 164))}
        out = []
        for k in range(frames):
            for q in range(4):
                r = s["vocoder_step"].run(None, {"conditioning": cond[k:k + 1, q * 80:(q + 1) * 80], "period": per[k + 1:k + 2], **st})
                st = dict(zip(st, r[1:])); out.append(r[0][0])
        return lfilter([1.0], [1.0, -0.85], np.concatenate(out)).astype(np.float32)


def speak(backend, text, voice="F_s4", speed=1.0):
    waves = []

    def read(piece):
        try:
            ph = phonemize(piece)
        except NothingToPronounce:
            return
        if len(ph.ids) <= 400:
            try:
                waves.append(backend(ph, VOICES[voice], speed)); return
            except ValueError:
                pass
        parts = shorter(piece)
        if not parts:
            raise ValueError(f"palavra longa demais: {piece[:40]!r}")
        for part in parts:
            read(part)

    for sentence in sentences(finished(text)):
        read(sentence)
    if not waves:
        raise NothingToPronounce("nada para ler")
    wave = np.concatenate(waves)
    peak = float(np.abs(wave).max())
    return wave * (PEAK / peak) if peak > PEAK else wave


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--backend", choices=["mlx", "onnx"], default="onnx")
    ap.add_argument("--voice", choices=list(VOICES), default="F_s4")
    ap.add_argument("--text"); ap.add_argument("--text-file")
    ap.add_argument("--speed", type=float, default=1.0)
    ap.add_argument("-o", "--output", default="saida.wav")
    a = ap.parse_args()
    text = Path(a.text_file).read_text() if a.text_file else a.text
    y = speak(MLXBackend() if a.backend == "mlx" else ONNXBackend(), text, a.voice, a.speed)
    sf.write(a.output, y, RATE); print(a.output, round(len(y) / RATE, 2), "s")