Text-to-Speech
MLX
ONNX
Portuguese
tts
pt-br
brazilian-portuguese
small-model
quantized
apple-silicon
pingo
fargan
Instructions to use semfence/pingo-ptbr with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use semfence/pingo-ptbr with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download semfence/pingo-ptbr --local-dir pingo-ptbr
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 9,542 Bytes
27d8618 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 | """Acoustic model in the style of PiVo (text -> 20 Opus features every 10 ms).
Architecture rebuilt from the PiVo-1.4M technical report and ONNX graphs (shapes and operators):
* encoder: token + voice embeddings; 6 blocks alternating GQA attention (RoPE, RMSNorm, gated-SiLU FFN)
and dilated temporal convolution (k=5, dilations 1,2,4); final RMSNorm.
* duration predictor (log1p of frames per token): 2 depthwise k=3 layers + pointwise.
* 4 phase signals per frame (progress, sin, cos, log-duration) enter through a projection.
* decoder: 2 causal attention blocks with a 64-frame window (RoPE only in the first) -> 20 features.
The code is ours; only the design and dimensions follow PiVo (Apache-2.0).
"""
import math
from dataclasses import dataclass, asdict
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class Config:
vocab: int = 128
voices: int = 2
width: int = 96
ff: int = 192
heads: int = 4
kv_heads: int = 2
text_layers: int = 6
decoder_layers: int = 2
kernel: int = 5
dur_width: int = 64
window: int = 64
feature_dim: int = 20
max_token_frames: int = 200
pad: int = 0
def to_dict(self):
return asdict(self)
class RMSNorm(nn.Module):
def __init__(self, n, eps=1e-6):
super().__init__()
self.weight, self.eps = nn.Parameter(torch.ones(n)), eps
def forward(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
def rope(x, pos):
"""x: (B,H,T,D), pos: (T,) absolute positions."""
d = x.shape[-1] // 2
inv = 1.0 / (10000 ** (torch.arange(d, device=x.device, dtype=torch.float32) / d))
ang = pos.float()[:, None] * inv[None]
cos, sin = ang.cos().to(x.dtype), ang.sin().to(x.dtype)
a, b = x[..., :d], x[..., d:]
return torch.cat((a * cos - b * sin, a * sin + b * cos), -1)
class Attention(nn.Module):
def __init__(self, c, use_rope=True):
super().__init__()
self.h, self.kv, self.hd = c.heads, c.kv_heads, c.width // c.heads
self.q = nn.Linear(c.width, c.heads * self.hd, bias=False)
self.k = nn.Linear(c.width, c.kv_heads * self.hd, bias=False)
self.v = nn.Linear(c.width, c.kv_heads * self.hd, bias=False)
self.o = nn.Linear(c.heads * self.hd, c.width, bias=False)
self.use_rope = use_rope
def forward(self, x, mask):
"""mask: (B or 1, 1, T, T) booleano, True = pode atender."""
b, t, _ = x.shape
q = self.q(x).view(b, t, self.h, self.hd).transpose(1, 2)
k = self.k(x).view(b, t, self.kv, self.hd).transpose(1, 2)
v = self.v(x).view(b, t, self.kv, self.hd).transpose(1, 2)
if self.use_rope:
pos = torch.arange(t, device=x.device)
q, k = rope(q, pos), rope(k, pos)
rep = self.h // self.kv
k, v = k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1)
y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
return self.o(y.transpose(1, 2).reshape(b, t, -1))
class GatedFFN(nn.Module):
def __init__(self, c):
super().__init__()
self.gate = nn.Linear(c.width, c.ff, bias=False)
self.up = nn.Linear(c.width, c.ff, bias=False)
self.down = nn.Linear(c.ff, c.width, bias=False)
def forward(self, x):
return self.down(F.silu(self.gate(x)) * self.up(x))
class AttnBlock(nn.Module):
def __init__(self, c, use_rope=True):
super().__init__()
self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width)
self.attention, self.ffn = Attention(c, use_rope), GatedFFN(c)
def forward(self, x, mask):
x = x + self.attention(self.norm1(x), mask)
return x + self.ffn(self.norm2(x))
class ConvBlock(nn.Module):
def __init__(self, c, dilation):
super().__init__()
self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width)
pad = dilation * (c.kernel - 1) // 2
self.conv = nn.Conv1d(c.width, c.width, c.kernel, padding=pad, dilation=dilation, bias=False)
def forward(self, x, keep):
h = self.norm1(x) * keep
h = self.conv(h.transpose(1, 2)).transpose(1, 2)
return x + F.silu(self.norm2(h))
class DurationPredictor(nn.Module):
def __init__(self, c):
super().__init__()
w = c.dur_width
self.in_proj = nn.Linear(c.width, w)
self.dw = nn.ModuleList([nn.Conv1d(w, w, 3, padding=1, groups=w) for _ in range(2)])
self.pw = nn.ModuleList([nn.Conv1d(w, w, 1) for _ in range(2)])
self.norms = nn.ModuleList([RMSNorm(w) for _ in range(2)])
self.out = nn.Linear(w, 1)
def forward(self, x, keep):
h = F.silu(self.in_proj(x)) * keep
for dw, pw, n in zip(self.dw, self.pw, self.norms):
y = pw(dw((n(h) * keep).transpose(1, 2))).transpose(1, 2)
h = (h + F.silu(y)) * keep
return self.out(h).squeeze(-1)
def frame_index(durations):
"""For each frame, the index of the token that generates it. -> idx (B,F), mask (B,F), total count (B,)."""
total = durations.sum(1)
n = int(total.max())
ends = torch.cumsum(durations, 1) # (B,L)
frames = torch.arange(n, device=durations.device)
idx = (frames[None, :, None] >= ends[:, None, :]).sum(-1).clamp(max=durations.shape[1] - 1)
return idx, frames[None] < total[:, None], ends - durations
def phase_features(durations, max_token_frames=200):
"""Per-frame phase signals. durations: (B,L) int -> (B,F,4), mask (B,F)."""
idx, mask, starts = frame_index(durations)
counts = durations.gather(1, idx).clamp(min=1).float()
st = starts.gather(1, idx).float()
frac = (torch.arange(idx.shape[1], device=idx.device)[None].float() - st + 0.5) / counts
out = torch.stack((2 * frac - 1, torch.sin(math.pi * frac), torch.cos(math.pi * frac),
torch.log1p(counts) / math.log1p(max_token_frames)), -1)
return out * mask.unsqueeze(-1), mask
def expand(encoded, durations):
"""Repeat each token encoding for its duration. -> (B,F,W), mask (B,F)."""
idx, mask, _ = frame_index(durations)
out = encoded.gather(1, idx.unsqueeze(-1).expand(-1, -1, encoded.shape[-1]))
return out * mask.unsqueeze(-1), mask
class Acoustic(nn.Module):
def __init__(self, c: Config = Config()):
super().__init__()
self.c = c
self.tokens = nn.Embedding(c.vocab, c.width, padding_idx=c.pad)
self.voices = nn.Embedding(c.voices, c.width)
self.stress = nn.Embedding(2, c.width)
self.text_blocks = nn.ModuleList(
[AttnBlock(c) if i % 2 == 0 else ConvBlock(c, 2 ** (i // 2)) for i in range(c.text_layers)])
self.text_norm = RMSNorm(c.width)
self.durations = DurationPredictor(c)
self.phase = nn.Linear(4, c.width, bias=False)
self.decoder = nn.ModuleList([AttnBlock(c, use_rope=(i == 0)) for i in range(c.decoder_layers)])
self.feature_norm = RMSNorm(c.width)
self.head = nn.Linear(c.width, c.feature_dim, bias=False)
# feature statistics: the head predicts normalized values; the export folds mean/std into the weights
self.register_buffer("f_mean", torch.zeros(c.feature_dim))
self.register_buffer("f_std", torch.ones(c.feature_dim))
def encode(self, tokens, stress, voice, lengths):
b, t = tokens.shape
keep = (torch.arange(t, device=tokens.device)[None] < lengths[:, None]).unsqueeze(-1).float()
x = (self.tokens(tokens) + self.stress(stress) + self.voices(voice)[:, None]) * keep
amask = (keep.squeeze(-1) > 0)[:, None, None, :].expand(b, 1, t, t)
for blk in self.text_blocks:
x = blk(x, amask) * keep if isinstance(blk, AttnBlock) else blk(x, keep) * keep
x = self.text_norm(x) * keep
return x, self.durations(x, keep) # predicted log1p(duration)
def decode(self, expanded, phase, fmask):
"""expanded (B,F,W), phase (B,F,4), fmask (B,F) -> (B,F,20)."""
b, n, _ = expanded.shape
i = torch.arange(n, device=expanded.device)
causal = (i[None] <= i[:, None]) & (i[:, None] - i[None] < self.c.window)
mask = (causal[None, None] & fmask[:, None, None, :]) | torch.eye(n, dtype=torch.bool, device=expanded.device)[None, None]
# (keeping the diagonal open avoids fully masked rows in padding frames, which give NaN in CUDA SDPA)
x = expanded + self.phase(phase)
for blk in self.decoder:
x = blk(x, mask)
return self.head(self.feature_norm(x))
def forward(self, tokens, stress, voice, lengths, durations):
enc, log_dur = self.encode(tokens, stress, voice, lengths)
expanded, fmask = expand(enc, durations)
phase, _ = phase_features(durations, self.c.max_token_frames)
return self.decode(expanded, phase, fmask), log_dur, fmask
if __name__ == "__main__":
for w, ff in ((96, 192), (128, 256)):
m = Acoustic(Config(width=w, ff=ff))
n = sum(p.numel() for p in m.parameters())
tok = torch.randint(1, 60, (2, 12)); lens = torch.tensor([12, 9]); voice = torch.tensor([0, 1])
dur = torch.randint(1, 6, (2, 12)); dur[1, 9:] = 0
feats, log_dur, fmask = m(tok, torch.zeros_like(tok), voice, lens, dur)
loss = feats.pow(2).mean() + log_dur.pow(2).mean(); loss.backward()
print(f"width {w}: {n:,} parameters | features {tuple(feats.shape)} | log_dur {tuple(log_dur.shape)}")
|