Text-to-Speech
MLX
ONNX
Portuguese
tts
pt-br
brazilian-portuguese
small-model
quantized
apple-silicon
pingo
fargan
Instructions to use semfence/pingo-ptbr with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use semfence/pingo-ptbr with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download semfence/pingo-ptbr --local-dir pingo-ptbr
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download training/model.py from semfence/pingo-ptbr: direct link, hf CLI and curl.
- Browser
- Download file 9.54 kB
-
https://huggingface.co/semfence/pingo-ptbr/resolve/main/training/model.py
- Command line
-
hf download hf://semfence/pingo-ptbr/training/model.py
-
curl -L -o model.py https://huggingface.co/semfence/pingo-ptbr/resolve/main/training/model.py
9.54 kB
| """Acoustic model in the style of PiVo (text -> 20 Opus features every 10 ms). | |
| Architecture rebuilt from the PiVo-1.4M technical report and ONNX graphs (shapes and operators): | |
| * encoder: token + voice embeddings; 6 blocks alternating GQA attention (RoPE, RMSNorm, gated-SiLU FFN) | |
| and dilated temporal convolution (k=5, dilations 1,2,4); final RMSNorm. | |
| * duration predictor (log1p of frames per token): 2 depthwise k=3 layers + pointwise. | |
| * 4 phase signals per frame (progress, sin, cos, log-duration) enter through a projection. | |
| * decoder: 2 causal attention blocks with a 64-frame window (RoPE only in the first) -> 20 features. | |
| The code is ours; only the design and dimensions follow PiVo (Apache-2.0). | |
| """ | |
| import math | |
| from dataclasses import dataclass, asdict | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class Config: | |
| vocab: int = 128 | |
| voices: int = 2 | |
| width: int = 96 | |
| ff: int = 192 | |
| heads: int = 4 | |
| kv_heads: int = 2 | |
| text_layers: int = 6 | |
| decoder_layers: int = 2 | |
| kernel: int = 5 | |
| dur_width: int = 64 | |
| window: int = 64 | |
| feature_dim: int = 20 | |
| max_token_frames: int = 200 | |
| pad: int = 0 | |
| def to_dict(self): | |
| return asdict(self) | |
| class RMSNorm(nn.Module): | |
| def __init__(self, n, eps=1e-6): | |
| super().__init__() | |
| self.weight, self.eps = nn.Parameter(torch.ones(n)), eps | |
| def forward(self, x): | |
| return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight | |
| def rope(x, pos): | |
| """x: (B,H,T,D), pos: (T,) absolute positions.""" | |
| d = x.shape[-1] // 2 | |
| inv = 1.0 / (10000 ** (torch.arange(d, device=x.device, dtype=torch.float32) / d)) | |
| ang = pos.float()[:, None] * inv[None] | |
| cos, sin = ang.cos().to(x.dtype), ang.sin().to(x.dtype) | |
| a, b = x[..., :d], x[..., d:] | |
| return torch.cat((a * cos - b * sin, a * sin + b * cos), -1) | |
| class Attention(nn.Module): | |
| def __init__(self, c, use_rope=True): | |
| super().__init__() | |
| self.h, self.kv, self.hd = c.heads, c.kv_heads, c.width // c.heads | |
| self.q = nn.Linear(c.width, c.heads * self.hd, bias=False) | |
| self.k = nn.Linear(c.width, c.kv_heads * self.hd, bias=False) | |
| self.v = nn.Linear(c.width, c.kv_heads * self.hd, bias=False) | |
| self.o = nn.Linear(c.heads * self.hd, c.width, bias=False) | |
| self.use_rope = use_rope | |
| def forward(self, x, mask): | |
| """mask: (B or 1, 1, T, T) booleano, True = pode atender.""" | |
| b, t, _ = x.shape | |
| q = self.q(x).view(b, t, self.h, self.hd).transpose(1, 2) | |
| k = self.k(x).view(b, t, self.kv, self.hd).transpose(1, 2) | |
| v = self.v(x).view(b, t, self.kv, self.hd).transpose(1, 2) | |
| if self.use_rope: | |
| pos = torch.arange(t, device=x.device) | |
| q, k = rope(q, pos), rope(k, pos) | |
| rep = self.h // self.kv | |
| k, v = k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1) | |
| y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask) | |
| return self.o(y.transpose(1, 2).reshape(b, t, -1)) | |
| class GatedFFN(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| self.gate = nn.Linear(c.width, c.ff, bias=False) | |
| self.up = nn.Linear(c.width, c.ff, bias=False) | |
| self.down = nn.Linear(c.ff, c.width, bias=False) | |
| def forward(self, x): | |
| return self.down(F.silu(self.gate(x)) * self.up(x)) | |
| class AttnBlock(nn.Module): | |
| def __init__(self, c, use_rope=True): | |
| super().__init__() | |
| self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width) | |
| self.attention, self.ffn = Attention(c, use_rope), GatedFFN(c) | |
| def forward(self, x, mask): | |
| x = x + self.attention(self.norm1(x), mask) | |
| return x + self.ffn(self.norm2(x)) | |
| class ConvBlock(nn.Module): | |
| def __init__(self, c, dilation): | |
| super().__init__() | |
| self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width) | |
| pad = dilation * (c.kernel - 1) // 2 | |
| self.conv = nn.Conv1d(c.width, c.width, c.kernel, padding=pad, dilation=dilation, bias=False) | |
| def forward(self, x, keep): | |
| h = self.norm1(x) * keep | |
| h = self.conv(h.transpose(1, 2)).transpose(1, 2) | |
| return x + F.silu(self.norm2(h)) | |
| class DurationPredictor(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| w = c.dur_width | |
| self.in_proj = nn.Linear(c.width, w) | |
| self.dw = nn.ModuleList([nn.Conv1d(w, w, 3, padding=1, groups=w) for _ in range(2)]) | |
| self.pw = nn.ModuleList([nn.Conv1d(w, w, 1) for _ in range(2)]) | |
| self.norms = nn.ModuleList([RMSNorm(w) for _ in range(2)]) | |
| self.out = nn.Linear(w, 1) | |
| def forward(self, x, keep): | |
| h = F.silu(self.in_proj(x)) * keep | |
| for dw, pw, n in zip(self.dw, self.pw, self.norms): | |
| y = pw(dw((n(h) * keep).transpose(1, 2))).transpose(1, 2) | |
| h = (h + F.silu(y)) * keep | |
| return self.out(h).squeeze(-1) | |
| def frame_index(durations): | |
| """For each frame, the index of the token that generates it. -> idx (B,F), mask (B,F), total count (B,).""" | |
| total = durations.sum(1) | |
| n = int(total.max()) | |
| ends = torch.cumsum(durations, 1) # (B,L) | |
| frames = torch.arange(n, device=durations.device) | |
| idx = (frames[None, :, None] >= ends[:, None, :]).sum(-1).clamp(max=durations.shape[1] - 1) | |
| return idx, frames[None] < total[:, None], ends - durations | |
| def phase_features(durations, max_token_frames=200): | |
| """Per-frame phase signals. durations: (B,L) int -> (B,F,4), mask (B,F).""" | |
| idx, mask, starts = frame_index(durations) | |
| counts = durations.gather(1, idx).clamp(min=1).float() | |
| st = starts.gather(1, idx).float() | |
| frac = (torch.arange(idx.shape[1], device=idx.device)[None].float() - st + 0.5) / counts | |
| out = torch.stack((2 * frac - 1, torch.sin(math.pi * frac), torch.cos(math.pi * frac), | |
| torch.log1p(counts) / math.log1p(max_token_frames)), -1) | |
| return out * mask.unsqueeze(-1), mask | |
| def expand(encoded, durations): | |
| """Repeat each token encoding for its duration. -> (B,F,W), mask (B,F).""" | |
| idx, mask, _ = frame_index(durations) | |
| out = encoded.gather(1, idx.unsqueeze(-1).expand(-1, -1, encoded.shape[-1])) | |
| return out * mask.unsqueeze(-1), mask | |
| class Acoustic(nn.Module): | |
| def __init__(self, c: Config = Config()): | |
| super().__init__() | |
| self.c = c | |
| self.tokens = nn.Embedding(c.vocab, c.width, padding_idx=c.pad) | |
| self.voices = nn.Embedding(c.voices, c.width) | |
| self.stress = nn.Embedding(2, c.width) | |
| self.text_blocks = nn.ModuleList( | |
| [AttnBlock(c) if i % 2 == 0 else ConvBlock(c, 2 ** (i // 2)) for i in range(c.text_layers)]) | |
| self.text_norm = RMSNorm(c.width) | |
| self.durations = DurationPredictor(c) | |
| self.phase = nn.Linear(4, c.width, bias=False) | |
| self.decoder = nn.ModuleList([AttnBlock(c, use_rope=(i == 0)) for i in range(c.decoder_layers)]) | |
| self.feature_norm = RMSNorm(c.width) | |
| self.head = nn.Linear(c.width, c.feature_dim, bias=False) | |
| # feature statistics: the head predicts normalized values; the export folds mean/std into the weights | |
| self.register_buffer("f_mean", torch.zeros(c.feature_dim)) | |
| self.register_buffer("f_std", torch.ones(c.feature_dim)) | |
| def encode(self, tokens, stress, voice, lengths): | |
| b, t = tokens.shape | |
| keep = (torch.arange(t, device=tokens.device)[None] < lengths[:, None]).unsqueeze(-1).float() | |
| x = (self.tokens(tokens) + self.stress(stress) + self.voices(voice)[:, None]) * keep | |
| amask = (keep.squeeze(-1) > 0)[:, None, None, :].expand(b, 1, t, t) | |
| for blk in self.text_blocks: | |
| x = blk(x, amask) * keep if isinstance(blk, AttnBlock) else blk(x, keep) * keep | |
| x = self.text_norm(x) * keep | |
| return x, self.durations(x, keep) # predicted log1p(duration) | |
| def decode(self, expanded, phase, fmask): | |
| """expanded (B,F,W), phase (B,F,4), fmask (B,F) -> (B,F,20).""" | |
| b, n, _ = expanded.shape | |
| i = torch.arange(n, device=expanded.device) | |
| causal = (i[None] <= i[:, None]) & (i[:, None] - i[None] < self.c.window) | |
| mask = (causal[None, None] & fmask[:, None, None, :]) | torch.eye(n, dtype=torch.bool, device=expanded.device)[None, None] | |
| # (keeping the diagonal open avoids fully masked rows in padding frames, which give NaN in CUDA SDPA) | |
| x = expanded + self.phase(phase) | |
| for blk in self.decoder: | |
| x = blk(x, mask) | |
| return self.head(self.feature_norm(x)) | |
| def forward(self, tokens, stress, voice, lengths, durations): | |
| enc, log_dur = self.encode(tokens, stress, voice, lengths) | |
| expanded, fmask = expand(enc, durations) | |
| phase, _ = phase_features(durations, self.c.max_token_frames) | |
| return self.decode(expanded, phase, fmask), log_dur, fmask | |
| if __name__ == "__main__": | |
| for w, ff in ((96, 192), (128, 256)): | |
| m = Acoustic(Config(width=w, ff=ff)) | |
| n = sum(p.numel() for p in m.parameters()) | |
| tok = torch.randint(1, 60, (2, 12)); lens = torch.tensor([12, 9]); voice = torch.tensor([0, 1]) | |
| dur = torch.randint(1, 6, (2, 12)); dur[1, 9:] = 0 | |
| feats, log_dur, fmask = m(tok, torch.zeros_like(tok), voice, lens, dur) | |
| loss = feats.pow(2).mean() + log_dur.pow(2).mean(); loss.backward() | |
| print(f"width {w}: {n:,} parameters | features {tuple(feats.shape)} | log_dur {tuple(log_dur.shape)}") | |