pingo-ptbr / training /model.py
itxtoledo's picture
Add pingo-ptbr: 0.87 MB pt-BR TTS (MLX + ONNX + web demo)
27d8618 verified
Raw History Blame Contribute Delete
9.54 kB
"""Acoustic model in the style of PiVo (text -> 20 Opus features every 10 ms).
Architecture rebuilt from the PiVo-1.4M technical report and ONNX graphs (shapes and operators):
* encoder: token + voice embeddings; 6 blocks alternating GQA attention (RoPE, RMSNorm, gated-SiLU FFN)
and dilated temporal convolution (k=5, dilations 1,2,4); final RMSNorm.
* duration predictor (log1p of frames per token): 2 depthwise k=3 layers + pointwise.
* 4 phase signals per frame (progress, sin, cos, log-duration) enter through a projection.
* decoder: 2 causal attention blocks with a 64-frame window (RoPE only in the first) -> 20 features.
The code is ours; only the design and dimensions follow PiVo (Apache-2.0).
"""
import math
from dataclasses import dataclass, asdict
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class Config:
vocab: int = 128
voices: int = 2
width: int = 96
ff: int = 192
heads: int = 4
kv_heads: int = 2
text_layers: int = 6
decoder_layers: int = 2
kernel: int = 5
dur_width: int = 64
window: int = 64
feature_dim: int = 20
max_token_frames: int = 200
pad: int = 0
def to_dict(self):
return asdict(self)
class RMSNorm(nn.Module):
def __init__(self, n, eps=1e-6):
super().__init__()
self.weight, self.eps = nn.Parameter(torch.ones(n)), eps
def forward(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
def rope(x, pos):
"""x: (B,H,T,D), pos: (T,) absolute positions."""
d = x.shape[-1] // 2
inv = 1.0 / (10000 ** (torch.arange(d, device=x.device, dtype=torch.float32) / d))
ang = pos.float()[:, None] * inv[None]
cos, sin = ang.cos().to(x.dtype), ang.sin().to(x.dtype)
a, b = x[..., :d], x[..., d:]
return torch.cat((a * cos - b * sin, a * sin + b * cos), -1)
class Attention(nn.Module):
def __init__(self, c, use_rope=True):
super().__init__()
self.h, self.kv, self.hd = c.heads, c.kv_heads, c.width // c.heads
self.q = nn.Linear(c.width, c.heads * self.hd, bias=False)
self.k = nn.Linear(c.width, c.kv_heads * self.hd, bias=False)
self.v = nn.Linear(c.width, c.kv_heads * self.hd, bias=False)
self.o = nn.Linear(c.heads * self.hd, c.width, bias=False)
self.use_rope = use_rope
def forward(self, x, mask):
"""mask: (B or 1, 1, T, T) booleano, True = pode atender."""
b, t, _ = x.shape
q = self.q(x).view(b, t, self.h, self.hd).transpose(1, 2)
k = self.k(x).view(b, t, self.kv, self.hd).transpose(1, 2)
v = self.v(x).view(b, t, self.kv, self.hd).transpose(1, 2)
if self.use_rope:
pos = torch.arange(t, device=x.device)
q, k = rope(q, pos), rope(k, pos)
rep = self.h // self.kv
k, v = k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1)
y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
return self.o(y.transpose(1, 2).reshape(b, t, -1))
class GatedFFN(nn.Module):
def __init__(self, c):
super().__init__()
self.gate = nn.Linear(c.width, c.ff, bias=False)
self.up = nn.Linear(c.width, c.ff, bias=False)
self.down = nn.Linear(c.ff, c.width, bias=False)
def forward(self, x):
return self.down(F.silu(self.gate(x)) * self.up(x))
class AttnBlock(nn.Module):
def __init__(self, c, use_rope=True):
super().__init__()
self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width)
self.attention, self.ffn = Attention(c, use_rope), GatedFFN(c)
def forward(self, x, mask):
x = x + self.attention(self.norm1(x), mask)
return x + self.ffn(self.norm2(x))
class ConvBlock(nn.Module):
def __init__(self, c, dilation):
super().__init__()
self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width)
pad = dilation * (c.kernel - 1) // 2
self.conv = nn.Conv1d(c.width, c.width, c.kernel, padding=pad, dilation=dilation, bias=False)
def forward(self, x, keep):
h = self.norm1(x) * keep
h = self.conv(h.transpose(1, 2)).transpose(1, 2)
return x + F.silu(self.norm2(h))
class DurationPredictor(nn.Module):
def __init__(self, c):
super().__init__()
w = c.dur_width
self.in_proj = nn.Linear(c.width, w)
self.dw = nn.ModuleList([nn.Conv1d(w, w, 3, padding=1, groups=w) for _ in range(2)])
self.pw = nn.ModuleList([nn.Conv1d(w, w, 1) for _ in range(2)])
self.norms = nn.ModuleList([RMSNorm(w) for _ in range(2)])
self.out = nn.Linear(w, 1)
def forward(self, x, keep):
h = F.silu(self.in_proj(x)) * keep
for dw, pw, n in zip(self.dw, self.pw, self.norms):
y = pw(dw((n(h) * keep).transpose(1, 2))).transpose(1, 2)
h = (h + F.silu(y)) * keep
return self.out(h).squeeze(-1)
def frame_index(durations):
"""For each frame, the index of the token that generates it. -> idx (B,F), mask (B,F), total count (B,)."""
total = durations.sum(1)
n = int(total.max())
ends = torch.cumsum(durations, 1) # (B,L)
frames = torch.arange(n, device=durations.device)
idx = (frames[None, :, None] >= ends[:, None, :]).sum(-1).clamp(max=durations.shape[1] - 1)
return idx, frames[None] < total[:, None], ends - durations
def phase_features(durations, max_token_frames=200):
"""Per-frame phase signals. durations: (B,L) int -> (B,F,4), mask (B,F)."""
idx, mask, starts = frame_index(durations)
counts = durations.gather(1, idx).clamp(min=1).float()
st = starts.gather(1, idx).float()
frac = (torch.arange(idx.shape[1], device=idx.device)[None].float() - st + 0.5) / counts
out = torch.stack((2 * frac - 1, torch.sin(math.pi * frac), torch.cos(math.pi * frac),
torch.log1p(counts) / math.log1p(max_token_frames)), -1)
return out * mask.unsqueeze(-1), mask
def expand(encoded, durations):
"""Repeat each token encoding for its duration. -> (B,F,W), mask (B,F)."""
idx, mask, _ = frame_index(durations)
out = encoded.gather(1, idx.unsqueeze(-1).expand(-1, -1, encoded.shape[-1]))
return out * mask.unsqueeze(-1), mask
class Acoustic(nn.Module):
def __init__(self, c: Config = Config()):
super().__init__()
self.c = c
self.tokens = nn.Embedding(c.vocab, c.width, padding_idx=c.pad)
self.voices = nn.Embedding(c.voices, c.width)
self.stress = nn.Embedding(2, c.width)
self.text_blocks = nn.ModuleList(
[AttnBlock(c) if i % 2 == 0 else ConvBlock(c, 2 ** (i // 2)) for i in range(c.text_layers)])
self.text_norm = RMSNorm(c.width)
self.durations = DurationPredictor(c)
self.phase = nn.Linear(4, c.width, bias=False)
self.decoder = nn.ModuleList([AttnBlock(c, use_rope=(i == 0)) for i in range(c.decoder_layers)])
self.feature_norm = RMSNorm(c.width)
self.head = nn.Linear(c.width, c.feature_dim, bias=False)
# feature statistics: the head predicts normalized values; the export folds mean/std into the weights
self.register_buffer("f_mean", torch.zeros(c.feature_dim))
self.register_buffer("f_std", torch.ones(c.feature_dim))
def encode(self, tokens, stress, voice, lengths):
b, t = tokens.shape
keep = (torch.arange(t, device=tokens.device)[None] < lengths[:, None]).unsqueeze(-1).float()
x = (self.tokens(tokens) + self.stress(stress) + self.voices(voice)[:, None]) * keep
amask = (keep.squeeze(-1) > 0)[:, None, None, :].expand(b, 1, t, t)
for blk in self.text_blocks:
x = blk(x, amask) * keep if isinstance(blk, AttnBlock) else blk(x, keep) * keep
x = self.text_norm(x) * keep
return x, self.durations(x, keep) # predicted log1p(duration)
def decode(self, expanded, phase, fmask):
"""expanded (B,F,W), phase (B,F,4), fmask (B,F) -> (B,F,20)."""
b, n, _ = expanded.shape
i = torch.arange(n, device=expanded.device)
causal = (i[None] <= i[:, None]) & (i[:, None] - i[None] < self.c.window)
mask = (causal[None, None] & fmask[:, None, None, :]) | torch.eye(n, dtype=torch.bool, device=expanded.device)[None, None]
# (keeping the diagonal open avoids fully masked rows in padding frames, which give NaN in CUDA SDPA)
x = expanded + self.phase(phase)
for blk in self.decoder:
x = blk(x, mask)
return self.head(self.feature_norm(x))
def forward(self, tokens, stress, voice, lengths, durations):
enc, log_dur = self.encode(tokens, stress, voice, lengths)
expanded, fmask = expand(enc, durations)
phase, _ = phase_features(durations, self.c.max_token_frames)
return self.decode(expanded, phase, fmask), log_dur, fmask
if __name__ == "__main__":
for w, ff in ((96, 192), (128, 256)):
m = Acoustic(Config(width=w, ff=ff))
n = sum(p.numel() for p in m.parameters())
tok = torch.randint(1, 60, (2, 12)); lens = torch.tensor([12, 9]); voice = torch.tensor([0, 1])
dur = torch.randint(1, 6, (2, 12)); dur[1, 9:] = 0
feats, log_dur, fmask = m(tok, torch.zeros_like(tok), voice, lens, dur)
loss = feats.pow(2).mean() + log_dur.pow(2).mean(); loss.backward()
print(f"width {w}: {n:,} parameters | features {tuple(feats.shape)} | log_dur {tuple(log_dur.shape)}")