File size: 9,542 Bytes
27d8618
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
"""Acoustic model in the style of PiVo (text -> 20 Opus features every 10 ms).

Architecture rebuilt from the PiVo-1.4M technical report and ONNX graphs (shapes and operators):
  * encoder: token + voice embeddings; 6 blocks alternating GQA attention (RoPE, RMSNorm, gated-SiLU FFN)
    and dilated temporal convolution (k=5, dilations 1,2,4); final RMSNorm.
  * duration predictor (log1p of frames per token): 2 depthwise k=3 layers + pointwise.
  * 4 phase signals per frame (progress, sin, cos, log-duration) enter through a projection.
  * decoder: 2 causal attention blocks with a 64-frame window (RoPE only in the first) -> 20 features.
The code is ours; only the design and dimensions follow PiVo (Apache-2.0).
"""
import math
from dataclasses import dataclass, asdict

import torch
import torch.nn as nn
import torch.nn.functional as F


@dataclass
class Config:
    vocab: int = 128
    voices: int = 2
    width: int = 96
    ff: int = 192
    heads: int = 4
    kv_heads: int = 2
    text_layers: int = 6
    decoder_layers: int = 2
    kernel: int = 5
    dur_width: int = 64
    window: int = 64
    feature_dim: int = 20
    max_token_frames: int = 200
    pad: int = 0

    def to_dict(self):
        return asdict(self)


class RMSNorm(nn.Module):
    def __init__(self, n, eps=1e-6):
        super().__init__()
        self.weight, self.eps = nn.Parameter(torch.ones(n)), eps

    def forward(self, x):
        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight


def rope(x, pos):
    """x: (B,H,T,D), pos: (T,) absolute positions."""
    d = x.shape[-1] // 2
    inv = 1.0 / (10000 ** (torch.arange(d, device=x.device, dtype=torch.float32) / d))
    ang = pos.float()[:, None] * inv[None]
    cos, sin = ang.cos().to(x.dtype), ang.sin().to(x.dtype)
    a, b = x[..., :d], x[..., d:]
    return torch.cat((a * cos - b * sin, a * sin + b * cos), -1)


class Attention(nn.Module):
    def __init__(self, c, use_rope=True):
        super().__init__()
        self.h, self.kv, self.hd = c.heads, c.kv_heads, c.width // c.heads
        self.q = nn.Linear(c.width, c.heads * self.hd, bias=False)
        self.k = nn.Linear(c.width, c.kv_heads * self.hd, bias=False)
        self.v = nn.Linear(c.width, c.kv_heads * self.hd, bias=False)
        self.o = nn.Linear(c.heads * self.hd, c.width, bias=False)
        self.use_rope = use_rope

    def forward(self, x, mask):
        """mask: (B or 1, 1, T, T) booleano, True = pode atender."""
        b, t, _ = x.shape
        q = self.q(x).view(b, t, self.h, self.hd).transpose(1, 2)
        k = self.k(x).view(b, t, self.kv, self.hd).transpose(1, 2)
        v = self.v(x).view(b, t, self.kv, self.hd).transpose(1, 2)
        if self.use_rope:
            pos = torch.arange(t, device=x.device)
            q, k = rope(q, pos), rope(k, pos)
        rep = self.h // self.kv
        k, v = k.repeat_interleave(rep, 1), v.repeat_interleave(rep, 1)
        y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
        return self.o(y.transpose(1, 2).reshape(b, t, -1))


class GatedFFN(nn.Module):
    def __init__(self, c):
        super().__init__()
        self.gate = nn.Linear(c.width, c.ff, bias=False)
        self.up = nn.Linear(c.width, c.ff, bias=False)
        self.down = nn.Linear(c.ff, c.width, bias=False)

    def forward(self, x):
        return self.down(F.silu(self.gate(x)) * self.up(x))


class AttnBlock(nn.Module):
    def __init__(self, c, use_rope=True):
        super().__init__()
        self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width)
        self.attention, self.ffn = Attention(c, use_rope), GatedFFN(c)

    def forward(self, x, mask):
        x = x + self.attention(self.norm1(x), mask)
        return x + self.ffn(self.norm2(x))


class ConvBlock(nn.Module):
    def __init__(self, c, dilation):
        super().__init__()
        self.norm1, self.norm2 = RMSNorm(c.width), RMSNorm(c.width)
        pad = dilation * (c.kernel - 1) // 2
        self.conv = nn.Conv1d(c.width, c.width, c.kernel, padding=pad, dilation=dilation, bias=False)

    def forward(self, x, keep):
        h = self.norm1(x) * keep
        h = self.conv(h.transpose(1, 2)).transpose(1, 2)
        return x + F.silu(self.norm2(h))


class DurationPredictor(nn.Module):
    def __init__(self, c):
        super().__init__()
        w = c.dur_width
        self.in_proj = nn.Linear(c.width, w)
        self.dw = nn.ModuleList([nn.Conv1d(w, w, 3, padding=1, groups=w) for _ in range(2)])
        self.pw = nn.ModuleList([nn.Conv1d(w, w, 1) for _ in range(2)])
        self.norms = nn.ModuleList([RMSNorm(w) for _ in range(2)])
        self.out = nn.Linear(w, 1)

    def forward(self, x, keep):
        h = F.silu(self.in_proj(x)) * keep
        for dw, pw, n in zip(self.dw, self.pw, self.norms):
            y = pw(dw((n(h) * keep).transpose(1, 2))).transpose(1, 2)
            h = (h + F.silu(y)) * keep
        return self.out(h).squeeze(-1)


def frame_index(durations):
    """For each frame, the index of the token that generates it. -> idx (B,F), mask (B,F), total count (B,)."""
    total = durations.sum(1)
    n = int(total.max())
    ends = torch.cumsum(durations, 1)                                    # (B,L)
    frames = torch.arange(n, device=durations.device)
    idx = (frames[None, :, None] >= ends[:, None, :]).sum(-1).clamp(max=durations.shape[1] - 1)
    return idx, frames[None] < total[:, None], ends - durations


def phase_features(durations, max_token_frames=200):
    """Per-frame phase signals. durations: (B,L) int -> (B,F,4), mask (B,F)."""
    idx, mask, starts = frame_index(durations)
    counts = durations.gather(1, idx).clamp(min=1).float()
    st = starts.gather(1, idx).float()
    frac = (torch.arange(idx.shape[1], device=idx.device)[None].float() - st + 0.5) / counts
    out = torch.stack((2 * frac - 1, torch.sin(math.pi * frac), torch.cos(math.pi * frac),
                       torch.log1p(counts) / math.log1p(max_token_frames)), -1)
    return out * mask.unsqueeze(-1), mask


def expand(encoded, durations):
    """Repeat each token encoding for its duration. -> (B,F,W), mask (B,F)."""
    idx, mask, _ = frame_index(durations)
    out = encoded.gather(1, idx.unsqueeze(-1).expand(-1, -1, encoded.shape[-1]))
    return out * mask.unsqueeze(-1), mask


class Acoustic(nn.Module):
    def __init__(self, c: Config = Config()):
        super().__init__()
        self.c = c
        self.tokens = nn.Embedding(c.vocab, c.width, padding_idx=c.pad)
        self.voices = nn.Embedding(c.voices, c.width)
        self.stress = nn.Embedding(2, c.width)
        self.text_blocks = nn.ModuleList(
            [AttnBlock(c) if i % 2 == 0 else ConvBlock(c, 2 ** (i // 2)) for i in range(c.text_layers)])
        self.text_norm = RMSNorm(c.width)
        self.durations = DurationPredictor(c)
        self.phase = nn.Linear(4, c.width, bias=False)
        self.decoder = nn.ModuleList([AttnBlock(c, use_rope=(i == 0)) for i in range(c.decoder_layers)])
        self.feature_norm = RMSNorm(c.width)
        self.head = nn.Linear(c.width, c.feature_dim, bias=False)
        # feature statistics: the head predicts normalized values; the export folds mean/std into the weights
        self.register_buffer("f_mean", torch.zeros(c.feature_dim))
        self.register_buffer("f_std", torch.ones(c.feature_dim))

    def encode(self, tokens, stress, voice, lengths):
        b, t = tokens.shape
        keep = (torch.arange(t, device=tokens.device)[None] < lengths[:, None]).unsqueeze(-1).float()
        x = (self.tokens(tokens) + self.stress(stress) + self.voices(voice)[:, None]) * keep
        amask = (keep.squeeze(-1) > 0)[:, None, None, :].expand(b, 1, t, t)
        for blk in self.text_blocks:
            x = blk(x, amask) * keep if isinstance(blk, AttnBlock) else blk(x, keep) * keep
        x = self.text_norm(x) * keep
        return x, self.durations(x, keep)            # predicted log1p(duration)

    def decode(self, expanded, phase, fmask):
        """expanded (B,F,W), phase (B,F,4), fmask (B,F) -> (B,F,20)."""
        b, n, _ = expanded.shape
        i = torch.arange(n, device=expanded.device)
        causal = (i[None] <= i[:, None]) & (i[:, None] - i[None] < self.c.window)
        mask = (causal[None, None] & fmask[:, None, None, :]) | torch.eye(n, dtype=torch.bool, device=expanded.device)[None, None]
        # (keeping the diagonal open avoids fully masked rows in padding frames, which give NaN in CUDA SDPA)
        x = expanded + self.phase(phase)
        for blk in self.decoder:
            x = blk(x, mask)
        return self.head(self.feature_norm(x))

    def forward(self, tokens, stress, voice, lengths, durations):
        enc, log_dur = self.encode(tokens, stress, voice, lengths)
        expanded, fmask = expand(enc, durations)
        phase, _ = phase_features(durations, self.c.max_token_frames)
        return self.decode(expanded, phase, fmask), log_dur, fmask


if __name__ == "__main__":
    for w, ff in ((96, 192), (128, 256)):
        m = Acoustic(Config(width=w, ff=ff))
        n = sum(p.numel() for p in m.parameters())
        tok = torch.randint(1, 60, (2, 12)); lens = torch.tensor([12, 9]); voice = torch.tensor([0, 1])
        dur = torch.randint(1, 6, (2, 12)); dur[1, 9:] = 0
        feats, log_dur, fmask = m(tok, torch.zeros_like(tok), voice, lens, dur)
        loss = feats.pow(2).mean() + log_dur.pow(2).mean(); loss.backward()
        print(f"width {w}: {n:,} parameters | features {tuple(feats.shape)} | log_dur {tuple(log_dur.shape)}")