aria-tts / fastpath.py
VoiceClimax
Streaming speech service on ZeroGPU
b1c28a5
Raw History Blame Contribute Delete
10.2 kB
"""Speed-ups for the fish-speech the engine decode step that keep its arithmetic.
Decoding one frame at batch size 1 is bound by kernel count, not by FLOPs: the eager
step launches about 7,600 kernels. These changes cut that number:
- Horizontal GEMM fusion: projections that read the same input are one matmul over
their concatenated weights (GatedDeltaNet q/k/v/z/b/a, attention q/k/v, the MoE
router with the shared expert's gate and up projections, the tower's w1/w3). Each
output element is the same dot product as before.
- Single-query attention in fp32 (scores, softmax and the value sum), instead of the
memory-efficient SDPA kernel, which takes 40-65 us for one query row. Prefill keeps
SDPA.
- Regional torch.compile of the repeated layers (40 backbone, 13 tower), with
emulate_precision_casts so fused kernels round to bfloat16 where eager PyTorch does.
Sampling stays eager, with PyTorch's own random number generator.
"""
from __future__ import annotations
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from fish_speech.models.text2semantic.fish_omni import layers as fish_layers
from fish_speech.models.text2semantic.fish_omni import tower as fish_tower
from fish_speech.models.text2semantic.fish_omni.layers import (
FullAttention,
GatedDeltaNet,
LayerContext,
apply_neox_rope,
chunk_gated_delta_rule,
recurrent_gated_delta_rule,
)
from fish_speech.models.text2semantic.fish_omni.moe import SparseMoeBlock
from fish_speech.models.text2semantic.fish_omni.tower import TowerMLP
_sdpa_grouped_attention = fish_layers.grouped_attention
def fused_weight(module: nn.Module, name: str, linears: list[nn.Linear]) -> None:
"""Replace linears (no bias) by one buffer holding their stacked weights."""
weight = torch.cat([linear.weight.detach() for linear in linears])
module.register_buffer(name, weight, persistent=False)
# ─── Fused forwards (same operations as fish_speech, one GEMM per input) ──────
def gdn_forward(self: GatedDeltaNet, x: torch.Tensor, context: LayerContext) -> torch.Tensor:
batch, length, _ = x.shape
projected = F.linear(x, self.in_proj_fused)
mixed, z, b, a = projected.split(
[self.conv_dim, self.value_dim, self.num_v_heads, self.num_v_heads], dim=-1
)
mixed = mixed.transpose(1, 2) # [B, C, T]
history = self.conv_kernel_size - 1
if context.continue_state:
previous = self.conv_state[:batch].to(mixed.dtype)
else:
previous = mixed.new_zeros(batch, self.conv_dim, history)
padded = torch.cat((previous, mixed), dim=-1)
if context.use_cache:
self.conv_state[:batch].copy_(padded[..., -history:])
mixed = F.silu(F.conv1d(padded, self.conv1d.weight, groups=self.conv_dim))
query, key, value = mixed.transpose(1, 2).split(
[self.key_dim, self.key_dim, self.value_dim], dim=-1
)
query = query.reshape(batch, length, self.num_k_heads, self.head_k_dim)
key = key.reshape(batch, length, self.num_k_heads, self.head_k_dim)
value = value.reshape(batch, length, self.num_v_heads, self.head_v_dim)
z = z.reshape(batch, length, self.num_v_heads, self.head_v_dim)
beta = torch.sigmoid(b.float())
decay = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias.float())
groups = self.num_v_heads // self.num_k_heads
if groups > 1:
query = query.repeat_interleave(groups, dim=2)
key = key.repeat_interleave(groups, dim=2)
initial_state = self.recurrent_state[:batch] if context.continue_state else None
rule = (
recurrent_gated_delta_rule
if context.continue_state and length == 1
else chunk_gated_delta_rule
)
core, final_state = rule(query, key, value, decay, beta, initial_state)
if context.use_cache:
self.recurrent_state[:batch].copy_(final_state)
core = self.norm(
core.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim)
).reshape(batch, length, self.value_dim)
return self.out_proj(core)
def attention_forward(self: FullAttention, x: torch.Tensor, context: LayerContext) -> torch.Tensor:
batch, length, _ = x.shape
q_width = self.num_heads * self.head_dim * 2
kv_width = self.num_kv_heads * self.head_dim
query_gate, key, value = F.linear(x, self.qkv_fused).split([q_width, kv_width, kv_width], dim=-1)
query, gate = query_gate.view(batch, length, self.num_heads, 2 * self.head_dim).chunk(2, dim=-1)
gate = gate.reshape(batch, length, self.num_heads * self.head_dim)
query = self.q_norm(query)
key = self.k_norm(key.reshape(batch, length, self.num_kv_heads, self.head_dim))
value = value.reshape(batch, length, self.num_kv_heads, self.head_dim)
query = apply_neox_rope(query, context.cos, context.sin).transpose(1, 2)
key = apply_neox_rope(key, context.cos, context.sin).transpose(1, 2)
value = value.transpose(1, 2)
if context.use_cache:
self.k_cache[:batch].index_copy_(2, context.input_pos, key)
self.v_cache[:batch].index_copy_(2, context.input_pos, value)
key = self.k_cache[:batch, :, : context.kv_len]
value = self.v_cache[:batch, :, : context.kv_len]
key_positions = torch.arange(context.kv_len, device=x.device)
mask = key_positions[None, :] <= context.input_pos[:, None]
is_causal = False
else:
mask = None
is_causal = length > 1
attended = fish_layers.grouped_attention(
query, key, value, mask, is_causal=is_causal, scale=self.head_dim**-0.5
)
attended = attended.transpose(1, 2).reshape(batch, length, self.num_heads * self.head_dim)
return self.o_proj(attended * torch.sigmoid(gate))
def moe_forward(self: SparseMoeBlock, x: torch.Tensor) -> torch.Tensor:
flat = x.reshape(-1, x.shape[-1])
experts = self.experts.num_experts
shared = self.shared_intermediate
router, shared_gate, gate, up = F.linear(flat, self.router_shared_fused).split(
[experts, 1, shared, shared], dim=-1
)
probabilities = torch.softmax(router.float(), dim=-1)
topk_weights, topk_ids = probabilities.topk(self.top_k, dim=-1)
if self.norm_topk_prob:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
out = self.experts(flat, topk_ids, topk_weights.to(flat.dtype))
shared_out = self.shared_expert.down_proj(F.silu(gate) * up)
return (out + torch.sigmoid(shared_gate) * shared_out).view_as(x)
def tower_mlp_forward(self: TowerMLP, x: torch.Tensor) -> torch.Tensor:
gate, up = F.linear(x, self.w13_fused).chunk(2, dim=-1)
return self.w2(F.silu(gate) * up)
def decode_attention(query, key, value, mask, *, is_causal=False, scale=None):
"""grouped_attention, with one query row computed in fp32 without SDPA."""
batch, heads, length, head_dim = query.shape
if length != 1 or is_causal:
return _sdpa_grouped_attention(query, key, value, mask, is_causal=is_causal, scale=scale)
kv_heads = key.shape[1]
scale = head_dim**-0.5 if scale is None else scale
folded = query.reshape(batch, kv_heads, heads // kv_heads, head_dim).float()
scores = torch.matmul(folded, key.float().transpose(-1, -2)) * scale
if mask is not None:
scores = scores.masked_fill(~mask, float("-inf"))
out = torch.matmul(scores.softmax(dim=-1), value.float())
return out.reshape(batch, heads, 1, head_dim).to(query.dtype)
# ─── Installation ─────────────────────────────────────────────────────────────
def fuse_projections(model) -> None:
"""Stack weights that share an input; works on CPU before the move to the GPU."""
for layer in model.backbone.model.layers:
if layer.layer_type == "full_attention":
attention = layer.self_attn
fused_weight(attention, "qkv_fused", [attention.q_proj, attention.k_proj, attention.v_proj])
del attention.q_proj, attention.k_proj, attention.v_proj
attention.forward = attention_forward.__get__(attention)
else:
gdn = layer.linear_attn
fused_weight(gdn, "in_proj_fused", [gdn.in_proj_qkv, gdn.in_proj_z, gdn.in_proj_b, gdn.in_proj_a])
del gdn.in_proj_qkv, gdn.in_proj_z, gdn.in_proj_b, gdn.in_proj_a
gdn.forward = gdn_forward.__get__(gdn)
moe = layer.mlp
moe.shared_intermediate = moe.shared_expert.gate_proj.out_features
fused_weight(moe, "router_shared_fused",
[moe.gate, moe.shared_expert_gate, moe.shared_expert.gate_proj, moe.shared_expert.up_proj])
del moe.gate, moe.shared_expert_gate, moe.shared_expert.gate_proj, moe.shared_expert.up_proj
moe.forward = moe_forward.__get__(moe)
tower = model.audio_decoder
for layer in (*tower.shared_layers, *tower.residual_layers):
mlp = layer.transformer_layer.feed_forward
fused_weight(mlp, "w13_fused", [mlp.w1, mlp.w3])
del mlp.w1, mlp.w3
mlp.forward = tower_mlp_forward.__get__(mlp)
def install_decode_attention() -> None:
fish_layers.grouped_attention = decode_attention
fish_tower.grouped_attention = decode_attention
def compile_layers(model) -> None:
"""Regional compilation: one compiled graph per layer kind, reused by every layer."""
import torch._inductor.config as inductor_config
inductor_config.emulate_precision_casts = True
torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 32)
tower = model.audio_decoder
for layer in (*model.backbone.model.layers, *tower.shared_layers, *tower.residual_layers):
eager = layer.forward
compiled = torch.compile(eager, dynamic=False)
def forward(x, *args, _eager=eager, _compiled=compiled, **kwargs):
# Prefill rows vary in number; only the one-row decode step is compiled.
rows = x.shape[-2] if x.dim() == 3 else x.shape[0]
return (_compiled if rows == 1 else _eager)(x, *args, **kwargs)
layer.forward = forward
def enabled() -> bool:
return os.environ.get("ENGINE_COMPILE", "1") != "0"