Download fastpath.py from vclmax/aria-tts: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/spaces/vclmax/aria-tts/resolve/main/fastpath.py
- Command line
-
hf download hf://spaces/vclmax/aria-tts/fastpath.py
-
curl -L -o fastpath.py https://huggingface.co/spaces/vclmax/aria-tts/resolve/main/fastpath.py
10.2 kB
| """Speed-ups for the fish-speech the engine decode step that keep its arithmetic. | |
| Decoding one frame at batch size 1 is bound by kernel count, not by FLOPs: the eager | |
| step launches about 7,600 kernels. These changes cut that number: | |
| - Horizontal GEMM fusion: projections that read the same input are one matmul over | |
| their concatenated weights (GatedDeltaNet q/k/v/z/b/a, attention q/k/v, the MoE | |
| router with the shared expert's gate and up projections, the tower's w1/w3). Each | |
| output element is the same dot product as before. | |
| - Single-query attention in fp32 (scores, softmax and the value sum), instead of the | |
| memory-efficient SDPA kernel, which takes 40-65 us for one query row. Prefill keeps | |
| SDPA. | |
| - Regional torch.compile of the repeated layers (40 backbone, 13 tower), with | |
| emulate_precision_casts so fused kernels round to bfloat16 where eager PyTorch does. | |
| Sampling stays eager, with PyTorch's own random number generator. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from fish_speech.models.text2semantic.fish_omni import layers as fish_layers | |
| from fish_speech.models.text2semantic.fish_omni import tower as fish_tower | |
| from fish_speech.models.text2semantic.fish_omni.layers import ( | |
| FullAttention, | |
| GatedDeltaNet, | |
| LayerContext, | |
| apply_neox_rope, | |
| chunk_gated_delta_rule, | |
| recurrent_gated_delta_rule, | |
| ) | |
| from fish_speech.models.text2semantic.fish_omni.moe import SparseMoeBlock | |
| from fish_speech.models.text2semantic.fish_omni.tower import TowerMLP | |
| _sdpa_grouped_attention = fish_layers.grouped_attention | |
| def fused_weight(module: nn.Module, name: str, linears: list[nn.Linear]) -> None: | |
| """Replace linears (no bias) by one buffer holding their stacked weights.""" | |
| weight = torch.cat([linear.weight.detach() for linear in linears]) | |
| module.register_buffer(name, weight, persistent=False) | |
| # βββ Fused forwards (same operations as fish_speech, one GEMM per input) ββββββ | |
| def gdn_forward(self: GatedDeltaNet, x: torch.Tensor, context: LayerContext) -> torch.Tensor: | |
| batch, length, _ = x.shape | |
| projected = F.linear(x, self.in_proj_fused) | |
| mixed, z, b, a = projected.split( | |
| [self.conv_dim, self.value_dim, self.num_v_heads, self.num_v_heads], dim=-1 | |
| ) | |
| mixed = mixed.transpose(1, 2) # [B, C, T] | |
| history = self.conv_kernel_size - 1 | |
| if context.continue_state: | |
| previous = self.conv_state[:batch].to(mixed.dtype) | |
| else: | |
| previous = mixed.new_zeros(batch, self.conv_dim, history) | |
| padded = torch.cat((previous, mixed), dim=-1) | |
| if context.use_cache: | |
| self.conv_state[:batch].copy_(padded[..., -history:]) | |
| mixed = F.silu(F.conv1d(padded, self.conv1d.weight, groups=self.conv_dim)) | |
| query, key, value = mixed.transpose(1, 2).split( | |
| [self.key_dim, self.key_dim, self.value_dim], dim=-1 | |
| ) | |
| query = query.reshape(batch, length, self.num_k_heads, self.head_k_dim) | |
| key = key.reshape(batch, length, self.num_k_heads, self.head_k_dim) | |
| value = value.reshape(batch, length, self.num_v_heads, self.head_v_dim) | |
| z = z.reshape(batch, length, self.num_v_heads, self.head_v_dim) | |
| beta = torch.sigmoid(b.float()) | |
| decay = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias.float()) | |
| groups = self.num_v_heads // self.num_k_heads | |
| if groups > 1: | |
| query = query.repeat_interleave(groups, dim=2) | |
| key = key.repeat_interleave(groups, dim=2) | |
| initial_state = self.recurrent_state[:batch] if context.continue_state else None | |
| rule = ( | |
| recurrent_gated_delta_rule | |
| if context.continue_state and length == 1 | |
| else chunk_gated_delta_rule | |
| ) | |
| core, final_state = rule(query, key, value, decay, beta, initial_state) | |
| if context.use_cache: | |
| self.recurrent_state[:batch].copy_(final_state) | |
| core = self.norm( | |
| core.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim) | |
| ).reshape(batch, length, self.value_dim) | |
| return self.out_proj(core) | |
| def attention_forward(self: FullAttention, x: torch.Tensor, context: LayerContext) -> torch.Tensor: | |
| batch, length, _ = x.shape | |
| q_width = self.num_heads * self.head_dim * 2 | |
| kv_width = self.num_kv_heads * self.head_dim | |
| query_gate, key, value = F.linear(x, self.qkv_fused).split([q_width, kv_width, kv_width], dim=-1) | |
| query, gate = query_gate.view(batch, length, self.num_heads, 2 * self.head_dim).chunk(2, dim=-1) | |
| gate = gate.reshape(batch, length, self.num_heads * self.head_dim) | |
| query = self.q_norm(query) | |
| key = self.k_norm(key.reshape(batch, length, self.num_kv_heads, self.head_dim)) | |
| value = value.reshape(batch, length, self.num_kv_heads, self.head_dim) | |
| query = apply_neox_rope(query, context.cos, context.sin).transpose(1, 2) | |
| key = apply_neox_rope(key, context.cos, context.sin).transpose(1, 2) | |
| value = value.transpose(1, 2) | |
| if context.use_cache: | |
| self.k_cache[:batch].index_copy_(2, context.input_pos, key) | |
| self.v_cache[:batch].index_copy_(2, context.input_pos, value) | |
| key = self.k_cache[:batch, :, : context.kv_len] | |
| value = self.v_cache[:batch, :, : context.kv_len] | |
| key_positions = torch.arange(context.kv_len, device=x.device) | |
| mask = key_positions[None, :] <= context.input_pos[:, None] | |
| is_causal = False | |
| else: | |
| mask = None | |
| is_causal = length > 1 | |
| attended = fish_layers.grouped_attention( | |
| query, key, value, mask, is_causal=is_causal, scale=self.head_dim**-0.5 | |
| ) | |
| attended = attended.transpose(1, 2).reshape(batch, length, self.num_heads * self.head_dim) | |
| return self.o_proj(attended * torch.sigmoid(gate)) | |
| def moe_forward(self: SparseMoeBlock, x: torch.Tensor) -> torch.Tensor: | |
| flat = x.reshape(-1, x.shape[-1]) | |
| experts = self.experts.num_experts | |
| shared = self.shared_intermediate | |
| router, shared_gate, gate, up = F.linear(flat, self.router_shared_fused).split( | |
| [experts, 1, shared, shared], dim=-1 | |
| ) | |
| probabilities = torch.softmax(router.float(), dim=-1) | |
| topk_weights, topk_ids = probabilities.topk(self.top_k, dim=-1) | |
| if self.norm_topk_prob: | |
| topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) | |
| out = self.experts(flat, topk_ids, topk_weights.to(flat.dtype)) | |
| shared_out = self.shared_expert.down_proj(F.silu(gate) * up) | |
| return (out + torch.sigmoid(shared_gate) * shared_out).view_as(x) | |
| def tower_mlp_forward(self: TowerMLP, x: torch.Tensor) -> torch.Tensor: | |
| gate, up = F.linear(x, self.w13_fused).chunk(2, dim=-1) | |
| return self.w2(F.silu(gate) * up) | |
| def decode_attention(query, key, value, mask, *, is_causal=False, scale=None): | |
| """grouped_attention, with one query row computed in fp32 without SDPA.""" | |
| batch, heads, length, head_dim = query.shape | |
| if length != 1 or is_causal: | |
| return _sdpa_grouped_attention(query, key, value, mask, is_causal=is_causal, scale=scale) | |
| kv_heads = key.shape[1] | |
| scale = head_dim**-0.5 if scale is None else scale | |
| folded = query.reshape(batch, kv_heads, heads // kv_heads, head_dim).float() | |
| scores = torch.matmul(folded, key.float().transpose(-1, -2)) * scale | |
| if mask is not None: | |
| scores = scores.masked_fill(~mask, float("-inf")) | |
| out = torch.matmul(scores.softmax(dim=-1), value.float()) | |
| return out.reshape(batch, heads, 1, head_dim).to(query.dtype) | |
| # βββ Installation βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def fuse_projections(model) -> None: | |
| """Stack weights that share an input; works on CPU before the move to the GPU.""" | |
| for layer in model.backbone.model.layers: | |
| if layer.layer_type == "full_attention": | |
| attention = layer.self_attn | |
| fused_weight(attention, "qkv_fused", [attention.q_proj, attention.k_proj, attention.v_proj]) | |
| del attention.q_proj, attention.k_proj, attention.v_proj | |
| attention.forward = attention_forward.__get__(attention) | |
| else: | |
| gdn = layer.linear_attn | |
| fused_weight(gdn, "in_proj_fused", [gdn.in_proj_qkv, gdn.in_proj_z, gdn.in_proj_b, gdn.in_proj_a]) | |
| del gdn.in_proj_qkv, gdn.in_proj_z, gdn.in_proj_b, gdn.in_proj_a | |
| gdn.forward = gdn_forward.__get__(gdn) | |
| moe = layer.mlp | |
| moe.shared_intermediate = moe.shared_expert.gate_proj.out_features | |
| fused_weight(moe, "router_shared_fused", | |
| [moe.gate, moe.shared_expert_gate, moe.shared_expert.gate_proj, moe.shared_expert.up_proj]) | |
| del moe.gate, moe.shared_expert_gate, moe.shared_expert.gate_proj, moe.shared_expert.up_proj | |
| moe.forward = moe_forward.__get__(moe) | |
| tower = model.audio_decoder | |
| for layer in (*tower.shared_layers, *tower.residual_layers): | |
| mlp = layer.transformer_layer.feed_forward | |
| fused_weight(mlp, "w13_fused", [mlp.w1, mlp.w3]) | |
| del mlp.w1, mlp.w3 | |
| mlp.forward = tower_mlp_forward.__get__(mlp) | |
| def install_decode_attention() -> None: | |
| fish_layers.grouped_attention = decode_attention | |
| fish_tower.grouped_attention = decode_attention | |
| def compile_layers(model) -> None: | |
| """Regional compilation: one compiled graph per layer kind, reused by every layer.""" | |
| import torch._inductor.config as inductor_config | |
| inductor_config.emulate_precision_casts = True | |
| torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 32) | |
| tower = model.audio_decoder | |
| for layer in (*model.backbone.model.layers, *tower.shared_layers, *tower.residual_layers): | |
| eager = layer.forward | |
| compiled = torch.compile(eager, dynamic=False) | |
| def forward(x, *args, _eager=eager, _compiled=compiled, **kwargs): | |
| # Prefill rows vary in number; only the one-row decode step is compiled. | |
| rows = x.shape[-2] if x.dim() == 3 else x.shape[0] | |
| return (_compiled if rows == 1 else _eager)(x, *args, **kwargs) | |
| layer.forward = forward | |
| def enabled() -> bool: | |
| return os.environ.get("ENGINE_COMPILE", "1") != "0" | |