"""Speed-ups for the fish-speech the engine decode step that keep its arithmetic. Decoding one frame at batch size 1 is bound by kernel count, not by FLOPs: the eager step launches about 7,600 kernels. These changes cut that number: - Horizontal GEMM fusion: projections that read the same input are one matmul over their concatenated weights (GatedDeltaNet q/k/v/z/b/a, attention q/k/v, the MoE router with the shared expert's gate and up projections, the tower's w1/w3). Each output element is the same dot product as before. - Single-query attention in fp32 (scores, softmax and the value sum), instead of the memory-efficient SDPA kernel, which takes 40-65 us for one query row. Prefill keeps SDPA. - Regional torch.compile of the repeated layers (40 backbone, 13 tower), with emulate_precision_casts so fused kernels round to bfloat16 where eager PyTorch does. Sampling stays eager, with PyTorch's own random number generator. """ from __future__ import annotations import os import torch import torch.nn as nn import torch.nn.functional as F from fish_speech.models.text2semantic.fish_omni import layers as fish_layers from fish_speech.models.text2semantic.fish_omni import tower as fish_tower from fish_speech.models.text2semantic.fish_omni.layers import ( FullAttention, GatedDeltaNet, LayerContext, apply_neox_rope, chunk_gated_delta_rule, recurrent_gated_delta_rule, ) from fish_speech.models.text2semantic.fish_omni.moe import SparseMoeBlock from fish_speech.models.text2semantic.fish_omni.tower import TowerMLP _sdpa_grouped_attention = fish_layers.grouped_attention def fused_weight(module: nn.Module, name: str, linears: list[nn.Linear]) -> None: """Replace linears (no bias) by one buffer holding their stacked weights.""" weight = torch.cat([linear.weight.detach() for linear in linears]) module.register_buffer(name, weight, persistent=False) # ─── Fused forwards (same operations as fish_speech, one GEMM per input) ────── def gdn_forward(self: GatedDeltaNet, x: torch.Tensor, context: LayerContext) -> torch.Tensor: batch, length, _ = x.shape projected = F.linear(x, self.in_proj_fused) mixed, z, b, a = projected.split( [self.conv_dim, self.value_dim, self.num_v_heads, self.num_v_heads], dim=-1 ) mixed = mixed.transpose(1, 2) # [B, C, T] history = self.conv_kernel_size - 1 if context.continue_state: previous = self.conv_state[:batch].to(mixed.dtype) else: previous = mixed.new_zeros(batch, self.conv_dim, history) padded = torch.cat((previous, mixed), dim=-1) if context.use_cache: self.conv_state[:batch].copy_(padded[..., -history:]) mixed = F.silu(F.conv1d(padded, self.conv1d.weight, groups=self.conv_dim)) query, key, value = mixed.transpose(1, 2).split( [self.key_dim, self.key_dim, self.value_dim], dim=-1 ) query = query.reshape(batch, length, self.num_k_heads, self.head_k_dim) key = key.reshape(batch, length, self.num_k_heads, self.head_k_dim) value = value.reshape(batch, length, self.num_v_heads, self.head_v_dim) z = z.reshape(batch, length, self.num_v_heads, self.head_v_dim) beta = torch.sigmoid(b.float()) decay = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias.float()) groups = self.num_v_heads // self.num_k_heads if groups > 1: query = query.repeat_interleave(groups, dim=2) key = key.repeat_interleave(groups, dim=2) initial_state = self.recurrent_state[:batch] if context.continue_state else None rule = ( recurrent_gated_delta_rule if context.continue_state and length == 1 else chunk_gated_delta_rule ) core, final_state = rule(query, key, value, decay, beta, initial_state) if context.use_cache: self.recurrent_state[:batch].copy_(final_state) core = self.norm( core.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim) ).reshape(batch, length, self.value_dim) return self.out_proj(core) def attention_forward(self: FullAttention, x: torch.Tensor, context: LayerContext) -> torch.Tensor: batch, length, _ = x.shape q_width = self.num_heads * self.head_dim * 2 kv_width = self.num_kv_heads * self.head_dim query_gate, key, value = F.linear(x, self.qkv_fused).split([q_width, kv_width, kv_width], dim=-1) query, gate = query_gate.view(batch, length, self.num_heads, 2 * self.head_dim).chunk(2, dim=-1) gate = gate.reshape(batch, length, self.num_heads * self.head_dim) query = self.q_norm(query) key = self.k_norm(key.reshape(batch, length, self.num_kv_heads, self.head_dim)) value = value.reshape(batch, length, self.num_kv_heads, self.head_dim) query = apply_neox_rope(query, context.cos, context.sin).transpose(1, 2) key = apply_neox_rope(key, context.cos, context.sin).transpose(1, 2) value = value.transpose(1, 2) if context.use_cache: self.k_cache[:batch].index_copy_(2, context.input_pos, key) self.v_cache[:batch].index_copy_(2, context.input_pos, value) key = self.k_cache[:batch, :, : context.kv_len] value = self.v_cache[:batch, :, : context.kv_len] key_positions = torch.arange(context.kv_len, device=x.device) mask = key_positions[None, :] <= context.input_pos[:, None] is_causal = False else: mask = None is_causal = length > 1 attended = fish_layers.grouped_attention( query, key, value, mask, is_causal=is_causal, scale=self.head_dim**-0.5 ) attended = attended.transpose(1, 2).reshape(batch, length, self.num_heads * self.head_dim) return self.o_proj(attended * torch.sigmoid(gate)) def moe_forward(self: SparseMoeBlock, x: torch.Tensor) -> torch.Tensor: flat = x.reshape(-1, x.shape[-1]) experts = self.experts.num_experts shared = self.shared_intermediate router, shared_gate, gate, up = F.linear(flat, self.router_shared_fused).split( [experts, 1, shared, shared], dim=-1 ) probabilities = torch.softmax(router.float(), dim=-1) topk_weights, topk_ids = probabilities.topk(self.top_k, dim=-1) if self.norm_topk_prob: topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) out = self.experts(flat, topk_ids, topk_weights.to(flat.dtype)) shared_out = self.shared_expert.down_proj(F.silu(gate) * up) return (out + torch.sigmoid(shared_gate) * shared_out).view_as(x) def tower_mlp_forward(self: TowerMLP, x: torch.Tensor) -> torch.Tensor: gate, up = F.linear(x, self.w13_fused).chunk(2, dim=-1) return self.w2(F.silu(gate) * up) def decode_attention(query, key, value, mask, *, is_causal=False, scale=None): """grouped_attention, with one query row computed in fp32 without SDPA.""" batch, heads, length, head_dim = query.shape if length != 1 or is_causal: return _sdpa_grouped_attention(query, key, value, mask, is_causal=is_causal, scale=scale) kv_heads = key.shape[1] scale = head_dim**-0.5 if scale is None else scale folded = query.reshape(batch, kv_heads, heads // kv_heads, head_dim).float() scores = torch.matmul(folded, key.float().transpose(-1, -2)) * scale if mask is not None: scores = scores.masked_fill(~mask, float("-inf")) out = torch.matmul(scores.softmax(dim=-1), value.float()) return out.reshape(batch, heads, 1, head_dim).to(query.dtype) # ─── Installation ───────────────────────────────────────────────────────────── def fuse_projections(model) -> None: """Stack weights that share an input; works on CPU before the move to the GPU.""" for layer in model.backbone.model.layers: if layer.layer_type == "full_attention": attention = layer.self_attn fused_weight(attention, "qkv_fused", [attention.q_proj, attention.k_proj, attention.v_proj]) del attention.q_proj, attention.k_proj, attention.v_proj attention.forward = attention_forward.__get__(attention) else: gdn = layer.linear_attn fused_weight(gdn, "in_proj_fused", [gdn.in_proj_qkv, gdn.in_proj_z, gdn.in_proj_b, gdn.in_proj_a]) del gdn.in_proj_qkv, gdn.in_proj_z, gdn.in_proj_b, gdn.in_proj_a gdn.forward = gdn_forward.__get__(gdn) moe = layer.mlp moe.shared_intermediate = moe.shared_expert.gate_proj.out_features fused_weight(moe, "router_shared_fused", [moe.gate, moe.shared_expert_gate, moe.shared_expert.gate_proj, moe.shared_expert.up_proj]) del moe.gate, moe.shared_expert_gate, moe.shared_expert.gate_proj, moe.shared_expert.up_proj moe.forward = moe_forward.__get__(moe) tower = model.audio_decoder for layer in (*tower.shared_layers, *tower.residual_layers): mlp = layer.transformer_layer.feed_forward fused_weight(mlp, "w13_fused", [mlp.w1, mlp.w3]) del mlp.w1, mlp.w3 mlp.forward = tower_mlp_forward.__get__(mlp) def install_decode_attention() -> None: fish_layers.grouped_attention = decode_attention fish_tower.grouped_attention = decode_attention def compile_layers(model) -> None: """Regional compilation: one compiled graph per layer kind, reused by every layer.""" import torch._inductor.config as inductor_config inductor_config.emulate_precision_casts = True torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 32) tower = model.audio_decoder for layer in (*model.backbone.model.layers, *tower.shared_layers, *tower.residual_layers): eager = layer.forward compiled = torch.compile(eager, dynamic=False) def forward(x, *args, _eager=eager, _compiled=compiled, **kwargs): # Prefill rows vary in number; only the one-row decode step is compiled. rows = x.shape[-2] if x.dim() == 3 else x.shape[0] return (_compiled if rows == 1 else _eager)(x, *args, **kwargs) layer.forward = forward def enabled() -> bool: return os.environ.get("ENGINE_COMPILE", "1") != "0"