Spaces:
Running on Zero
Running on Zero
File size: 4,677 Bytes
ba6c090 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | """Automatic selection and inference wrapper for the bundled Mamba-3 MIMO kernels.
Includes Mamba-3 forward integration adapted by Continuous Intelligence:
https://github.com/state-spaces/mamba/tree/a14b1dff0454a3bc27d9eb31355dc01e4b2490ec
Copyright (c) 2026, Dao AI Lab, Goombalab.
Upstream portions are licensed under Apache-2.0; see NOTICE.
"""
from __future__ import annotations
import warnings
import torch
import torch.nn.functional as F
from einops import rearrange
from torch import Tensor, nn
from ._mamba import Mamba3
def configure_backend(model: nn.Module) -> None:
"""Enable fused inference only for supported hardware and cell geometry."""
model.backend = "torch"
device = next(model.parameters()).device
if device.type != "cuda" or torch.version.hip is not None:
return
cells = [cell for cell in model.modules() if isinstance(cell, Mamba3)]
if torch.cuda.get_device_capability(device)[0] < 8 or not cells:
return
# Float32 tiles need more shared memory than bf16. Use smaller chunks on
# Ampere and consumer GPUs.
chunk_budget = (
64
if torch.cuda.get_device_properties(device).shared_memory_per_block_optin >= 196608
else 32
)
if any(
not cell.is_mimo
or cell.mimo_rank not in (1, 2, 4, 8)
or cell.d_state not in (32, 64, 128)
or cell.headdim not in (32, 64, 128)
or max(8, chunk_budget // cell.mimo_rank) * cell.mimo_rank > chunk_budget
for cell in cells
):
return
try:
from .._vendor.mamba3.mamba3_mimo_fwd import mamba_mimo_forward # noqa: F401
except (ImportError, OSError) as error:
warnings.warn(
f"Fused Mamba-3 is unavailable; using portable PyTorch on CUDA: {error}",
RuntimeWarning,
stacklevel=2,
)
return
for cell in cells:
cell.chunk_size = max(8, chunk_budget // cell.mimo_rank)
cell.fused = True
model.backend = "mamba3"
def mamba3_forward(cell: Mamba3, u: Tensor) -> Tensor:
"""Run the forward path, including rank-scaled chunk padding."""
from .._vendor.mamba3.angle_dt import angle_dt_fwd
from .._vendor.mamba3.layernorm_gated import _layer_norm_fwd
from .._vendor.mamba3.mamba3_mimo_fwd import mamba_mimo_forward
from .._vendor.mamba3.mamba3_mimo_utils import compute_dacs_segsum_triton
seq = u.shape[1]
chunk = cell.chunk_size
u = F.pad(u, (0, 0, 0, (-seq) % chunk))
bc = cell.d_state * cell.mimo_rank
z, x, B, C, dd_dt, dd_A, trap, angles = torch.split(
cell.in_proj(u),
[
cell.d_inner,
cell.d_inner,
bc,
bc,
cell.nheads,
cell.nheads,
cell.nheads,
cell.num_rope_angles,
],
dim=-1,
)
x = rearrange(x, "b l (h p) -> b l h p", p=cell.headdim)
z = rearrange(z, "b l (h p) -> b l h p", p=cell.headdim)
B = rearrange(B, "b l (r g n) -> b l r g n", r=cell.mimo_rank, g=1)
C = rearrange(C, "b l (r g n) -> b l r g n", r=cell.mimo_rank, g=1)
# Keep the kernel's reduction and rounding order, including 1 / sqrt.
B = _layer_norm_fwd(
B.reshape(-1, cell.d_state),
cell.B_norm.weight,
None,
cell.B_norm.eps,
is_rms_norm=True,
)[0].view_as(B)
C = _layer_norm_fwd(
C.reshape(-1, cell.d_state),
cell.C_norm.weight,
None,
cell.C_norm.eps,
is_rms_norm=True,
)[0].view_as(C)
A = (-F.softplus(dd_A.float())).clamp(max=-1e-4)
dt = F.softplus(dd_dt + cell.dt_bias)
adt = rearrange(A * dt, "b l h -> b h l").contiguous()
dt = rearrange(dt, "b l h -> b h l").contiguous()
angles = angles[:, :, None].expand(-1, -1, cell.nheads, -1).float().contiguous()
with torch.cuda.device(u.device):
cumulative, _ = angle_dt_fwd(angles, dt, chunk_size=chunk, return_output_state=True)
da_cs, da_cs_rev, segsum = compute_dacs_segsum_triton(adt, chunk)
y, _, _ = mamba_mimo_forward(
C.contiguous(),
B.contiguous(),
x.contiguous(),
cell.C_bias,
cell.B_bias,
cell.mimo_x,
cell.mimo_o,
z.contiguous(),
cell.D,
cell.mimo_z,
cumulative,
da_cs,
da_cs_rev,
dt,
rearrange(trap, "b l h -> b h l").contiguous(),
segsum,
chunk_size=chunk,
rotary_dim_divisor=4,
dtype=x.dtype,
)
return cell.out_proj(rearrange(y, "b l h p -> b l (h p)").to(u.dtype))[:, :seq]
|