File size: 4,677 Bytes
ba6c090
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""Automatic selection and inference wrapper for the bundled Mamba-3 MIMO kernels.

Includes Mamba-3 forward integration adapted by Continuous Intelligence:
https://github.com/state-spaces/mamba/tree/a14b1dff0454a3bc27d9eb31355dc01e4b2490ec

Copyright (c) 2026, Dao AI Lab, Goombalab.
Upstream portions are licensed under Apache-2.0; see NOTICE.
"""

from __future__ import annotations

import warnings

import torch
import torch.nn.functional as F
from einops import rearrange
from torch import Tensor, nn

from ._mamba import Mamba3


def configure_backend(model: nn.Module) -> None:
    """Enable fused inference only for supported hardware and cell geometry."""
    model.backend = "torch"
    device = next(model.parameters()).device
    if device.type != "cuda" or torch.version.hip is not None:
        return
    cells = [cell for cell in model.modules() if isinstance(cell, Mamba3)]
    if torch.cuda.get_device_capability(device)[0] < 8 or not cells:
        return
    # Float32 tiles need more shared memory than bf16. Use smaller chunks on
    # Ampere and consumer GPUs.
    chunk_budget = (
        64
        if torch.cuda.get_device_properties(device).shared_memory_per_block_optin >= 196608
        else 32
    )
    if any(
        not cell.is_mimo
        or cell.mimo_rank not in (1, 2, 4, 8)
        or cell.d_state not in (32, 64, 128)
        or cell.headdim not in (32, 64, 128)
        or max(8, chunk_budget // cell.mimo_rank) * cell.mimo_rank > chunk_budget
        for cell in cells
    ):
        return
    try:
        from .._vendor.mamba3.mamba3_mimo_fwd import mamba_mimo_forward  # noqa: F401
    except (ImportError, OSError) as error:
        warnings.warn(
            f"Fused Mamba-3 is unavailable; using portable PyTorch on CUDA: {error}",
            RuntimeWarning,
            stacklevel=2,
        )
        return
    for cell in cells:
        cell.chunk_size = max(8, chunk_budget // cell.mimo_rank)
        cell.fused = True
    model.backend = "mamba3"


def mamba3_forward(cell: Mamba3, u: Tensor) -> Tensor:
    """Run the forward path, including rank-scaled chunk padding."""
    from .._vendor.mamba3.angle_dt import angle_dt_fwd
    from .._vendor.mamba3.layernorm_gated import _layer_norm_fwd
    from .._vendor.mamba3.mamba3_mimo_fwd import mamba_mimo_forward
    from .._vendor.mamba3.mamba3_mimo_utils import compute_dacs_segsum_triton

    seq = u.shape[1]
    chunk = cell.chunk_size
    u = F.pad(u, (0, 0, 0, (-seq) % chunk))
    bc = cell.d_state * cell.mimo_rank
    z, x, B, C, dd_dt, dd_A, trap, angles = torch.split(
        cell.in_proj(u),
        [
            cell.d_inner,
            cell.d_inner,
            bc,
            bc,
            cell.nheads,
            cell.nheads,
            cell.nheads,
            cell.num_rope_angles,
        ],
        dim=-1,
    )
    x = rearrange(x, "b l (h p) -> b l h p", p=cell.headdim)
    z = rearrange(z, "b l (h p) -> b l h p", p=cell.headdim)
    B = rearrange(B, "b l (r g n) -> b l r g n", r=cell.mimo_rank, g=1)
    C = rearrange(C, "b l (r g n) -> b l r g n", r=cell.mimo_rank, g=1)
    # Keep the kernel's reduction and rounding order, including 1 / sqrt.
    B = _layer_norm_fwd(
        B.reshape(-1, cell.d_state),
        cell.B_norm.weight,
        None,
        cell.B_norm.eps,
        is_rms_norm=True,
    )[0].view_as(B)
    C = _layer_norm_fwd(
        C.reshape(-1, cell.d_state),
        cell.C_norm.weight,
        None,
        cell.C_norm.eps,
        is_rms_norm=True,
    )[0].view_as(C)
    A = (-F.softplus(dd_A.float())).clamp(max=-1e-4)
    dt = F.softplus(dd_dt + cell.dt_bias)
    adt = rearrange(A * dt, "b l h -> b h l").contiguous()
    dt = rearrange(dt, "b l h -> b h l").contiguous()
    angles = angles[:, :, None].expand(-1, -1, cell.nheads, -1).float().contiguous()
    with torch.cuda.device(u.device):
        cumulative, _ = angle_dt_fwd(angles, dt, chunk_size=chunk, return_output_state=True)
        da_cs, da_cs_rev, segsum = compute_dacs_segsum_triton(adt, chunk)
        y, _, _ = mamba_mimo_forward(
            C.contiguous(),
            B.contiguous(),
            x.contiguous(),
            cell.C_bias,
            cell.B_bias,
            cell.mimo_x,
            cell.mimo_o,
            z.contiguous(),
            cell.D,
            cell.mimo_z,
            cumulative,
            da_cs,
            da_cs_rev,
            dt,
            rearrange(trap, "b l h -> b h l").contiguous(),
            segsum,
            chunk_size=chunk,
            rotary_dim_divisor=4,
            dtype=x.dtype,
        )
    return cell.out_proj(rearrange(y, "b l h p -> b l (h p)").to(u.dtype))[:, :seq]