zefancainlp's picture
download
raw
9.28 kB
"""
TriMul “outgoing” forward kernel (AlphaFold3).
The implementation follows the PyTorch reference:
1. Layer‑Norm on the last dimension (dim).
2. Linear projections to a hidden dimension (hidden_dim) for left/right.
3. Optional pair‑wise mask (unsqueezed to hidden_dim).
4. Gate linear → sigmoid → element‑wise gating.
5. Core contraction:
out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d]
which is a batched mat‑mul left @ rightᵀ for each hidden channel.
The heavy N³ work is done by a custom Triton kernel that processes all
(batch·hidden_dim) matrix‑multiplications in parallel.
6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back
to the original dimension.
All heavy arithmetic (projections, mask, gates, contraction, final linear)
is performed in float16 (FP16) for speed on H100 tensor‑cores, while the
final output is returned in float32.
"""
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
# ----------------------------------------------------------------------
# Triton kernel: batched mat‑mul of (bc, M, K) @ (bc, N, K)ᵀ → (bc, M, N)
# ----------------------------------------------------------------------
@triton.jit
def batched_matmul_kernel(
A_ptr, B_ptr, C_ptr, # pointers
M, N, K, # matrix sizes
stride_a_bc, stride_am, stride_ak, # A strides
stride_b_bc, stride_bk, stride_bn, # B strides (B is read as (bc, K, N))
stride_c_bc, stride_cm, stride_cn, # C strides
BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr
):
pid_m = tl.program_id(0) # block index over M
pid_n = tl.program_id(1) # block index over N
pid_bc = tl.program_id(2) # block index over batch*hidden
# Offsets within the block
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
# Pointers for the current block of A and B
a_ptrs = A_ptr + pid_bc * stride_a_bc \
+ offs_m[:, None] * stride_am \
+ offs_k[None, :] * stride_ak
# B is accessed as (bc, K, N) → original B layout (bc, N, K)
b_ptrs = B_ptr + pid_bc * stride_b_bc \
+ offs_k[:, None] * stride_bk \
+ offs_n[None, :] * stride_bn
# Accumulator in FP32 for higher precision
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
# Loop over K dimension
for _ in range(tl.cdiv(K, BLOCK_K)):
a = tl.load(a_ptrs,
mask=(offs_m[:, None] < M) & (offs_k[None, :] < K),
other=0.0)
b = tl.load(b_ptrs,
mask=(offs_k[:, None] < K) & (offs_n[None, :] < N),
other=0.0)
# a : (BLOCK_M, BLOCK_K) b : (BLOCK_K, BLOCK_N)
acc += tl.dot(a, b) # promotion to FP32 happens automatically
a_ptrs += BLOCK_K * stride_ak
b_ptrs += BLOCK_K * stride_bk
# Write result back (store as FP16)
c_ptrs = C_ptr + pid_bc * stride_c_bc \
+ offs_m[:, None] * stride_cm \
+ offs_n[None, :] * stride_cn
tl.store(c_ptrs,
acc.to(tl.float16),
mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))
def batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:
"""
Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K)
using the Triton kernel above. All tensors must be FP16 and contiguous.
"""
assert A.dtype == torch.float16 and B.dtype == torch.float16
bc, M, K = A.shape
_, N, K2 = B.shape
assert K == K2, "Inner dimensions must match"
# Output tensor
C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16)
# Strides (in elements, not bytes)
stride_a_bc, stride_am, stride_ak = A.stride()
stride_b_bc, stride_bn, stride_bk = B.stride() # B is (bc, N, K) → (bc, row, col)
stride_c_bc, stride_cm, stride_cn = C.stride()
# Block sizes – tuned for typical N up to 1024 on H100
BLOCK_M = 128
BLOCK_N = 128
BLOCK_K = 32
grid = (
(M + BLOCK_M - 1) // BLOCK_M,
(N + BLOCK_N - 1) // BLOCK_N,
bc,
)
batched_matmul_kernel[grid](
A, B, C,
M, N, K,
stride_a_bc, stride_am, stride_ak,
stride_b_bc, stride_bk, stride_bn,
stride_c_bc, stride_cm, stride_cn,
BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K,
)
return C
# ----------------------------------------------------------------------
# Entry point used by the test harness
# ----------------------------------------------------------------------
def custom_kernel(data):
"""
Arguments
---------
data : tuple
(input_tensor, mask, weights, config)
Returns
-------
torch.Tensor
Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim],
dtype torch.float32.
"""
input_tensor, mask, weights, config = data
device = input_tensor.device
eps = 1e-5
dim = config["dim"]
hidden_dim = config["hidden_dim"]
use_mask = not config.get("nomask", True)
# ------------------------------------------------------------------
# Extract and cast weights
# ------------------------------------------------------------------
# LayerNorm on input
norm_weight = weights["norm.weight"].to(torch.float32)
norm_bias = weights["norm.bias"].to(torch.float32)
# Linear projections (float16 for speed)
left_proj_w = weights["left_proj.weight"].to(torch.float16)
right_proj_w = weights["right_proj.weight"].to(torch.float16)
# Gates (float16)
left_gate_w = weights["left_gate.weight"].to(torch.float16)
right_gate_w = weights["right_gate.weight"].to(torch.float16)
out_gate_w = weights["out_gate.weight"].to(torch.float16)
# Output norm
to_out_norm_w = weights["to_out_norm.weight"].to(torch.float16)
to_out_norm_b = weights["to_out_norm.bias"].to(torch.float16)
# Final linear
to_out_w = weights["to_out.weight"].to(torch.float16)
# ------------------------------------------------------------------
# 1) Input layer‑norm (keep in FP32 for numerical stability)
# ------------------------------------------------------------------
x_norm = F.layer_norm(
input_tensor,
normalized_shape=(dim,),
weight=norm_weight,
bias=norm_bias,
eps=eps,
)
# Cast to FP16 for the remaining heavy ops
x_norm_h = x_norm.to(torch.float16)
# ------------------------------------------------------------------
# 2) Linear projections
# ------------------------------------------------------------------
left = F.linear(x_norm_h, left_proj_w) # [B, N, N, hidden_dim]
right = F.linear(x_norm_h, right_proj_w)
# ------------------------------------------------------------------
# 3) Optional mask
# ------------------------------------------------------------------
if use_mask and mask is not None:
# mask shape: [B, N, N]; broadcast over hidden_dim
mask_h = mask.to(torch.float16).unsqueeze(-1) # → [B, N, N, 1]
left = left * mask_h
right = right * mask_h
# ------------------------------------------------------------------
# 4) Gates (linear + sigmoid) and gating
# ------------------------------------------------------------------
left_gate = F.linear(x_norm_h, left_gate_w).sigmoid()
right_gate = F.linear(x_norm_h, right_gate_w).sigmoid()
out_gate = F.linear(x_norm_h, out_gate_w).sigmoid()
left = left * left_gate
right = right * right_gate
# ------------------------------------------------------------------
# 5) Core contraction: batched mat‑mul using Triton
# left, right : [B, N, N, hidden_dim] → permute to [B, hidden, N, N]
# ------------------------------------------------------------------
B, N, _, _ = left.shape
left_perm = left.permute(0, 3, 1, 2).contiguous() # [B, hidden, N, N]
right_perm = right.permute(0, 3, 1, 2).contiguous()
# Collapse batch×hidden into a single leading dimension
bc = B * hidden_dim
left_flat = left_perm.view(bc, N, N) # (bc, M=N, K=N)
right_flat = right_perm.view(bc, N, N) # (bc, N, K=N)
# Triton batched mat‑mul (FP16)
out_flat = batched_matmul_triton(left_flat, right_flat) # (bc, N, N), FP16
# Reshape back to [B, N, N, hidden_dim]
out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous()
# ------------------------------------------------------------------
# 6) Output layer‑norm, out‑gate, and final linear
# ------------------------------------------------------------------
out_norm = F.layer_norm(
out,
normalized_shape=(hidden_dim,),
weight=to_out_norm_w,
bias=to_out_norm_b,
eps=eps,
)
out_gated = out_norm * out_gate # both FP16
out_final = F.linear(out_gated, to_out_w) # → [B, N, N, dim] (FP16)
# Return FP32 tensor as required
return out_final.to(torch.float32)

Xet Storage Details

Size:
9.28 kB
·
Xet hash:
eb9366ce6c575b59c028e51caafcbe141c65ff2414af94a7a78ddf43d0f02a96

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.