zefancainlp's picture
download
raw
7.61 kB
"""
TriMul “outgoing” forward kernel (AlphaFold3).
The implementation follows the PyTorch reference:
1. Layer‑Norm on the last dimension (dim).
2. Linear projections to a hidden dimension (hidden_dim) for left/right.
3. Optional pair‑wise mask (unsqueezed to hidden_dim).
4. Gate linear → sigmoid → element‑wise gating.
5. Core contraction:
out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d]
which is a batched mat‑mul left @ rightᵀ for each hidden channel.
The heavy N³ work is done by a custom Triton kernel that processes all
(batch·hidden_dim) matrix‑multiplications in parallel.
6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back
to the original dimension.
All heavy arithmetic (projections, mask, gates, contraction, final linear)
is performed in float16 (FP16) for speed on H100 tensor‑cores, while the
final output is returned in float32.
"""
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
@triton.jit
def batched_matmul_kernel(A_ptr, B_ptr, C_ptr, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
pid_bc = tl.program_id(2)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
a_ptrs = A_ptr + pid_bc * stride_a_bc + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak
b_ptrs = B_ptr + pid_bc * stride_b_bc + offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(tl.cdiv(K, BLOCK_K)):
a = tl.load(a_ptrs, mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)
b = tl.load(b_ptrs, mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), other=0.0)
acc += tl.dot(a, b)
a_ptrs += BLOCK_K * stride_ak
b_ptrs += BLOCK_K * stride_bk
c_ptrs = C_ptr + pid_bc * stride_c_bc + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
tl.store(c_ptrs, acc.to(tl.float16), mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))
def batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:
"""
Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K)
using the Triton kernel above. All tensors must be FP16 and contiguous.
"""
assert A.dtype == torch.float16 and B.dtype == torch.float16
bc, M, K = A.shape
_, N, K2 = B.shape
assert K == K2, 'Inner dimensions must match'
C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16)
stride_a_bc, stride_am, stride_ak = A.stride()
stride_b_bc, stride_bn, stride_bk = B.stride()
stride_c_bc, stride_cm, stride_cn = C.stride()
BLOCK_M = 128
BLOCK_N = 128
BLOCK_K = 32
grid = ((M + BLOCK_M - 1) // BLOCK_M, (N + BLOCK_N - 1) // BLOCK_N, bc)
batched_matmul_kernel[grid](A, B, C, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K)
return C
def pfg_s0_extract_and_cast_weights(weights):
"""Extract and cast weights"""
norm_weight = weights['norm.weight'].to(torch.float32)
norm_bias = weights['norm.bias'].to(torch.float32)
left_proj_w = weights['left_proj.weight'].to(torch.float16)
right_proj_w = weights['right_proj.weight'].to(torch.float16)
left_gate_w = weights['left_gate.weight'].to(torch.float16)
right_gate_w = weights['right_gate.weight'].to(torch.float16)
out_gate_w = weights['out_gate.weight'].to(torch.float16)
to_out_norm_w = weights['to_out_norm.weight'].to(torch.float16)
to_out_norm_b = weights['to_out_norm.bias'].to(torch.float16)
to_out_w = weights['to_out.weight'].to(torch.float16)
return (norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w)
def pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim):
"""1) Input layer‑norm (keep in FP32 for numerical stability)"""
x_norm = F.layer_norm(input_tensor, normalized_shape=(dim,), weight=norm_weight, bias=norm_bias, eps=eps)
x_norm_h = x_norm.to(torch.float16)
return x_norm_h
def pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w):
"""2) Linear projections"""
left = F.linear(x_norm_h, left_proj_w)
right = F.linear(x_norm_h, right_proj_w)
return (left, right)
def pfg_s0_optional_mask(use_mask, left, right, mask):
"""3) Optional mask"""
if use_mask and mask is not None:
mask_h = mask.to(torch.float16).unsqueeze(-1)
left = left * mask_h
right = right * mask_h
return (left, right)
def pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right):
"""4) Gates (linear + sigmoid) and gating"""
left_gate = F.linear(x_norm_h, left_gate_w).sigmoid()
right_gate = F.linear(x_norm_h, right_gate_w).sigmoid()
out_gate = F.linear(x_norm_h, out_gate_w).sigmoid()
left = left * left_gate
right = right * right_gate
return (out_gate, left, right)
def pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim):
"""5) Core contraction: batched mat‑mul using Triton"""
B, N, _, _ = left.shape
left_perm = left.permute(0, 3, 1, 2).contiguous()
right_perm = right.permute(0, 3, 1, 2).contiguous()
bc = B * hidden_dim
left_flat = left_perm.view(bc, N, N)
right_flat = right_perm.view(bc, N, N)
out_flat = batched_matmul_triton(left_flat, right_flat)
out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous()
return out
def pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w):
"""6) Output layer‑norm, out‑gate, and final linear"""
out_norm = F.layer_norm(out, normalized_shape=(hidden_dim,), weight=to_out_norm_w, bias=to_out_norm_b, eps=eps)
out_gated = out_norm * out_gate
out_final = F.linear(out_gated, to_out_w)
return out_final.to(torch.float32)
def custom_kernel(data):
"""
Arguments
---------
data : tuple
(input_tensor, mask, weights, config)
Returns
-------
torch.Tensor
Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim],
dtype torch.float32.
"""
input_tensor, mask, weights, config = data
device = input_tensor.device
eps = 1e-05
dim = config['dim']
hidden_dim = config['hidden_dim']
use_mask = not config.get('nomask', True)
norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w = pfg_s0_extract_and_cast_weights(weights)
x_norm_h = pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim)
left, right = pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w)
left, right = pfg_s0_optional_mask(use_mask, left, right, mask)
out_gate, left, right = pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right)
out = pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim)
return pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w)

Xet Storage Details

Size:
7.61 kB
·
Xet hash:
4c71d10e9d9e08ca85b2144c3aad8396916720457824127e3f902823d6eb547a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.