Buckets:
zefancainlp/xiaofan_backup / kernel /online /split_step0_api3_20261105 /inputs /kernel.Step0.original.py
| """ | |
| TriMul “outgoing” forward kernel (AlphaFold3). | |
| The implementation follows the PyTorch reference: | |
| 1. Layer‑Norm on the last dimension (dim). | |
| 2. Linear projections to a hidden dimension (hidden_dim) for left/right. | |
| 3. Optional pair‑wise mask (unsqueezed to hidden_dim). | |
| 4. Gate linear → sigmoid → element‑wise gating. | |
| 5. Core contraction: | |
| out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d] | |
| which is a batched mat‑mul left @ rightᵀ for each hidden channel. | |
| The heavy N³ work is done by a custom Triton kernel that processes all | |
| (batch·hidden_dim) matrix‑multiplications in parallel. | |
| 6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back | |
| to the original dimension. | |
| All heavy arithmetic (projections, mask, gates, contraction, final linear) | |
| is performed in float16 (FP16) for speed on H100 tensor‑cores, while the | |
| final output is returned in float32. | |
| """ | |
| import torch | |
| import torch.nn.functional as F | |
| import triton | |
| import triton.language as tl | |
| # ---------------------------------------------------------------------- | |
| # Triton kernel: batched mat‑mul of (bc, M, K) @ (bc, N, K)ᵀ → (bc, M, N) | |
| # ---------------------------------------------------------------------- | |
| def batched_matmul_kernel( | |
| A_ptr, B_ptr, C_ptr, # pointers | |
| M, N, K, # matrix sizes | |
| stride_a_bc, stride_am, stride_ak, # A strides | |
| stride_b_bc, stride_bk, stride_bn, # B strides (B is read as (bc, K, N)) | |
| stride_c_bc, stride_cm, stride_cn, # C strides | |
| BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr | |
| ): | |
| pid_m = tl.program_id(0) # block index over M | |
| pid_n = tl.program_id(1) # block index over N | |
| pid_bc = tl.program_id(2) # block index over batch*hidden | |
| # Offsets within the block | |
| offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) | |
| offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) | |
| offs_k = tl.arange(0, BLOCK_K) | |
| # Pointers for the current block of A and B | |
| a_ptrs = A_ptr + pid_bc * stride_a_bc \ | |
| + offs_m[:, None] * stride_am \ | |
| + offs_k[None, :] * stride_ak | |
| # B is accessed as (bc, K, N) → original B layout (bc, N, K) | |
| b_ptrs = B_ptr + pid_bc * stride_b_bc \ | |
| + offs_k[:, None] * stride_bk \ | |
| + offs_n[None, :] * stride_bn | |
| # Accumulator in FP32 for higher precision | |
| acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) | |
| # Loop over K dimension | |
| for _ in range(tl.cdiv(K, BLOCK_K)): | |
| a = tl.load(a_ptrs, | |
| mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), | |
| other=0.0) | |
| b = tl.load(b_ptrs, | |
| mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), | |
| other=0.0) | |
| # a : (BLOCK_M, BLOCK_K) b : (BLOCK_K, BLOCK_N) | |
| acc += tl.dot(a, b) # promotion to FP32 happens automatically | |
| a_ptrs += BLOCK_K * stride_ak | |
| b_ptrs += BLOCK_K * stride_bk | |
| # Write result back (store as FP16) | |
| c_ptrs = C_ptr + pid_bc * stride_c_bc \ | |
| + offs_m[:, None] * stride_cm \ | |
| + offs_n[None, :] * stride_cn | |
| tl.store(c_ptrs, | |
| acc.to(tl.float16), | |
| mask=(offs_m[:, None] < M) & (offs_n[None, :] < N)) | |
| def batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K) | |
| using the Triton kernel above. All tensors must be FP16 and contiguous. | |
| """ | |
| assert A.dtype == torch.float16 and B.dtype == torch.float16 | |
| bc, M, K = A.shape | |
| _, N, K2 = B.shape | |
| assert K == K2, "Inner dimensions must match" | |
| # Output tensor | |
| C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16) | |
| # Strides (in elements, not bytes) | |
| stride_a_bc, stride_am, stride_ak = A.stride() | |
| stride_b_bc, stride_bn, stride_bk = B.stride() # B is (bc, N, K) → (bc, row, col) | |
| stride_c_bc, stride_cm, stride_cn = C.stride() | |
| # Block sizes – tuned for typical N up to 1024 on H100 | |
| BLOCK_M = 128 | |
| BLOCK_N = 128 | |
| BLOCK_K = 32 | |
| grid = ( | |
| (M + BLOCK_M - 1) // BLOCK_M, | |
| (N + BLOCK_N - 1) // BLOCK_N, | |
| bc, | |
| ) | |
| batched_matmul_kernel[grid]( | |
| A, B, C, | |
| M, N, K, | |
| stride_a_bc, stride_am, stride_ak, | |
| stride_b_bc, stride_bk, stride_bn, | |
| stride_c_bc, stride_cm, stride_cn, | |
| BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, | |
| ) | |
| return C | |
| # ---------------------------------------------------------------------- | |
| # Entry point used by the test harness | |
| # ---------------------------------------------------------------------- | |
| def custom_kernel(data): | |
| """ | |
| Arguments | |
| --------- | |
| data : tuple | |
| (input_tensor, mask, weights, config) | |
| Returns | |
| ------- | |
| torch.Tensor | |
| Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim], | |
| dtype torch.float32. | |
| """ | |
| input_tensor, mask, weights, config = data | |
| device = input_tensor.device | |
| eps = 1e-5 | |
| dim = config["dim"] | |
| hidden_dim = config["hidden_dim"] | |
| use_mask = not config.get("nomask", True) | |
| # ------------------------------------------------------------------ | |
| # Extract and cast weights | |
| # ------------------------------------------------------------------ | |
| # LayerNorm on input | |
| norm_weight = weights["norm.weight"].to(torch.float32) | |
| norm_bias = weights["norm.bias"].to(torch.float32) | |
| # Linear projections (float16 for speed) | |
| left_proj_w = weights["left_proj.weight"].to(torch.float16) | |
| right_proj_w = weights["right_proj.weight"].to(torch.float16) | |
| # Gates (float16) | |
| left_gate_w = weights["left_gate.weight"].to(torch.float16) | |
| right_gate_w = weights["right_gate.weight"].to(torch.float16) | |
| out_gate_w = weights["out_gate.weight"].to(torch.float16) | |
| # Output norm | |
| to_out_norm_w = weights["to_out_norm.weight"].to(torch.float16) | |
| to_out_norm_b = weights["to_out_norm.bias"].to(torch.float16) | |
| # Final linear | |
| to_out_w = weights["to_out.weight"].to(torch.float16) | |
| # ------------------------------------------------------------------ | |
| # 1) Input layer‑norm (keep in FP32 for numerical stability) | |
| # ------------------------------------------------------------------ | |
| x_norm = F.layer_norm( | |
| input_tensor, | |
| normalized_shape=(dim,), | |
| weight=norm_weight, | |
| bias=norm_bias, | |
| eps=eps, | |
| ) | |
| # Cast to FP16 for the remaining heavy ops | |
| x_norm_h = x_norm.to(torch.float16) | |
| # ------------------------------------------------------------------ | |
| # 2) Linear projections | |
| # ------------------------------------------------------------------ | |
| left = F.linear(x_norm_h, left_proj_w) # [B, N, N, hidden_dim] | |
| right = F.linear(x_norm_h, right_proj_w) | |
| # ------------------------------------------------------------------ | |
| # 3) Optional mask | |
| # ------------------------------------------------------------------ | |
| if use_mask and mask is not None: | |
| # mask shape: [B, N, N]; broadcast over hidden_dim | |
| mask_h = mask.to(torch.float16).unsqueeze(-1) # → [B, N, N, 1] | |
| left = left * mask_h | |
| right = right * mask_h | |
| # ------------------------------------------------------------------ | |
| # 4) Gates (linear + sigmoid) and gating | |
| # ------------------------------------------------------------------ | |
| left_gate = F.linear(x_norm_h, left_gate_w).sigmoid() | |
| right_gate = F.linear(x_norm_h, right_gate_w).sigmoid() | |
| out_gate = F.linear(x_norm_h, out_gate_w).sigmoid() | |
| left = left * left_gate | |
| right = right * right_gate | |
| # ------------------------------------------------------------------ | |
| # 5) Core contraction: batched mat‑mul using Triton | |
| # left, right : [B, N, N, hidden_dim] → permute to [B, hidden, N, N] | |
| # ------------------------------------------------------------------ | |
| B, N, _, _ = left.shape | |
| left_perm = left.permute(0, 3, 1, 2).contiguous() # [B, hidden, N, N] | |
| right_perm = right.permute(0, 3, 1, 2).contiguous() | |
| # Collapse batch×hidden into a single leading dimension | |
| bc = B * hidden_dim | |
| left_flat = left_perm.view(bc, N, N) # (bc, M=N, K=N) | |
| right_flat = right_perm.view(bc, N, N) # (bc, N, K=N) | |
| # Triton batched mat‑mul (FP16) | |
| out_flat = batched_matmul_triton(left_flat, right_flat) # (bc, N, N), FP16 | |
| # Reshape back to [B, N, N, hidden_dim] | |
| out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous() | |
| # ------------------------------------------------------------------ | |
| # 6) Output layer‑norm, out‑gate, and final linear | |
| # ------------------------------------------------------------------ | |
| out_norm = F.layer_norm( | |
| out, | |
| normalized_shape=(hidden_dim,), | |
| weight=to_out_norm_w, | |
| bias=to_out_norm_b, | |
| eps=eps, | |
| ) | |
| out_gated = out_norm * out_gate # both FP16 | |
| out_final = F.linear(out_gated, to_out_w) # → [B, N, N, dim] (FP16) | |
| # Return FP32 tensor as required | |
| return out_final.to(torch.float32) |
Xet Storage Details
- Size:
- 9.28 kB
- Xet hash:
- eb9366ce6c575b59c028e51caafcbe141c65ff2414af94a7a78ddf43d0f02a96
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.