Buckets:
zefancainlp/xiaofan_backup / kernel /online /split_step0_api3_20261105 /inputs /kernel.Step0.split.py
| """ | |
| TriMul “outgoing” forward kernel (AlphaFold3). | |
| The implementation follows the PyTorch reference: | |
| 1. Layer‑Norm on the last dimension (dim). | |
| 2. Linear projections to a hidden dimension (hidden_dim) for left/right. | |
| 3. Optional pair‑wise mask (unsqueezed to hidden_dim). | |
| 4. Gate linear → sigmoid → element‑wise gating. | |
| 5. Core contraction: | |
| out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d] | |
| which is a batched mat‑mul left @ rightᵀ for each hidden channel. | |
| The heavy N³ work is done by a custom Triton kernel that processes all | |
| (batch·hidden_dim) matrix‑multiplications in parallel. | |
| 6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back | |
| to the original dimension. | |
| All heavy arithmetic (projections, mask, gates, contraction, final linear) | |
| is performed in float16 (FP16) for speed on H100 tensor‑cores, while the | |
| final output is returned in float32. | |
| """ | |
| import torch | |
| import torch.nn.functional as F | |
| import triton | |
| import triton.language as tl | |
| def batched_matmul_kernel(A_ptr, B_ptr, C_ptr, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr): | |
| pid_m = tl.program_id(0) | |
| pid_n = tl.program_id(1) | |
| pid_bc = tl.program_id(2) | |
| offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) | |
| offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) | |
| offs_k = tl.arange(0, BLOCK_K) | |
| a_ptrs = A_ptr + pid_bc * stride_a_bc + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak | |
| b_ptrs = B_ptr + pid_bc * stride_b_bc + offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn | |
| acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) | |
| for _ in range(tl.cdiv(K, BLOCK_K)): | |
| a = tl.load(a_ptrs, mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0) | |
| b = tl.load(b_ptrs, mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), other=0.0) | |
| acc += tl.dot(a, b) | |
| a_ptrs += BLOCK_K * stride_ak | |
| b_ptrs += BLOCK_K * stride_bk | |
| c_ptrs = C_ptr + pid_bc * stride_c_bc + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn | |
| tl.store(c_ptrs, acc.to(tl.float16), mask=(offs_m[:, None] < M) & (offs_n[None, :] < N)) | |
| def batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K) | |
| using the Triton kernel above. All tensors must be FP16 and contiguous. | |
| """ | |
| assert A.dtype == torch.float16 and B.dtype == torch.float16 | |
| bc, M, K = A.shape | |
| _, N, K2 = B.shape | |
| assert K == K2, 'Inner dimensions must match' | |
| C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16) | |
| stride_a_bc, stride_am, stride_ak = A.stride() | |
| stride_b_bc, stride_bn, stride_bk = B.stride() | |
| stride_c_bc, stride_cm, stride_cn = C.stride() | |
| BLOCK_M = 128 | |
| BLOCK_N = 128 | |
| BLOCK_K = 32 | |
| grid = ((M + BLOCK_M - 1) // BLOCK_M, (N + BLOCK_N - 1) // BLOCK_N, bc) | |
| batched_matmul_kernel[grid](A, B, C, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K) | |
| return C | |
| def pfg_s0_extract_and_cast_weights(weights): | |
| """Extract and cast weights""" | |
| norm_weight = weights['norm.weight'].to(torch.float32) | |
| norm_bias = weights['norm.bias'].to(torch.float32) | |
| left_proj_w = weights['left_proj.weight'].to(torch.float16) | |
| right_proj_w = weights['right_proj.weight'].to(torch.float16) | |
| left_gate_w = weights['left_gate.weight'].to(torch.float16) | |
| right_gate_w = weights['right_gate.weight'].to(torch.float16) | |
| out_gate_w = weights['out_gate.weight'].to(torch.float16) | |
| to_out_norm_w = weights['to_out_norm.weight'].to(torch.float16) | |
| to_out_norm_b = weights['to_out_norm.bias'].to(torch.float16) | |
| to_out_w = weights['to_out.weight'].to(torch.float16) | |
| return (norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w) | |
| def pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim): | |
| """1) Input layer‑norm (keep in FP32 for numerical stability)""" | |
| x_norm = F.layer_norm(input_tensor, normalized_shape=(dim,), weight=norm_weight, bias=norm_bias, eps=eps) | |
| x_norm_h = x_norm.to(torch.float16) | |
| return x_norm_h | |
| def pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w): | |
| """2) Linear projections""" | |
| left = F.linear(x_norm_h, left_proj_w) | |
| right = F.linear(x_norm_h, right_proj_w) | |
| return (left, right) | |
| def pfg_s0_optional_mask(use_mask, left, right, mask): | |
| """3) Optional mask""" | |
| if use_mask and mask is not None: | |
| mask_h = mask.to(torch.float16).unsqueeze(-1) | |
| left = left * mask_h | |
| right = right * mask_h | |
| return (left, right) | |
| def pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right): | |
| """4) Gates (linear + sigmoid) and gating""" | |
| left_gate = F.linear(x_norm_h, left_gate_w).sigmoid() | |
| right_gate = F.linear(x_norm_h, right_gate_w).sigmoid() | |
| out_gate = F.linear(x_norm_h, out_gate_w).sigmoid() | |
| left = left * left_gate | |
| right = right * right_gate | |
| return (out_gate, left, right) | |
| def pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim): | |
| """5) Core contraction: batched mat‑mul using Triton""" | |
| B, N, _, _ = left.shape | |
| left_perm = left.permute(0, 3, 1, 2).contiguous() | |
| right_perm = right.permute(0, 3, 1, 2).contiguous() | |
| bc = B * hidden_dim | |
| left_flat = left_perm.view(bc, N, N) | |
| right_flat = right_perm.view(bc, N, N) | |
| out_flat = batched_matmul_triton(left_flat, right_flat) | |
| out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous() | |
| return out | |
| def pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w): | |
| """6) Output layer‑norm, out‑gate, and final linear""" | |
| out_norm = F.layer_norm(out, normalized_shape=(hidden_dim,), weight=to_out_norm_w, bias=to_out_norm_b, eps=eps) | |
| out_gated = out_norm * out_gate | |
| out_final = F.linear(out_gated, to_out_w) | |
| return out_final.to(torch.float32) | |
| def custom_kernel(data): | |
| """ | |
| Arguments | |
| --------- | |
| data : tuple | |
| (input_tensor, mask, weights, config) | |
| Returns | |
| ------- | |
| torch.Tensor | |
| Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim], | |
| dtype torch.float32. | |
| """ | |
| input_tensor, mask, weights, config = data | |
| device = input_tensor.device | |
| eps = 1e-05 | |
| dim = config['dim'] | |
| hidden_dim = config['hidden_dim'] | |
| use_mask = not config.get('nomask', True) | |
| norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w = pfg_s0_extract_and_cast_weights(weights) | |
| x_norm_h = pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim) | |
| left, right = pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w) | |
| left, right = pfg_s0_optional_mask(use_mask, left, right, mask) | |
| out_gate, left, right = pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right) | |
| out = pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim) | |
| return pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w) | |
Xet Storage Details
- Size:
- 7.61 kB
- Xet hash:
- 4c71d10e9d9e08ca85b2144c3aad8396916720457824127e3f902823d6eb547a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.