zefancainlp's picture
download
raw
24.8 kB
{
"schema": "ttt-shared-step0-v1",
"task": "trimul",
"policy_updates": 0,
"best_candidate": {
"type": "State",
"id": "39b8fbaa-4ea0-50f0-8389-de25bdf06dcb",
"timestep": 0,
"code": "\"\"\"\nTriMul “outgoing” forward kernel (AlphaFold3).\n\nThe implementation follows the PyTorch reference:\n\n1. Layer‑Norm on the last dimension (dim).\n2. Linear projections to a hidden dimension (hidden_dim) for left/right.\n3. Optional pair‑wise mask (unsqueezed to hidden_dim).\n4. Gate linear → sigmoid → element‑wise gating.\n5. Core contraction:\n out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d]\n which is a batched mat‑mul left @ rightᵀ for each hidden channel.\n The heavy N³ work is done by a custom Triton kernel that processes all\n (batch·hidden_dim) matrix‑multiplications in parallel.\n6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back\n to the original dimension.\n\nAll heavy arithmetic (projections, mask, gates, contraction, final linear)\nis performed in float16 (FP16) for speed on H100 tensor‑cores, while the\nfinal output is returned in float32.\n\"\"\"\nimport torch\nimport torch.nn.functional as F\nimport triton\nimport triton.language as tl\n\n@triton.jit\ndef batched_matmul_kernel(A_ptr, B_ptr, C_ptr, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):\n pid_m = tl.program_id(0)\n pid_n = tl.program_id(1)\n pid_bc = tl.program_id(2)\n offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)\n offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)\n offs_k = tl.arange(0, BLOCK_K)\n a_ptrs = A_ptr + pid_bc * stride_a_bc + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak\n b_ptrs = B_ptr + pid_bc * stride_b_bc + offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn\n acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)\n for _ in range(tl.cdiv(K, BLOCK_K)):\n a = tl.load(a_ptrs, mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)\n b = tl.load(b_ptrs, mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), other=0.0)\n acc += tl.dot(a, b)\n a_ptrs += BLOCK_K * stride_ak\n b_ptrs += BLOCK_K * stride_bk\n c_ptrs = C_ptr + pid_bc * stride_c_bc + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn\n tl.store(c_ptrs, acc.to(tl.float16), mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))\n\ndef batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:\n \"\"\"\n Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K)\n using the Triton kernel above. All tensors must be FP16 and contiguous.\n \"\"\"\n assert A.dtype == torch.float16 and B.dtype == torch.float16\n bc, M, K = A.shape\n _, N, K2 = B.shape\n assert K == K2, 'Inner dimensions must match'\n C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16)\n stride_a_bc, stride_am, stride_ak = A.stride()\n stride_b_bc, stride_bn, stride_bk = B.stride()\n stride_c_bc, stride_cm, stride_cn = C.stride()\n BLOCK_M = 128\n BLOCK_N = 128\n BLOCK_K = 32\n grid = ((M + BLOCK_M - 1) // BLOCK_M, (N + BLOCK_N - 1) // BLOCK_N, bc)\n batched_matmul_kernel[grid](A, B, C, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K)\n return C\n\ndef pfg_s0_extract_and_cast_weights(weights):\n \"\"\"Extract and cast weights\"\"\"\n norm_weight = weights['norm.weight'].to(torch.float32)\n norm_bias = weights['norm.bias'].to(torch.float32)\n left_proj_w = weights['left_proj.weight'].to(torch.float16)\n right_proj_w = weights['right_proj.weight'].to(torch.float16)\n left_gate_w = weights['left_gate.weight'].to(torch.float16)\n right_gate_w = weights['right_gate.weight'].to(torch.float16)\n out_gate_w = weights['out_gate.weight'].to(torch.float16)\n to_out_norm_w = weights['to_out_norm.weight'].to(torch.float16)\n to_out_norm_b = weights['to_out_norm.bias'].to(torch.float16)\n to_out_w = weights['to_out.weight'].to(torch.float16)\n return (norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w)\n\ndef pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim):\n \"\"\"1) Input layer‑norm (keep in FP32 for numerical stability)\"\"\"\n x_norm = F.layer_norm(input_tensor, normalized_shape=(dim,), weight=norm_weight, bias=norm_bias, eps=eps)\n x_norm_h = x_norm.to(torch.float16)\n return x_norm_h\n\ndef pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w):\n \"\"\"2) Linear projections\"\"\"\n left = F.linear(x_norm_h, left_proj_w)\n right = F.linear(x_norm_h, right_proj_w)\n return (left, right)\n\ndef pfg_s0_optional_mask(use_mask, left, right, mask):\n \"\"\"3) Optional mask\"\"\"\n if use_mask and mask is not None:\n mask_h = mask.to(torch.float16).unsqueeze(-1)\n left = left * mask_h\n right = right * mask_h\n return (left, right)\n\ndef pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right):\n \"\"\"4) Gates (linear + sigmoid) and gating\"\"\"\n left_gate = F.linear(x_norm_h, left_gate_w).sigmoid()\n right_gate = F.linear(x_norm_h, right_gate_w).sigmoid()\n out_gate = F.linear(x_norm_h, out_gate_w).sigmoid()\n left = left * left_gate\n right = right * right_gate\n return (out_gate, left, right)\n\ndef pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim):\n \"\"\"5) Core contraction: batched mat‑mul using Triton\"\"\"\n B, N, _, _ = left.shape\n left_perm = left.permute(0, 3, 1, 2).contiguous()\n right_perm = right.permute(0, 3, 1, 2).contiguous()\n bc = B * hidden_dim\n left_flat = left_perm.view(bc, N, N)\n right_flat = right_perm.view(bc, N, N)\n out_flat = batched_matmul_triton(left_flat, right_flat)\n out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous()\n return out\n\ndef pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w):\n \"\"\"6) Output layer‑norm, out‑gate, and final linear\"\"\"\n out_norm = F.layer_norm(out, normalized_shape=(hidden_dim,), weight=to_out_norm_w, bias=to_out_norm_b, eps=eps)\n out_gated = out_norm * out_gate\n out_final = F.linear(out_gated, to_out_w)\n return out_final.to(torch.float32)\n\ndef custom_kernel(data):\n \"\"\"\n Arguments\n ---------\n data : tuple\n (input_tensor, mask, weights, config)\n\n Returns\n -------\n torch.Tensor\n Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim],\n dtype torch.float32.\n \"\"\"\n input_tensor, mask, weights, config = data\n device = input_tensor.device\n eps = 1e-05\n dim = config['dim']\n hidden_dim = config['hidden_dim']\n use_mask = not config.get('nomask', True)\n norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w = pfg_s0_extract_and_cast_weights(weights)\n x_norm_h = pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim)\n left, right = pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w)\n left, right = pfg_s0_optional_mask(use_mask, left, right, mask)\n out_gate, left, right = pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right)\n out = pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim)\n return pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w)\n",
"value": -5479.77083400601,
"construction": []
},
"baseline_sampler": {
"states": [
{
"type": "State",
"id": "39b8fbaa-4ea0-50f0-8389-de25bdf06dcb",
"timestep": 0,
"code": "\"\"\"\nTriMul “outgoing” forward kernel (AlphaFold3).\n\nThe implementation follows the PyTorch reference:\n\n1. Layer‑Norm on the last dimension (dim).\n2. Linear projections to a hidden dimension (hidden_dim) for left/right.\n3. Optional pair‑wise mask (unsqueezed to hidden_dim).\n4. Gate linear → sigmoid → element‑wise gating.\n5. Core contraction:\n out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d]\n which is a batched mat‑mul left @ rightᵀ for each hidden channel.\n The heavy N³ work is done by a custom Triton kernel that processes all\n (batch·hidden_dim) matrix‑multiplications in parallel.\n6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back\n to the original dimension.\n\nAll heavy arithmetic (projections, mask, gates, contraction, final linear)\nis performed in float16 (FP16) for speed on H100 tensor‑cores, while the\nfinal output is returned in float32.\n\"\"\"\nimport torch\nimport torch.nn.functional as F\nimport triton\nimport triton.language as tl\n\n@triton.jit\ndef batched_matmul_kernel(A_ptr, B_ptr, C_ptr, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):\n pid_m = tl.program_id(0)\n pid_n = tl.program_id(1)\n pid_bc = tl.program_id(2)\n offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)\n offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)\n offs_k = tl.arange(0, BLOCK_K)\n a_ptrs = A_ptr + pid_bc * stride_a_bc + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak\n b_ptrs = B_ptr + pid_bc * stride_b_bc + offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn\n acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)\n for _ in range(tl.cdiv(K, BLOCK_K)):\n a = tl.load(a_ptrs, mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)\n b = tl.load(b_ptrs, mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), other=0.0)\n acc += tl.dot(a, b)\n a_ptrs += BLOCK_K * stride_ak\n b_ptrs += BLOCK_K * stride_bk\n c_ptrs = C_ptr + pid_bc * stride_c_bc + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn\n tl.store(c_ptrs, acc.to(tl.float16), mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))\n\ndef batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:\n \"\"\"\n Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K)\n using the Triton kernel above. All tensors must be FP16 and contiguous.\n \"\"\"\n assert A.dtype == torch.float16 and B.dtype == torch.float16\n bc, M, K = A.shape\n _, N, K2 = B.shape\n assert K == K2, 'Inner dimensions must match'\n C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16)\n stride_a_bc, stride_am, stride_ak = A.stride()\n stride_b_bc, stride_bn, stride_bk = B.stride()\n stride_c_bc, stride_cm, stride_cn = C.stride()\n BLOCK_M = 128\n BLOCK_N = 128\n BLOCK_K = 32\n grid = ((M + BLOCK_M - 1) // BLOCK_M, (N + BLOCK_N - 1) // BLOCK_N, bc)\n batched_matmul_kernel[grid](A, B, C, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K)\n return C\n\ndef pfg_s0_extract_and_cast_weights(weights):\n \"\"\"Extract and cast weights\"\"\"\n norm_weight = weights['norm.weight'].to(torch.float32)\n norm_bias = weights['norm.bias'].to(torch.float32)\n left_proj_w = weights['left_proj.weight'].to(torch.float16)\n right_proj_w = weights['right_proj.weight'].to(torch.float16)\n left_gate_w = weights['left_gate.weight'].to(torch.float16)\n right_gate_w = weights['right_gate.weight'].to(torch.float16)\n out_gate_w = weights['out_gate.weight'].to(torch.float16)\n to_out_norm_w = weights['to_out_norm.weight'].to(torch.float16)\n to_out_norm_b = weights['to_out_norm.bias'].to(torch.float16)\n to_out_w = weights['to_out.weight'].to(torch.float16)\n return (norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w)\n\ndef pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim):\n \"\"\"1) Input layer‑norm (keep in FP32 for numerical stability)\"\"\"\n x_norm = F.layer_norm(input_tensor, normalized_shape=(dim,), weight=norm_weight, bias=norm_bias, eps=eps)\n x_norm_h = x_norm.to(torch.float16)\n return x_norm_h\n\ndef pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w):\n \"\"\"2) Linear projections\"\"\"\n left = F.linear(x_norm_h, left_proj_w)\n right = F.linear(x_norm_h, right_proj_w)\n return (left, right)\n\ndef pfg_s0_optional_mask(use_mask, left, right, mask):\n \"\"\"3) Optional mask\"\"\"\n if use_mask and mask is not None:\n mask_h = mask.to(torch.float16).unsqueeze(-1)\n left = left * mask_h\n right = right * mask_h\n return (left, right)\n\ndef pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right):\n \"\"\"4) Gates (linear + sigmoid) and gating\"\"\"\n left_gate = F.linear(x_norm_h, left_gate_w).sigmoid()\n right_gate = F.linear(x_norm_h, right_gate_w).sigmoid()\n out_gate = F.linear(x_norm_h, out_gate_w).sigmoid()\n left = left * left_gate\n right = right * right_gate\n return (out_gate, left, right)\n\ndef pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim):\n \"\"\"5) Core contraction: batched mat‑mul using Triton\"\"\"\n B, N, _, _ = left.shape\n left_perm = left.permute(0, 3, 1, 2).contiguous()\n right_perm = right.permute(0, 3, 1, 2).contiguous()\n bc = B * hidden_dim\n left_flat = left_perm.view(bc, N, N)\n right_flat = right_perm.view(bc, N, N)\n out_flat = batched_matmul_triton(left_flat, right_flat)\n out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous()\n return out\n\ndef pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w):\n \"\"\"6) Output layer‑norm, out‑gate, and final linear\"\"\"\n out_norm = F.layer_norm(out, normalized_shape=(hidden_dim,), weight=to_out_norm_w, bias=to_out_norm_b, eps=eps)\n out_gated = out_norm * out_gate\n out_final = F.linear(out_gated, to_out_w)\n return out_final.to(torch.float32)\n\ndef custom_kernel(data):\n \"\"\"\n Arguments\n ---------\n data : tuple\n (input_tensor, mask, weights, config)\n\n Returns\n -------\n torch.Tensor\n Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim],\n dtype torch.float32.\n \"\"\"\n input_tensor, mask, weights, config = data\n device = input_tensor.device\n eps = 1e-05\n dim = config['dim']\n hidden_dim = config['hidden_dim']\n use_mask = not config.get('nomask', True)\n norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w = pfg_s0_extract_and_cast_weights(weights)\n x_norm_h = pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim)\n left, right = pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w)\n left, right = pfg_s0_optional_mask(use_mask, left, right, mask)\n out_gate, left, right = pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right)\n out = pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim)\n return pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w)\n",
"value": -5479.77083400601,
"construction": []
}
],
"initial_states": []
},
"candidates": [
{
"state": {
"type": "State",
"id": "39b8fbaa-4ea0-50f0-8389-de25bdf06dcb",
"timestep": 0,
"code": "\"\"\"\nTriMul “outgoing” forward kernel (AlphaFold3).\n\nThe implementation follows the PyTorch reference:\n\n1. Layer‑Norm on the last dimension (dim).\n2. Linear projections to a hidden dimension (hidden_dim) for left/right.\n3. Optional pair‑wise mask (unsqueezed to hidden_dim).\n4. Gate linear → sigmoid → element‑wise gating.\n5. Core contraction:\n out[b,i,j,d] = Σₖ left[b,i,k,d] * right[b,j,k,d]\n which is a batched mat‑mul left @ rightᵀ for each hidden channel.\n The heavy N³ work is done by a custom Triton kernel that processes all\n (batch·hidden_dim) matrix‑multiplications in parallel.\n6. Layer‑Norm on hidden_dim, gated by an output‑gate, and a final linear back\n to the original dimension.\n\nAll heavy arithmetic (projections, mask, gates, contraction, final linear)\nis performed in float16 (FP16) for speed on H100 tensor‑cores, while the\nfinal output is returned in float32.\n\"\"\"\nimport torch\nimport torch.nn.functional as F\nimport triton\nimport triton.language as tl\n\n@triton.jit\ndef batched_matmul_kernel(A_ptr, B_ptr, C_ptr, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):\n pid_m = tl.program_id(0)\n pid_n = tl.program_id(1)\n pid_bc = tl.program_id(2)\n offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)\n offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)\n offs_k = tl.arange(0, BLOCK_K)\n a_ptrs = A_ptr + pid_bc * stride_a_bc + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak\n b_ptrs = B_ptr + pid_bc * stride_b_bc + offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn\n acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)\n for _ in range(tl.cdiv(K, BLOCK_K)):\n a = tl.load(a_ptrs, mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)\n b = tl.load(b_ptrs, mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), other=0.0)\n acc += tl.dot(a, b)\n a_ptrs += BLOCK_K * stride_ak\n b_ptrs += BLOCK_K * stride_bk\n c_ptrs = C_ptr + pid_bc * stride_c_bc + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn\n tl.store(c_ptrs, acc.to(tl.float16), mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))\n\ndef batched_matmul_triton(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:\n \"\"\"\n Compute C = A @ Bᵀ for batched tensors A,B of shape (bc, M, K)\n using the Triton kernel above. All tensors must be FP16 and contiguous.\n \"\"\"\n assert A.dtype == torch.float16 and B.dtype == torch.float16\n bc, M, K = A.shape\n _, N, K2 = B.shape\n assert K == K2, 'Inner dimensions must match'\n C = torch.empty((bc, M, N), device=A.device, dtype=torch.float16)\n stride_a_bc, stride_am, stride_ak = A.stride()\n stride_b_bc, stride_bn, stride_bk = B.stride()\n stride_c_bc, stride_cm, stride_cn = C.stride()\n BLOCK_M = 128\n BLOCK_N = 128\n BLOCK_K = 32\n grid = ((M + BLOCK_M - 1) // BLOCK_M, (N + BLOCK_N - 1) // BLOCK_N, bc)\n batched_matmul_kernel[grid](A, B, C, M, N, K, stride_a_bc, stride_am, stride_ak, stride_b_bc, stride_bk, stride_bn, stride_c_bc, stride_cm, stride_cn, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K)\n return C\n\ndef pfg_s0_extract_and_cast_weights(weights):\n \"\"\"Extract and cast weights\"\"\"\n norm_weight = weights['norm.weight'].to(torch.float32)\n norm_bias = weights['norm.bias'].to(torch.float32)\n left_proj_w = weights['left_proj.weight'].to(torch.float16)\n right_proj_w = weights['right_proj.weight'].to(torch.float16)\n left_gate_w = weights['left_gate.weight'].to(torch.float16)\n right_gate_w = weights['right_gate.weight'].to(torch.float16)\n out_gate_w = weights['out_gate.weight'].to(torch.float16)\n to_out_norm_w = weights['to_out_norm.weight'].to(torch.float16)\n to_out_norm_b = weights['to_out_norm.bias'].to(torch.float16)\n to_out_w = weights['to_out.weight'].to(torch.float16)\n return (norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w)\n\ndef pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim):\n \"\"\"1) Input layer‑norm (keep in FP32 for numerical stability)\"\"\"\n x_norm = F.layer_norm(input_tensor, normalized_shape=(dim,), weight=norm_weight, bias=norm_bias, eps=eps)\n x_norm_h = x_norm.to(torch.float16)\n return x_norm_h\n\ndef pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w):\n \"\"\"2) Linear projections\"\"\"\n left = F.linear(x_norm_h, left_proj_w)\n right = F.linear(x_norm_h, right_proj_w)\n return (left, right)\n\ndef pfg_s0_optional_mask(use_mask, left, right, mask):\n \"\"\"3) Optional mask\"\"\"\n if use_mask and mask is not None:\n mask_h = mask.to(torch.float16).unsqueeze(-1)\n left = left * mask_h\n right = right * mask_h\n return (left, right)\n\ndef pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right):\n \"\"\"4) Gates (linear + sigmoid) and gating\"\"\"\n left_gate = F.linear(x_norm_h, left_gate_w).sigmoid()\n right_gate = F.linear(x_norm_h, right_gate_w).sigmoid()\n out_gate = F.linear(x_norm_h, out_gate_w).sigmoid()\n left = left * left_gate\n right = right * right_gate\n return (out_gate, left, right)\n\ndef pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim):\n \"\"\"5) Core contraction: batched mat‑mul using Triton\"\"\"\n B, N, _, _ = left.shape\n left_perm = left.permute(0, 3, 1, 2).contiguous()\n right_perm = right.permute(0, 3, 1, 2).contiguous()\n bc = B * hidden_dim\n left_flat = left_perm.view(bc, N, N)\n right_flat = right_perm.view(bc, N, N)\n out_flat = batched_matmul_triton(left_flat, right_flat)\n out = out_flat.view(B, hidden_dim, N, N).permute(0, 2, 3, 1).contiguous()\n return out\n\ndef pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w):\n \"\"\"6) Output layer‑norm, out‑gate, and final linear\"\"\"\n out_norm = F.layer_norm(out, normalized_shape=(hidden_dim,), weight=to_out_norm_w, bias=to_out_norm_b, eps=eps)\n out_gated = out_norm * out_gate\n out_final = F.linear(out_gated, to_out_w)\n return out_final.to(torch.float32)\n\ndef custom_kernel(data):\n \"\"\"\n Arguments\n ---------\n data : tuple\n (input_tensor, mask, weights, config)\n\n Returns\n -------\n torch.Tensor\n Output of the TriMul forward pass, shape [bs, seq_len, seq_len, dim],\n dtype torch.float32.\n \"\"\"\n input_tensor, mask, weights, config = data\n device = input_tensor.device\n eps = 1e-05\n dim = config['dim']\n hidden_dim = config['hidden_dim']\n use_mask = not config.get('nomask', True)\n norm_weight, norm_bias, left_proj_w, right_proj_w, left_gate_w, right_gate_w, out_gate_w, to_out_norm_w, to_out_norm_b, to_out_w = pfg_s0_extract_and_cast_weights(weights)\n x_norm_h = pfg_s0_input_layer_norm_keep_in_fp32_for_numerical_stability(input_tensor, norm_weight, norm_bias, eps, dim)\n left, right = pfg_s0_linear_projections(x_norm_h, left_proj_w, right_proj_w)\n left, right = pfg_s0_optional_mask(use_mask, left, right, mask)\n out_gate, left, right = pfg_s0_gates_linear_sigmoid_and_gating(x_norm_h, left_gate_w, right_gate_w, out_gate_w, left, right)\n out = pfg_s0_core_contraction_batched_mat_mul_using_triton(left, right, hidden_dim)\n return pfg_s0_output_layer_norm_out_gate_and_final_linear(out, to_out_norm_w, to_out_norm_b, eps, hidden_dim, out_gate, to_out_w)\n",
"value": -5479.77083400601,
"construction": []
},
"parent": null
}
],
"provenance": {
"original_program_sha256": "d4fe30c2270c484dace78e1462a1d1d3a2e172fe8aefd0212f682e33bdc18174",
"split_program_sha256": "edc2fba210584be16adab1883fd54586fd1a25b976f6ceb9c4969cf7a3473b7c",
"original_historical_us": 5472.479937439972,
"root_score_remeasured": true,
"original_ast_restored_by_reinlining": true,
"operations_unchanged": true,
"starting_policy": "fresh base",
"trained_weights_imported": false
}
}

Xet Storage Details

Size:
24.8 kB
·
Xet hash:
68f48150fdd0392c25116df02319f9852055127265c2c1be87ed2c7d8f0eecc8

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.