zefancainlp's picture
download
raw
229 kB
{"sources": {"0": "import pathlib as _gp_path, tempfile as _gp_temp, sys as _gp_sys, importlib as _gp_import\n_gp_directory=_gp_temp.TemporaryDirectory(prefix='program_components_')\n_gp_root=_gp_path.Path(_gp_directory.name)\n_gp_files={'__init__.py': '', 'c_finish/output_tail.py': \"import torch\\nimport triton\\nfrom .tail_numeric import _as_tail\\nfrom .paper_numeric import _ln_gate_out_linear_fused_kernel\\n\\n\\ndef finish_fast(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n nw=weights['to_out_norm.weight'].to(torch.float16).contiguous()\\n nb=weights['to_out_norm.bias'].to(torch.float16).contiguous()\\n weight=weights['to_out.weight'].to(torch.float16).contiguous()\\n output=torch.empty((m,c),dtype=torch.float32,device=x.device)\\n # Keep the good code's strided view. Do not force an extra contiguous copy.\\n hidden=product.view(b,h,n,n).permute(0,2,3,1)\\n _as_tail(hidden,gate,nw,nb,weight,output,1e-5,1,h,compact=False)\\n return output.view(b,n,n,c)\\n\\n\\ndef finish_paper(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n weight=weights['to_out.weight'].t().contiguous().to(torch.float16)\\n output=torch.empty((b,n,n,c),dtype=torch.float32,device=x.device)\\n _ln_gate_out_linear_fused_kernel[(triton.cdiv(m,64),)](\\n product.view(-1),gate.view(-1),weights['to_out_norm.weight'],weights['to_out_norm.bias'],\\n weight,output,b,n,h,c,1e-5,BLOCK_M=64,BLOCK_H=h,BLOCK_D=64,num_warps=4)\\n return output\\n\", 'c_finish/tail_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\ndef _as_segments(x):\\n (shape, strides) = (tuple(x.shape), tuple(x.stride()))\\n start = len(shape) - 2\\n while start > 0 and strides[start - 1] == strides[start] * shape[start]:\\n start -= 1\\n rows = 1\\n for size in shape[start:-1]:\\n rows *= size\\n for prefix in _as_itertools.product(*(range(s) for s in shape[:start])):\\n view = x[prefix] if prefix else x\\n yield (prefix, view.as_strided((rows, shape[-1]), (strides[-2], strides[-1])))\\n\\n\\ndef _as_norm(v, gamma, beta, gate=None, eps=1e-05, compact=False):\\n if compact:\\n v = v.contiguous()\\n if gate is None:\\n gate = _as_torch.ones((), dtype=v.dtype, device=v.device).expand(v.shape)\\n else:\\n gate = gate.reshape(v.shape)\\n y = _as_torch.empty_strided(v.shape, v.stride(), dtype=_as_torch.float16, device=v.device)\\n gamma = gamma.reshape(-1)\\n beta = beta.reshape(-1)\\n for (prefix, a) in _as_segments(v):\\n g = (gate[prefix] if prefix else gate).reshape(a.shape)\\n q = y[prefix] if prefix else y\\n z = q.as_strided(a.shape, a.stride())\\n _as_norm_launch(a, g, gamma, beta, z, eps)\\n return y\\n\\n\\ndef _as_tail(v, gate, gamma, beta, weight, output, eps, wh, wn, compact=False):\\n h = v.shape[-1]\\n w = weight.as_strided((h, output.shape[-1]), (wh, wn))\\n y = _as_norm(v, gamma, beta, gate, eps, compact)\\n result = output.view(*y.shape[:-1], output.shape[-1])\\n for (prefix, a) in _as_segments(y):\\n o = (result[prefix] if prefix else result).reshape(a.shape[0], output.shape[-1])\\n _as_gemm_launch(a, w, o)\\n\\n\\n@_as_n_triton.jit\\ndef _as_n_layernorm_gate_kernel(out_ptr, out_gate_ptr, norm_w_ptr, norm_b_ptr, out_norm_ptr, M, H: _as_n_tl.constexpr, stride_out_m, stride_out_h, stride_gate_m, stride_gate_h, stride_norm_w, stride_norm_b, eps: _as_n_tl.constexpr, BLOCK_M: _as_n_tl.constexpr, BLOCK_H: _as_n_tl.constexpr):\\n pid = _as_n_tl.program_id(0)\\n offs_m = pid * BLOCK_M + _as_n_tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n sumsq = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n sum_ += _as_n_tl.sum(out_val, axis=1)\\n sumsq += _as_n_tl.sum(out_val * out_val, axis=1)\\n mean = sum_ / H\\n var = sumsq / H - mean * mean\\n inv_std = 1.0 / _as_n_tl.sqrt(var + eps)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n gate_val = _as_n_tl.load(out_gate_ptr + offs_m[:, None] * stride_gate_m + offs_h[None, :] * stride_gate_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n w = _as_n_tl.load(norm_w_ptr + offs_h * stride_norm_w, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n b = _as_n_tl.load(norm_b_ptr + offs_h * stride_norm_b, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n normalized = (out_val - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n gated = normalized * gate_val\\n _as_n_tl.store(out_norm_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, gated.to(_as_n_tl.float16), mask=mask_m[:, None] & mask_h[None, :])\\n\\n\\ndef _as_norm_launch(v, g, gamma, beta, o, eps):\\n _as_n_layernorm_gate_kernel[_as_triton.cdiv(v.shape[0], 128),](v, g, gamma, beta, o, v.shape[0], v.shape[1], v.stride(0), v.stride(1), g.stride(0), g.stride(1), gamma.stride(0), beta.stride(0), eps, 128, 64)\\n\\n\\n@_as_g_triton.jit\\ndef _as_g_linear_gemm_kernel(a_ptr, b_ptr, c_ptr, M, N, K, stride_am, stride_ak, stride_bk, stride_bn, stride_cm, stride_cn, BLOCK_M: _as_g_tl.constexpr, BLOCK_N: _as_g_tl.constexpr, BLOCK_K: _as_g_tl.constexpr):\\n \"\"\"C = A @ B where A is (M, K) and B is (K, N).\"\"\"\\n pid = _as_g_tl.program_id(0)\\n num_blocks_m = _as_g_tl.cdiv(M, BLOCK_M)\\n num_blocks_n = _as_g_tl.cdiv(N, BLOCK_N)\\n block_m = pid // num_blocks_n\\n block_n = pid % num_blocks_n\\n offs_m = block_m * BLOCK_M + _as_g_tl.arange(0, BLOCK_M)\\n offs_n = block_n * BLOCK_N + _as_g_tl.arange(0, BLOCK_N)\\n mask_m = offs_m < M\\n mask_n = offs_n < N\\n acc = _as_g_tl.zeros((BLOCK_M, BLOCK_N), dtype=_as_g_tl.float32)\\n for k in range(0, _as_g_tl.cdiv(K, BLOCK_K)):\\n k_offset = k * BLOCK_K\\n offs_k = k_offset + _as_g_tl.arange(0, BLOCK_K)\\n a = _as_g_tl.load(a_ptr + (offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak), mask=mask_m[:, None] & (offs_k < K), other=0.0)\\n b = _as_g_tl.load(b_ptr + (offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn), mask=(offs_k < K)[:, None] & mask_n[None, :], other=0.0)\\n acc += _as_g_tl.dot(a, b)\\n c_ptrs = c_ptr + (offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn)\\n _as_g_tl.store(c_ptrs, acc, mask=mask_m[:, None] & mask_n[None, :])\\n\\n\\ndef _as_gemm_launch(a, b, o):\\n _as_g_linear_gemm_kernel[_as_triton.cdiv(a.shape[0], 128) * _as_triton.cdiv(b.shape[1], 64),](a, b, o, a.shape[0], b.shape[1], a.shape[1], a.stride(0), a.stride(1), b.stride(0), b.stride(1), o.stride(0), o.stride(1), 128, 64, 32)\\n\\n', 'c_finish/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_finish/__init__.py': '\\nfrom .output_tail import finish_fast as run\\n', 'c_normalize/input_norm.py': \"import torch\\nimport triton\\nfrom .norm_numeric import component_front_layernorm_fused_kernel\\nfrom .paper_numeric import _row_layernorm_fp16\\n\\n\\ndef normalize_fast(data,packed):\\n x=data[0];b,n,_,c=x.shape;m=b*n*n\\n flat=x.contiguous().view(m,c);normalized=torch.empty_like(flat,dtype=torch.float16,device=x.device)\\n component_front_layernorm_fused_kernel[(triton.cdiv(m,128),)](\\n flat,normalized,packed['norm.weight'],packed['norm.bias'],m,c,1e-5,c,1,c,1,\\n BLOCK_M=128,BLOCK_C=min(c,64))\\n return normalized.view(b,n,n,c)\\n\\n\\ndef normalize_paper(data,packed):\\n # The paper's own LN wrapper owns its weight dtype and launch configuration.\\n return _row_layernorm_fp16(data[0],data[2]['norm.weight'],data[2]['norm.bias'],eps=1e-5)\\n\", 'c_normalize/norm_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@triton.jit\\ndef component_front_layernorm_fused_kernel(x_ptr, out_ptr, norm_w_ptr, norm_b_ptr, M: tl.constexpr, C: tl.constexpr, eps: tl.constexpr, stride_x_m, stride_x_c, stride_out_m, stride_out_c, BLOCK_M: tl.constexpr, BLOCK_C: tl.constexpr):\\n \"\"\"\\n Compute LayerNorm over the last dimension (C) of a (M, C) matrix.\\n Input is float32, output is float16. The weight and bias are float16.\\n \"\"\"\\n pid = tl.program_id(0)\\n offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n sumsq = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n sum_ += tl.sum(x, axis=1)\\n sumsq += tl.sum(x * x, axis=1)\\n mean = sum_ / C\\n var = sumsq / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n w = tl.load(norm_w_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n b = tl.load(norm_b_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n out = (x - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n tl.store(out_ptr + offs_m[:, None] * stride_out_m + offs_c[None, :] * stride_out_c, out.to(tl.float16), mask=mask_m[:, None] & mask_c[None, :])\\n\\n', 'c_normalize/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_normalize/__init__.py': '\\nfrom .input_norm import normalize_fast as run\\n', 'c_project/projection.py': \"import torch\\nimport triton\\nfrom .projection_numeric import component_front_fused_proj_gate_outgate_kernel_4d\\nfrom .paper_numeric import _proj_gate_mask_kernel\\n\\n\\ndef project_fast(data,normalized,packed):\\n x,mask,_,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n flat=normalized.view(m,c).contiguous()\\n mask=torch.ones((b,n,n),dtype=torch.float16,device=x.device)if mask is None else mask.to(torch.float16)\\n mask=mask.reshape(-1).contiguous()\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((m,h),dtype=torch.float16,device=x.device)\\n component_front_fused_proj_gate_outgate_kernel_4d[(triton.cdiv(m,64),triton.cdiv(h,min(h,64)))](\\n flat,mask,packed['left_proj.weight'],packed['left_gate.weight'],packed['right_proj.weight'],\\n packed['right_gate.weight'],packed['out_gate.weight'],left,right,gate,m,h,c,b,n,c,1,c,1,\\n BLOCK_M=64,BLOCK_H=min(h,64),BLOCK_K=32)\\n return left,right,gate\\n\\n\\ndef project_paper(data,normalized,packed):\\n x,mask,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n # This kernel consumes C,H-packed weights, unlike the fast H,C-packed one.\\n names=['left_proj.weight','left_gate.weight','right_proj.weight','right_gate.weight','out_gate.weight']\\n wp=[weights[name].t().contiguous().to(torch.float16)for name in names]\\n masked=mask is not None\\n mask_flat=mask.reshape(m).to(torch.float16).contiguous()if masked else torch.empty(0,dtype=torch.float16,device=x.device)\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((b,n,n,h),dtype=torch.float16,device=x.device)\\n _proj_gate_mask_kernel[(triton.cdiv(m,64),triton.cdiv(h,64))](\\n normalized,mask_flat,*wp,left,right,gate,m,n,c,h,\\n BLOCK_M=64,BLOCK_H=64,BLOCK_K=32,MASKED=int(masked),num_warps=4)\\n return left,right,gate.reshape(m,h)\\n\", 'c_project/projection_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@_bundle_447b911a47b3_triton.jit\\ndef component_front_fused_proj_gate_outgate_kernel_4d(x_ptr, mask_ptr, proj_left_w_ptr, gate_left_w_ptr, proj_right_w_ptr, gate_right_w_ptr, out_gate_w_ptr, out_left_ptr, out_right_ptr, out_gate_ptr, M: _bundle_447b911a47b3_tl.constexpr, H: _bundle_447b911a47b3_tl.constexpr, K: _bundle_447b911a47b3_tl.constexpr, B: _bundle_447b911a47b3_tl.constexpr, N: _bundle_447b911a47b3_tl.constexpr, stride_x_m: _bundle_447b911a47b3_tl.constexpr, stride_x_k: _bundle_447b911a47b3_tl.constexpr, stride_w_h: _bundle_447b911a47b3_tl.constexpr, stride_w_k: _bundle_447b911a47b3_tl.constexpr, BLOCK_M: _bundle_447b911a47b3_tl.constexpr, BLOCK_H: _bundle_447b911a47b3_tl.constexpr, BLOCK_K: _bundle_447b911a47b3_tl.constexpr):\\n \"\"\"Fused projection + gating for left/right tensors and out\u2011gate.\\n\\n Arguments\\n ---------\\n x_ptr : pointer to the flattened input tensor of shape (M, K)\\n mask_ptr : pointer to the flattened mask of shape (M,)\\n *_proj_w_ptr, *_gate_w_ptr : pointers to the weight matrices of shape (H, K)\\n out_gate_w_ptr : pointer to the weight matrix for the out\u2011gate (H, K)\\n left_out_ptr, right_out_ptr : output buffers of shape (B, H, N, N)\\n out_gate_out_ptr : output buffer of shape (M, H) (flattened out\u2011gate)\\n M : total number of rows = B * N * N\\n H : hidden dimension\\n K : input channel dimension (C)\\n B : batch size\\n N : sequence length\\n stride_x_m, stride_x_k : strides for the input tensor\\n stride_w_h, stride_w_k : strides for the weight matrices\\n BLOCK_* : tiling constants\\n \"\"\"\\n _bundle_447b911a47b3_local_pid_m = _bundle_447b911a47b3_tl.program_id(0)\\n _bundle_447b911a47b3_local_pid_h = _bundle_447b911a47b3_tl.program_id(1)\\n _bundle_447b911a47b3_local_offs_m = _bundle_447b911a47b3_local_pid_m * BLOCK_M + _bundle_447b911a47b3_tl.arange(0, BLOCK_M)\\n _bundle_447b911a47b3_local_offs_h = _bundle_447b911a47b3_local_pid_h * BLOCK_H + _bundle_447b911a47b3_tl.arange(0, BLOCK_H)\\n _bundle_447b911a47b3_local_mask_m = _bundle_447b911a47b3_local_offs_m < M\\n _bundle_447b911a47b3_local_mask_h = _bundle_447b911a47b3_local_offs_h < H\\n _bundle_447b911a47b3_local_mask_mh = _bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_h[None, :]\\n _bundle_447b911a47b3_local_m = _bundle_447b911a47b3_tl.load(mask_ptr + _bundle_447b911a47b3_local_offs_m, mask=_bundle_447b911a47b3_local_mask_m, other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_lgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_outgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n for _bundle_447b911a47b3_local_k in range(0, _bundle_447b911a47b3_tl.cdiv(K, BLOCK_K)):\\n _bundle_447b911a47b3_local_offs_k = _bundle_447b911a47b3_local_k * BLOCK_K + _bundle_447b911a47b3_tl.arange(0, BLOCK_K)\\n _bundle_447b911a47b3_local_mask_k = _bundle_447b911a47b3_local_offs_k < K\\n _bundle_447b911a47b3_local_x = _bundle_447b911a47b3_tl.load(x_ptr + _bundle_447b911a47b3_local_offs_m[:, None] * stride_x_m + _bundle_447b911a47b3_local_offs_k[None, :] * stride_x_k, mask=_bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_k[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lproj = _bundle_447b911a47b3_tl.load(proj_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lgate = _bundle_447b911a47b3_tl.load(gate_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rproj = _bundle_447b911a47b3_tl.load(proj_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rgate = _bundle_447b911a47b3_tl.load(gate_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_outgate = _bundle_447b911a47b3_tl.load(out_gate_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lproj)\\n _bundle_447b911a47b3_local_acc_lgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lgate)\\n _bundle_447b911a47b3_local_acc_rproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rproj)\\n _bundle_447b911a47b3_local_acc_rgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rgate)\\n _bundle_447b911a47b3_local_acc_outgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_outgate)\\n _bundle_447b911a47b3_local_gate_l = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_lgate))\\n _bundle_447b911a47b3_local_gate_r = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_rgate))\\n _bundle_447b911a47b3_local_gate_out = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_outgate))\\n _bundle_447b911a47b3_local_left_val = _bundle_447b911a47b3_local_acc_lproj * _bundle_447b911a47b3_local_gate_l * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_right_val = _bundle_447b911a47b3_local_acc_rproj * _bundle_447b911a47b3_local_gate_r * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_N2 = N * N\\n _bundle_447b911a47b3_local_b = _bundle_447b911a47b3_local_offs_m // _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_rem = _bundle_447b911a47b3_local_offs_m % _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_i = _bundle_447b911a47b3_local_rem // N\\n _bundle_447b911a47b3_local_k_idx = _bundle_447b911a47b3_local_rem % N\\n _bundle_447b911a47b3_local_base = _bundle_447b911a47b3_local_b * (H * _bundle_447b911a47b3_local_N2) + _bundle_447b911a47b3_local_i * N + _bundle_447b911a47b3_local_k_idx\\n _bundle_447b911a47b3_local_offset = _bundle_447b911a47b3_local_base[:, None] + (_bundle_447b911a47b3_local_offs_h * _bundle_447b911a47b3_local_N2)[None, :]\\n _bundle_447b911a47b3_local_left_ptrs = out_left_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_local_right_ptrs = out_right_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_left_ptrs, _bundle_447b911a47b3_local_left_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_right_ptrs, _bundle_447b911a47b3_local_right_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_local_out_gate_offsets = _bundle_447b911a47b3_local_offs_m[:, None] * H + _bundle_447b911a47b3_local_offs_h[None, :]\\n _bundle_447b911a47b3_local_out_gate_ptrs = out_gate_ptr + _bundle_447b911a47b3_local_out_gate_offsets\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_out_gate_ptrs, _bundle_447b911a47b3_local_gate_out.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n\\n', 'c_project/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_project/__init__.py': '\\nfrom .projection import project_fast as run\\n', 'entry.py': \"\\nfrom .c_project import run as _gm_project\\n\\nfrom .c_normalize import run as _gm_normalize\\n\\nfrom .c_finish import run as _gm_finish\\nimport torch\\ndef custom_kernel(data):\\n w=data[2]\\n packed={name:w[name].to(torch.float16).contiguous()for name in ['norm.weight','norm.bias','left_proj.weight','right_proj.weight','left_gate.weight','right_gate.weight','out_gate.weight']}\\n normalized=_gm_normalize(data,packed)\\n left,right,gate=_gm_project(data,normalized,packed)\\n b,h,n,_=left.shape\\n product=torch.bmm(left.view(b*h,n,n),right.view(b*h,n,n).transpose(1,2))\\n return _gm_finish(data,product,gate)\\n__all__=['custom_kernel']\\n\"}\nfor _gp_name,_gp_text in _gp_files.items():\n _gp_file=_gp_root/'_gm_91ca2e5bf1bfb5d1b0c5'/_gp_name\n _gp_file.parent.mkdir(parents=True,exist_ok=True)\n _gp_file.write_text(_gp_text,encoding='utf-8')\n_gp_sys.path.insert(0,str(_gp_root))\n_gp_entry=_gp_import.import_module('_gm_91ca2e5bf1bfb5d1b0c5'+'.entry')\n\nfor _gp_symbol in getattr(_gp_entry,'__all__',[]):\n globals()[_gp_symbol]=getattr(_gp_entry,_gp_symbol)\n\n__source_binding_identities__={'_gm_project': 'c8ed0952bbc05ee3f4b59a5b9ad8f0b1802de271fca3170e2c391c84b4f3d017', '_gm_normalize': '6d449693f25b00075237ab4067336a98f6103599850dd06fd7b0ab0c1cd46994', '_gm_finish': '977da3d0268e7082d556995b378e7058d880cabcba6d4613c6ac793fd3dc980b'}\n__source_binding_requires__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_memoizable_bindings__={'_gm_project': True, '_gm_normalize': True, '_gm_finish': True}\n__source_state_locations__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_control_domains__={}\ndef _sg_install_bindings():\n pass\n_gm_project = _gp_entry._gm_project\n_gm_normalize = _gp_entry._gm_normalize\n_gm_finish = _gp_entry._gm_finish\nimport torch\n\ndef custom_kernel(data):\n _sg_install_bindings()\n w = data[2]\n packed = {name: w[name].to(torch.float16).contiguous() for name in ['norm.weight', 'norm.bias', 'left_proj.weight', 'right_proj.weight', 'left_gate.weight', 'right_gate.weight', 'out_gate.weight']}\n normalized = _gm_normalize(data, packed)\n (left, right, gate) = _gm_project(data, normalized, packed)\n (b, h, n, _) = left.shape\n product = torch.bmm(left.view(b * h, n, n), right.view(b * h, n, n).transpose(1, 2))\n return _gm_finish(data, product, gate)\n__all__ = ['custom_kernel']\n_gm_project._source_binding_identity = 'c8ed0952bbc05ee3f4b59a5b9ad8f0b1802de271fca3170e2c391c84b4f3d017'\n_gm_project._source_memoizable = True\n_gm_project._source_state_locations = []\n_gm_normalize._source_binding_identity = '6d449693f25b00075237ab4067336a98f6103599850dd06fd7b0ab0c1cd46994'\n_gm_normalize._source_memoizable = True\n_gm_normalize._source_state_locations = []\n_gm_finish._source_binding_identity = '977da3d0268e7082d556995b378e7058d880cabcba6d4613c6ac793fd3dc980b'\n_gm_finish._source_memoizable = True\n_gm_finish._source_state_locations = []\n", "1": "import pathlib as _gp_path, tempfile as _gp_temp, sys as _gp_sys, importlib as _gp_import\n_gp_directory=_gp_temp.TemporaryDirectory(prefix='program_components_')\n_gp_root=_gp_path.Path(_gp_directory.name)\n_gp_files={'__init__.py': '', 'c_finish/output_tail.py': \"import torch\\nimport triton\\nfrom .tail_numeric import _as_tail\\nfrom .paper_numeric import _ln_gate_out_linear_fused_kernel\\n\\n\\ndef finish_fast(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n nw=weights['to_out_norm.weight'].to(torch.float16).contiguous()\\n nb=weights['to_out_norm.bias'].to(torch.float16).contiguous()\\n weight=weights['to_out.weight'].to(torch.float16).contiguous()\\n output=torch.empty((m,c),dtype=torch.float32,device=x.device)\\n # Keep the good code's strided view. Do not force an extra contiguous copy.\\n hidden=product.view(b,h,n,n).permute(0,2,3,1)\\n _as_tail(hidden,gate,nw,nb,weight,output,1e-5,1,h,compact=False)\\n return output.view(b,n,n,c)\\n\\n\\ndef finish_paper(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n weight=weights['to_out.weight'].t().contiguous().to(torch.float16)\\n output=torch.empty((b,n,n,c),dtype=torch.float32,device=x.device)\\n _ln_gate_out_linear_fused_kernel[(triton.cdiv(m,64),)](\\n product.view(-1),gate.view(-1),weights['to_out_norm.weight'],weights['to_out_norm.bias'],\\n weight,output,b,n,h,c,1e-5,BLOCK_M=64,BLOCK_H=h,BLOCK_D=64,num_warps=4)\\n return output\\n\", 'c_finish/tail_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\ndef _as_segments(x):\\n (shape, strides) = (tuple(x.shape), tuple(x.stride()))\\n start = len(shape) - 2\\n while start > 0 and strides[start - 1] == strides[start] * shape[start]:\\n start -= 1\\n rows = 1\\n for size in shape[start:-1]:\\n rows *= size\\n for prefix in _as_itertools.product(*(range(s) for s in shape[:start])):\\n view = x[prefix] if prefix else x\\n yield (prefix, view.as_strided((rows, shape[-1]), (strides[-2], strides[-1])))\\n\\n\\ndef _as_norm(v, gamma, beta, gate=None, eps=1e-05, compact=False):\\n if compact:\\n v = v.contiguous()\\n if gate is None:\\n gate = _as_torch.ones((), dtype=v.dtype, device=v.device).expand(v.shape)\\n else:\\n gate = gate.reshape(v.shape)\\n y = _as_torch.empty_strided(v.shape, v.stride(), dtype=_as_torch.float16, device=v.device)\\n gamma = gamma.reshape(-1)\\n beta = beta.reshape(-1)\\n for (prefix, a) in _as_segments(v):\\n g = (gate[prefix] if prefix else gate).reshape(a.shape)\\n q = y[prefix] if prefix else y\\n z = q.as_strided(a.shape, a.stride())\\n _as_norm_launch(a, g, gamma, beta, z, eps)\\n return y\\n\\n\\ndef _as_tail(v, gate, gamma, beta, weight, output, eps, wh, wn, compact=False):\\n h = v.shape[-1]\\n w = weight.as_strided((h, output.shape[-1]), (wh, wn))\\n y = _as_norm(v, gamma, beta, gate, eps, compact)\\n result = output.view(*y.shape[:-1], output.shape[-1])\\n for (prefix, a) in _as_segments(y):\\n o = (result[prefix] if prefix else result).reshape(a.shape[0], output.shape[-1])\\n _as_gemm_launch(a, w, o)\\n\\n\\n@_as_n_triton.jit\\ndef _as_n_layernorm_gate_kernel(out_ptr, out_gate_ptr, norm_w_ptr, norm_b_ptr, out_norm_ptr, M, H: _as_n_tl.constexpr, stride_out_m, stride_out_h, stride_gate_m, stride_gate_h, stride_norm_w, stride_norm_b, eps: _as_n_tl.constexpr, BLOCK_M: _as_n_tl.constexpr, BLOCK_H: _as_n_tl.constexpr):\\n pid = _as_n_tl.program_id(0)\\n offs_m = pid * BLOCK_M + _as_n_tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n sumsq = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n sum_ += _as_n_tl.sum(out_val, axis=1)\\n sumsq += _as_n_tl.sum(out_val * out_val, axis=1)\\n mean = sum_ / H\\n var = sumsq / H - mean * mean\\n inv_std = 1.0 / _as_n_tl.sqrt(var + eps)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n gate_val = _as_n_tl.load(out_gate_ptr + offs_m[:, None] * stride_gate_m + offs_h[None, :] * stride_gate_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n w = _as_n_tl.load(norm_w_ptr + offs_h * stride_norm_w, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n b = _as_n_tl.load(norm_b_ptr + offs_h * stride_norm_b, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n normalized = (out_val - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n gated = normalized * gate_val\\n _as_n_tl.store(out_norm_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, gated.to(_as_n_tl.float16), mask=mask_m[:, None] & mask_h[None, :])\\n\\n\\ndef _as_norm_launch(v, g, gamma, beta, o, eps):\\n _as_n_layernorm_gate_kernel[_as_triton.cdiv(v.shape[0], 128),](v, g, gamma, beta, o, v.shape[0], v.shape[1], v.stride(0), v.stride(1), g.stride(0), g.stride(1), gamma.stride(0), beta.stride(0), eps, 128, 64)\\n\\n\\n@_as_g_triton.jit\\ndef _as_g_linear_gemm_kernel(a_ptr, b_ptr, c_ptr, M, N, K, stride_am, stride_ak, stride_bk, stride_bn, stride_cm, stride_cn, BLOCK_M: _as_g_tl.constexpr, BLOCK_N: _as_g_tl.constexpr, BLOCK_K: _as_g_tl.constexpr):\\n \"\"\"C = A @ B where A is (M, K) and B is (K, N).\"\"\"\\n pid = _as_g_tl.program_id(0)\\n num_blocks_m = _as_g_tl.cdiv(M, BLOCK_M)\\n num_blocks_n = _as_g_tl.cdiv(N, BLOCK_N)\\n block_m = pid // num_blocks_n\\n block_n = pid % num_blocks_n\\n offs_m = block_m * BLOCK_M + _as_g_tl.arange(0, BLOCK_M)\\n offs_n = block_n * BLOCK_N + _as_g_tl.arange(0, BLOCK_N)\\n mask_m = offs_m < M\\n mask_n = offs_n < N\\n acc = _as_g_tl.zeros((BLOCK_M, BLOCK_N), dtype=_as_g_tl.float32)\\n for k in range(0, _as_g_tl.cdiv(K, BLOCK_K)):\\n k_offset = k * BLOCK_K\\n offs_k = k_offset + _as_g_tl.arange(0, BLOCK_K)\\n a = _as_g_tl.load(a_ptr + (offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak), mask=mask_m[:, None] & (offs_k < K), other=0.0)\\n b = _as_g_tl.load(b_ptr + (offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn), mask=(offs_k < K)[:, None] & mask_n[None, :], other=0.0)\\n acc += _as_g_tl.dot(a, b)\\n c_ptrs = c_ptr + (offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn)\\n _as_g_tl.store(c_ptrs, acc, mask=mask_m[:, None] & mask_n[None, :])\\n\\n\\ndef _as_gemm_launch(a, b, o):\\n _as_g_linear_gemm_kernel[_as_triton.cdiv(a.shape[0], 128) * _as_triton.cdiv(b.shape[1], 64),](a, b, o, a.shape[0], b.shape[1], a.shape[1], a.stride(0), a.stride(1), b.stride(0), b.stride(1), o.stride(0), o.stride(1), 128, 64, 32)\\n\\n', 'c_finish/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_finish/__init__.py': '\\nfrom .output_tail import finish_paper as run\\n', 'c_normalize/input_norm.py': \"import torch\\nimport triton\\nfrom .norm_numeric import component_front_layernorm_fused_kernel\\nfrom .paper_numeric import _row_layernorm_fp16\\n\\n\\ndef normalize_fast(data,packed):\\n x=data[0];b,n,_,c=x.shape;m=b*n*n\\n flat=x.contiguous().view(m,c);normalized=torch.empty_like(flat,dtype=torch.float16,device=x.device)\\n component_front_layernorm_fused_kernel[(triton.cdiv(m,128),)](\\n flat,normalized,packed['norm.weight'],packed['norm.bias'],m,c,1e-5,c,1,c,1,\\n BLOCK_M=128,BLOCK_C=min(c,64))\\n return normalized.view(b,n,n,c)\\n\\n\\ndef normalize_paper(data,packed):\\n # The paper's own LN wrapper owns its weight dtype and launch configuration.\\n return _row_layernorm_fp16(data[0],data[2]['norm.weight'],data[2]['norm.bias'],eps=1e-5)\\n\", 'c_normalize/norm_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@triton.jit\\ndef component_front_layernorm_fused_kernel(x_ptr, out_ptr, norm_w_ptr, norm_b_ptr, M: tl.constexpr, C: tl.constexpr, eps: tl.constexpr, stride_x_m, stride_x_c, stride_out_m, stride_out_c, BLOCK_M: tl.constexpr, BLOCK_C: tl.constexpr):\\n \"\"\"\\n Compute LayerNorm over the last dimension (C) of a (M, C) matrix.\\n Input is float32, output is float16. The weight and bias are float16.\\n \"\"\"\\n pid = tl.program_id(0)\\n offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n sumsq = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n sum_ += tl.sum(x, axis=1)\\n sumsq += tl.sum(x * x, axis=1)\\n mean = sum_ / C\\n var = sumsq / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n w = tl.load(norm_w_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n b = tl.load(norm_b_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n out = (x - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n tl.store(out_ptr + offs_m[:, None] * stride_out_m + offs_c[None, :] * stride_out_c, out.to(tl.float16), mask=mask_m[:, None] & mask_c[None, :])\\n\\n', 'c_normalize/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_normalize/__init__.py': '\\nfrom .input_norm import normalize_fast as run\\n', 'c_project/projection.py': \"import torch\\nimport triton\\nfrom .projection_numeric import component_front_fused_proj_gate_outgate_kernel_4d\\nfrom .paper_numeric import _proj_gate_mask_kernel\\n\\n\\ndef project_fast(data,normalized,packed):\\n x,mask,_,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n flat=normalized.view(m,c).contiguous()\\n mask=torch.ones((b,n,n),dtype=torch.float16,device=x.device)if mask is None else mask.to(torch.float16)\\n mask=mask.reshape(-1).contiguous()\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((m,h),dtype=torch.float16,device=x.device)\\n component_front_fused_proj_gate_outgate_kernel_4d[(triton.cdiv(m,64),triton.cdiv(h,min(h,64)))](\\n flat,mask,packed['left_proj.weight'],packed['left_gate.weight'],packed['right_proj.weight'],\\n packed['right_gate.weight'],packed['out_gate.weight'],left,right,gate,m,h,c,b,n,c,1,c,1,\\n BLOCK_M=64,BLOCK_H=min(h,64),BLOCK_K=32)\\n return left,right,gate\\n\\n\\ndef project_paper(data,normalized,packed):\\n x,mask,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n # This kernel consumes C,H-packed weights, unlike the fast H,C-packed one.\\n names=['left_proj.weight','left_gate.weight','right_proj.weight','right_gate.weight','out_gate.weight']\\n wp=[weights[name].t().contiguous().to(torch.float16)for name in names]\\n masked=mask is not None\\n mask_flat=mask.reshape(m).to(torch.float16).contiguous()if masked else torch.empty(0,dtype=torch.float16,device=x.device)\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((b,n,n,h),dtype=torch.float16,device=x.device)\\n _proj_gate_mask_kernel[(triton.cdiv(m,64),triton.cdiv(h,64))](\\n normalized,mask_flat,*wp,left,right,gate,m,n,c,h,\\n BLOCK_M=64,BLOCK_H=64,BLOCK_K=32,MASKED=int(masked),num_warps=4)\\n return left,right,gate.reshape(m,h)\\n\", 'c_project/projection_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@_bundle_447b911a47b3_triton.jit\\ndef component_front_fused_proj_gate_outgate_kernel_4d(x_ptr, mask_ptr, proj_left_w_ptr, gate_left_w_ptr, proj_right_w_ptr, gate_right_w_ptr, out_gate_w_ptr, out_left_ptr, out_right_ptr, out_gate_ptr, M: _bundle_447b911a47b3_tl.constexpr, H: _bundle_447b911a47b3_tl.constexpr, K: _bundle_447b911a47b3_tl.constexpr, B: _bundle_447b911a47b3_tl.constexpr, N: _bundle_447b911a47b3_tl.constexpr, stride_x_m: _bundle_447b911a47b3_tl.constexpr, stride_x_k: _bundle_447b911a47b3_tl.constexpr, stride_w_h: _bundle_447b911a47b3_tl.constexpr, stride_w_k: _bundle_447b911a47b3_tl.constexpr, BLOCK_M: _bundle_447b911a47b3_tl.constexpr, BLOCK_H: _bundle_447b911a47b3_tl.constexpr, BLOCK_K: _bundle_447b911a47b3_tl.constexpr):\\n \"\"\"Fused projection + gating for left/right tensors and out\u2011gate.\\n\\n Arguments\\n ---------\\n x_ptr : pointer to the flattened input tensor of shape (M, K)\\n mask_ptr : pointer to the flattened mask of shape (M,)\\n *_proj_w_ptr, *_gate_w_ptr : pointers to the weight matrices of shape (H, K)\\n out_gate_w_ptr : pointer to the weight matrix for the out\u2011gate (H, K)\\n left_out_ptr, right_out_ptr : output buffers of shape (B, H, N, N)\\n out_gate_out_ptr : output buffer of shape (M, H) (flattened out\u2011gate)\\n M : total number of rows = B * N * N\\n H : hidden dimension\\n K : input channel dimension (C)\\n B : batch size\\n N : sequence length\\n stride_x_m, stride_x_k : strides for the input tensor\\n stride_w_h, stride_w_k : strides for the weight matrices\\n BLOCK_* : tiling constants\\n \"\"\"\\n _bundle_447b911a47b3_local_pid_m = _bundle_447b911a47b3_tl.program_id(0)\\n _bundle_447b911a47b3_local_pid_h = _bundle_447b911a47b3_tl.program_id(1)\\n _bundle_447b911a47b3_local_offs_m = _bundle_447b911a47b3_local_pid_m * BLOCK_M + _bundle_447b911a47b3_tl.arange(0, BLOCK_M)\\n _bundle_447b911a47b3_local_offs_h = _bundle_447b911a47b3_local_pid_h * BLOCK_H + _bundle_447b911a47b3_tl.arange(0, BLOCK_H)\\n _bundle_447b911a47b3_local_mask_m = _bundle_447b911a47b3_local_offs_m < M\\n _bundle_447b911a47b3_local_mask_h = _bundle_447b911a47b3_local_offs_h < H\\n _bundle_447b911a47b3_local_mask_mh = _bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_h[None, :]\\n _bundle_447b911a47b3_local_m = _bundle_447b911a47b3_tl.load(mask_ptr + _bundle_447b911a47b3_local_offs_m, mask=_bundle_447b911a47b3_local_mask_m, other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_lgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_outgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n for _bundle_447b911a47b3_local_k in range(0, _bundle_447b911a47b3_tl.cdiv(K, BLOCK_K)):\\n _bundle_447b911a47b3_local_offs_k = _bundle_447b911a47b3_local_k * BLOCK_K + _bundle_447b911a47b3_tl.arange(0, BLOCK_K)\\n _bundle_447b911a47b3_local_mask_k = _bundle_447b911a47b3_local_offs_k < K\\n _bundle_447b911a47b3_local_x = _bundle_447b911a47b3_tl.load(x_ptr + _bundle_447b911a47b3_local_offs_m[:, None] * stride_x_m + _bundle_447b911a47b3_local_offs_k[None, :] * stride_x_k, mask=_bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_k[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lproj = _bundle_447b911a47b3_tl.load(proj_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lgate = _bundle_447b911a47b3_tl.load(gate_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rproj = _bundle_447b911a47b3_tl.load(proj_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rgate = _bundle_447b911a47b3_tl.load(gate_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_outgate = _bundle_447b911a47b3_tl.load(out_gate_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lproj)\\n _bundle_447b911a47b3_local_acc_lgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lgate)\\n _bundle_447b911a47b3_local_acc_rproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rproj)\\n _bundle_447b911a47b3_local_acc_rgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rgate)\\n _bundle_447b911a47b3_local_acc_outgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_outgate)\\n _bundle_447b911a47b3_local_gate_l = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_lgate))\\n _bundle_447b911a47b3_local_gate_r = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_rgate))\\n _bundle_447b911a47b3_local_gate_out = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_outgate))\\n _bundle_447b911a47b3_local_left_val = _bundle_447b911a47b3_local_acc_lproj * _bundle_447b911a47b3_local_gate_l * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_right_val = _bundle_447b911a47b3_local_acc_rproj * _bundle_447b911a47b3_local_gate_r * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_N2 = N * N\\n _bundle_447b911a47b3_local_b = _bundle_447b911a47b3_local_offs_m // _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_rem = _bundle_447b911a47b3_local_offs_m % _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_i = _bundle_447b911a47b3_local_rem // N\\n _bundle_447b911a47b3_local_k_idx = _bundle_447b911a47b3_local_rem % N\\n _bundle_447b911a47b3_local_base = _bundle_447b911a47b3_local_b * (H * _bundle_447b911a47b3_local_N2) + _bundle_447b911a47b3_local_i * N + _bundle_447b911a47b3_local_k_idx\\n _bundle_447b911a47b3_local_offset = _bundle_447b911a47b3_local_base[:, None] + (_bundle_447b911a47b3_local_offs_h * _bundle_447b911a47b3_local_N2)[None, :]\\n _bundle_447b911a47b3_local_left_ptrs = out_left_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_local_right_ptrs = out_right_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_left_ptrs, _bundle_447b911a47b3_local_left_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_right_ptrs, _bundle_447b911a47b3_local_right_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_local_out_gate_offsets = _bundle_447b911a47b3_local_offs_m[:, None] * H + _bundle_447b911a47b3_local_offs_h[None, :]\\n _bundle_447b911a47b3_local_out_gate_ptrs = out_gate_ptr + _bundle_447b911a47b3_local_out_gate_offsets\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_out_gate_ptrs, _bundle_447b911a47b3_local_gate_out.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n\\n', 'c_project/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_project/__init__.py': '\\nfrom .projection import project_fast as run\\n', 'entry.py': \"\\nfrom .c_project import run as _gm_project\\n\\nfrom .c_normalize import run as _gm_normalize\\n\\nfrom .c_finish import run as _gm_finish\\nimport torch\\ndef custom_kernel(data):\\n w=data[2]\\n packed={name:w[name].to(torch.float16).contiguous()for name in ['norm.weight','norm.bias','left_proj.weight','right_proj.weight','left_gate.weight','right_gate.weight','out_gate.weight']}\\n normalized=_gm_normalize(data,packed)\\n left,right,gate=_gm_project(data,normalized,packed)\\n b,h,n,_=left.shape\\n product=torch.bmm(left.view(b*h,n,n),right.view(b*h,n,n).transpose(1,2))\\n return _gm_finish(data,product,gate)\\n__all__=['custom_kernel']\\n\"}\nfor _gp_name,_gp_text in _gp_files.items():\n _gp_file=_gp_root/'_gm_2d92355748620d7e4522'/_gp_name\n _gp_file.parent.mkdir(parents=True,exist_ok=True)\n _gp_file.write_text(_gp_text,encoding='utf-8')\n_gp_sys.path.insert(0,str(_gp_root))\n_gp_entry=_gp_import.import_module('_gm_2d92355748620d7e4522'+'.entry')\n\nfor _gp_symbol in getattr(_gp_entry,'__all__',[]):\n globals()[_gp_symbol]=getattr(_gp_entry,_gp_symbol)\n\n__source_binding_identities__={'_gm_project': 'c8ed0952bbc05ee3f4b59a5b9ad8f0b1802de271fca3170e2c391c84b4f3d017', '_gm_normalize': '6d449693f25b00075237ab4067336a98f6103599850dd06fd7b0ab0c1cd46994', '_gm_finish': '06fdfde73bd8e7e935a20beb509a77a8ec1e47a6176a85433b3ab2beb0bb0587'}\n__source_binding_requires__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_memoizable_bindings__={'_gm_project': True, '_gm_normalize': True, '_gm_finish': True}\n__source_state_locations__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_control_domains__={}\ndef _sg_install_bindings():\n pass\n_gm_project = _gp_entry._gm_project\n_gm_normalize = _gp_entry._gm_normalize\n_gm_finish = _gp_entry._gm_finish\nimport torch\n\ndef custom_kernel(data):\n _sg_install_bindings()\n w = data[2]\n packed = {name: w[name].to(torch.float16).contiguous() for name in ['norm.weight', 'norm.bias', 'left_proj.weight', 'right_proj.weight', 'left_gate.weight', 'right_gate.weight', 'out_gate.weight']}\n normalized = _gm_normalize(data, packed)\n (left, right, gate) = _gm_project(data, normalized, packed)\n (b, h, n, _) = left.shape\n product = torch.bmm(left.view(b * h, n, n), right.view(b * h, n, n).transpose(1, 2))\n return _gm_finish(data, product, gate)\n__all__ = ['custom_kernel']\n_gm_project._source_binding_identity = 'c8ed0952bbc05ee3f4b59a5b9ad8f0b1802de271fca3170e2c391c84b4f3d017'\n_gm_project._source_memoizable = True\n_gm_project._source_state_locations = []\n_gm_normalize._source_binding_identity = '6d449693f25b00075237ab4067336a98f6103599850dd06fd7b0ab0c1cd46994'\n_gm_normalize._source_memoizable = True\n_gm_normalize._source_state_locations = []\n_gm_finish._source_binding_identity = '06fdfde73bd8e7e935a20beb509a77a8ec1e47a6176a85433b3ab2beb0bb0587'\n_gm_finish._source_memoizable = True\n_gm_finish._source_state_locations = []\n", "2": "import pathlib as _gp_path, tempfile as _gp_temp, sys as _gp_sys, importlib as _gp_import\n_gp_directory=_gp_temp.TemporaryDirectory(prefix='program_components_')\n_gp_root=_gp_path.Path(_gp_directory.name)\n_gp_files={'__init__.py': '', 'c_finish/output_tail.py': \"import torch\\nimport triton\\nfrom .tail_numeric import _as_tail\\nfrom .paper_numeric import _ln_gate_out_linear_fused_kernel\\n\\n\\ndef finish_fast(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n nw=weights['to_out_norm.weight'].to(torch.float16).contiguous()\\n nb=weights['to_out_norm.bias'].to(torch.float16).contiguous()\\n weight=weights['to_out.weight'].to(torch.float16).contiguous()\\n output=torch.empty((m,c),dtype=torch.float32,device=x.device)\\n # Keep the good code's strided view. Do not force an extra contiguous copy.\\n hidden=product.view(b,h,n,n).permute(0,2,3,1)\\n _as_tail(hidden,gate,nw,nb,weight,output,1e-5,1,h,compact=False)\\n return output.view(b,n,n,c)\\n\\n\\ndef finish_paper(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n weight=weights['to_out.weight'].t().contiguous().to(torch.float16)\\n output=torch.empty((b,n,n,c),dtype=torch.float32,device=x.device)\\n _ln_gate_out_linear_fused_kernel[(triton.cdiv(m,64),)](\\n product.view(-1),gate.view(-1),weights['to_out_norm.weight'],weights['to_out_norm.bias'],\\n weight,output,b,n,h,c,1e-5,BLOCK_M=64,BLOCK_H=h,BLOCK_D=64,num_warps=4)\\n return output\\n\", 'c_finish/tail_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\ndef _as_segments(x):\\n (shape, strides) = (tuple(x.shape), tuple(x.stride()))\\n start = len(shape) - 2\\n while start > 0 and strides[start - 1] == strides[start] * shape[start]:\\n start -= 1\\n rows = 1\\n for size in shape[start:-1]:\\n rows *= size\\n for prefix in _as_itertools.product(*(range(s) for s in shape[:start])):\\n view = x[prefix] if prefix else x\\n yield (prefix, view.as_strided((rows, shape[-1]), (strides[-2], strides[-1])))\\n\\n\\ndef _as_norm(v, gamma, beta, gate=None, eps=1e-05, compact=False):\\n if compact:\\n v = v.contiguous()\\n if gate is None:\\n gate = _as_torch.ones((), dtype=v.dtype, device=v.device).expand(v.shape)\\n else:\\n gate = gate.reshape(v.shape)\\n y = _as_torch.empty_strided(v.shape, v.stride(), dtype=_as_torch.float16, device=v.device)\\n gamma = gamma.reshape(-1)\\n beta = beta.reshape(-1)\\n for (prefix, a) in _as_segments(v):\\n g = (gate[prefix] if prefix else gate).reshape(a.shape)\\n q = y[prefix] if prefix else y\\n z = q.as_strided(a.shape, a.stride())\\n _as_norm_launch(a, g, gamma, beta, z, eps)\\n return y\\n\\n\\ndef _as_tail(v, gate, gamma, beta, weight, output, eps, wh, wn, compact=False):\\n h = v.shape[-1]\\n w = weight.as_strided((h, output.shape[-1]), (wh, wn))\\n y = _as_norm(v, gamma, beta, gate, eps, compact)\\n result = output.view(*y.shape[:-1], output.shape[-1])\\n for (prefix, a) in _as_segments(y):\\n o = (result[prefix] if prefix else result).reshape(a.shape[0], output.shape[-1])\\n _as_gemm_launch(a, w, o)\\n\\n\\n@_as_n_triton.jit\\ndef _as_n_layernorm_gate_kernel(out_ptr, out_gate_ptr, norm_w_ptr, norm_b_ptr, out_norm_ptr, M, H: _as_n_tl.constexpr, stride_out_m, stride_out_h, stride_gate_m, stride_gate_h, stride_norm_w, stride_norm_b, eps: _as_n_tl.constexpr, BLOCK_M: _as_n_tl.constexpr, BLOCK_H: _as_n_tl.constexpr):\\n pid = _as_n_tl.program_id(0)\\n offs_m = pid * BLOCK_M + _as_n_tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n sumsq = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n sum_ += _as_n_tl.sum(out_val, axis=1)\\n sumsq += _as_n_tl.sum(out_val * out_val, axis=1)\\n mean = sum_ / H\\n var = sumsq / H - mean * mean\\n inv_std = 1.0 / _as_n_tl.sqrt(var + eps)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n gate_val = _as_n_tl.load(out_gate_ptr + offs_m[:, None] * stride_gate_m + offs_h[None, :] * stride_gate_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n w = _as_n_tl.load(norm_w_ptr + offs_h * stride_norm_w, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n b = _as_n_tl.load(norm_b_ptr + offs_h * stride_norm_b, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n normalized = (out_val - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n gated = normalized * gate_val\\n _as_n_tl.store(out_norm_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, gated.to(_as_n_tl.float16), mask=mask_m[:, None] & mask_h[None, :])\\n\\n\\ndef _as_norm_launch(v, g, gamma, beta, o, eps):\\n _as_n_layernorm_gate_kernel[_as_triton.cdiv(v.shape[0], 128),](v, g, gamma, beta, o, v.shape[0], v.shape[1], v.stride(0), v.stride(1), g.stride(0), g.stride(1), gamma.stride(0), beta.stride(0), eps, 128, 64)\\n\\n\\n@_as_g_triton.jit\\ndef _as_g_linear_gemm_kernel(a_ptr, b_ptr, c_ptr, M, N, K, stride_am, stride_ak, stride_bk, stride_bn, stride_cm, stride_cn, BLOCK_M: _as_g_tl.constexpr, BLOCK_N: _as_g_tl.constexpr, BLOCK_K: _as_g_tl.constexpr):\\n \"\"\"C = A @ B where A is (M, K) and B is (K, N).\"\"\"\\n pid = _as_g_tl.program_id(0)\\n num_blocks_m = _as_g_tl.cdiv(M, BLOCK_M)\\n num_blocks_n = _as_g_tl.cdiv(N, BLOCK_N)\\n block_m = pid // num_blocks_n\\n block_n = pid % num_blocks_n\\n offs_m = block_m * BLOCK_M + _as_g_tl.arange(0, BLOCK_M)\\n offs_n = block_n * BLOCK_N + _as_g_tl.arange(0, BLOCK_N)\\n mask_m = offs_m < M\\n mask_n = offs_n < N\\n acc = _as_g_tl.zeros((BLOCK_M, BLOCK_N), dtype=_as_g_tl.float32)\\n for k in range(0, _as_g_tl.cdiv(K, BLOCK_K)):\\n k_offset = k * BLOCK_K\\n offs_k = k_offset + _as_g_tl.arange(0, BLOCK_K)\\n a = _as_g_tl.load(a_ptr + (offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak), mask=mask_m[:, None] & (offs_k < K), other=0.0)\\n b = _as_g_tl.load(b_ptr + (offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn), mask=(offs_k < K)[:, None] & mask_n[None, :], other=0.0)\\n acc += _as_g_tl.dot(a, b)\\n c_ptrs = c_ptr + (offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn)\\n _as_g_tl.store(c_ptrs, acc, mask=mask_m[:, None] & mask_n[None, :])\\n\\n\\ndef _as_gemm_launch(a, b, o):\\n _as_g_linear_gemm_kernel[_as_triton.cdiv(a.shape[0], 128) * _as_triton.cdiv(b.shape[1], 64),](a, b, o, a.shape[0], b.shape[1], a.shape[1], a.stride(0), a.stride(1), b.stride(0), b.stride(1), o.stride(0), o.stride(1), 128, 64, 32)\\n\\n', 'c_finish/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_finish/__init__.py': '\\nfrom .output_tail import finish_fast as run\\n', 'c_normalize/input_norm.py': \"import torch\\nimport triton\\nfrom .norm_numeric import component_front_layernorm_fused_kernel\\nfrom .paper_numeric import _row_layernorm_fp16\\n\\n\\ndef normalize_fast(data,packed):\\n x=data[0];b,n,_,c=x.shape;m=b*n*n\\n flat=x.contiguous().view(m,c);normalized=torch.empty_like(flat,dtype=torch.float16,device=x.device)\\n component_front_layernorm_fused_kernel[(triton.cdiv(m,128),)](\\n flat,normalized,packed['norm.weight'],packed['norm.bias'],m,c,1e-5,c,1,c,1,\\n BLOCK_M=128,BLOCK_C=min(c,64))\\n return normalized.view(b,n,n,c)\\n\\n\\ndef normalize_paper(data,packed):\\n # The paper's own LN wrapper owns its weight dtype and launch configuration.\\n return _row_layernorm_fp16(data[0],data[2]['norm.weight'],data[2]['norm.bias'],eps=1e-5)\\n\", 'c_normalize/norm_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@triton.jit\\ndef component_front_layernorm_fused_kernel(x_ptr, out_ptr, norm_w_ptr, norm_b_ptr, M: tl.constexpr, C: tl.constexpr, eps: tl.constexpr, stride_x_m, stride_x_c, stride_out_m, stride_out_c, BLOCK_M: tl.constexpr, BLOCK_C: tl.constexpr):\\n \"\"\"\\n Compute LayerNorm over the last dimension (C) of a (M, C) matrix.\\n Input is float32, output is float16. The weight and bias are float16.\\n \"\"\"\\n pid = tl.program_id(0)\\n offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n sumsq = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n sum_ += tl.sum(x, axis=1)\\n sumsq += tl.sum(x * x, axis=1)\\n mean = sum_ / C\\n var = sumsq / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n w = tl.load(norm_w_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n b = tl.load(norm_b_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n out = (x - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n tl.store(out_ptr + offs_m[:, None] * stride_out_m + offs_c[None, :] * stride_out_c, out.to(tl.float16), mask=mask_m[:, None] & mask_c[None, :])\\n\\n', 'c_normalize/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_normalize/__init__.py': '\\nfrom .input_norm import normalize_paper as run\\n', 'c_project/projection.py': \"import torch\\nimport triton\\nfrom .projection_numeric import component_front_fused_proj_gate_outgate_kernel_4d\\nfrom .paper_numeric import _proj_gate_mask_kernel\\n\\n\\ndef project_fast(data,normalized,packed):\\n x,mask,_,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n flat=normalized.view(m,c).contiguous()\\n mask=torch.ones((b,n,n),dtype=torch.float16,device=x.device)if mask is None else mask.to(torch.float16)\\n mask=mask.reshape(-1).contiguous()\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((m,h),dtype=torch.float16,device=x.device)\\n component_front_fused_proj_gate_outgate_kernel_4d[(triton.cdiv(m,64),triton.cdiv(h,min(h,64)))](\\n flat,mask,packed['left_proj.weight'],packed['left_gate.weight'],packed['right_proj.weight'],\\n packed['right_gate.weight'],packed['out_gate.weight'],left,right,gate,m,h,c,b,n,c,1,c,1,\\n BLOCK_M=64,BLOCK_H=min(h,64),BLOCK_K=32)\\n return left,right,gate\\n\\n\\ndef project_paper(data,normalized,packed):\\n x,mask,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n # This kernel consumes C,H-packed weights, unlike the fast H,C-packed one.\\n names=['left_proj.weight','left_gate.weight','right_proj.weight','right_gate.weight','out_gate.weight']\\n wp=[weights[name].t().contiguous().to(torch.float16)for name in names]\\n masked=mask is not None\\n mask_flat=mask.reshape(m).to(torch.float16).contiguous()if masked else torch.empty(0,dtype=torch.float16,device=x.device)\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((b,n,n,h),dtype=torch.float16,device=x.device)\\n _proj_gate_mask_kernel[(triton.cdiv(m,64),triton.cdiv(h,64))](\\n normalized,mask_flat,*wp,left,right,gate,m,n,c,h,\\n BLOCK_M=64,BLOCK_H=64,BLOCK_K=32,MASKED=int(masked),num_warps=4)\\n return left,right,gate.reshape(m,h)\\n\", 'c_project/projection_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@_bundle_447b911a47b3_triton.jit\\ndef component_front_fused_proj_gate_outgate_kernel_4d(x_ptr, mask_ptr, proj_left_w_ptr, gate_left_w_ptr, proj_right_w_ptr, gate_right_w_ptr, out_gate_w_ptr, out_left_ptr, out_right_ptr, out_gate_ptr, M: _bundle_447b911a47b3_tl.constexpr, H: _bundle_447b911a47b3_tl.constexpr, K: _bundle_447b911a47b3_tl.constexpr, B: _bundle_447b911a47b3_tl.constexpr, N: _bundle_447b911a47b3_tl.constexpr, stride_x_m: _bundle_447b911a47b3_tl.constexpr, stride_x_k: _bundle_447b911a47b3_tl.constexpr, stride_w_h: _bundle_447b911a47b3_tl.constexpr, stride_w_k: _bundle_447b911a47b3_tl.constexpr, BLOCK_M: _bundle_447b911a47b3_tl.constexpr, BLOCK_H: _bundle_447b911a47b3_tl.constexpr, BLOCK_K: _bundle_447b911a47b3_tl.constexpr):\\n \"\"\"Fused projection + gating for left/right tensors and out\u2011gate.\\n\\n Arguments\\n ---------\\n x_ptr : pointer to the flattened input tensor of shape (M, K)\\n mask_ptr : pointer to the flattened mask of shape (M,)\\n *_proj_w_ptr, *_gate_w_ptr : pointers to the weight matrices of shape (H, K)\\n out_gate_w_ptr : pointer to the weight matrix for the out\u2011gate (H, K)\\n left_out_ptr, right_out_ptr : output buffers of shape (B, H, N, N)\\n out_gate_out_ptr : output buffer of shape (M, H) (flattened out\u2011gate)\\n M : total number of rows = B * N * N\\n H : hidden dimension\\n K : input channel dimension (C)\\n B : batch size\\n N : sequence length\\n stride_x_m, stride_x_k : strides for the input tensor\\n stride_w_h, stride_w_k : strides for the weight matrices\\n BLOCK_* : tiling constants\\n \"\"\"\\n _bundle_447b911a47b3_local_pid_m = _bundle_447b911a47b3_tl.program_id(0)\\n _bundle_447b911a47b3_local_pid_h = _bundle_447b911a47b3_tl.program_id(1)\\n _bundle_447b911a47b3_local_offs_m = _bundle_447b911a47b3_local_pid_m * BLOCK_M + _bundle_447b911a47b3_tl.arange(0, BLOCK_M)\\n _bundle_447b911a47b3_local_offs_h = _bundle_447b911a47b3_local_pid_h * BLOCK_H + _bundle_447b911a47b3_tl.arange(0, BLOCK_H)\\n _bundle_447b911a47b3_local_mask_m = _bundle_447b911a47b3_local_offs_m < M\\n _bundle_447b911a47b3_local_mask_h = _bundle_447b911a47b3_local_offs_h < H\\n _bundle_447b911a47b3_local_mask_mh = _bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_h[None, :]\\n _bundle_447b911a47b3_local_m = _bundle_447b911a47b3_tl.load(mask_ptr + _bundle_447b911a47b3_local_offs_m, mask=_bundle_447b911a47b3_local_mask_m, other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_lgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_outgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n for _bundle_447b911a47b3_local_k in range(0, _bundle_447b911a47b3_tl.cdiv(K, BLOCK_K)):\\n _bundle_447b911a47b3_local_offs_k = _bundle_447b911a47b3_local_k * BLOCK_K + _bundle_447b911a47b3_tl.arange(0, BLOCK_K)\\n _bundle_447b911a47b3_local_mask_k = _bundle_447b911a47b3_local_offs_k < K\\n _bundle_447b911a47b3_local_x = _bundle_447b911a47b3_tl.load(x_ptr + _bundle_447b911a47b3_local_offs_m[:, None] * stride_x_m + _bundle_447b911a47b3_local_offs_k[None, :] * stride_x_k, mask=_bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_k[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lproj = _bundle_447b911a47b3_tl.load(proj_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lgate = _bundle_447b911a47b3_tl.load(gate_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rproj = _bundle_447b911a47b3_tl.load(proj_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rgate = _bundle_447b911a47b3_tl.load(gate_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_outgate = _bundle_447b911a47b3_tl.load(out_gate_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lproj)\\n _bundle_447b911a47b3_local_acc_lgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lgate)\\n _bundle_447b911a47b3_local_acc_rproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rproj)\\n _bundle_447b911a47b3_local_acc_rgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rgate)\\n _bundle_447b911a47b3_local_acc_outgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_outgate)\\n _bundle_447b911a47b3_local_gate_l = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_lgate))\\n _bundle_447b911a47b3_local_gate_r = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_rgate))\\n _bundle_447b911a47b3_local_gate_out = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_outgate))\\n _bundle_447b911a47b3_local_left_val = _bundle_447b911a47b3_local_acc_lproj * _bundle_447b911a47b3_local_gate_l * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_right_val = _bundle_447b911a47b3_local_acc_rproj * _bundle_447b911a47b3_local_gate_r * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_N2 = N * N\\n _bundle_447b911a47b3_local_b = _bundle_447b911a47b3_local_offs_m // _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_rem = _bundle_447b911a47b3_local_offs_m % _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_i = _bundle_447b911a47b3_local_rem // N\\n _bundle_447b911a47b3_local_k_idx = _bundle_447b911a47b3_local_rem % N\\n _bundle_447b911a47b3_local_base = _bundle_447b911a47b3_local_b * (H * _bundle_447b911a47b3_local_N2) + _bundle_447b911a47b3_local_i * N + _bundle_447b911a47b3_local_k_idx\\n _bundle_447b911a47b3_local_offset = _bundle_447b911a47b3_local_base[:, None] + (_bundle_447b911a47b3_local_offs_h * _bundle_447b911a47b3_local_N2)[None, :]\\n _bundle_447b911a47b3_local_left_ptrs = out_left_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_local_right_ptrs = out_right_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_left_ptrs, _bundle_447b911a47b3_local_left_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_right_ptrs, _bundle_447b911a47b3_local_right_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_local_out_gate_offsets = _bundle_447b911a47b3_local_offs_m[:, None] * H + _bundle_447b911a47b3_local_offs_h[None, :]\\n _bundle_447b911a47b3_local_out_gate_ptrs = out_gate_ptr + _bundle_447b911a47b3_local_out_gate_offsets\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_out_gate_ptrs, _bundle_447b911a47b3_local_gate_out.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n\\n', 'c_project/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_project/__init__.py': '\\nfrom .projection import project_fast as run\\n', 'entry.py': \"\\nfrom .c_project import run as _gm_project\\n\\nfrom .c_normalize import run as _gm_normalize\\n\\nfrom .c_finish import run as _gm_finish\\nimport torch\\ndef custom_kernel(data):\\n w=data[2]\\n packed={name:w[name].to(torch.float16).contiguous()for name in ['norm.weight','norm.bias','left_proj.weight','right_proj.weight','left_gate.weight','right_gate.weight','out_gate.weight']}\\n normalized=_gm_normalize(data,packed)\\n left,right,gate=_gm_project(data,normalized,packed)\\n b,h,n,_=left.shape\\n product=torch.bmm(left.view(b*h,n,n),right.view(b*h,n,n).transpose(1,2))\\n return _gm_finish(data,product,gate)\\n__all__=['custom_kernel']\\n\"}\nfor _gp_name,_gp_text in _gp_files.items():\n _gp_file=_gp_root/'_gm_88e66a6ab5d6d93bc507'/_gp_name\n _gp_file.parent.mkdir(parents=True,exist_ok=True)\n _gp_file.write_text(_gp_text,encoding='utf-8')\n_gp_sys.path.insert(0,str(_gp_root))\n_gp_entry=_gp_import.import_module('_gm_88e66a6ab5d6d93bc507'+'.entry')\n\nfor _gp_symbol in getattr(_gp_entry,'__all__',[]):\n globals()[_gp_symbol]=getattr(_gp_entry,_gp_symbol)\n\n__source_binding_identities__={'_gm_project': 'c8ed0952bbc05ee3f4b59a5b9ad8f0b1802de271fca3170e2c391c84b4f3d017', '_gm_normalize': 'd9b2d433ff8415f20ac37127cfa841378a77b1d070d941ca49851cfd9f5c3371', '_gm_finish': '977da3d0268e7082d556995b378e7058d880cabcba6d4613c6ac793fd3dc980b'}\n__source_binding_requires__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_memoizable_bindings__={'_gm_project': True, '_gm_normalize': True, '_gm_finish': True}\n__source_state_locations__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_control_domains__={}\ndef _sg_install_bindings():\n pass\n_gm_project = _gp_entry._gm_project\n_gm_normalize = _gp_entry._gm_normalize\n_gm_finish = _gp_entry._gm_finish\nimport torch\n\ndef custom_kernel(data):\n _sg_install_bindings()\n w = data[2]\n packed = {name: w[name].to(torch.float16).contiguous() for name in ['norm.weight', 'norm.bias', 'left_proj.weight', 'right_proj.weight', 'left_gate.weight', 'right_gate.weight', 'out_gate.weight']}\n normalized = _gm_normalize(data, packed)\n (left, right, gate) = _gm_project(data, normalized, packed)\n (b, h, n, _) = left.shape\n product = torch.bmm(left.view(b * h, n, n), right.view(b * h, n, n).transpose(1, 2))\n return _gm_finish(data, product, gate)\n__all__ = ['custom_kernel']\n_gm_project._source_binding_identity = 'c8ed0952bbc05ee3f4b59a5b9ad8f0b1802de271fca3170e2c391c84b4f3d017'\n_gm_project._source_memoizable = True\n_gm_project._source_state_locations = []\n_gm_normalize._source_binding_identity = 'd9b2d433ff8415f20ac37127cfa841378a77b1d070d941ca49851cfd9f5c3371'\n_gm_normalize._source_memoizable = True\n_gm_normalize._source_state_locations = []\n_gm_finish._source_binding_identity = '977da3d0268e7082d556995b378e7058d880cabcba6d4613c6ac793fd3dc980b'\n_gm_finish._source_memoizable = True\n_gm_finish._source_state_locations = []\n", "3": "import pathlib as _gp_path, tempfile as _gp_temp, sys as _gp_sys, importlib as _gp_import\n_gp_directory=_gp_temp.TemporaryDirectory(prefix='program_components_')\n_gp_root=_gp_path.Path(_gp_directory.name)\n_gp_files={'__init__.py': '', 'c_finish/output_tail.py': \"import torch\\nimport triton\\nfrom .tail_numeric import _as_tail\\nfrom .paper_numeric import _ln_gate_out_linear_fused_kernel\\n\\n\\ndef finish_fast(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n nw=weights['to_out_norm.weight'].to(torch.float16).contiguous()\\n nb=weights['to_out_norm.bias'].to(torch.float16).contiguous()\\n weight=weights['to_out.weight'].to(torch.float16).contiguous()\\n output=torch.empty((m,c),dtype=torch.float32,device=x.device)\\n # Keep the good code's strided view. Do not force an extra contiguous copy.\\n hidden=product.view(b,h,n,n).permute(0,2,3,1)\\n _as_tail(hidden,gate,nw,nb,weight,output,1e-5,1,h,compact=False)\\n return output.view(b,n,n,c)\\n\\n\\ndef finish_paper(data,product,gate):\\n x,_,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n weight=weights['to_out.weight'].t().contiguous().to(torch.float16)\\n output=torch.empty((b,n,n,c),dtype=torch.float32,device=x.device)\\n _ln_gate_out_linear_fused_kernel[(triton.cdiv(m,64),)](\\n product.view(-1),gate.view(-1),weights['to_out_norm.weight'],weights['to_out_norm.bias'],\\n weight,output,b,n,h,c,1e-5,BLOCK_M=64,BLOCK_H=h,BLOCK_D=64,num_warps=4)\\n return output\\n\", 'c_finish/tail_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\ndef _as_segments(x):\\n (shape, strides) = (tuple(x.shape), tuple(x.stride()))\\n start = len(shape) - 2\\n while start > 0 and strides[start - 1] == strides[start] * shape[start]:\\n start -= 1\\n rows = 1\\n for size in shape[start:-1]:\\n rows *= size\\n for prefix in _as_itertools.product(*(range(s) for s in shape[:start])):\\n view = x[prefix] if prefix else x\\n yield (prefix, view.as_strided((rows, shape[-1]), (strides[-2], strides[-1])))\\n\\n\\ndef _as_norm(v, gamma, beta, gate=None, eps=1e-05, compact=False):\\n if compact:\\n v = v.contiguous()\\n if gate is None:\\n gate = _as_torch.ones((), dtype=v.dtype, device=v.device).expand(v.shape)\\n else:\\n gate = gate.reshape(v.shape)\\n y = _as_torch.empty_strided(v.shape, v.stride(), dtype=_as_torch.float16, device=v.device)\\n gamma = gamma.reshape(-1)\\n beta = beta.reshape(-1)\\n for (prefix, a) in _as_segments(v):\\n g = (gate[prefix] if prefix else gate).reshape(a.shape)\\n q = y[prefix] if prefix else y\\n z = q.as_strided(a.shape, a.stride())\\n _as_norm_launch(a, g, gamma, beta, z, eps)\\n return y\\n\\n\\ndef _as_tail(v, gate, gamma, beta, weight, output, eps, wh, wn, compact=False):\\n h = v.shape[-1]\\n w = weight.as_strided((h, output.shape[-1]), (wh, wn))\\n y = _as_norm(v, gamma, beta, gate, eps, compact)\\n result = output.view(*y.shape[:-1], output.shape[-1])\\n for (prefix, a) in _as_segments(y):\\n o = (result[prefix] if prefix else result).reshape(a.shape[0], output.shape[-1])\\n _as_gemm_launch(a, w, o)\\n\\n\\n@_as_n_triton.jit\\ndef _as_n_layernorm_gate_kernel(out_ptr, out_gate_ptr, norm_w_ptr, norm_b_ptr, out_norm_ptr, M, H: _as_n_tl.constexpr, stride_out_m, stride_out_h, stride_gate_m, stride_gate_h, stride_norm_w, stride_norm_b, eps: _as_n_tl.constexpr, BLOCK_M: _as_n_tl.constexpr, BLOCK_H: _as_n_tl.constexpr):\\n pid = _as_n_tl.program_id(0)\\n offs_m = pid * BLOCK_M + _as_n_tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n sumsq = _as_n_tl.zeros((BLOCK_M,), dtype=_as_n_tl.float32)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n sum_ += _as_n_tl.sum(out_val, axis=1)\\n sumsq += _as_n_tl.sum(out_val * out_val, axis=1)\\n mean = sum_ / H\\n var = sumsq / H - mean * mean\\n inv_std = 1.0 / _as_n_tl.sqrt(var + eps)\\n for h in range(0, H, BLOCK_H):\\n offs_h = h + _as_n_tl.arange(0, BLOCK_H)\\n mask_h = offs_h < H\\n out_val = _as_n_tl.load(out_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n gate_val = _as_n_tl.load(out_gate_ptr + offs_m[:, None] * stride_gate_m + offs_h[None, :] * stride_gate_h, mask=mask_m[:, None] & mask_h[None, :], other=0.0).to(_as_n_tl.float32)\\n w = _as_n_tl.load(norm_w_ptr + offs_h * stride_norm_w, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n b = _as_n_tl.load(norm_b_ptr + offs_h * stride_norm_b, mask=mask_h, other=0.0).to(_as_n_tl.float32)\\n normalized = (out_val - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n gated = normalized * gate_val\\n _as_n_tl.store(out_norm_ptr + offs_m[:, None] * stride_out_m + offs_h[None, :] * stride_out_h, gated.to(_as_n_tl.float16), mask=mask_m[:, None] & mask_h[None, :])\\n\\n\\ndef _as_norm_launch(v, g, gamma, beta, o, eps):\\n _as_n_layernorm_gate_kernel[_as_triton.cdiv(v.shape[0], 128),](v, g, gamma, beta, o, v.shape[0], v.shape[1], v.stride(0), v.stride(1), g.stride(0), g.stride(1), gamma.stride(0), beta.stride(0), eps, 128, 64)\\n\\n\\n@_as_g_triton.jit\\ndef _as_g_linear_gemm_kernel(a_ptr, b_ptr, c_ptr, M, N, K, stride_am, stride_ak, stride_bk, stride_bn, stride_cm, stride_cn, BLOCK_M: _as_g_tl.constexpr, BLOCK_N: _as_g_tl.constexpr, BLOCK_K: _as_g_tl.constexpr):\\n \"\"\"C = A @ B where A is (M, K) and B is (K, N).\"\"\"\\n pid = _as_g_tl.program_id(0)\\n num_blocks_m = _as_g_tl.cdiv(M, BLOCK_M)\\n num_blocks_n = _as_g_tl.cdiv(N, BLOCK_N)\\n block_m = pid // num_blocks_n\\n block_n = pid % num_blocks_n\\n offs_m = block_m * BLOCK_M + _as_g_tl.arange(0, BLOCK_M)\\n offs_n = block_n * BLOCK_N + _as_g_tl.arange(0, BLOCK_N)\\n mask_m = offs_m < M\\n mask_n = offs_n < N\\n acc = _as_g_tl.zeros((BLOCK_M, BLOCK_N), dtype=_as_g_tl.float32)\\n for k in range(0, _as_g_tl.cdiv(K, BLOCK_K)):\\n k_offset = k * BLOCK_K\\n offs_k = k_offset + _as_g_tl.arange(0, BLOCK_K)\\n a = _as_g_tl.load(a_ptr + (offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak), mask=mask_m[:, None] & (offs_k < K), other=0.0)\\n b = _as_g_tl.load(b_ptr + (offs_k[:, None] * stride_bk + offs_n[None, :] * stride_bn), mask=(offs_k < K)[:, None] & mask_n[None, :], other=0.0)\\n acc += _as_g_tl.dot(a, b)\\n c_ptrs = c_ptr + (offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn)\\n _as_g_tl.store(c_ptrs, acc, mask=mask_m[:, None] & mask_n[None, :])\\n\\n\\ndef _as_gemm_launch(a, b, o):\\n _as_g_linear_gemm_kernel[_as_triton.cdiv(a.shape[0], 128) * _as_triton.cdiv(b.shape[1], 64),](a, b, o, a.shape[0], b.shape[1], a.shape[1], a.stride(0), a.stride(1), b.stride(0), b.stride(1), o.stride(0), o.stride(1), 128, 64, 32)\\n\\n', 'c_finish/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_finish/__init__.py': '\\nfrom .output_tail import finish_fast as run\\n', 'c_normalize/input_norm.py': \"import torch\\nimport triton\\nfrom .norm_numeric import component_front_layernorm_fused_kernel\\nfrom .paper_numeric import _row_layernorm_fp16\\n\\n\\ndef normalize_fast(data,packed):\\n x=data[0];b,n,_,c=x.shape;m=b*n*n\\n flat=x.contiguous().view(m,c);normalized=torch.empty_like(flat,dtype=torch.float16,device=x.device)\\n component_front_layernorm_fused_kernel[(triton.cdiv(m,128),)](\\n flat,normalized,packed['norm.weight'],packed['norm.bias'],m,c,1e-5,c,1,c,1,\\n BLOCK_M=128,BLOCK_C=min(c,64))\\n return normalized.view(b,n,n,c)\\n\\n\\ndef normalize_paper(data,packed):\\n # The paper's own LN wrapper owns its weight dtype and launch configuration.\\n return _row_layernorm_fp16(data[0],data[2]['norm.weight'],data[2]['norm.bias'],eps=1e-5)\\n\", 'c_normalize/norm_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@triton.jit\\ndef component_front_layernorm_fused_kernel(x_ptr, out_ptr, norm_w_ptr, norm_b_ptr, M: tl.constexpr, C: tl.constexpr, eps: tl.constexpr, stride_x_m, stride_x_c, stride_out_m, stride_out_c, BLOCK_M: tl.constexpr, BLOCK_C: tl.constexpr):\\n \"\"\"\\n Compute LayerNorm over the last dimension (C) of a (M, C) matrix.\\n Input is float32, output is float16. The weight and bias are float16.\\n \"\"\"\\n pid = tl.program_id(0)\\n offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)\\n mask_m = offs_m < M\\n sum_ = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n sumsq = tl.zeros((BLOCK_M,), dtype=tl.float32)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n sum_ += tl.sum(x, axis=1)\\n sumsq += tl.sum(x * x, axis=1)\\n mean = sum_ / C\\n var = sumsq / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n for k in range(0, tl.cdiv(C, BLOCK_C)):\\n offs_c = k * BLOCK_C + tl.arange(0, BLOCK_C)\\n mask_c = offs_c < C\\n x = tl.load(x_ptr + offs_m[:, None] * stride_x_m + offs_c[None, :] * stride_x_c, mask=mask_m[:, None] & mask_c[None, :], other=0.0).to(tl.float32)\\n w = tl.load(norm_w_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n b = tl.load(norm_b_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32)\\n out = (x - mean[:, None]) * inv_std[:, None] * w[None, :] + b[None, :]\\n tl.store(out_ptr + offs_m[:, None] * stride_out_m + offs_c[None, :] * stride_out_c, out.to(tl.float16), mask=mask_m[:, None] & mask_c[None, :])\\n\\n', 'c_normalize/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_normalize/__init__.py': '\\nfrom .input_norm import normalize_fast as run\\n', 'c_project/projection.py': \"import torch\\nimport triton\\nfrom .projection_numeric import component_front_fused_proj_gate_outgate_kernel_4d\\nfrom .paper_numeric import _proj_gate_mask_kernel\\n\\n\\ndef project_fast(data,normalized,packed):\\n x,mask,_,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n flat=normalized.view(m,c).contiguous()\\n mask=torch.ones((b,n,n),dtype=torch.float16,device=x.device)if mask is None else mask.to(torch.float16)\\n mask=mask.reshape(-1).contiguous()\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((m,h),dtype=torch.float16,device=x.device)\\n component_front_fused_proj_gate_outgate_kernel_4d[(triton.cdiv(m,64),triton.cdiv(h,min(h,64)))](\\n flat,mask,packed['left_proj.weight'],packed['left_gate.weight'],packed['right_proj.weight'],\\n packed['right_gate.weight'],packed['out_gate.weight'],left,right,gate,m,h,c,b,n,c,1,c,1,\\n BLOCK_M=64,BLOCK_H=min(h,64),BLOCK_K=32)\\n return left,right,gate\\n\\n\\ndef project_paper(data,normalized,packed):\\n x,mask,weights,cfg=data;b,n,_,c=x.shape;h=cfg['hidden_dim'];m=b*n*n\\n # This kernel consumes C,H-packed weights, unlike the fast H,C-packed one.\\n names=['left_proj.weight','left_gate.weight','right_proj.weight','right_gate.weight','out_gate.weight']\\n wp=[weights[name].t().contiguous().to(torch.float16)for name in names]\\n masked=mask is not None\\n mask_flat=mask.reshape(m).to(torch.float16).contiguous()if masked else torch.empty(0,dtype=torch.float16,device=x.device)\\n left=torch.empty((b,h,n,n),dtype=torch.float16,device=x.device);right=torch.empty_like(left)\\n gate=torch.empty((b,n,n,h),dtype=torch.float16,device=x.device)\\n _proj_gate_mask_kernel[(triton.cdiv(m,64),triton.cdiv(h,64))](\\n normalized,mask_flat,*wp,left,right,gate,m,n,c,h,\\n BLOCK_M=64,BLOCK_H=64,BLOCK_K=32,MASKED=int(masked),num_warps=4)\\n return left,right,gate.reshape(m,h)\\n\", 'c_project/projection_numeric.py': 'import torch\\nimport triton\\nimport triton.language as tl\\nimport torch as _as_torch\\nimport triton as _as_triton\\nimport itertools as _as_itertools\\nimport torch as _as_n_torch\\nimport triton as _as_n_triton\\nimport triton.language as _as_n_tl\\nimport math as _as_g_math\\nimport torch as _as_g_torch\\nimport triton as _as_g_triton\\nimport triton.language as _as_g_tl\\nfrom torch.nn import functional as _as_g_F\\nimport torch as _bundle_447b911a47b3_torch\\nimport triton as _bundle_447b911a47b3_triton\\nimport triton.language as _bundle_447b911a47b3_tl\\n\\n@_bundle_447b911a47b3_triton.jit\\ndef component_front_fused_proj_gate_outgate_kernel_4d(x_ptr, mask_ptr, proj_left_w_ptr, gate_left_w_ptr, proj_right_w_ptr, gate_right_w_ptr, out_gate_w_ptr, out_left_ptr, out_right_ptr, out_gate_ptr, M: _bundle_447b911a47b3_tl.constexpr, H: _bundle_447b911a47b3_tl.constexpr, K: _bundle_447b911a47b3_tl.constexpr, B: _bundle_447b911a47b3_tl.constexpr, N: _bundle_447b911a47b3_tl.constexpr, stride_x_m: _bundle_447b911a47b3_tl.constexpr, stride_x_k: _bundle_447b911a47b3_tl.constexpr, stride_w_h: _bundle_447b911a47b3_tl.constexpr, stride_w_k: _bundle_447b911a47b3_tl.constexpr, BLOCK_M: _bundle_447b911a47b3_tl.constexpr, BLOCK_H: _bundle_447b911a47b3_tl.constexpr, BLOCK_K: _bundle_447b911a47b3_tl.constexpr):\\n \"\"\"Fused projection + gating for left/right tensors and out\u2011gate.\\n\\n Arguments\\n ---------\\n x_ptr : pointer to the flattened input tensor of shape (M, K)\\n mask_ptr : pointer to the flattened mask of shape (M,)\\n *_proj_w_ptr, *_gate_w_ptr : pointers to the weight matrices of shape (H, K)\\n out_gate_w_ptr : pointer to the weight matrix for the out\u2011gate (H, K)\\n left_out_ptr, right_out_ptr : output buffers of shape (B, H, N, N)\\n out_gate_out_ptr : output buffer of shape (M, H) (flattened out\u2011gate)\\n M : total number of rows = B * N * N\\n H : hidden dimension\\n K : input channel dimension (C)\\n B : batch size\\n N : sequence length\\n stride_x_m, stride_x_k : strides for the input tensor\\n stride_w_h, stride_w_k : strides for the weight matrices\\n BLOCK_* : tiling constants\\n \"\"\"\\n _bundle_447b911a47b3_local_pid_m = _bundle_447b911a47b3_tl.program_id(0)\\n _bundle_447b911a47b3_local_pid_h = _bundle_447b911a47b3_tl.program_id(1)\\n _bundle_447b911a47b3_local_offs_m = _bundle_447b911a47b3_local_pid_m * BLOCK_M + _bundle_447b911a47b3_tl.arange(0, BLOCK_M)\\n _bundle_447b911a47b3_local_offs_h = _bundle_447b911a47b3_local_pid_h * BLOCK_H + _bundle_447b911a47b3_tl.arange(0, BLOCK_H)\\n _bundle_447b911a47b3_local_mask_m = _bundle_447b911a47b3_local_offs_m < M\\n _bundle_447b911a47b3_local_mask_h = _bundle_447b911a47b3_local_offs_h < H\\n _bundle_447b911a47b3_local_mask_mh = _bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_h[None, :]\\n _bundle_447b911a47b3_local_m = _bundle_447b911a47b3_tl.load(mask_ptr + _bundle_447b911a47b3_local_offs_m, mask=_bundle_447b911a47b3_local_mask_m, other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_lgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rproj = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_rgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n _bundle_447b911a47b3_local_acc_outgate = _bundle_447b911a47b3_tl.zeros((BLOCK_M, BLOCK_H), dtype=_bundle_447b911a47b3_tl.float32)\\n for _bundle_447b911a47b3_local_k in range(0, _bundle_447b911a47b3_tl.cdiv(K, BLOCK_K)):\\n _bundle_447b911a47b3_local_offs_k = _bundle_447b911a47b3_local_k * BLOCK_K + _bundle_447b911a47b3_tl.arange(0, BLOCK_K)\\n _bundle_447b911a47b3_local_mask_k = _bundle_447b911a47b3_local_offs_k < K\\n _bundle_447b911a47b3_local_x = _bundle_447b911a47b3_tl.load(x_ptr + _bundle_447b911a47b3_local_offs_m[:, None] * stride_x_m + _bundle_447b911a47b3_local_offs_k[None, :] * stride_x_k, mask=_bundle_447b911a47b3_local_mask_m[:, None] & _bundle_447b911a47b3_local_mask_k[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lproj = _bundle_447b911a47b3_tl.load(proj_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_lgate = _bundle_447b911a47b3_tl.load(gate_left_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rproj = _bundle_447b911a47b3_tl.load(proj_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_rgate = _bundle_447b911a47b3_tl.load(gate_right_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_w_outgate = _bundle_447b911a47b3_tl.load(out_gate_w_ptr + _bundle_447b911a47b3_local_offs_k[:, None] * stride_w_k + _bundle_447b911a47b3_local_offs_h[None, :] * stride_w_h, mask=_bundle_447b911a47b3_local_mask_k[:, None] & _bundle_447b911a47b3_local_mask_h[None, :], other=0.0)\\n _bundle_447b911a47b3_local_acc_lproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lproj)\\n _bundle_447b911a47b3_local_acc_lgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_lgate)\\n _bundle_447b911a47b3_local_acc_rproj += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rproj)\\n _bundle_447b911a47b3_local_acc_rgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_rgate)\\n _bundle_447b911a47b3_local_acc_outgate += _bundle_447b911a47b3_tl.dot(_bundle_447b911a47b3_local_x, _bundle_447b911a47b3_local_w_outgate)\\n _bundle_447b911a47b3_local_gate_l = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_lgate))\\n _bundle_447b911a47b3_local_gate_r = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_rgate))\\n _bundle_447b911a47b3_local_gate_out = 1.0 / (1.0 + _bundle_447b911a47b3_tl.exp(-_bundle_447b911a47b3_local_acc_outgate))\\n _bundle_447b911a47b3_local_left_val = _bundle_447b911a47b3_local_acc_lproj * _bundle_447b911a47b3_local_gate_l * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_right_val = _bundle_447b911a47b3_local_acc_rproj * _bundle_447b911a47b3_local_gate_r * _bundle_447b911a47b3_local_m[:, None]\\n _bundle_447b911a47b3_local_N2 = N * N\\n _bundle_447b911a47b3_local_b = _bundle_447b911a47b3_local_offs_m // _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_rem = _bundle_447b911a47b3_local_offs_m % _bundle_447b911a47b3_local_N2\\n _bundle_447b911a47b3_local_i = _bundle_447b911a47b3_local_rem // N\\n _bundle_447b911a47b3_local_k_idx = _bundle_447b911a47b3_local_rem % N\\n _bundle_447b911a47b3_local_base = _bundle_447b911a47b3_local_b * (H * _bundle_447b911a47b3_local_N2) + _bundle_447b911a47b3_local_i * N + _bundle_447b911a47b3_local_k_idx\\n _bundle_447b911a47b3_local_offset = _bundle_447b911a47b3_local_base[:, None] + (_bundle_447b911a47b3_local_offs_h * _bundle_447b911a47b3_local_N2)[None, :]\\n _bundle_447b911a47b3_local_left_ptrs = out_left_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_local_right_ptrs = out_right_ptr + _bundle_447b911a47b3_local_offset\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_left_ptrs, _bundle_447b911a47b3_local_left_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_right_ptrs, _bundle_447b911a47b3_local_right_val.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n _bundle_447b911a47b3_local_out_gate_offsets = _bundle_447b911a47b3_local_offs_m[:, None] * H + _bundle_447b911a47b3_local_offs_h[None, :]\\n _bundle_447b911a47b3_local_out_gate_ptrs = out_gate_ptr + _bundle_447b911a47b3_local_out_gate_offsets\\n _bundle_447b911a47b3_tl.store(_bundle_447b911a47b3_local_out_gate_ptrs, _bundle_447b911a47b3_local_gate_out.to(_bundle_447b911a47b3_tl.float16), mask=_bundle_447b911a47b3_local_mask_mh)\\n\\n', 'c_project/paper_numeric.py': 'from typing import Tuple, Dict\\nimport torch\\nimport triton\\nimport triton.language as tl\\n\\n@triton.jit\\ndef _row_ln_fp16_kernel(\\n X_ptr, Y_ptr, # (M, C) input / output\\n w_ptr, b_ptr, # LN weight & bias (fp32)\\n M, C: tl.constexpr, # rows, columns (C is compile-time constant)\\n eps,\\n BLOCK_M: tl.constexpr,\\n BLOCK_C: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M)\\n row_mask = rows < M\\n\\n # ---------- mean / var (fp32) ----------\\n sum_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n sumsq_val = tl.zeros([BLOCK_M], dtype=tl.float32)\\n\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_C)\\n\\n sum_val += tl.sum(x, axis=1)\\n sumsq_val += tl.sum(x * x, axis=1)\\n\\n mean = sum_val / C\\n var = sumsq_val / C - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ---------- normalize + affine (fp16) ----------\\n for c in range(0, C, BLOCK_C):\\n cur_c = c + tl.arange(0, BLOCK_C)\\n col_mask = cur_c < C\\n x = tl.load(\\n X_ptr + rows[:, None] * C + cur_c[None, :],\\n mask=row_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ).to(tl.float32)\\n\\n y = (x - mean[:, None]) * inv_std[:, None]\\n\\n w = tl.load(w_ptr + cur_c, mask=col_mask, other=0.0)\\n b = tl.load(b_ptr + cur_c, mask=col_mask, other=0.0)\\n\\n y = y * w[None, :] + b[None, :]\\n tl.store(\\n Y_ptr + rows[:, None] * C + cur_c[None, :],\\n y.to(tl.float16),\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n\\ndef _row_layernorm_fp16(\\n x: torch.Tensor,\\n weight: torch.Tensor,\\n bias: torch.Tensor,\\n eps: float = 1e-5,\\n) -> torch.Tensor:\\n \"\"\"Row-wise LayerNorm over the last dim -> FP16 output.\"\"\"\\n B, N, _, C = x.shape\\n M = B * N * N\\n x_flat = x.view(M, C).contiguous()\\n y_flat = torch.empty((M, C), dtype=torch.float16, device=x.device)\\n\\n BLOCK_M = 128\\n BLOCK_C = 128\\n grid = lambda meta: (triton.cdiv(M, meta[\"BLOCK_M\"]),)\\n\\n _row_ln_fp16_kernel[grid](\\n x_flat,\\n y_flat,\\n weight,\\n bias,\\n M,\\n C,\\n eps,\\n BLOCK_M=BLOCK_M,\\n BLOCK_C=BLOCK_C,\\n num_warps=8,\\n )\\n return y_flat.view(B, N, N, C)\\n\\n\\n@triton.jit\\ndef _proj_gate_mask_kernel(\\n x_ptr, # (M, C) fp16\\n mask_ptr, # (M,) fp16 (if MASKED==1)\\n left_proj_w_ptr, # (C, H) fp16\\n left_gate_w_ptr, # (C, H) fp16\\n right_proj_w_ptr, # (C, H) fp16\\n right_gate_w_ptr, # (C, H) fp16\\n out_gate_w_ptr, # (C, H) fp16\\n left_ptr, # (B, H, N, N) fp16\\n right_ptr, # (B, H, N, N) fp16\\n out_gate_ptr, # (B, N, N, H) fp16\\n M, N, C: tl.constexpr, H: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_K: tl.constexpr,\\n MASKED: tl.constexpr,\\n):\\n pid_m = tl.program_id(0) # row block\\n pid_h = tl.program_id(1) # hidden block\\n\\n row_start = pid_m * BLOCK_M\\n hid_start = pid_h * BLOCK_H\\n\\n rows = row_start + tl.arange(0, BLOCK_M) # (BLOCK_M,)\\n hids = hid_start + tl.arange(0, BLOCK_H) # (BLOCK_H,)\\n\\n row_mask = rows < M\\n hid_mask = hids < H\\n\\n # ---------------- mask (scalar per row) ----------------\\n if MASKED:\\n mask_val = tl.load(mask_ptr + rows, mask=row_mask, other=0.0).to(tl.float32) # (BLOCK_M,)\\n else:\\n mask_val = tl.full([BLOCK_M], 1.0, dtype=tl.float32)\\n\\n # ---------------- accumulators (fp32) ------------------\\n acc_lp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left proj\\n acc_lg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # left gate\\n acc_rp = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right proj\\n acc_rg = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # right gate\\n acc_og = tl.zeros((BLOCK_M, BLOCK_H), dtype=tl.float32) # out gate\\n\\n for k in range(0, C, BLOCK_K):\\n cur_k = k + tl.arange(0, BLOCK_K)\\n k_mask = cur_k < C\\n\\n # input tile (fp16 -> fp32)\\n a = tl.load(\\n x_ptr + rows[:, None] * C + cur_k[None, :],\\n mask=row_mask[:, None] & k_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_M, BLOCK_K) fp16\\n\\n # weight tiles (C,H) column-major\\n w_lp = tl.load(\\n left_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_lg = tl.load(\\n left_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rp = tl.load(\\n right_proj_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_rg = tl.load(\\n right_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n w_og = tl.load(\\n out_gate_w_ptr + cur_k[:, None] * H + hids[None, :],\\n mask=k_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n )\\n\\n # fp16*fp16 -> fp32 dot products\\n acc_lp += tl.dot(a, w_lp)\\n acc_lg += tl.dot(a, w_lg)\\n acc_rp += tl.dot(a, w_rp)\\n acc_rg += tl.dot(a, w_rg)\\n acc_og += tl.dot(a, w_og)\\n\\n # ---------------- sigmoid gates -------------------------\\n left_gate = 1.0 / (1.0 + tl.exp(-acc_lg))\\n right_gate = 1.0 / (1.0 + tl.exp(-acc_rg))\\n out_gate = 1.0 / (1.0 + tl.exp(-acc_og))\\n\\n # ---------------- apply mask and per-row gates ----------\\n left_out = acc_lp * left_gate * mask_val[:, None]\\n right_out = acc_rp * right_gate * mask_val[:, None]\\n\\n # ---------------- store left/right (B,H,N,N) -------------\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n k_idx = rem - i_idx * N\\n\\n # layout for left/right: (B, H, N, N) -> flat index:\\n off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + k_idx[:, None]\\n\\n tl.store(\\n left_ptr + off,\\n left_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n tl.store(\\n right_ptr + off,\\n right_out.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n # ---------------- store out_gate (B,N,N,H) ---------------\\n off_gate = rows[:, None] * H + hids[None, :]\\n tl.store(\\n out_gate_ptr + off_gate,\\n out_gate.to(tl.float16),\\n mask=row_mask[:, None] & hid_mask[None, :],\\n )\\n\\n\\n@triton.jit\\ndef _ln_gate_out_linear_fused_kernel(\\n hidden_ptr, # (B*H*N*N,) fp16 flattened\\n out_gate_ptr, # (B*N*N*H,) fp16 flattened\\n ln_w_ptr, ln_b_ptr, # (H,) fp32\\n w_out_ptr, # (H, D) fp16\\n out_ptr, # (B, N, N, D) fp32\\n B, N, H, D: tl.constexpr,\\n eps: tl.constexpr,\\n BLOCK_M: tl.constexpr,\\n BLOCK_H: tl.constexpr,\\n BLOCK_D: tl.constexpr,\\n):\\n pid = tl.program_id(0)\\n row_start = pid * BLOCK_M\\n rows = row_start + tl.arange(0, BLOCK_M) # flat index for (b,i,j)\\n row_mask = rows < (B * N * N)\\n\\n N_sq = N * N\\n b_idx = rows // N_sq\\n rem = rows - b_idx * N_sq\\n i_idx = rem // N\\n j_idx = rem - i_idx * N\\n\\n # ----- load hidden slice (BLOCK_M, BLOCK_H) ------------\\n hids = tl.arange(0, BLOCK_H)\\n hid_mask = hids < H\\n\\n hidden_off = ((b_idx[:, None] * H + hids[None, :]) * N_sq) + i_idx[:, None] * N + j_idx[:, None]\\n hidden_tile = tl.load(\\n hidden_ptr + hidden_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ) # fp16\\n hidden_fp32 = hidden_tile.to(tl.float32)\\n\\n # ----- mean / var across H (fp32) -----\\n sum_val = tl.sum(hidden_fp32, axis=1) # (BLOCK_M,)\\n sumsq_val = tl.sum(hidden_fp32 * hidden_fp32, axis=1) # (BLOCK_M,)\\n mean = sum_val / H\\n var = sumsq_val / H - mean * mean\\n inv_std = 1.0 / tl.sqrt(var + eps)\\n\\n # ----- LayerNorm (fp32) -----\\n w_ln = tl.load(ln_w_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n b_ln = tl.load(ln_b_ptr + hids, mask=hid_mask, other=0.0) # (H,)\\n hidden_norm = (hidden_fp32 - mean[:, None]) * inv_std[:, None]\\n hidden_norm = hidden_norm * w_ln[None, :] + b_ln[None, :] # (BLOCK_M, BLOCK_H)\\n\\n # ----- out-gate (fp32) -----\\n out_gate_off = rows[:, None] * H + hids[None, :]\\n out_gate_tile = tl.load(\\n out_gate_ptr + out_gate_off,\\n mask=row_mask[:, None] & hid_mask[None, :],\\n other=0.0,\\n ).to(tl.float32) # (BLOCK_M, BLOCK_H)\\n\\n gated = hidden_norm * out_gate_tile # (BLOCK_M, BLOCK_H)\\n\\n # ----- final linear projection (fp16 matmul, fp32 acc) -----\\n gated_fp16 = gated.to(tl.float16)\\n\\n for d0 in range(0, D, BLOCK_D):\\n cols = d0 + tl.arange(0, BLOCK_D)\\n col_mask = cols < D\\n\\n w_out = tl.load(\\n w_out_ptr + hids[:, None] * D + cols[None, :],\\n mask=hid_mask[:, None] & col_mask[None, :],\\n other=0.0,\\n ) # (BLOCK_H, BLOCK_D) fp16\\n\\n out = tl.dot(gated_fp16, w_out) # (BLOCK_M, BLOCK_D) fp32\\n\\n tl.store(\\n out_ptr + rows[:, None] * D + cols[None, :],\\n out,\\n mask=row_mask[:, None] & col_mask[None, :],\\n )\\n\\n', 'c_project/__init__.py': '\\nfrom .projection import project_paper as run\\n', 'entry.py': \"\\nfrom .c_project import run as _gm_project\\n\\nfrom .c_normalize import run as _gm_normalize\\n\\nfrom .c_finish import run as _gm_finish\\nimport torch\\ndef custom_kernel(data):\\n w=data[2]\\n packed={name:w[name].to(torch.float16).contiguous()for name in ['norm.weight','norm.bias','left_proj.weight','right_proj.weight','left_gate.weight','right_gate.weight','out_gate.weight']}\\n normalized=_gm_normalize(data,packed)\\n left,right,gate=_gm_project(data,normalized,packed)\\n b,h,n,_=left.shape\\n product=torch.bmm(left.view(b*h,n,n),right.view(b*h,n,n).transpose(1,2))\\n return _gm_finish(data,product,gate)\\n__all__=['custom_kernel']\\n\"}\nfor _gp_name,_gp_text in _gp_files.items():\n _gp_file=_gp_root/'_gm_ebde169c6f5e57090557'/_gp_name\n _gp_file.parent.mkdir(parents=True,exist_ok=True)\n _gp_file.write_text(_gp_text,encoding='utf-8')\n_gp_sys.path.insert(0,str(_gp_root))\n_gp_entry=_gp_import.import_module('_gm_ebde169c6f5e57090557'+'.entry')\n\nfor _gp_symbol in getattr(_gp_entry,'__all__',[]):\n globals()[_gp_symbol]=getattr(_gp_entry,_gp_symbol)\n\n__source_binding_identities__={'_gm_project': '280d2de8c3f65f3a89a26095c99aea9d7dcab9643ae247de2bfc3d6f31cda79c', '_gm_normalize': '6d449693f25b00075237ab4067336a98f6103599850dd06fd7b0ab0c1cd46994', '_gm_finish': '977da3d0268e7082d556995b378e7058d880cabcba6d4613c6ac793fd3dc980b'}\n__source_binding_requires__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_memoizable_bindings__={'_gm_project': True, '_gm_normalize': True, '_gm_finish': True}\n__source_state_locations__={'_gm_project': [], '_gm_normalize': [], '_gm_finish': []}\n__source_control_domains__={}\ndef _sg_install_bindings():\n pass\n_gm_project = _gp_entry._gm_project\n_gm_normalize = _gp_entry._gm_normalize\n_gm_finish = _gp_entry._gm_finish\nimport torch\n\ndef custom_kernel(data):\n _sg_install_bindings()\n w = data[2]\n packed = {name: w[name].to(torch.float16).contiguous() for name in ['norm.weight', 'norm.bias', 'left_proj.weight', 'right_proj.weight', 'left_gate.weight', 'right_gate.weight', 'out_gate.weight']}\n normalized = _gm_normalize(data, packed)\n (left, right, gate) = _gm_project(data, normalized, packed)\n (b, h, n, _) = left.shape\n product = torch.bmm(left.view(b * h, n, n), right.view(b * h, n, n).transpose(1, 2))\n return _gm_finish(data, product, gate)\n__all__ = ['custom_kernel']\n_gm_project._source_binding_identity = '280d2de8c3f65f3a89a26095c99aea9d7dcab9643ae247de2bfc3d6f31cda79c'\n_gm_project._source_memoizable = True\n_gm_project._source_state_locations = []\n_gm_normalize._source_binding_identity = '6d449693f25b00075237ab4067336a98f6103599850dd06fd7b0ab0c1cd46994'\n_gm_normalize._source_memoizable = True\n_gm_normalize._source_state_locations = []\n_gm_finish._source_binding_identity = '977da3d0268e7082d556995b378e7058d880cabcba6d4613c6ac793fd3dc980b'\n_gm_finish._source_memoizable = True\n_gm_finish._source_state_locations = []\n"}, "manifest": {"sources": {"0": "/data/xiaofan/continuation-graph-fusion-20261103-v4/cohort_kernel/source_0.py", "1": "/data/xiaofan/continuation-graph-fusion-20261103-v4/cohort_kernel/source_1.py", "2": "/data/xiaofan/continuation-graph-fusion-20261103-v4/cohort_kernel/source_2.py", "3": "/data/xiaofan/continuation-graph-fusion-20261103-v4/cohort_kernel/source_3.py"}, "entry": "custom_kernel", "base": "0", "source_choices": [{"normalize": "good", "project": "good", "finish": "good"}, {"finish": "reference", "normalize": "good", "project": "good"}, {"finish": "good", "normalize": "reference", "project": "good"}, {"finish": "good", "normalize": "good", "project": "reference"}], "known_joint_program_not_in_sources": true, "native_adapter": "kernel_case.KernelGraphOracle"}}

Xet Storage Details

Size:
229 kB
·
Xet hash:
11f618b6d2ae470ea2843844b4cbe4b9152ba1774388e0f5169fa1085da1b2be

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.